From 8798a7b827818f104fa8da395864b0c950bd5253 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 12 Sep 2019 17:33:54 -0400 Subject: [PATCH 01/39] initial totally broken mid-merge commit Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 17 -- include/envoy/config/grpc_mux.h | 54 ++-- include/envoy/config/subscription.h | 2 +- include/envoy/config/xds_grpc_context.h | 42 --- source/common/config/BUILD | 68 ++--- .../common/config/delta_subscription_impl.cc | 107 -------- .../common/config/delta_subscription_impl.h | 94 ------- source/common/config/grpc_mux_impl.cc | 251 ------------------ source/common/config/grpc_mux_impl.h | 132 --------- .../config/grpc_mux_subscription_impl.cc | 100 ------- .../config/grpc_mux_subscription_impl.h | 50 ---- source/common/config/grpc_stream.h | 2 +- .../common/config/grpc_subscription_impl.cc | 77 ++++++ source/common/config/grpc_subscription_impl.h | 80 ++++-- source/common/config/new_grpc_mux_impl.cc | 222 ++++++++++++++++ source/common/config/new_grpc_mux_impl.h | 168 ++++++++++++ source/common/config/pausable_ack_queue.cc | 66 +++++ source/common/config/pausable_ack_queue.h | 43 +++ source/common/config/subscription_state.h | 64 +++++ ...n_state.cc => subscription_state_delta.cc} | 107 +++----- ...ion_state.h => subscription_state_delta.h} | 45 ++-- .../common/config/subscription_state_sotw.cc | 146 ++++++++++ .../config/delta_subscription_impl_test.cc | 74 ++++-- .../config/delta_subscription_state_test.cc | 101 +++---- .../config/delta_subscription_test_harness.h | 43 ++- .../filesystem_subscription_test_harness.h | 15 +- test/common/config/grpc_mux_impl_test.cc | 2 +- test/common/config/grpc_stream_test.cc | 4 +- .../config/grpc_subscription_impl_test.cc | 10 +- .../config/grpc_subscription_test_harness.h | 16 +- .../config/http_subscription_test_harness.h | 4 +- .../config/subscription_factory_impl_test.cc | 2 +- test/common/config/subscription_impl_test.cc | 51 ++-- .../common/config/subscription_test_harness.h | 4 +- test/common/grpc/grpc_client_integration.h | 40 +-- .../upstream/cluster_manager_impl_test.cc | 12 +- test/integration/ads_integration.cc | 4 +- test/integration/ads_integration.h | 23 +- test/integration/ads_integration_test.cc | 54 ++-- test/mocks/config/BUILD | 1 - 40 files changed, 1217 insertions(+), 1180 deletions(-) delete mode 100644 include/envoy/config/xds_grpc_context.h delete mode 100644 source/common/config/delta_subscription_impl.cc delete mode 100644 source/common/config/delta_subscription_impl.h delete mode 100644 source/common/config/grpc_mux_impl.cc delete mode 100644 source/common/config/grpc_mux_impl.h delete mode 100644 source/common/config/grpc_mux_subscription_impl.cc delete mode 100644 source/common/config/grpc_mux_subscription_impl.h create mode 100644 source/common/config/grpc_subscription_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.h create mode 100644 source/common/config/pausable_ack_queue.cc create mode 100644 source/common/config/pausable_ack_queue.h create mode 100644 source/common/config/subscription_state.h rename source/common/config/{delta_subscription_state.cc => subscription_state_delta.cc} (66%) rename source/common/config/{delta_subscription_state.h => subscription_state_delta.h} (70%) create mode 100644 source/common/config/subscription_state_sotw.cc diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 001d3cbe46af..7df8a214104f 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -65,20 +65,3 @@ envoy_cc_library( "//source/common/protobuf", ], ) - -envoy_cc_library( - name = "watch_map_interface", - hdrs = ["watch_map.h"], - deps = [ - ":subscription_interface", - ], -) - -envoy_cc_library( - name = "xds_grpc_context_interface", - hdrs = ["xds_grpc_context.h"], - deps = [ - ":subscription_interface", - "//source/common/protobuf", - ], -) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 1d9920e2d5e3..328795dc4ee3 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -67,6 +67,8 @@ class GrpcMuxWatch { using GrpcMuxWatchPtr = std::unique_ptr; +struct Watch; + /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. @@ -80,21 +82,6 @@ class GrpcMux { */ virtual void start() PURE; - /** - * Start a configuration subscription asynchronously for some API type and resources. - * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster. - * @param resources set of resource names to watch for. If this is empty, then all - * resources for type_url will result in callbacks. - * @param callbacks the callbacks to be notified of configuration updates. These must be valid - * until GrpcMuxWatch is destroyed. - * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes - * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. - */ - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) PURE; - /** * Pause discovery requests for a given API type. This is useful when we're processing an update * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery @@ -111,6 +98,12 @@ class GrpcMux { */ virtual void resume(const std::string& type_url) PURE; + virtual Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; + virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; + /** * Retrieves the current pause state as set by pause()/resume(). * @param type_url type URL corresponding to xDS API, e.g. @@ -121,6 +114,37 @@ class GrpcMux { }; using GrpcMuxPtr = std::unique_ptr; +using GrpcMuxSharedPtr = std::shared_ptr; + +/** + * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. + */ +template class GrpcStreamCallbacks { +public: + virtual ~GrpcStreamCallbacks() = default; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void onStreamEstablished() PURE; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void onEstablishmentFailure() PURE; + + /** + * For the GrpcStream to pass received protos to the context. + */ + virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + + /** + * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + */ + virtual void onWriteable() PURE; +}; } // namespace Config } // namespace Envoy diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index ee21da758d10..009c4a265dc6 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -89,7 +89,7 @@ class Subscription { * @param resources vector of resource names to fetch. It's a (not unordered_)set so that it can * be passed to std::set_difference, which must be given sorted collections. */ - virtual void updateResources(const std::set& update_to_these_names) PURE; + virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; }; using SubscriptionPtr = std::unique_ptr; diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h deleted file mode 100644 index 312d3e0650f7..000000000000 --- a/include/envoy/config/xds_grpc_context.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" - -#include "common/protobuf/protobuf.h" - -namespace Envoy { -namespace Config { - -/** - * A grouping of callbacks that an XdsGrpcContext should provide to its GrpcStream. - */ -template class GrpcStreamCallbacks { -public: - virtual ~GrpcStreamCallbacks() = default; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to the - * gRPC stream having been successfully established. - */ - virtual void onStreamEstablished() PURE; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to - * failure to establish the gRPC stream. - */ - virtual void onEstablishmentFailure() PURE; - - /** - * For the GrpcStream to pass received protos to the context. - */ - virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; - - /** - * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. - */ - virtual void onWriteable() PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/BUILD b/source/common/config/BUILD index f3bcaacfdbb7..8c4133ea5331 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -71,8 +71,8 @@ envoy_cc_library( srcs = ["delta_subscription_impl.cc"], hdrs = ["delta_subscription_impl.h"], deps = [ - ":delta_subscription_state_lib", ":grpc_stream_lib", + ":new_grpc_mux_lib", ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/grpc:async_client_interface", @@ -90,6 +90,7 @@ envoy_cc_library( srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ + ":pausable_ack_queue_lib", "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", "//source/common/common:assert_lib", @@ -125,7 +126,6 @@ envoy_cc_library( ":utility_lib", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:backoff_lib", @@ -174,46 +174,16 @@ envoy_cc_library( ) envoy_cc_library( - name = "grpc_mux_lib", - srcs = ["grpc_mux_impl.cc"], - hdrs = ["grpc_mux_impl.h"], + name = "new_grpc_mux_lib", + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], deps = [ + ":delta_subscription_state_lib", ":grpc_stream_lib", - ":utility_lib", - "//include/envoy/config:grpc_mux_interface", - "//include/envoy/config:subscription_interface", - "//include/envoy/upstream:cluster_manager_interface", - "//source/common/common:minimal_logger_lib", - "//source/common/protobuf", - ], -) - -envoy_cc_library( - name = "grpc_mux_subscription_lib", - srcs = ["grpc_mux_subscription_impl.cc"], - hdrs = ["grpc_mux_subscription_impl.h"], - deps = [ - "//include/envoy/config:grpc_mux_interface", - "//include/envoy/config:subscription_interface", - "//include/envoy/event:dispatcher_interface", - "//source/common/common:assert_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/grpc:common_lib", - "//source/common/protobuf", - "@envoy_api//envoy/api/v2:discovery_cc", - ], -) - -envoy_cc_library( - name = "grpc_subscription_lib", - hdrs = ["grpc_subscription_impl.h"], - deps = [ - ":grpc_mux_lib", - ":grpc_mux_subscription_lib", - "//include/envoy/config:subscription_interface", + ":pausable_ack_queue_lib", + ":watch_map_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", - "@envoy_api//envoy/api/v2/core:base_cc", ], ) @@ -261,6 +231,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "pausable_ack_queue_lib", + srcs = ["pausable_ack_queue.cc"], + hdrs = ["pausable_ack_queue.h"], + deps = [ + "//source/common/common:assert_lib", + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + envoy_cc_library( name = "protobuf_link_hacks", hdrs = ["protobuf_link_hacks.h"], @@ -300,12 +280,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "resources_lib", - hdrs = ["resources.h"], - deps = ["//source/common/singleton:const_singleton"], -) - envoy_cc_library( name = "remote_data_fetcher_lib", srcs = ["remote_data_fetcher.cc"], @@ -319,6 +293,12 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "resources_lib", + hdrs = ["resources.h"], + deps = ["//source/common/singleton:const_singleton"], +) + envoy_cc_library( name = "runtime_utility_lib", srcs = ["runtime_utility.cc"], diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc deleted file mode 100644 index 8bc69f04f4eb..000000000000 --- a/source/common/config/delta_subscription_impl.cc +++ /dev/null @@ -1,107 +0,0 @@ -#include "common/config/delta_subscription_impl.h" - -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/token_bucket_impl.h" -#include "common/config/utility.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -DeltaSubscriptionImpl::DeltaSubscriptionImpl( - const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - absl::string_view type_url, Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, SubscriptionCallbacks& callbacks, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings), - type_url_(type_url), local_info_(local_info), callbacks_(callbacks), stats_(stats), - dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} - -void DeltaSubscriptionImpl::pause() { state_->pause(); } -void DeltaSubscriptionImpl::resume() { - state_->resume(); - trySendDiscoveryRequests(); -} - -// Config::Subscription -void DeltaSubscriptionImpl::start(const std::set& resource_names) { - state_ = std::make_unique( - type_url_, resource_names, callbacks_, local_info_, init_fetch_timeout_, dispatcher_, stats_); - grpc_stream_.establishNewStream(); - updateResources(resource_names); -} - -void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - state_->updateResourceInterest(update_to_these_names); - // Tell the server about our new interests, if there are any. - trySendDiscoveryRequests(); -} - -// Config::GrpcStreamCallbacks -void DeltaSubscriptionImpl::onStreamEstablished() { - state_->markStreamFresh(); - trySendDiscoveryRequests(); -} - -void DeltaSubscriptionImpl::onEstablishmentFailure() { state_->handleEstablishmentFailure(); } - -void DeltaSubscriptionImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url_, - message->system_version_info()); - kickOffAck(state_->handleResponse(*message)); -} - -void DeltaSubscriptionImpl::onWriteable() { trySendDiscoveryRequests(); } - -void DeltaSubscriptionImpl::kickOffAck(UpdateAck ack) { - ack_queue_.push(ack); - trySendDiscoveryRequests(); -} - -// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check -// whether we *want* to send a DeltaDiscoveryRequest). -bool DeltaSubscriptionImpl::canSendDiscoveryRequest() { - if (state_->paused()) { - ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url_); - return false; - } else if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a DiscoveryRequest for {}.", type_url_); - return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} DiscoveryRequest hit rate limit; will try later.", type_url_); - return false; - } - return true; -} - -// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or -// a subscription update. (Does not check whether we *can* send a DeltaDiscoveryRequest). -bool DeltaSubscriptionImpl::wantToSendDiscoveryRequest() { - return !ack_queue_.empty() || state_->subscriptionUpdatePending(); -} - -void DeltaSubscriptionImpl::trySendDiscoveryRequests() { - while (wantToSendDiscoveryRequest() && canSendDiscoveryRequest()) { - envoy::api::v2::DeltaDiscoveryRequest request = state_->getNextRequest(); - if (!ack_queue_.empty()) { - const UpdateAck& ack = ack_queue_.front(); - request.set_response_nonce(ack.nonce_); - if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { - // Don't needlessly make the field present-but-empty if status is ok. - request.mutable_error_detail()->CopyFrom(ack.error_detail_); - } - ack_queue_.pop(); - } - ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url_, request.DebugString()); - grpc_stream_.sendMessage(request); - } - grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h deleted file mode 100644 index dbee51c688c3..000000000000 --- a/source/common/config/delta_subscription_impl.h +++ /dev/null @@ -1,94 +0,0 @@ -#pragma once - -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" -#include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" -#include "envoy/local_info/local_info.h" - -#include "common/common/logger.h" -#include "common/config/delta_subscription_state.h" -#include "common/config/grpc_stream.h" -#include "common/grpc/common.h" - -namespace Envoy { -namespace Config { - -/** - * Manages the logic of a (non-aggregated) delta xDS subscription. - * TODO(fredlas) add aggregation support. The plan is for that to happen in XdsGrpcContext, - * which this class will then "have a" rather than "be a". - * TODO(kyessenov) implement skip_subsequent_node for delta xDS subscription. - */ -class DeltaSubscriptionImpl : public Subscription, - public GrpcStreamCallbacks, - public Logger::Loggable { -public: - DeltaSubscriptionImpl(const LocalInfo::LocalInfo& local_info, - Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - absl::string_view type_url, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - SubscriptionCallbacks& callbacks, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout); - - void pause(); - void resume(); - - // Config::Subscription - void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; - - // Config::GrpcStreamCallbacks - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; - - void onWriteable() override; - -private: - void kickOffAck(UpdateAck ack); - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(); - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send a DeltaDiscoveryRequest). - bool wantToSendDiscoveryRequest(); - - void trySendDiscoveryRequests(); - - GrpcStream - grpc_stream_; - - const std::string type_url_; - - // An item in the queue represents a DeltaDiscoveryRequest that must be sent. If an item is not - // empty, it is the ACK (nonce + error_detail) to set on that request. An empty entry should - // still send a request; it just won't have an ACK. - // - // More details: DeltaDiscoveryRequest plays two independent roles: - // 1) informing the server of what resources we're interested in, and - // 2) acknowledging resources the server has sent us. - // Each entry in this queue was added for exactly one of those purposes, but since the - // subscription interest is tracked separately, in a non-queue way, subscription changes can get - // mixed in with an ACK request. In that case, the entry that the subscription change originally - // queued up *does* still get sent, just empty and pointless. (TODO(fredlas) we would like to skip - // those no-op requests). - std::queue ack_queue_; - - const LocalInfo::LocalInfo& local_info_; - SubscriptionCallbacks& callbacks_; - SubscriptionStats stats_; - Event::Dispatcher& dispatcher_; - std::chrono::milliseconds init_fetch_timeout_; - - std::unique_ptr state_; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc deleted file mode 100644 index c1eda78b6dd6..000000000000 --- a/source/common/config/grpc_mux_impl.cc +++ /dev/null @@ -1,251 +0,0 @@ -#include "common/config/grpc_mux_impl.h" - -#include - -#include "common/config/utility.h" -#include "common/protobuf/protobuf.h" - -namespace Envoy { -namespace Config { - -GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, - Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings), - local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), - first_stream_request_(true) { - Config::Utility::checkLocalInfo("ads", local_info); -} - -GrpcMuxImpl::~GrpcMuxImpl() { - for (const auto& api_state : api_state_) { - for (auto watch : api_state.second.watches_) { - watch->clear(); - } - } -} - -void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - -void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { - if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); - return; // Drop this request; the reconnect will enqueue a new one. - } - - ApiState& api_state = api_state_[type_url]; - if (api_state.paused_) { - ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest(), setting pending.", type_url); - api_state.pending_ = true; - return; // Drop this request; the unpause will enqueue a new one. - } - - auto& request = api_state.request_; - request.mutable_resource_names()->Clear(); - - // Maintain a set to avoid dupes. - std::unordered_set resources; - for (const auto* watch : api_state.watches_) { - for (const std::string& resource : watch->resources_) { - if (resources.count(resource) == 0) { - resources.emplace(resource); - request.add_resource_names(resource); - } - } - } - - if (skip_subsequent_node_ && !first_stream_request_) { - request.clear_node(); - } - ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); - grpc_stream_.sendMessage(request); - first_stream_request_ = false; - - // clear error_detail after the request is sent if it exists. - if (api_state_[type_url].request_.has_error_detail()) { - api_state_[type_url].request_.clear_error_detail(); - } -} - -GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) { - auto watch = - std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); - ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); - - // Lazily kick off the requests based on first subscription. This has the - // convenient side-effect that we order messages on the channel based on - // Envoy's internal dependency ordering. - // TODO(gsagula): move TokenBucketImpl params to a config. - if (!api_state_[type_url].subscribed_) { - api_state_[type_url].request_.set_type_url(type_url); - api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); - api_state_[type_url].subscribed_ = true; - subscriptions_.emplace_back(type_url); - } - - // This will send an updated request on each subscription. - // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. - // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we - // only send a single RDS/EDS update after the CDS/LDS update. - queueDiscoveryRequest(type_url); - - return watch; -} - -void GrpcMuxImpl::pause(const std::string& type_url) { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(!api_state.paused_); - ASSERT(!api_state.pending_); - api_state.paused_ = true; -} - -void GrpcMuxImpl::resume(const std::string& type_url) { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(api_state.paused_); - api_state.paused_ = false; - - if (api_state.pending_) { - ASSERT(api_state.subscribed_); - queueDiscoveryRequest(type_url); - api_state.pending_ = false; - } -} - -bool GrpcMuxImpl::paused(const std::string& type_url) const { - auto entry = api_state_.find(type_url); - if (entry == api_state_.end()) { - return false; - } - return entry->second.paused_; -} - -void GrpcMuxImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - const std::string& type_url = message->type_url(); - ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); - if (api_state_.count(type_url) == 0) { - ENVOY_LOG(warn, "Ignoring the message for type URL {} as it has no current subscribers.", - type_url); - // TODO(yuval-k): This should never happen. consider dropping the stream as this is a - // protocol violation - return; - } - if (api_state_[type_url].watches_.empty()) { - // update the nonce as we are processing this response. - api_state_[type_url].request_.set_response_nonce(message->nonce()); - if (message->resources().empty()) { - // No watches and no resources. This can happen when envoy unregisters from a - // resource that's removed from the server as well. For example, a deleted cluster - // triggers un-watching the ClusterLoadAssignment watch, and at the same time the - // xDS server sends an empty list of ClusterLoadAssignment resources. we'll accept - // this update. no need to send a discovery request, as we don't watch for anything. - api_state_[type_url].request_.set_version_info(message->version_info()); - } else { - // No watches and we have resources - this should not happen. send a NACK (by not - // updating the version). - ENVOY_LOG(warn, "Ignoring unwatched type URL {}", type_url); - queueDiscoveryRequest(type_url); - } - return; - } - try { - // To avoid O(n^2) explosion (e.g. when we have 1000s of EDS watches), we - // build a map here from resource name to resource and then walk watches_. - // We have to walk all watches (and need an efficient map as a result) to - // ensure we deliver empty config updates when a resource is dropped. - std::unordered_map resources; - GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; - for (const auto& resource : message->resources()) { - if (type_url != resource.type_url()) { - throw EnvoyException(fmt::format("{} does not match {} type URL in DiscoveryResponse {}", - resource.type_url(), type_url, message->DebugString())); - } - const std::string resource_name = callbacks.resourceName(resource); - resources.emplace(resource_name, resource); - } - for (auto watch : api_state_[type_url].watches_) { - // onConfigUpdate should be called in all cases for single watch xDS (Cluster and - // Listener) even if the message does not have resources so that update_empty stat - // is properly incremented and state-of-the-world semantics are maintained. - if (watch->resources_.empty()) { - watch->callbacks_.onConfigUpdate(message->resources(), message->version_info()); - continue; - } - Protobuf::RepeatedPtrField found_resources; - for (const auto& watched_resource_name : watch->resources_) { - auto it = resources.find(watched_resource_name); - if (it != resources.end()) { - found_resources.Add()->MergeFrom(it->second); - } - } - // onConfigUpdate should be called only on watches(clusters/routes) that have - // updates in the message for EDS/RDS. - if (!found_resources.empty()) { - watch->callbacks_.onConfigUpdate(found_resources, message->version_info()); - } - } - // TODO(mattklein123): In the future if we start tracking per-resource versions, we - // would do that tracking here. - api_state_[type_url].request_.set_version_info(message->version_info()); - } catch (const EnvoyException& e) { - for (auto watch : api_state_[type_url].watches_) { - watch->callbacks_.onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); - } - ::google::rpc::Status* error_detail = api_state_[type_url].request_.mutable_error_detail(); - error_detail->set_code(Grpc::Status::GrpcStatus::Internal); - error_detail->set_message(e.what()); - } - api_state_[type_url].request_.set_response_nonce(message->nonce()); - queueDiscoveryRequest(type_url); -} - -void GrpcMuxImpl::onWriteable() { drainRequests(); } - -void GrpcMuxImpl::onStreamEstablished() { - first_stream_request_ = true; - for (const auto& type_url : subscriptions_) { - queueDiscoveryRequest(type_url); - } -} - -void GrpcMuxImpl::onEstablishmentFailure() { - for (const auto& api_state : api_state_) { - for (auto watch : api_state.second.watches_) { - watch->callbacks_.onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); - } - } -} - -void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { - request_queue_.push(queue_item); - drainRequests(); -} - -void GrpcMuxImpl::clearRequestQueue() { - grpc_stream_.maybeUpdateQueueSizeStat(0); - // TODO(fredlas) when we have C++17: request_queue_ = {}; - while (!request_queue_.empty()) { - request_queue_.pop(); - } -} - -void GrpcMuxImpl::drainRequests() { - while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { - // Process the request, if rate limiting is not enabled at all or if it is under rate limit. - sendDiscoveryRequest(request_queue_.front()); - request_queue_.pop(); - } - grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h deleted file mode 100644 index ae245c48d7fd..000000000000 --- a/source/common/config/grpc_mux_impl.h +++ /dev/null @@ -1,132 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/common/time.h" -#include "envoy/config/grpc_mux.h" -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/grpc/status.h" -#include "envoy/upstream/cluster_manager.h" - -#include "common/common/cleanup.h" -#include "common/common/logger.h" -#include "common/config/grpc_stream.h" -#include "common/config/utility.h" - -namespace Envoy { -namespace Config { - -/** - * ADS API implementation that fetches via gRPC. - */ -class GrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - public Logger::Loggable { -public: - GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); - ~GrpcMuxImpl() override; - - void start() override; - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks) override; - void pause(const std::string& type_url) override; - void resume(const std::string& type_url) override; - bool paused(const std::string& type_url) const override; - - void sendDiscoveryRequest(const std::string& type_url); - - // Config::GrpcStreamCallbacks - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void onDiscoveryResponse(std::unique_ptr&& message) override; - void onWriteable() override; - - GrpcStream& - grpcStreamForTest() { - return grpc_stream_; - } - -private: - void setRetryTimer(); - - struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { - GrpcMuxWatchImpl(const std::set& resources, GrpcMuxCallbacks& callbacks, - const std::string& type_url, GrpcMuxImpl& parent) - : RaiiListElement(parent.api_state_[type_url].watches_, this), - resources_(resources), callbacks_(callbacks), type_url_(type_url), parent_(parent), - inserted_(true) {} - ~GrpcMuxWatchImpl() override { - if (inserted_) { - erase(); - if (!resources_.empty()) { - parent_.sendDiscoveryRequest(type_url_); - } - } - } - - void clear() { - inserted_ = false; - cancel(); - } - - std::set resources_; - GrpcMuxCallbacks& callbacks_; - const std::string type_url_; - GrpcMuxImpl& parent_; - - private: - bool inserted_; - }; - - // Per muxed API state. - struct ApiState { - // Watches on the returned resources for the API; - std::list watches_; - // Current DiscoveryRequest for API. - envoy::api::v2::DiscoveryRequest request_; - // Paused via pause()? - bool paused_{}; - // Was a DiscoveryRequest elided during a pause? - bool pending_{}; - // Has this API been tracked in subscriptions_? - bool subscribed_{}; - }; - - // Request queue management logic. - void queueDiscoveryRequest(const std::string& queue_item); - void clearRequestQueue(); - void drainRequests(); - - GrpcStream grpc_stream_; - const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; - bool first_stream_request_; - std::unordered_map api_state_; - // Envoy's dependency ordering. - std::list subscriptions_; - - // A queue to store requests while rate limited. Note that when requests cannot be sent due to the - // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - // This string is a type URL. - std::queue request_queue_; -}; - -class NullGrpcMuxImpl : public GrpcMux { -public: - void start() override {} - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - void pause(const std::string&) override {} - void resume(const std::string&) override {} - bool paused(const std::string&) const override { return false; } -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc deleted file mode 100644 index a181b4efa174..000000000000 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ /dev/null @@ -1,100 +0,0 @@ -#include "common/config/grpc_mux_subscription_impl.h" - -#include "common/common/assert.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, - SubscriptionCallbacks& callbacks, - SubscriptionStats stats, - absl::string_view type_url, - Event::Dispatcher& dispatcher, - std::chrono::milliseconds init_fetch_timeout) - : grpc_mux_(grpc_mux), callbacks_(callbacks), stats_(stats), type_url_(type_url), - dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} - -// Config::Subscription -void GrpcMuxSubscriptionImpl::start(const std::set& resources) { - if (init_fetch_timeout_.count() > 0) { - init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, - nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } - - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); - // The attempt stat here is maintained for the purposes of having consistency between ADS and - // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an - // "attempt" for a given xDS API combined by ADS is not really that meaningful. - stats_.update_attempt_.inc(); -} - -void GrpcMuxSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - // First destroy the watch, so that this subscribe doesn't send a request for both the - // previously watched resources and the new ones (we may have lost interest in some of the - // previously watched ones). - watch_.reset(); - watch_ = grpc_mux_.subscribe(type_url_, update_to_these_names, *this); - stats_.update_attempt_.inc(); -} - -// Config::GrpcMuxCallbacks -void GrpcMuxSubscriptionImpl::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { - disableInitFetchTimeoutTimer(); - // TODO(mattklein123): In the future if we start tracking per-resource versions, we need to - // supply those versions to onConfigUpdate() along with the xDS response ("system") - // version_info. This way, both types of versions can be tracked and exposed for debugging by - // the configuration update targets. - callbacks_.onConfigUpdate(resources, version_info); - stats_.update_success_.inc(); - stats_.update_attempt_.inc(); - stats_.version_.set(HashUtil::xxHash64(version_info)); - ENVOY_LOG(debug, "gRPC config for {} accepted with {} resources with version {}", type_url_, - resources.size(), version_info); -} - -void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, - const EnvoyException* e) { - switch (reason) { - case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: - stats_.update_failure_.inc(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); - break; - case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: - stats_.init_fetch_timeout_.inc(); - disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); - break; - case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: - // We expect Envoy exception to be thrown when update is rejected. - ASSERT(e != nullptr); - disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); - ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); - break; - } - stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(reason, e); -} - -std::string GrpcMuxSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { - return callbacks_.resourceName(resource); -} - -void GrpcMuxSubscriptionImpl::disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h deleted file mode 100644 index 9fb4cd76407c..000000000000 --- a/source/common/config/grpc_mux_subscription_impl.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/config/grpc_mux.h" -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" - -#include "common/common/logger.h" - -namespace Envoy { -namespace Config { - -/** - * Adapter from typed Subscription to untyped GrpcMux. Also handles per-xDS API stats/logging. - */ -class GrpcMuxSubscriptionImpl : public Subscription, - GrpcMuxCallbacks, - Logger::Loggable { -public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionCallbacks& callbacks, - SubscriptionStats stats, absl::string_view type_url, - Event::Dispatcher& dispatcher, - std::chrono::milliseconds init_fetch_timeout); - - // Config::Subscription - void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; - - // Config::GrpcMuxCallbacks - void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override; - void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, - const EnvoyException* e) override; - std::string resourceName(const ProtobufWkt::Any& resource) override; - -private: - void disableInitFetchTimeoutTimer(); - - GrpcMux& grpc_mux_; - SubscriptionCallbacks& callbacks_; - SubscriptionStats stats_; - const std::string type_url_; - GrpcMuxWatchPtr watch_{}; - Event::Dispatcher& dispatcher_; - std::chrono::milliseconds init_fetch_timeout_; - Event::TimerPtr init_fetch_timeout_timer_; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 9a182c2637ff..1be922742930 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/xds_grpc_context.h" +#include "envoy/config/grpc_mux.h" #include "envoy/grpc/async_client.h" #include "common/common/backoff_strategy.h" diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc new file mode 100644 index 000000000000..ddfd22d51173 --- /dev/null +++ b/source/common/config/grpc_subscription_impl.cc @@ -0,0 +1,77 @@ +#include "common/config/delta_subscription_impl.h" + +namespace Envoy { +namespace Config { + +DeltaSubscriptionImpl::DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) + : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} + +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { + if (watch_) { + context_->removeWatch(type_url_, watch_); + } +} + +void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } + +void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } + +// Config::DeltaSubscription +void DeltaSubscriptionImpl::start(const std::set& resources) { + // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, + // whereas non-ADS xDS users must call it themselves. + if (!is_aggregated_) { + context_->start(); + } + watch_ = context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); + stats_.update_attempt_.inc(); +} + +void DeltaSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { + watch_ = context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, + init_fetch_timeout_); + stats_.update_attempt_.inc(); +} + +// Config::SubscriptionCallbacks +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + callbacks_.onConfigUpdate(resources, version_info); + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(version_info)); +} +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(system_version_info)); +} +void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { + stats_.update_attempt_.inc(); + if (reason == ConfigUpdateFailureReason::FetchTimedout) { + stats_.init_fetch_timeout_.inc(); + } else if (e == nullptr) { + stats_.update_failure_.inc(); + } else { + stats_.update_rejected_.inc(); + } + callbacks_.onConfigUpdateFailed(reason, e); +} +std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { + return callbacks_.resourceName(resource); +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 94e920843102..b45be7c4da85 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -1,48 +1,70 @@ #pragma once -#include "envoy/api/v2/core/base.pb.h" #include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/grpc/async_client.h" -#include "common/config/grpc_mux_impl.h" -#include "common/config/grpc_mux_subscription_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/utility.h" namespace Envoy { namespace Config { -class GrpcSubscriptionImpl : public Config::Subscription { +// DeltaSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with +// an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) +// NewGrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in +// various resources via start() and updateResourceInterest(). It receives updates to those +// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own +// Subscription object for the same type_url; NewGrpcMuxImpl maintains a subscription to the +// union of interested resources, and delivers to the users just the resource updates that they +// are "watching" for. +// +// DeltaSubscriptionImpl and NewGrpcMuxImpl are both built to provide both regular xDS and ADS, +// distinguished by whether multiple DeltaSubscriptionImpls are sharing a single +// NewGrpcMuxImpl. (And by the gRPC method string, but that's taken care of over in +// SubscriptionFactory). +// +// Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the DeltaSubscriptionImpl) upon a +// config update. The idea is, DeltaSubscriptionImpl presents itself to WatchMap as the +// SubscriptionCallbacks, and then passes (after incrementing stats) all callbacks through to +// callbacks_, which are the real SubscriptionCallbacks. +class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { public: - GrpcSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, - const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats stats, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - std::chrono::milliseconds init_fetch_timeout, bool skip_subsequent_node) - : callbacks_(callbacks), - grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, - rate_limit_settings, skip_subsequent_node), - grpc_mux_subscription_(grpc_mux_, callbacks_, stats, type_url, dispatcher, - init_fetch_timeout) {} + // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only in + // the latter case. + DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); + ~DeltaSubscriptionImpl() override; + + void pause(); + void resume(); // Config::Subscription - void start(const std::set& resource_names) override { - // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. - grpc_mux_subscription_.start(resource_names); - grpc_mux_.start(); - } + void start(const std::set& resource_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; - void updateResources(const std::set& update_to_these_names) override { - grpc_mux_subscription_.updateResources(update_to_these_names); - } + // Config::SubscriptionCallbacks (all pass through to callbacks_!) + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override; - GrpcMuxImpl& grpcMux() { return grpc_mux_; } + GrpcMuxSharedPtr getContextForTest() { return context_; } private: - Config::SubscriptionCallbacks& callbacks_; - GrpcMuxImpl grpc_mux_; - GrpcMuxSubscriptionImpl grpc_mux_subscription_; + GrpcMuxSharedPtr context_; + const std::string type_url_; + SubscriptionCallbacks& callbacks_; + SubscriptionStats stats_; + // NOTE: if another subscription of the same type_url has already been started, this value will be + // ignored in favor of the other subscription's. + std::chrono::milliseconds init_fetch_timeout_; + Watch* watch_{}; + const bool is_aggregated_; }; } // namespace Config diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc new file mode 100644 index 000000000000..276211ab610e --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.cc @@ -0,0 +1,222 @@ +#include "common/config/new_grpc_mux_impl.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +NewGrpcMuxImpl::NewGrpcMuxImpl(Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info) {} + +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch, resources); + return watch; + } +} + +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + ENVOY_LOG(error, "removeWatch() called for non-existent subscription {}.", type_url); + return; + } + entry->second->watch_map_.removeWatch(watch); +} + +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } + +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} + +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + pausable_ack_queue_.push(std::move(sub->second->sub_state_.handleResponse(*message))); + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); + } + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } + +void NewGrpcMuxImpl::start() { establishGrpcStream(); } + +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + Watch* watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); + return; + } + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our new interests, if there are any. + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", + next_request_type_url); + // It's safe to assume the front of the ACK queue is of this type, because that's the only + // way whoWantsToSendDiscoveryRequest() could return something for a non-existent + // subscription. + pausable_ack_queue_.pop(); + continue; + } + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + sendGrpcMessage(sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + sendGrpcMessage(sub->second->sub_state_.getNextRequestAckless()); + } + } + maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + if (pausable_ack_queue_.paused(type_url)) { + ASSERT(false, + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). " + "Returning false, but your xDS might be about to get head-of-line blocked " + "- permanently, if the pause is never undone.", + type_url)); + return false; + } else if (!grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!rateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } + } + return absl::nullopt; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h new file mode 100644 index 000000000000..c30f998e91c0 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.h @@ -0,0 +1,168 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" + +#include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" + +namespace Envoy { +namespace Config { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +// TODO(fredlas) name left as "NewGrpcMuxImpl" for easier reviewing in comparison to the +// work done so far. Should be changed. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { +public: + NewGrpcMuxImpl(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info); + + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; + + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; + + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onWriteable() override; + void start() override; + +protected: + // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes + // that underlie it) are templated on protobufs. That means that a single implementation that + // supports different types of protobufs cannot use polymorphism to share code. The workaround: + // the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is + // seen here in the base class as calls to abstract functions, to be provided by those derived + // classes. + virtual void establishGrpcStream() PURE; + virtual void sendGrpcMessage(void* msg_proto_ptr) PURE; + virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; + virtual bool grpcStreamAvailable() const PURE; + virtual bool rateLimitAllowsDrain() PURE; + +private: + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); + + void trySendDiscoveryRequests(); + + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMap watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; +}; + +class GrpcMuxDelta : public NewGrpcMuxImpl { + GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : NewGrpcMuxImpl(dispatcher, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + + void establishGrpcStream() override { grpc_stream_.establishNewStream(); } + void sendGrpcMessage(void* msg_proto_ptr) override { + auto* typed_proto_ptr = static_cast(msg_proto_ptr); + grpc_stream_.sendMessage(*typed_proto_ptr); + delete typed_proto_ptr; + } + void maybeUpdateQueueSizeStat(uint64_t size) override { + grpc_stream_.maybeUpdateQueueSizeStat(size); + } + bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } + bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + + GrpcStream + grpc_stream_; +}; + +class GrpcMuxSotw : public NewGrpcMuxImpl { + NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : NewGrpcMuxImpl(dispatcher, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + + void establishGrpcStream() override { grpc_stream_.establishNewStream(); } + void sendGrpcMessage(void* msg_proto_ptr) override { + auto* typed_proto_ptr = static_cast(msg_proto_ptr); + grpc_stream_.sendMessage(*typed_proto_ptr); + delete typed_proto_ptr; + } + void maybeUpdateQueueSizeStat(uint64_t size) override { + grpc_stream_.maybeUpdateQueueSizeStat(size); + } + bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } + bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + + GrpcStream grpc_stream_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc new file mode 100644 index 000000000000..34951e38b15c --- /dev/null +++ b/source/common/config/pausable_ack_queue.cc @@ -0,0 +1,66 @@ +#include "common/config/pausable_ack_queue.h" + +#include + +#include "common/common/assert.h" + +namespace Envoy { +namespace Config { + +void PausableAckQueue::push(UpdateAck x) { storage_.push_back(std::move(x)); } + +size_t PausableAckQueue::size() const { return storage_.size(); } + +bool PausableAckQueue::empty() { + for (const auto& entry : storage_) { + if (!paused_[entry.type_url_]) { + return false; + } + } + return true; +} + +const UpdateAck& PausableAckQueue::front() { + for (const auto& entry : storage_) { + if (!paused_[entry.type_url_]) { + return entry; + } + } + RELEASE_ASSERT(!storage_.empty(), "front() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void PausableAckQueue::pop() { + for (auto it = storage_.begin(); it != storage_.end(); ++it) { + if (!paused_[it->type_url_]) { + storage_.erase(it); + return; + } + } + RELEASE_ASSERT(!storage_.empty(), "pop() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} + +void PausableAckQueue::pause(const std::string& type_url) { + // It's ok to pause a subscription that doesn't exist yet. + auto& pause_entry = paused_[type_url]; + ASSERT(!pause_entry); + pause_entry = true; +} + +void PausableAckQueue::resume(const std::string& type_url) { + auto& pause_entry = paused_[type_url]; + ASSERT(pause_entry); + pause_entry = false; +} + +bool PausableAckQueue::paused(const std::string& type_url) const { + auto entry = paused_.find(type_url); + if (entry == paused_.end()) { + return false; + } + return entry->second; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h new file mode 100644 index 000000000000..46222a8b2e3c --- /dev/null +++ b/source/common/config/pausable_ack_queue.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include "envoy/api/v2/discovery.pb.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Config { + +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + +// There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for +// subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. +// We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() +// to choose the first element whose type_url isn't paused. +class PausableAckQueue { +public: + void push(UpdateAck x); + size_t size() const; + bool empty(); + const UpdateAck& front(); + void pop(); + void pause(const std::string& type_url); + void resume(const std::string& type_url); + bool paused(const std::string& type_url) const; + +private: + // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give + // the pause state its own map. (Map key is type_url.) + absl::flat_hash_map paused_; + std::list storage_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h new file mode 100644 index 000000000000..7fe69ccf769c --- /dev/null +++ b/source/common/config/subscription_state.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/pure.h" +#include "envoy/config/subscription.h" + +#include "common/protobuf/protobuf.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Config { + +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + +class SubscriptionState { +public: + virtual ~SubscriptionState() = default; + + // Update which resources we're interested in subscribing to. + virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; + + // Whether there was a change in our subscription interest we have yet to inform the server of. + virtual bool subscriptionUpdatePending() const PURE; + + virtual void markStreamFresh() PURE; + + // Argument should have been static_cast from GrpcStream's ResponseProto type. + virtual UpdateAck handleResponse(const Protobuf::Message& message) PURE; + + virtual void handleEstablishmentFailure() PURE; + + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the + // pointer actually is. + void* getNextRequestAckless(); + // The WithAck version first calls the Ackless version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the + // pointer actually is. + void* getNextRequestWithAck(const UpdateAck& ack); +}; + +class SubscriptionStateFactory { +public: + virtual ~SubscriptionStateFactory() = default; + virtual SubscriptionState makeSubscriptionState(const std::string& type_url, + const std::set& resource_names, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + SubscriptionStats& stats) PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/subscription_state_delta.cc similarity index 66% rename from source/common/config/delta_subscription_state.cc rename to source/common/config/subscription_state_delta.cc index bec633841aff..ea0b56fb0577 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/subscription_state_delta.cc @@ -1,78 +1,43 @@ -#include "common/config/delta_subscription_state.h" - #include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/delta_subscription_state.h" namespace Envoy { namespace Config { DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, - const std::set& resource_names, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, - SubscriptionStats& stats) + Event::Dispatcher& dispatcher) : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout), stats_(stats) { - // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery - // request to be queued if it returns true. We don't need to do that because we know that the - // subscription gRPC stream is not yet established, and establishment causes a request. - updateResourceInterest(resource_names); - setInitFetchTimeout(dispatcher); -} - -void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) { + init_fetch_timeout_(init_fetch_timeout) { if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - stats_.init_fetch_timeout_.inc(); ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, - nullptr); + callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } } -void DeltaSubscriptionState::pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; -} - -void DeltaSubscriptionState::resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; -} - -// Returns true if there is any meaningful change in our subscription interest, worth reporting to -// the server. -void DeltaSubscriptionState::updateResourceInterest( - const std::set& update_to_these_names) { - std::vector cur_added; - std::vector cur_removed; - - std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - resource_names_.begin(), resource_names_.end(), - std::inserter(cur_added, cur_added.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), update_to_these_names.begin(), - update_to_these_names.end(), std::inserter(cur_removed, cur_removed.begin())); - +void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) { for (const auto& a : cur_added) { setResourceWaitingForServer(a); - // Removed->added requires us to keep track of it as a "new" addition, since our user may have - // forgotten its copy of the resource after instructing us to remove it, and so needs to be - // reminded of it. + // If interest in a resource is removed-then-added (all before a discovery request + // can be sent), we must treat it as a "new" addition: our user may have forgotten its + // copy of the resource after instructing us to remove it, and need to be reminded of it. names_removed_.erase(a); names_added_.insert(a); } for (const auto& r : cur_removed) { setLostInterestInResource(r); - // Ideally, when a resource is added-then-removed in between requests, we would avoid putting - // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, - // the removed-then-added case *does* need to go in the request, and due to how we accomplish - // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" - // has to be treated as equivalent to just "add"). + // Ideally, when interest in a resource is added-then-removed in between requests, + // we would avoid putting a superfluous "unsubscribe [resource that was never subscribed]" + // in the request. However, the removed-then-added case *does* need to go in the request, + // and due to how we accomplish that, it's difficult to distinguish remove-add-remove from + // add-remove (because "remove-add" has to be treated as equivalent to just "add"). names_added_.erase(r); names_removed_.insert(r); } @@ -89,8 +54,7 @@ UpdateAck DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message) { // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. - UpdateAck ack(message.nonce()); - stats_.update_attempt_.inc(); + UpdateAck ack(message.nonce(), type_url_); try { handleGoodResponse(message); } catch (const EnvoyException& e) { @@ -108,6 +72,12 @@ void DeltaSubscriptionState::handleGoodResponse( throw EnvoyException( fmt::format("duplicate name {} found among added/updated resources", resource.name())); } + if (message.type_url() != resource.resource().type_url()) { + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DeltaDiscoveryResponse {}", + resource.resource().type_url(), message.type_url(), + message.DebugString())); + } } for (const auto& name : message.removed_resources()) { if (!names_added_removed.insert(name).second) { @@ -115,7 +85,6 @@ void DeltaSubscriptionState::handleGoodResponse( fmt::format("duplicate name {} found in the union of added+removed resources", name)); } } - callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), message.system_version_info()); for (const auto& resource : message.resources()) { @@ -134,8 +103,6 @@ void DeltaSubscriptionState::handleGoodResponse( setResourceWaitingForServer(resource_name); } } - stats_.update_success_.inc(); - stats_.version_.set(HashUtil::xxHash64(message.system_version_info())); ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, message.resources().size(), message.removed_resources().size()); } @@ -145,21 +112,19 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void DeltaSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); - stats_.update_failure_.inc(); - stats_.update_attempt_.inc(); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } -envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { - envoy::api::v2::DeltaDiscoveryRequest request; +envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal() { + auto* request = new envoy::api::v2::DeltaDiscoveryRequest; if (!any_request_sent_yet_in_current_stream_) { any_request_sent_yet_in_current_stream_ = true; // initial_resource_versions "must be populated for first request in a stream". @@ -170,7 +135,7 @@ envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { // Resources we are interested in, but are still waiting to get any version of from the // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) if (!resource.second.waitingForServer()) { - (*request.mutable_initial_resource_versions())[resource.first] = resource.second.version(); + (*request->mutable_initial_resource_versions())[resource.first] = resource.second.version(); } // As mentioned above, fill resource_names_subscribe with everything, including names we // have yet to receive any resource for. @@ -179,14 +144,26 @@ envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequest() { names_removed_.clear(); } std::copy(names_added_.begin(), names_added_.end(), - Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_subscribe())); std::copy(names_removed_.begin(), names_removed_.end(), - Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_unsubscribe())); + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_unsubscribe())); names_added_.clear(); names_removed_.clear(); - request.set_type_url(type_url_); - request.mutable_node()->MergeFrom(local_info_.node()); + request->set_type_url(type_url_); + request->mutable_node()->MergeFrom(local_info_.node()); + return request; +} + +void* DeltaSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } + +void* DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DeltaDiscoveryRequest* request = getNextRequestInternal(); + request->set_response_nonce(ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request->mutable_error_detail()->CopyFrom(ack.error_detail_); + } return request; } diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/subscription_state_delta.h similarity index 70% rename from source/common/config/delta_subscription_state.h rename to source/common/config/subscription_state_delta.h index f21e0b895b9e..ccd78e5e0ad0 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/subscription_state_delta.h @@ -7,34 +7,26 @@ #include "envoy/local_info/local_info.h" #include "common/common/assert.h" -#include "common/common/hash.h" #include "common/common/logger.h" +#include "common/config/pausable_ack_queue.h" namespace Envoy { namespace Config { -struct UpdateAck { - UpdateAck(absl::string_view nonce) : nonce_(nonce) {} - std::string nonce_; - ::google::rpc::Status error_detail_; -}; - -// Tracks the xDS protocol state of an individual ongoing delta xDS session. +// Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. +// There can be multiple DeltaSubscriptionStates active. They will always all be +// blissfully unaware of each other's existence, even when their messages are +// being multiplexed together by ADS. class DeltaSubscriptionState : public Logger::Loggable { public: - DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, - SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, + DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, SubscriptionStats& stats); - - void setInitFetchTimeout(Event::Dispatcher& dispatcher); - - void pause(); - void resume(); - bool paused() const { return paused_; } + Event::Dispatcher& dispatcher); // Update which resources we're interested in subscribing to. - void updateResourceInterest(const std::set& update_to_these_names); + void updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed); // Whether there was a change in our subscription interest we have yet to inform the server of. bool subscriptionUpdatePending() const; @@ -45,9 +37,20 @@ class DeltaSubscriptionState : public Logger::Loggable { void handleEstablishmentFailure(); - envoy::api::v2::DeltaDiscoveryRequest getNextRequest(); + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestAckless(); + // The WithAck version first calls the Ackless version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestWithAck(const UpdateAck& ack); + + DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; + DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; private: + // Returns a new'd pointer, meant to be owned by the caller. + envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal(); void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); void handleBadResponse(const EnvoyException& e, UpdateAck& ack); void disableInitFetchTimeoutTimer(); @@ -86,12 +89,12 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set resource_names_; const std::string type_url_; + // callbacks_ is expected to be a WatchMap. SubscriptionCallbacks& callbacks_; const LocalInfo::LocalInfo& local_info_; std::chrono::milliseconds init_fetch_timeout_; Event::TimerPtr init_fetch_timeout_timer_; - bool paused_{}; bool any_request_sent_yet_in_current_stream_{}; // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. @@ -99,8 +102,6 @@ class DeltaSubscriptionState : public Logger::Loggable { // Feel free to change to unordered if you can figure out how to make it work. std::set names_added_; std::set names_removed_; - - SubscriptionStats& stats_; }; } // namespace Config diff --git a/source/common/config/subscription_state_sotw.cc b/source/common/config/subscription_state_sotw.cc new file mode 100644 index 000000000000..0816001882c4 --- /dev/null +++ b/source/common/config/subscription_state_sotw.cc @@ -0,0 +1,146 @@ +#include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/sotw_subscription_state.h" + +namespace Envoy { +namespace Config { + +SubscriptionStateSotw::SubscriptionStateSotw(const std::string& type_url, + SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher) + : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout) { + setInitFetchTimeout(dispatcher); +} + +void SubscriptionStateSotw::setInitFetchTimeout(Event::Dispatcher& dispatcher) { + if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); + } +} + +void SubscriptionStateSotw::pause() { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); + ASSERT(!paused_); + paused_ = true; +} + +void SubscriptionStateSotw::resume() { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); + ASSERT(paused_); + paused_ = false; +} + +void SubscriptionStateSotw::updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) { + for (const auto& a : cur_added) { + names_tracked_.insert(a); + } + for (const auto& r : cur_removed) { + names_tracked_.erase(r); + } + update_pending_ = true; +} + +// Not having sent any requests yet counts as an "update pending" since you're supposed to resend +// the entirety of your interest at the start of a stream, even if nothing has changed. +bool SubscriptionStateSotw::subscriptionUpdatePending() const { return update_pending_; } + +UpdateAck SubscriptionStateSotw::handleResponse(const envoy::api::v2::DiscoveryResponse& message) { + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + UpdateAck ack(message.nonce(), type_url_); + try { + handleGoodResponse(message); + } catch (const EnvoyException& e) { + handleBadResponse(e, ack); + } + return ack; +} + +void SubscriptionStateSotw::handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message) { + disableInitFetchTimeoutTimer(); + callbacks_.onConfigUpdate(message.resources(), message.version_info()); + // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. + last_good_version_info_ = message.version_info(); + ENVOY_LOG(debug, "Config update for {} accepted with {} resources", type_url_, + message.resources().size()); +} + +void SubscriptionStateSotw::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); + ack.error_detail_.set_message(e.what()); + disableInitFetchTimeoutTimer(); + ENVOY_LOG(warn, "Config update for {} rejected: {}", type_url_, e.what()); + callbacks_.onConfigUpdateFailed(&e); +} + +void SubscriptionStateSotw::handleEstablishmentFailure() { + disableInitFetchTimeoutTimer(); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); + callbacks_.onConfigUpdateFailed(nullptr); +} + +envoy::api::v2::DiscoveryRequest SubscriptionStateSotw::getNextRequestAckless() { + envoy::api::v2::DiscoveryRequest request; + + std::copy(names_tracked_.begin(), names_tracked_.end(), + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names())); + + request.set_type_url(type_url_); + request.mutable_node()->MergeFrom(local_info_.node()); + update_pending_ = false; + return request; +} + +envoy::api::v2::DiscoveryRequest +SubscriptionStateSotw::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DiscoveryRequest request = getNextRequestAckless(); + request.set_response_nonce(ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request.mutable_error_detail()->CopyFrom(ack.error_detail_); + // TODO TODO? if last_good_version_info_ != nullopt + request.set_version_info(last_good_version_info_); + } else { + // TODO TODO in SotW, seems like previous response's version_info should accompany the + // response_nonce.... + // ........ hmmmm..... i think everything is fine if we can add a version_info field to + // UpdateAck (even though unusued for delta) + request.set_version_info(ack.version_info_) + } + return request; +} + +void SubscriptionStateSotw::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + +class SubscriptionStateSotw : public Logger::Loggable { + const std::string type_url_; + // callbacks_ is expected to be a WatchMap. + SubscriptionCallbacks& callbacks_; + const LocalInfo::LocalInfo& local_info_; + std::chrono::milliseconds init_fetch_timeout_; + Event::TimerPtr init_fetch_timeout_timer_; + std::string last_good_version_info_; // should this be absl::optional to distinguish none and ""? + + bool paused_{}; + bool update_pending_{true}; // Should send a request upon subscription start. + + SubscriptionStateSotw(const SubscriptionStateSotw&) = delete; + SubscriptionStateSotw& operator=(const SubscriptionStateSotw&) = delete; +}; + +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 26d7daca0df3..f8127f93cfdf 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -9,20 +9,25 @@ namespace { class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { protected: DeltaSubscriptionImplTest() = default; + + // We need to destroy the subscription before the test's destruction, because the subscription's + // destructor removes its watch from the NewGrpcMuxImpl, and that removal process involves + // some things held by the test fixture. + void TearDown() override { doSubscriptionTearDown(); } }; TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { startSubscription({"name1", "name2", "name3"}); expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2", "name3"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name4"}); + subscription_->updateResourceInterest({"name4"}); } // Checks that after a pause(), no requests are sent until resume(). @@ -36,11 +41,11 @@ TEST_F(DeltaSubscriptionImplTest, PauseHoldsRequest) { expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); // If not for the pause, these updates would make the expectSendMessage fail due to too many // messages being sent. - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); subscription_->resume(); } @@ -64,8 +69,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). @@ -76,8 +83,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version2A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version2A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). @@ -88,8 +97,10 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1B"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1B")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)) @@ -106,13 +117,36 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { // in the correct order. } -TEST_F(DeltaSubscriptionImplTest, NoGrpcStream) { - // Have to call start() to get state_ populated (which this test needs to not segfault), but - // start() also tries to start the GrpcStream. So, have that attempt return nullptr. - EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(nullptr)); - EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)).Times(0); - subscription_->start({"name1"}); - subscription_->updateResources({"name1", "name2"}); +TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { + Stats::IsolatedStoreImpl stats_store; + SubscriptionStats stats(Utility::generateStats(stats_store)); + + envoy::api::v2::core::Node node; + node.set_id("fo0"); + NiceMock local_info; + EXPECT_CALL(local_info, node()).WillRepeatedly(testing::ReturnRef(node)); + + NiceMock dispatcher; + NiceMock random; + Envoy::Config::RateLimitSettings rate_limit_settings; + NiceMock> callbacks; + auto* async_client = new Grpc::MockAsyncClient(); + + const Protobuf::MethodDescriptor* method_descriptor = + Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); + std::shared_ptr xds_context = std::make_shared( + std::unique_ptr(async_client), dispatcher, *method_descriptor, random, + stats_store, rate_limit_settings, local_info); + + std::unique_ptr subscription = std::make_unique( + xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, stats, + std::chrono::milliseconds(12345), false); + + EXPECT_CALL(*async_client, startRaw(_, _, _)).WillOnce(Return(nullptr)); + + subscription->start({"name1"}); + subscription->updateResourceInterest({"name1", "name2"}); } } // namespace diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 83975471c114..2a96f2b0953f 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -22,10 +22,9 @@ const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; class DeltaSubscriptionStateTest : public testing::Test { protected: DeltaSubscriptionStateTest() - : stats_(Utility::generateStats(store_)), - state_(TypeUrl, {"name1", "name2", "name3"}, callbacks_, local_info_, - std::chrono::milliseconds(0U), dispatcher_, stats_) { - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + : state_(TypeUrl, callbacks_, local_info_, std::chrono::milliseconds(0U), dispatcher_) { + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1", "name2", "name3")); } @@ -62,8 +61,6 @@ class DeltaSubscriptionStateTest : public testing::Test { NiceMock> callbacks_; NiceMock local_info_; NiceMock dispatcher_; - Stats::IsolatedStoreImpl store_; - SubscriptionStats stats_; // We start out interested in three resources: name1, name2, and name3. DeltaSubscriptionState state_; }; @@ -82,14 +79,14 @@ populateRepeatedResource(std::vector> items) // Basic gaining/losing interest in resources should lead to (un)subscriptions. TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { { - state_.updateResourceInterest({"name2", "name3", "name4"}); // drop name1, add name4 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1")); } { - state_.updateResourceInterest({"name1", "name2"}); // add back name1, drop name3 and 4 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); } @@ -105,9 +102,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { // interpret the resource_names_subscribe field as "send these resources even if you think Envoy // already has them". TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { - state_.updateResourceInterest({"name1", "name2"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name3")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } @@ -121,29 +118,29 @@ TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { // should be like this. What *is* important: the server must happily and cleanly ignore // "unsubscribe from [resource name I have never before referred to]" requests. TEST_F(DeltaSubscriptionStateTest, AddThenRemove) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name4")); } // add/remove/add == add. TEST_F(DeltaSubscriptionStateTest, AddRemoveAdd) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } // remove/add/remove == remove. TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { - state_.updateResourceInterest({"name1", "name2"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name1", "name2"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + state_.updateSubscriptionInterest({}, {"name3"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3")); } @@ -151,19 +148,19 @@ TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { // Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are // removed/added/removed. End result should be 4 added and 1,2,3 removed. TEST_F(DeltaSubscriptionStateTest, BothAddAndRemove) { - state_.updateResourceInterest({"name4"}); - state_.updateResourceInterest({"name1", "name2", "name3"}); - state_.updateResourceInterest({"name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2", "name3")); } TEST_F(DeltaSubscriptionStateTest, CumulativeUpdates) { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); - state_.updateResourceInterest({"name1", "name2", "name3", "name4", "name5"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({"name5"}, {}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); } @@ -217,7 +214,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); EXPECT_EQ(cur_request.initial_resource_versions().end(), @@ -232,7 +229,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove2.Add() = "name2"; deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ(cur_request.initial_resource_versions().end(), cur_request.initial_resource_versions().find("name2")); @@ -246,15 +243,15 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name3"; deliverDiscoveryResponse({}, remove1_3, "debugversion3"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } { // ...but our own map should remember our interest. In particular, losing interest in a // resource should cause its name to appear in the next request's resource_names_unsubscribe. - state_.updateResourceInterest({"name3", "name4"}); // note the lack of 1 and 2 - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); } @@ -273,9 +270,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); - state_.updateResourceInterest({"name2", "name3", "name4"}); // drop name1, add name4 - state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + state_.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); // Regarding the resource_names_subscribe field: // name1: do not include: we lost interest. // name2: yes do include: we're interested and we have a version of it. @@ -296,7 +293,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); @@ -304,7 +301,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { // Then, after updating the resources but not reconnecting the stream, verify that initial // versions are not sent. { - state_.updateResourceInterest({"name1", "name2", "name3", "name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); // The xDS server updates our resources, and gives us our newly requested one too. Protobuf::RepeatedPtrField add_all = populateRepeatedResource({{"name1", "version1B"}, @@ -312,7 +309,7 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {"name3", "version3B"}, {"name4", "version4A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion2"); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } } @@ -321,28 +318,16 @@ TEST_F(DeltaSubscriptionStateTest, CheckUpdatePending) { // Note that the test fixture ctor causes the first request to be "sent", so we start in the // middle of a stream, with our initially interested resources having been requested already. EXPECT_FALSE(state_.subscriptionUpdatePending()); - state_.updateResourceInterest({"name1", "name2", "name3"}); // no change + state_.updateSubscriptionInterest({}, {}); // no change EXPECT_FALSE(state_.subscriptionUpdatePending()); state_.markStreamFresh(); - EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream - state_.updateResourceInterest({"name1", "name2"}); // one removed + EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream + state_.updateSubscriptionInterest({}, {"name3"}); // one removed EXPECT_TRUE(state_.subscriptionUpdatePending()); - state_.updateResourceInterest({"name1", "name2", "name3"}); // one added + state_.updateSubscriptionInterest({"name3"}, {}); // one added EXPECT_TRUE(state_.subscriptionUpdatePending()); } -TEST_F(DeltaSubscriptionStateTest, PauseAndResume) { - EXPECT_FALSE(state_.paused()); - state_.pause(); - EXPECT_TRUE(state_.paused()); - state_.resume(); - EXPECT_FALSE(state_.paused()); - state_.pause(); - EXPECT_TRUE(state_.paused()); - state_.resume(); - EXPECT_FALSE(state_.paused()); -} - // The next three tests test that duplicate resource names (whether additions or removals) cause // DeltaSubscriptionState to reject the update without even trying to hand it to the consuming API's // onConfigUpdate(). diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 4414aa77489c..35e8a4bbef06 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "common/config/delta_subscription_impl.h" #include "common/grpc/common.h" @@ -32,15 +34,30 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); + xds_context_ = std::make_shared( + std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, + random_, stats_store_, rate_limit_settings_, local_info_); subscription_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, - *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, random_, stats_store_, - rate_limit_settings_, callbacks_, stats_, init_fetch_timeout); + xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + init_fetch_timeout, false); + EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + } + + void doSubscriptionTearDown() override { + if (subscription_started_) { + EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); + subscription_.reset(); + } } ~DeltaSubscriptionTestHarness() override { while (!nonce_acks_required_.empty()) { - EXPECT_FALSE(nonce_acks_sent_.empty()); + if (nonce_acks_sent_.empty()) { + // It's not enough to EXPECT_FALSE(nonce_acks_sent_.empty()), we need to skip the following + // EXPECT_EQ, otherwise the undefined .front() can get pretty bad. + EXPECT_FALSE(nonce_acks_sent_.empty()); + break; + } EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); nonce_acks_required_.pop(); nonce_acks_sent_.pop(); @@ -49,7 +66,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { } void startSubscription(const std::set& cluster_names) override { - EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + subscription_started_ = true; last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names); @@ -104,12 +121,11 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) override { - std::unique_ptr response( - new envoy::api::v2::DeltaDiscoveryResponse()); - + auto response = std::make_unique(); last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); response->set_nonce(last_response_nonce_); response->set_system_version_info(version); + response->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); Protobuf::RepeatedPtrField typed_resources; for (const auto& cluster : cluster_names) { @@ -132,11 +148,12 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - subscription_->onDiscoveryResponse(std::move(response)); + static_cast(subscription_->getContextForTest().get()) + ->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { std::set sub; std::set unsub; @@ -147,7 +164,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { std::inserter(unsub, unsub.begin())); expectSendMessage(sub, unsub, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } @@ -157,7 +174,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); } void expectDisableInitFetchTimeoutTimer() override { @@ -172,6 +189,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; + std::shared_ptr xds_context_; std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; @@ -181,6 +199,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock> callbacks_; std::queue nonce_acks_required_; std::queue nonce_acks_sent_; + bool subscription_started_{}; }; } // namespace diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 901b5cf29b31..1958a4270bd6 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -43,11 +43,11 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { subscription_.start(cluster_names); } - void updateResources(const std::set& cluster_names) override { - subscription_.updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) override { + subscription_.updateResourceInterest(cluster_names); } - void updateFile(const std::string json, bool run_dispatcher = true) { + void updateFile(const std::string& json, bool run_dispatcher = true) { // Write JSON contents to file, rename to path_ and run dispatcher to catch // inotify. const std::string temp_path = TestEnvironment::writeStringToFileForTest("eds.json.tmp", json); @@ -94,13 +94,10 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { version); } - void expectConfigUpdateFailed() override { - // initial_fetch_timeout not implemented - } + void expectConfigUpdateFailed() override { stats_.update_failure_.inc(); } - void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { - UNREFERENCED_PARAMETER(timeout); - // initial_fetch_timeout not implemented + void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds) override { + // initial_fetch_timeout not implemented. } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index c2493c0934f5..1d1df3f57931 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -89,7 +89,7 @@ class GrpcMuxImplTestBase : public testing::Test { } NiceMock dispatcher_; - Runtime::MockRandomGenerator random_; + NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; std::unique_ptr grpc_mux_; diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc index 783f188eefd6..e54f901ddee6 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/common/config/grpc_stream_test.cc @@ -43,7 +43,7 @@ class GrpcStreamTest : public testing::Test { // Tests that establishNewStream() establishes it, a second call does nothing, and a third call // after the stream was disconnected re-establishes it. -TEST_F(GrpcStreamTest, EstablishNewStream) { +TEST_F(GrpcStreamTest, EstablishStream) { EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); // Successful establishment { @@ -52,7 +52,7 @@ TEST_F(GrpcStreamTest, EstablishNewStream) { grpc_stream_.establishNewStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } - // Idempotency: do nothing (other than logging a warning) if already connected + // Idempotent { EXPECT_CALL(*async_client_, startRaw(_, _, _)).Times(0); EXPECT_CALL(callbacks_, onStreamEstablished()).Times(0); diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 7a1ca435985d..8a33cf7a894e 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -23,7 +23,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. - subscription_->updateResources({"cluster2"}); + subscription_->updateResourceInterest({"cluster2"}); // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); @@ -42,8 +42,8 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); - subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, - ""); + subscription_->grpcMux()->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, + ""); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); verifyControlPlaneStats(0); @@ -61,14 +61,14 @@ TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); // First with the initial, empty version update to "0". - updateResources({"cluster2"}); + updateResourceInterest({"cluster2"}); EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); EXPECT_TRUE(statsAre(3, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); EXPECT_TRUE(statsAre(4, 1, 1, 0, 0, 7148434200721666028)); // Now with version "0" update to "1". - updateResources({"cluster3"}); + updateResourceInterest({"cluster3"}); EXPECT_TRUE(statsAre(5, 1, 1, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster3"}, "1", false); EXPECT_TRUE(statsAre(6, 1, 2, 0, 0, 7148434200721666028)); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 9e68838ee646..788aa7293055 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -114,16 +114,17 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::GrpcStatus::Internal, "bad config"); } - subscription_->grpcMux().onDiscoveryResponse(std::move(response)); + subscription_->grpcMux()->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { // The "watch" mechanism means that updates that lose interest in a resource // will first generate a request for [still watched resources, i.e. without newly unwatched // ones] before generating the request for all of cluster_names. // TODO(fredlas) this unnecessary second request will stop happening once the watch mechanism is // no longer internally used by GrpcSubscriptionImpl. + // TODO TODO i think this should now be resolved std::set both; for (const auto& n : cluster_names) { if (last_cluster_names_.find(n) != last_cluster_names_.end()) { @@ -132,17 +133,22 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { } expectSendMessage(both, version_); expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)) + .WillOnce([this](ConfigUpdateFailureReason reason, const EnvoyException*) { + if (reason == ConfigUpdateFailureReason::FetchTimedout) { + stats_.init_fetch_timeout_.inc(); + } + }); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { init_timeout_timer_ = new Event::MockTimer(&dispatcher_); - EXPECT_CALL(*init_timeout_timer_, enableTimer(std::chrono::milliseconds(timeout), _)); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); } void expectDisableInitFetchTimeoutTimer() override { diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 1d41ee38019c..1f00c95cd87d 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -106,10 +106,10 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { subscription_->start(cluster_names); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { cluster_names_ = cluster_names; expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); timer_cb_(); } diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index c5cd9c4c771a..49f102993ce2 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -38,7 +38,7 @@ class SubscriptionFactoryTest : public testing::Test { return SubscriptionFactoryImpl(local_info_, dispatcher_, cm_, random_, validation_visitor_, *api_) .subscriptionFromConfigSource(config, Config::TypeUrl::get().ClusterLoadAssignment, - stats_store_, callbacks_); + stats_store_, callbacks_, false); } Upstream::MockClusterManager cm_; diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 622a268c90cb..6beb74812412 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -39,12 +39,14 @@ class SubscriptionImplTest : public testing::TestWithParam { } } + void TearDown() override { test_harness_->doSubscriptionTearDown(); } + void startSubscription(const std::set& cluster_names) { test_harness_->startSubscription(cluster_names); } - void updateResources(const std::set& cluster_names) { - test_harness_->updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) { + test_harness_->updateResourceInterest(cluster_names); } void expectSendMessage(const std::set& cluster_names, const std::string& version, @@ -90,68 +92,71 @@ INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplInitFetchTimeoutT // Validate basic request-response succeeds. TEST_P(SubscriptionImplTest, InitialRequestResponse) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); } // Validate that multiple streamed updates succeed. TEST_P(SubscriptionImplTest, ResponseStream) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 2, 0, 0, 0, 13237225503670494420U); + EXPECT_TRUE(statsAre(3, 2, 0, 0, 0, 13237225503670494420U)); } // Validate that the client can reject a config. TEST_P(SubscriptionImplTest, RejectConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); } // Validate that the client can reject a config and accept the same config later. TEST_P(SubscriptionImplTest, RejectAcceptConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(3, 1, 1, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, 7148434200721666028)); } // Validate that the client can reject a config and accept another config later. TEST_P(SubscriptionImplTest, RejectAcceptNextConfig) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); - statsAre(2, 0, 1, 0, 0, 0); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); - statsAre(3, 1, 1, 0, 0, 13237225503670494420U); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, 13237225503670494420U)); } // Validate that stream updates send a message with the updated resources. TEST_P(SubscriptionImplTest, UpdateResources) { startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); - statsAre(2, 1, 0, 0, 0, 7148434200721666028); - updateResources({"cluster2"}); - statsAre(3, 1, 0, 0, 0, 7148434200721666028); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, 7148434200721666028)); + updateResourceInterest({"cluster2"}); + EXPECT_TRUE(statsAre(3, 1, 0, 0, 0, 7148434200721666028)); } // Validate that initial fetch timer is created and calls callback on timeout TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { + if (GetParam() == SubscriptionType::Filesystem) { + return; // initial_fetch_timeout not implemented for filesystem. + } InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectConfigUpdateFailed(); callInitFetchTimeoutCb(); - statsAre(1, 0, 0, 0, 1, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } // Validate that initial fetch timer is disabled on config update @@ -159,7 +164,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); } @@ -169,7 +174,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - statsAre(1, 0, 0, 0, 0, 0); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectDisableInitFetchTimeoutTimer(); deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 6a094116324d..0f3b6c44da35 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -30,7 +30,7 @@ class SubscriptionTestHarness { * Update cluster names to be delivered via EDS. * @param cluster_names cluster names. */ - virtual void updateResources(const std::set& cluster_names) PURE; + virtual void updateResourceInterest(const std::set& cluster_names) PURE; /** * Expect that an update request is sent by the Subscription implementation. @@ -94,6 +94,8 @@ class SubscriptionTestHarness { virtual void callInitFetchTimeoutCb() PURE; + virtual void doSubscriptionTearDown() {} + Stats::IsolatedStoreImpl stats_store_; SubscriptionStats stats_; }; diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index bdfc0c6ae1ba..86017cf821d1 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -53,34 +53,23 @@ class GrpcClientIntegrationParamTest ClientType clientType() const override { return std::get<1>(GetParam()); } }; -class DeltaSotwGrpcClientIntegrationParamTest +class DeltaSotwIntegrationParamTest : public BaseGrpcClientIntegrationParamTest, - public testing::TestWithParam> { + public testing::TestWithParam< + std::tuple> { public: - static std::string protocolTestParamsToString( - const ::testing::TestParamInfo>& - p) { - return fmt::format("{}_{}", + ~DeltaSotwIntegrationParamTest() override = default; + static std::string + protocolTestParamsToString(const ::testing::TestParamInfo< + std::tuple>& p) { + return fmt::format("{}_{}_{}", std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", std::get<1>(p.param) == ClientType::GoogleGrpc ? "GoogleGrpc" : "EnvoyGrpc", - std::get<2>(p.param) ? "Delta" : "StateOfTheWorld"); + std::get<2>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); } Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); } ClientType clientType() const override { return std::get<1>(GetParam()); } - bool isDelta() { return std::get<2>(GetParam()); } -}; - -class DeltaSotwIntegrationParamTest - : public testing::TestWithParam> { -public: - static std::string protocolTestParamsToString( - const ::testing::TestParamInfo>& p) { - return fmt::format("{}_{}_{}", - std::get<0>(p.param) == Network::Address::IpVersion::v4 ? "IPv4" : "IPv6", - std::get<1>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); - } - Network::Address::IpVersion ipVersion() const { return std::get<0>(GetParam()); } - SotwOrDelta sotwOrDelta() const { return std::get<1>(GetParam()); } + SotwOrDelta sotwOrDelta() const { return std::get<2>(GetParam()); } }; // Skip tests based on gRPC client type. @@ -102,19 +91,16 @@ class DeltaSotwIntegrationParamTest #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ - testing::Bool()) + testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc)) #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc), testing::Bool()) -#endif // ENVOY_GOOGLE_GRPC - -#define DELTA_INTEGRATION_PARAMS \ - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ + testing::Values(Grpc::ClientType::EnvoyGrpc), \ testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) +#endif // ENVOY_GOOGLE_GRPC } // namespace Grpc } // namespace Envoy diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 1534ba3f3b2d..da72909457d6 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -101,7 +101,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { return std::make_pair(result.first, ThreadAwareLoadBalancerPtr(result.second)); } - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, ClusterManager&) override { + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, bool, ClusterManager&) override { return CdsApiPtr{createCds_()}; } @@ -2994,12 +2994,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithoutEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(false)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); @@ -3021,12 +3016,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(true)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))).Times(0); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))).Times(0); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index 4e2590e4b41d..a2073cac79e8 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -19,7 +19,9 @@ using testing::AssertionResult; namespace Envoy { AdsIntegrationTest::AdsIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 628550fb9cea..69aa472c2f9b 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -13,35 +13,40 @@ #include "test/integration/http_integration.h" namespace Envoy { -static const std::string& AdsIntegrationConfig() { - CONSTRUCT_ON_FIRST_USE(std::string, R"EOF( +static const std::string& AdsIntegrationConfig(const std::string& api_type) { + CONSTRUCT_ON_FIRST_USE(std::string, fmt::format( + R"EOF( dynamic_resources: - lds_config: {ads: {}} - cds_config: {ads: {}} + lds_config: + ads: {{}} + cds_config: + ads: {{}} ads_config: - api_type: GRPC + api_type: {} set_node_on_first_message_only: true static_resources: clusters: name: dummy_cluster - connect_timeout: { seconds: 5 } + connect_timeout: + seconds: 5 type: STATIC hosts: socket_address: address: 127.0.0.1 port_value: 0 lb_policy: ROUND_ROBIN - http2_protocol_options: {} + http2_protocol_options: {{}} admin: access_log_path: /dev/null address: socket_address: address: 127.0.0.1 port_value: 0 -)EOF"); +)EOF", + api_type)); } -class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { +class AdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsIntegrationTest(); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index e2cb9250c0e6..91e916ed451f 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -26,7 +26,8 @@ using testing::AssertionResult; namespace Envoy { -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate basic config delivery and upgrade. TEST_P(AdsIntegrationTest, Basic) { @@ -111,6 +112,7 @@ TEST_P(AdsIntegrationTest, Failure) { {"route_config_0"}, {}, {})); test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + makeSingleRequest(); } @@ -261,7 +263,7 @@ TEST_P(AdsIntegrationTest, DuplicateWarmingClusters) { {buildListener("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", - {"cluster_0"}, {"cluster_0"}, {})); + {"cluster_0"}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {"route_config_0"}, {})); sendDiscoveryResponse( @@ -321,30 +323,32 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { makeSingleRequest(); EXPECT_FALSE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // Send the first warming cluster. sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_1")}, {buildCluster("warming_cluster_1")}, {"cluster_0"}, "2"); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); EXPECT_TRUE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); // We would've got a Cluster discovery request with version 2 here, had the CDS not been paused. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_2", "warming_cluster_1"}, {"warming_cluster_2"}, {})); EXPECT_TRUE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // Finish warming the clusters. sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, @@ -357,7 +361,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { // Validate that clusters are warmed. test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); EXPECT_FALSE( - test_server_->server().clusterManager().adsMux().paused(Config::TypeUrl::get().Cluster)); + test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); @@ -412,9 +416,9 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", @@ -425,7 +429,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_1")}, - {buildClusterLoadAssignment("warming_cluster_1")}, {"cluster_0"}, "2"); + {buildClusterLoadAssignment("warming_cluster_1")}, {}, "2"); // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response. @@ -446,7 +450,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_2")}, - {buildClusterLoadAssignment("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + {buildClusterLoadAssignment("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); } @@ -580,13 +584,16 @@ TEST_P(AdsIntegrationTest, RdsAfterLdsInvalidated) { test_server_->waitForCounterGe("listener_manager.listener_create_success", 2); } -class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -609,8 +616,8 @@ class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsFailIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsFailIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate that we don't crash on failed ADS stream. TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { @@ -621,13 +628,16 @@ TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { xds_stream_->finishGrpcStream(Grpc::Status::Internal); } -class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), AdsIntegrationConfig()) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + AdsIntegrationConfig( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -658,8 +668,8 @@ class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsConfigIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsConfigIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // This is s regression validating that we don't crash on EDS static Cluster that uses ADS. TEST_P(AdsConfigIntegrationTest, EdsClusterWithAdsConfigSource) { diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 7cd7830ad794..69698c7dc732 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -16,7 +16,6 @@ envoy_cc_mock( "//include/envoy/config:config_provider_manager_interface", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//source/common/config:config_provider_lib", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", From 2a2d53d6c4953930b367b7105d8a2b158c4752a4 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 13 Sep 2019 17:46:19 -0400 Subject: [PATCH 02/39] xds sotw and delta unification almost entirely compiling Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 10 - include/envoy/upstream/cluster_manager.h | 2 +- source/common/config/BUILD | 59 ++++-- ...e_delta.cc => delta_subscription_state.cc} | 73 ++++--- ...ate_delta.h => delta_subscription_state.h} | 55 ++--- .../config/filesystem_subscription_impl.cc | 2 +- .../config/filesystem_subscription_impl.h | 2 +- ...{new_grpc_mux_impl.cc => grpc_mux_impl.cc} | 189 +++++++++--------- .../{new_grpc_mux_impl.h => grpc_mux_impl.h} | 122 ++++++----- .../common/config/grpc_subscription_impl.cc | 32 +-- source/common/config/grpc_subscription_impl.h | 42 ++-- .../common/config/http_subscription_impl.cc | 3 +- source/common/config/http_subscription_impl.h | 2 +- source/common/config/pausable_ack_queue.h | 10 +- .../common/config/sotw_subscription_state.cc | 120 +++++++++++ .../common/config/sotw_subscription_state.h | 83 ++++++++ .../config/subscription_factory_impl.cc | 64 +++--- source/common/config/subscription_state.cc | 36 ++++ source/common/config/subscription_state.h | 58 ++++-- .../common/config/subscription_state_sotw.cc | 146 -------------- source/common/config/update_ack.h | 21 ++ source/common/router/scoped_rds.cc | 12 +- source/common/upstream/cds_api_impl.cc | 8 +- .../common/upstream/cluster_manager_impl.cc | 41 ++-- source/common/upstream/cluster_manager_impl.h | 4 +- source/server/lds_api.cc | 8 +- source/server/server.cc | 8 +- test/common/config/BUILD | 180 ++++++++--------- .../config/delta_subscription_state_test.cc | 109 +++++----- .../config/subscription_factory_impl_test.cc | 2 +- test/integration/cds_integration_test.cc | 3 +- test/mocks/config/mocks.cc | 9 - test/mocks/config/mocks.h | 21 +- test/mocks/upstream/mocks.cc | 3 +- test/mocks/upstream/mocks.h | 4 +- 35 files changed, 867 insertions(+), 676 deletions(-) rename source/common/config/{subscription_state_delta.cc => delta_subscription_state.cc} (77%) rename source/common/config/{subscription_state_delta.h => delta_subscription_state.h} (73%) rename source/common/config/{new_grpc_mux_impl.cc => grpc_mux_impl.cc} (55%) rename source/common/config/{new_grpc_mux_impl.h => grpc_mux_impl.h} (59%) create mode 100644 source/common/config/sotw_subscription_state.cc create mode 100644 source/common/config/sotw_subscription_state.h create mode 100644 source/common/config/subscription_state.cc delete mode 100644 source/common/config/subscription_state_sotw.cc create mode 100644 source/common/config/update_ack.h diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 328795dc4ee3..d6ccdfe6e3ef 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -57,16 +57,6 @@ class GrpcMuxCallbacks { virtual std::string resourceName(const ProtobufWkt::Any& resource) PURE; }; -/** - * Handle on an muxed gRPC subscription. The subscription is canceled on destruction. - */ -class GrpcMuxWatch { -public: - virtual ~GrpcMuxWatch() = default; -}; - -using GrpcMuxWatchPtr = std::unique_ptr; - struct Watch; /** diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 7dfde10cc03c..676bc78a8f7c 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -187,7 +187,7 @@ class ClusterManager { * * @return GrpcMux& ADS API provider referencee. */ - virtual Config::GrpcMux& adsMux() PURE; + virtual Config::GrpcMuxSharedPtr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 8c4133ea5331..6b39043909d0 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -67,12 +67,12 @@ envoy_cc_library( ) envoy_cc_library( - name = "delta_subscription_lib", - srcs = ["delta_subscription_impl.cc"], - hdrs = ["delta_subscription_impl.h"], + name = "grpc_subscription_lib", + srcs = ["grpc_subscription_impl.cc"], + hdrs = ["grpc_subscription_impl.h"], deps = [ + ":grpc_mux_lib", ":grpc_stream_lib", - ":new_grpc_mux_lib", ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/grpc:async_client_interface", @@ -90,19 +90,38 @@ envoy_cc_library( srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ - ":pausable_ack_queue_lib", - "//include/envoy/config:subscription_interface", - "//include/envoy/event:dispatcher_interface", - "//source/common/common:assert_lib", - "//source/common/common:backoff_lib", - "//source/common/common:minimal_logger_lib", - "//source/common/common:token_bucket_impl_lib", + ":subscription_state_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + +envoy_cc_library( + name = "sotw_subscription_state_lib", + srcs = ["sotw_subscription_state.cc"], + hdrs = ["sotw_subscription_state.h"], + deps = [ + ":subscription_state_lib", "//source/common/grpc:common_lib", "//source/common/protobuf", "@envoy_api//envoy/api/v2:discovery_cc", ], ) +envoy_cc_library( + name = "subscription_state_lib", + srcs = ["subscription_state.cc"], + hdrs = ["subscription_state.h"], + deps = [ + ":update_ack_lib", + "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/local_info:local_info_interface", + "//source/common/common:minimal_logger_lib", + ], +) + envoy_cc_library( name = "filesystem_subscription_lib", srcs = ["filesystem_subscription_impl.cc"], @@ -174,13 +193,14 @@ envoy_cc_library( ) envoy_cc_library( - name = "new_grpc_mux_lib", - srcs = ["new_grpc_mux_impl.cc"], - hdrs = ["new_grpc_mux_impl.h"], + name = "grpc_mux_lib", + srcs = ["grpc_mux_impl.cc"], + hdrs = ["grpc_mux_impl.h"], deps = [ ":delta_subscription_state_lib", ":grpc_stream_lib", ":pausable_ack_queue_lib", + ":sotw_subscription_state_lib", ":watch_map_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", @@ -236,6 +256,7 @@ envoy_cc_library( srcs = ["pausable_ack_queue.cc"], hdrs = ["pausable_ack_queue.h"], deps = [ + ":update_ack_lib", "//source/common/common:assert_lib", "@envoy_api//envoy/api/v2:discovery_cc", ], @@ -311,9 +332,7 @@ envoy_cc_library( srcs = ["subscription_factory_impl.cc"], hdrs = ["subscription_factory_impl.h"], deps = [ - ":delta_subscription_lib", ":filesystem_subscription_lib", - ":grpc_mux_subscription_lib", ":grpc_subscription_lib", ":http_subscription_lib", ":type_to_endpoint_lib", @@ -342,6 +361,14 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "update_ack_lib", + hdrs = ["update_ack.h"], + deps = [ + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + envoy_cc_library( name = "utility_lib", srcs = ["utility.cc"], diff --git a/source/common/config/subscription_state_delta.cc b/source/common/config/delta_subscription_state.cc similarity index 77% rename from source/common/config/subscription_state_delta.cc rename to source/common/config/delta_subscription_state.cc index ea0b56fb0577..6a706b689e3c 100644 --- a/source/common/config/subscription_state_delta.cc +++ b/source/common/config/delta_subscription_state.cc @@ -1,6 +1,7 @@ +#include "common/config/delta_subscription_state.h" + #include "common/common/assert.h" #include "common/common/hash.h" -#include "common/config/delta_subscription_state.h" namespace Envoy { namespace Config { @@ -9,16 +10,27 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout) { - if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { - init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } + Event::Dispatcher& dispatcher, + bool skip_subsequent_node) + : SubscriptionState(type_url, callbacks, local_info, init_fetch_timeout, dispatcher, + skip_subsequent_node) {} + +DeltaSubscriptionState::~DeltaSubscriptionState() {} + +DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node) + : dispatcher_(dispatcher), local_info_(local_info), + skip_subsequent_node_(skip_subsequent_node) {} + +DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() {} + +std::unique_ptr +DeltaSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + return std::make_unique( + type_url, callbacks, local_info_, init_fetch_timeout, dispatcher_, skip_subsequent_node_); } void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, @@ -50,13 +62,13 @@ bool DeltaSubscriptionState::subscriptionUpdatePending() const { !any_request_sent_yet_in_current_stream_; } -UpdateAck -DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message) { +UpdateAck DeltaSubscriptionState::handleResponse(const void* reponse_proto_ptr) { + auto* response = static_cast(reponse_proto_ptr); // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. - UpdateAck ack(message.nonce(), type_url_); + UpdateAck ack(response->nonce(), type_url()); try { - handleGoodResponse(message); + handleGoodResponse(*response); } catch (const EnvoyException& e) { handleBadResponse(e, ack); } @@ -85,8 +97,8 @@ void DeltaSubscriptionState::handleGoodResponse( fmt::format("duplicate name {} found in the union of added+removed resources", name)); } } - callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), - message.system_version_info()); + callbacks().onConfigUpdate(message.resources(), message.removed_resources(), + message.system_version_info()); for (const auto& resource : message.resources()) { setResourceVersion(resource.name(), resource.version()); } @@ -103,7 +115,7 @@ void DeltaSubscriptionState::handleGoodResponse( setResourceWaitingForServer(resource_name); } } - ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, + ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url(), message.resources().size(), message.removed_resources().size()); } @@ -112,18 +124,18 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); + ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url(), e.what()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void DeltaSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, - nullptr); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); } -envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal() { +envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInternal() { auto* request = new envoy::api::v2::DeltaDiscoveryRequest; if (!any_request_sent_yet_in_current_stream_) { any_request_sent_yet_in_current_stream_ = true; @@ -150,8 +162,10 @@ envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal() { names_added_.clear(); names_removed_.clear(); - request->set_type_url(type_url_); - request->mutable_node()->MergeFrom(local_info_.node()); + request->set_type_url(type_url()); + if (!any_request_sent_yet_in_current_stream_ || !skip_subsequent_node()) { + request->mutable_node()->MergeFrom(local_info().node()); + } return request; } @@ -167,13 +181,6 @@ void* DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { return request; } -void DeltaSubscriptionState::disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } -} - void DeltaSubscriptionState::setResourceVersion(const std::string& resource_name, const std::string& resource_version) { resource_versions_[resource_name] = ResourceVersion(resource_version); diff --git a/source/common/config/subscription_state_delta.h b/source/common/config/delta_subscription_state.h similarity index 73% rename from source/common/config/subscription_state_delta.h rename to source/common/config/delta_subscription_state.h index ccd78e5e0ad0..0df0045369f6 100644 --- a/source/common/config/subscription_state_delta.h +++ b/source/common/config/delta_subscription_state.h @@ -1,49 +1,47 @@ #pragma once #include "envoy/api/v2/discovery.pb.h" -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" -#include "envoy/local_info/local_info.h" #include "common/common/assert.h" #include "common/common/logger.h" -#include "common/config/pausable_ack_queue.h" +#include "common/config/subscription_state.h" + +#include "absl/types/optional.h" namespace Envoy { namespace Config { -// Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. -// There can be multiple DeltaSubscriptionStates active. They will always all be -// blissfully unaware of each other's existence, even when their messages are -// being multiplexed together by ADS. -class DeltaSubscriptionState : public Logger::Loggable { +// Tracks the state of a delta xDS-over-gRPC protocol session. +class DeltaSubscriptionState : public SubscriptionState { public: DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher); + Event::Dispatcher& dispatcher, bool skip_subsequent_node); + ~DeltaSubscriptionState() override; // Update which resources we're interested in subscribing to. void updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed); + const std::set& cur_removed) override; // Whether there was a change in our subscription interest we have yet to inform the server of. - bool subscriptionUpdatePending() const; + bool subscriptionUpdatePending() const override; - void markStreamFresh() { any_request_sent_yet_in_current_stream_ = false; } + void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } - UpdateAck handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); + // message is expected to be a envoy::api::v2::DeltaDiscoveryResponse. + UpdateAck handleResponse(const void* reponse_proto_ptr) override; - void handleEstablishmentFailure(); + void handleEstablishmentFailure() override; // Returns the next gRPC request proto to be sent off to the server, based on this object's // understanding of the current protocol state, and new resources that Envoy wants to request. // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestAckless(); + void* getNextRequestAckless() override; // The WithAck version first calls the Ackless version, then adds in the passed-in ack. // Returns a new'd pointer, meant to be owned by the caller. - void* getNextRequestWithAck(const UpdateAck& ack); + void* getNextRequestWithAck(const UpdateAck& ack) override; DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; @@ -53,7 +51,6 @@ class DeltaSubscriptionState : public Logger::Loggable { envoy::api::v2::DeltaDiscoveryRequest* getNextRequestInternal(); void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); void handleBadResponse(const EnvoyException& e, UpdateAck& ack); - void disableInitFetchTimeoutTimer(); class ResourceVersion { public: @@ -88,13 +85,6 @@ class DeltaSubscriptionState : public Logger::Loggable { // iterator into just its keys, e.g. for use in std::set_difference. std::set resource_names_; - const std::string type_url_; - // callbacks_ is expected to be a WatchMap. - SubscriptionCallbacks& callbacks_; - const LocalInfo::LocalInfo& local_info_; - std::chrono::milliseconds init_fetch_timeout_; - Event::TimerPtr init_fetch_timeout_timer_; - bool any_request_sent_yet_in_current_stream_{}; // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. @@ -104,5 +94,20 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set names_removed_; }; +class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { +public: + DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + ~DeltaSubscriptionStateFactory() override; + std::unique_ptr + makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + +private: + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; +}; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index 1bef8eafe8f6..80a693c69d9c 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -27,7 +27,7 @@ void FilesystemSubscriptionImpl::start(const std::set&) { refresh(); } -void FilesystemSubscriptionImpl::updateResources(const std::set&) { +void FilesystemSubscriptionImpl::updateResourceInterest(const std::set&) { // Bump stats for consistence behavior with other xDS. stats_.update_attempt_.inc(); } diff --git a/source/common/config/filesystem_subscription_impl.h b/source/common/config/filesystem_subscription_impl.h index 6ff089208dc8..e61f3317c08e 100644 --- a/source/common/config/filesystem_subscription_impl.h +++ b/source/common/config/filesystem_subscription_impl.h @@ -28,7 +28,7 @@ class FilesystemSubscriptionImpl : public Config::Subscription, // We report all discovered resources in the watched file, so the resource names arguments are // unused, and updateResources is a no-op (other than updating a stat). void start(const std::set&) override; - void updateResources(const std::set&) override; + void updateResourceInterest(const std::set&) override; private: void refresh(); diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc similarity index 55% rename from source/common/config/new_grpc_mux_impl.cc rename to source/common/config/grpc_mux_impl.cc index 276211ab610e..a8943d631813 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "common/config/new_grpc_mux_impl.h" +#include "common/config/grpc_mux_impl.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" @@ -10,14 +10,13 @@ namespace Envoy { namespace Config { -NewGrpcMuxImpl::NewGrpcMuxImpl(Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info) {} +GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory) + : subscription_state_factory_(std::move(subscription_state_factory)) {} -Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { +Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { if (watch == nullptr) { return addWatch(type_url, resources, callbacks, init_fetch_timeout); } else { @@ -26,61 +25,100 @@ Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watc } } -void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { +void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { updateWatch(type_url, watch, {}); - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - ENVOY_LOG(error, "removeWatch() called for non-existent subscription {}.", type_url); - return; - } - entry->second->watch_map_.removeWatch(watch); + watchMapFor(type_url).removeWatch(watch); } -void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } +void GrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } -void NewGrpcMuxImpl::resume(const std::string& type_url) { +void GrpcMuxImpl::resume(const std::string& type_url) { pausable_ack_queue_.resume(type_url); trySendDiscoveryRequests(); } -bool NewGrpcMuxImpl::paused(const std::string& type_url) const { +bool GrpcMuxImpl::paused(const std::string& type_url) const { return pausable_ack_queue_.paused(type_url); } -void NewGrpcMuxImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), - message->system_version_info()); - auto sub = subscriptions_.find(message->type_url()); +void GrpcMuxImpl::start() { establishGrpcStream(); } + +Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto watch_map = watch_maps_.find(type_url); + if (watch_map == watch_maps_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + Watch* watch = watch_map->second->addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto* sub = subscriptionStateFor(type_url); + WatchMap& watch_map = watchMapFor(type_url); + + auto added_removed = watch_map.updateWatchInterest(watch, resources); + sub->updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + + // Tell the server about our new interests, if there are any. + if (sub->subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void GrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + watch_maps_.emplace(type_url, std::make_unique()); + subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( + type_url, *watch_maps_[type_url], init_fetch_timeout)); + subscription_ordering_.emplace_back(type_url); +} + +SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { + auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " - "subscription {}.", - message->system_version_info(), message->type_url()); - return; + throw EnvoyException("Tried to look up state for non-existent subscription " + type_url); } - pausable_ack_queue_.push(std::move(sub->second->sub_state_.handleResponse(*message))); - trySendDiscoveryRequests(); + return sub->second.get(); +} + +WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { + auto watch_map = watch_maps_.find(type_url); + if (watch_map == watch_maps_.end()) { + throw EnvoyException("Tried to look up WatchMap for non-existent subscription " + type_url); + } + return *watch_map->second; } -void NewGrpcMuxImpl::onStreamEstablished() { +void GrpcMuxImpl::handleEstablishedStream() { for (auto& sub : subscriptions_) { - sub.second->sub_state_.markStreamFresh(); + sub.second->markStreamFresh(); } trySendDiscoveryRequests(); } -void NewGrpcMuxImpl::onEstablishmentFailure() { +void GrpcMuxImpl::handleStreamEstablishmentFailure() { // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a // crash, the iteration needs to dance around a little: collect pointers to all // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; do { for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = &sub.second->sub_state_; + all_subscribed[sub.first] = sub.second.get(); } for (auto& sub : all_subscribed) { if (already_called.insert(sub).second) { // insert succeeded ==> not already called @@ -90,53 +128,21 @@ void NewGrpcMuxImpl::onEstablishmentFailure() { } while (all_subscribed.size() != subscriptions_.size()); } -void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } - -void NewGrpcMuxImpl::start() { establishGrpcStream(); } - -Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } - - Watch* watch = entry->second->watch_map_.addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch, resources); - return watch; -} - -// Updates the list of resource names watched by the given watch. If an added name is new across -// the whole subscription, or if a removed name has no other watch interested in it, then the -// subscription will enqueue and attempt to send an appropriate discovery request. -void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); +void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, + const void* response_proto_ptr) { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); + ENVOY_LOG(warn, + "The server sent an xDS response proto with type_url {}, which we have " + "not subscribed to. Ignoring.", + type_url); return; } - auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); - sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - // Tell the server about our new interests, if there are any. - if (sub->second->sub_state_.subscriptionUpdatePending()) { - trySendDiscoveryRequests(); - } -} - -void NewGrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { - subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, - dispatcher_, local_info_)); - subscription_ordering_.emplace_back(type_url); + pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); + trySendDiscoveryRequests(); } -void NewGrpcMuxImpl::trySendDiscoveryRequests() { +void GrpcMuxImpl::trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); @@ -146,16 +152,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { // If so, which one (by type_url)? std::string next_request_type_url = maybe_request_type.value(); // If we don't have a subscription object for this request's type_url, drop the request. - auto sub = subscriptions_.find(next_request_type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", - next_request_type_url); - // It's safe to assume the front of the ACK queue is of this type, because that's the only - // way whoWantsToSendDiscoveryRequest() could return something for a non-existent - // subscription. - pausable_ack_queue_.pop(); - continue; - } + auto* sub = subscriptionStateFor(next_request_type_url); // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { break; @@ -164,10 +161,13 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { if (!pausable_ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's of the type_url that we're wanting to send. - sendGrpcMessage(sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + UpdateAck ack = pausable_ack_queue_.front(); pausable_ack_queue_.pop(); + // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. + sendGrpcMessage(sub->getNextRequestWithAck(ack)); } else { - sendGrpcMessage(sub->second->sub_state_.getNextRequestAckless()); + // getNextRequestAckless() returns a raw unowned pointer, which sendGrpcMessage deletes. + sendGrpcMessage(sub->getNextRequestAckless()); } } maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); @@ -175,7 +175,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). -bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { +bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { if (pausable_ack_queue_.paused(type_url)) { ASSERT(false, fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " @@ -200,7 +200,7 @@ bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { // First, prioritizes ACKs over non-ACK subscription interest updates. // Then, prioritizes non-ACK updates in the order the various types // of subscriptions were activated. -absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { +absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from pausable_ack_queue_ if possible, before looking at pending updates. if (!pausable_ack_queue_.empty()) { @@ -209,10 +209,9 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // If we're looking to send multiple non-ACK requests, send them in the order that their // subscriptions were initiated. for (const auto& sub_type : subscription_ordering_) { - auto sub = subscriptions_.find(sub_type); - if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && - !pausable_ack_queue_.paused(sub_type)) { - return sub->first; + auto* sub = subscriptionStateFor(sub_type); + if (sub->subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { + return sub_type; } } return absl::nullopt; diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h similarity index 59% rename from source/common/config/new_grpc_mux_impl.h rename to source/common/config/grpc_mux_impl.h index c30f998e91c0..042b9b8173f0 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -9,6 +9,7 @@ #include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" #include "common/config/pausable_ack_queue.h" +#include "common/config/sotw_subscription_state.h" #include "common/config/watch_map.h" #include "common/grpc/common.h" @@ -20,13 +21,11 @@ namespace Config { // This class owns the GrpcStream used to talk to the server, maintains queuing // logic to properly order the subscription(s)' various messages, and allows // starting/stopping/pausing of the subscriptions. -// TODO(fredlas) name left as "NewGrpcMuxImpl" for easier reviewing in comparison to the +// TODO(fredlas) name left as "GrpcMuxImpl" for easier reviewing in comparison to the // work done so far. Should be changed. -class NewGrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - Logger::Loggable { +class GrpcMuxImpl : public GrpcMux, Logger::Loggable { public: - NewGrpcMuxImpl(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info); + GrpcMuxImpl(std::unique_ptr subscription_state_factory); Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, @@ -36,12 +35,6 @@ class NewGrpcMuxImpl : public GrpcMux, void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; - - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void onWriteable() override; void start() override; protected: @@ -52,11 +45,19 @@ class NewGrpcMuxImpl : public GrpcMux, // seen here in the base class as calls to abstract functions, to be provided by those derived // classes. virtual void establishGrpcStream() PURE; + // Deletes msg_proto_ptr. virtual void sendGrpcMessage(void* msg_proto_ptr) PURE; virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; virtual bool grpcStreamAvailable() const PURE; virtual bool rateLimitAllowsDrain() PURE; + SubscriptionState* subscriptionStateFor(const std::string& type_url); + WatchMap& watchMapFor(const std::string& type_url); + void handleEstablishedStream(); + void handleStreamEstablishmentFailure(); + void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); + void trySendDiscoveryRequests(); + private: Watch* addWatch(const std::string& type_url, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); @@ -69,8 +70,6 @@ class NewGrpcMuxImpl : public GrpcMux, void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - void trySendDiscoveryRequests(); - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). bool canSendDiscoveryRequest(const std::string& type_url); @@ -80,48 +79,50 @@ class NewGrpcMuxImpl : public GrpcMux, // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). // First, prioritizes ACKs over non-ACK subscription interest updates. // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated. + // of subscriptions were activated (as tracked by subscription_ordering_). absl::optional whoWantsToSendDiscoveryRequest(); - Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All // of our different resource types' ACKs are mixed together in this queue. See class for // description of how it interacts with pause() and resume(). PausableAckQueue pausable_ack_queue_; - struct SubscriptionStuff { - SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} - - WatchMap watch_map_; - DeltaSubscriptionState sub_state_; - const std::chrono::milliseconds init_fetch_timeout_; + // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing + // delta or SotW xDS is determined by whether this is a {Delta,Sotw}SubscriptionStateFactory. + std::unique_ptr subscription_state_factory_; - SubscriptionStuff(const SubscriptionStuff&) = delete; - SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; - }; // Map key is type_url. - absl::flat_hash_map> subscriptions_; + // Only addWatch() should insert into these maps. + absl::flat_hash_map> subscriptions_; + absl::flat_hash_map> watch_maps_; - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in - // the order of Envoy's dependency ordering). + // Determines the order of initial discovery requests. (Assumes that subscriptions are added + // to this GrpcMux in the order of Envoy's dependency ordering). std::list subscription_ordering_; }; -class GrpcMuxDelta : public NewGrpcMuxImpl { - GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : NewGrpcMuxImpl(dispatcher, local_info), +class GrpcMuxDelta : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher, local_info, + skip_subsequent_node)), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} + // GrpcStreamCallbacks + void onStreamEstablished() override { handleEstablishedStream(); } + void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } + void onWriteable() override { trySendDiscoveryRequests(); } + void + onDiscoveryResponse(std::unique_ptr&& message) override { + genericHandleResponse(message->type_url(), message.get()); + } + +protected: void establishGrpcStream() override { grpc_stream_.establishNewStream(); } void sendGrpcMessage(void* msg_proto_ptr) override { auto* typed_proto_ptr = static_cast(msg_proto_ptr); @@ -138,17 +139,27 @@ class GrpcMuxDelta : public NewGrpcMuxImpl { grpc_stream_; }; -class GrpcMuxSotw : public NewGrpcMuxImpl { - NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, - Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : NewGrpcMuxImpl(dispatcher, local_info), +class GrpcMuxSotw : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher, local_info, + skip_subsequent_node)), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} + // GrpcStreamCallbacks + void onStreamEstablished() override { handleEstablishedStream(); } + void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } + void onWriteable() override { trySendDiscoveryRequests(); } + void onDiscoveryResponse(std::unique_ptr&& message) override { + genericHandleResponse(message->type_url(), message.get()); + } + +protected: void establishGrpcStream() override { grpc_stream_.establishNewStream(); } void sendGrpcMessage(void* msg_proto_ptr) override { auto* typed_proto_ptr = static_cast(msg_proto_ptr); @@ -164,5 +175,22 @@ class GrpcMuxSotw : public NewGrpcMuxImpl { GrpcStream grpc_stream_; }; +class NullGrpcMuxImpl : public GrpcMux { +public: + void start() override {} + + void pause(const std::string&) override {} + void resume(const std::string&) override {} + bool paused(const std::string&) const override { return false; } + + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void removeWatch(const std::string&, Watch*) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } +}; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index ddfd22d51173..49c03a2aba32 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -1,28 +1,28 @@ -#include "common/config/delta_subscription_impl.h" +#include "common/config/grpc_subscription_impl.h" namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, - SubscriptionCallbacks& callbacks, - SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout, - bool is_aggregated) +GrpcSubscriptionImpl::GrpcSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} -DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { +GrpcSubscriptionImpl::~GrpcSubscriptionImpl() { if (watch_) { context_->removeWatch(type_url_, watch_); } } -void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } +void GrpcSubscriptionImpl::pause() { context_->pause(type_url_); } -void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } +void GrpcSubscriptionImpl::resume() { context_->resume(type_url_); } // Config::DeltaSubscription -void DeltaSubscriptionImpl::start(const std::set& resources) { +void GrpcSubscriptionImpl::start(const std::set& resources) { // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, // whereas non-ADS xDS users must call it themselves. if (!is_aggregated_) { @@ -32,7 +32,7 @@ void DeltaSubscriptionImpl::start(const std::set& resources) { stats_.update_attempt_.inc(); } -void DeltaSubscriptionImpl::updateResourceInterest( +void GrpcSubscriptionImpl::updateResourceInterest( const std::set& update_to_these_names) { watch_ = context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, init_fetch_timeout_); @@ -40,7 +40,7 @@ void DeltaSubscriptionImpl::updateResourceInterest( } // Config::SubscriptionCallbacks -void DeltaSubscriptionImpl::onConfigUpdate( +void GrpcSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { callbacks_.onConfigUpdate(resources, version_info); @@ -48,7 +48,7 @@ void DeltaSubscriptionImpl::onConfigUpdate( stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } -void DeltaSubscriptionImpl::onConfigUpdate( +void GrpcSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { @@ -57,8 +57,8 @@ void DeltaSubscriptionImpl::onConfigUpdate( stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } -void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, - const EnvoyException* e) { +void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { stats_.update_attempt_.inc(); if (reason == ConfigUpdateFailureReason::FetchTimedout) { stats_.init_fetch_timeout_.inc(); @@ -69,7 +69,7 @@ void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reaso } callbacks_.onConfigUpdateFailed(reason, e); } -std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { +std::string GrpcSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { return callbacks_.resourceName(resource); } diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index b45be7c4da85..598a25577a93 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -2,40 +2,38 @@ #include "envoy/config/subscription.h" -#include "common/config/new_grpc_mux_impl.h" +#include "common/config/grpc_mux_impl.h" #include "common/config/utility.h" namespace Envoy { namespace Config { -// DeltaSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with +// GrpcSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with // an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) -// NewGrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in +// GrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in // various resources via start() and updateResourceInterest(). It receives updates to those // resources via the SubscriptionCallbacks it provides. Multiple users can each have their own -// Subscription object for the same type_url; NewGrpcMuxImpl maintains a subscription to the +// Subscription object for the same type_url; GrpcMuxImpl maintains a subscription to the // union of interested resources, and delivers to the users just the resource updates that they // are "watching" for. // -// DeltaSubscriptionImpl and NewGrpcMuxImpl are both built to provide both regular xDS and ADS, -// distinguished by whether multiple DeltaSubscriptionImpls are sharing a single -// NewGrpcMuxImpl. (And by the gRPC method string, but that's taken care of over in -// SubscriptionFactory). +// GrpcSubscriptionImpl and GrpcMuxImpl are both built to provide both regular xDS and ADS, +// distinguished by whether multiple GrpcSubscriptionImpls are sharing a single NewGrpcMuxImpl. +// (Also distinguished by the gRPC method string, but that's taken care of in SubscriptionFactory). // -// Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it -// can write to SubscriptionStats (which needs to live out here in the DeltaSubscriptionImpl) upon a -// config update. The idea is, DeltaSubscriptionImpl presents itself to WatchMap as the -// SubscriptionCallbacks, and then passes (after incrementing stats) all callbacks through to -// callbacks_, which are the real SubscriptionCallbacks. -class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { +// Why does GrpcSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the GrpcSubscriptionImpl) upon a +// config update. GrpcSubscriptionImpl presents itself to WatchMap as the SubscriptionCallbacks, +// and then, after incrementing stats, passes through to the real callbacks_. +class GrpcSubscriptionImpl : public Subscription, public SubscriptionCallbacks { public: - // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether - // it's all ours. The practical difference is that we ourselves must call start() on it only in - // the latter case. - DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); - ~DeltaSubscriptionImpl() override; + // is_aggregated: whether our GrpcMux is also providing ADS to other Subscriptions, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only if + // we are the sole owner. + GrpcSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); + ~GrpcSubscriptionImpl() override; void pause(); void resume(); @@ -62,7 +60,7 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks SubscriptionStats stats_; // NOTE: if another subscription of the same type_url has already been started, this value will be // ignored in favor of the other subscription's. - std::chrono::milliseconds init_fetch_timeout_; + const std::chrono::milliseconds init_fetch_timeout_; Watch* watch_{}; const bool is_aggregated_; }; diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index 908250c79495..27fd6b53c428 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -52,7 +52,8 @@ void HttpSubscriptionImpl::start(const std::set& resource_names) { initialize(); } -void HttpSubscriptionImpl::updateResources(const std::set& update_to_these_names) { +void HttpSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { Protobuf::RepeatedPtrField resources_vector(update_to_these_names.begin(), update_to_these_names.end()); request_.mutable_resource_names()->Swap(&resources_vector); diff --git a/source/common/config/http_subscription_impl.h b/source/common/config/http_subscription_impl.h index 452ba132582a..2b334f0f29e0 100644 --- a/source/common/config/http_subscription_impl.h +++ b/source/common/config/http_subscription_impl.h @@ -31,7 +31,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Http::RestApiFetcher void createRequest(Http::Message& request) override; diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h index 46222a8b2e3c..875ff3b2f23b 100644 --- a/source/common/config/pausable_ack_queue.h +++ b/source/common/config/pausable_ack_queue.h @@ -2,21 +2,13 @@ #include -#include "envoy/api/v2/discovery.pb.h" +#include "common/config/update_ack.h" #include "absl/container/flat_hash_map.h" namespace Envoy { namespace Config { -struct UpdateAck { - UpdateAck(absl::string_view nonce, absl::string_view type_url) - : nonce_(nonce), type_url_(type_url) {} - std::string nonce_; - std::string type_url_; - ::google::rpc::Status error_detail_; -}; - // There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for // subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. // We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc new file mode 100644 index 000000000000..94757847ac39 --- /dev/null +++ b/source/common/config/sotw_subscription_state.cc @@ -0,0 +1,120 @@ +#include "common/config/sotw_subscription_state.h" + +#include "common/common/assert.h" +#include "common/common/hash.h" + +namespace Envoy { +namespace Config { + +SotwSubscriptionState::SotwSubscriptionState(const std::string& type_url, + SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, + bool skip_subsequent_node) + : SubscriptionState(type_url, callbacks, local_info, init_fetch_timeout, dispatcher, + skip_subsequent_node) {} + +SotwSubscriptionState::~SotwSubscriptionState() {} + +SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node) + : dispatcher_(dispatcher), local_info_(local_info), + skip_subsequent_node_(skip_subsequent_node) {} + +SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() {} + +std::unique_ptr +SotwSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + return std::make_unique( + type_url, callbacks, local_info_, init_fetch_timeout, dispatcher_, skip_subsequent_node_); +} + +void SotwSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) { + for (const auto& a : cur_added) { + names_tracked_.insert(a); + } + for (const auto& r : cur_removed) { + names_tracked_.erase(r); + } + update_pending_ = true; +} + +// Not having sent any requests yet counts as an "update pending" since you're supposed to resend +// the entirety of your interest at the start of a stream, even if nothing has changed. +bool SotwSubscriptionState::subscriptionUpdatePending() const { return update_pending_; } + +UpdateAck SotwSubscriptionState::handleResponse(const void* reponse_proto_ptr) { + auto* response = static_cast(reponse_proto_ptr); + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + UpdateAck ack(response->nonce(), type_url()); + try { + handleGoodResponse(*response); + } catch (const EnvoyException& e) { + handleBadResponse(e, ack); + } + return ack; +} + +void SotwSubscriptionState::handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message) { + disableInitFetchTimeoutTimer(); + callbacks().onConfigUpdate(message.resources(), message.version_info()); + // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. + last_good_version_info_ = message.version_info(); + ENVOY_LOG(debug, "Config update for {} accepted with {} resources", type_url(), + message.resources().size()); +} + +void SotwSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); + ack.error_detail_.set_message(e.what()); + disableInitFetchTimeoutTimer(); + ENVOY_LOG(warn, "Config update for {} rejected: {}", type_url(), e.what()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); +} + +void SotwSubscriptionState::handleEstablishmentFailure() { + disableInitFetchTimeoutTimer(); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); +} + +envoy::api::v2::DiscoveryRequest* SotwSubscriptionState::getNextRequestInternal() { + auto* request = new envoy::api::v2::DiscoveryRequest; + + std::copy(names_tracked_.begin(), names_tracked_.end(), + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names())); + + request->set_type_url(type_url()); + if (!any_request_sent_yet_in_current_stream_ || !skip_subsequent_node()) { + request->mutable_node()->MergeFrom(local_info().node()); + } + any_request_sent_yet_in_current_stream_ = true; + update_pending_ = false; + return request; +} + +void* SotwSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } + +void* SotwSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DiscoveryRequest* request = getNextRequestInternal(); + request->set_response_nonce(ack.nonce_); + if (last_good_version_info_.has_value()) { + request->set_version_info(last_good_version_info_.value()); + } + if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request->mutable_error_detail()->CopyFrom(ack.error_detail_); + } + return request; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h new file mode 100644 index 000000000000..c2b3814cbeb7 --- /dev/null +++ b/source/common/config/sotw_subscription_state.h @@ -0,0 +1,83 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/grpc/status.h" + +#include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/subscription_state.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Config { + +// Tracks the state of a "state-of-the-world" (i.e. not delta) xDS-over-gRPC protocol session. +class SotwSubscriptionState : public SubscriptionState { +public: + SotwSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, + bool skip_subsequent_node); + ~SotwSubscriptionState() override; + + // Update which resources we're interested in subscribing to. + void updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) override; + + // Whether there was a change in our subscription interest we have yet to inform the server of. + bool subscriptionUpdatePending() const override; + + void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } + + // message is expected to be a envoy::api::v2::DiscoveryResponse. + UpdateAck handleResponse(const void* reponse_proto_ptr) override; + + void handleEstablishmentFailure() override; + + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestAckless() override; + // The WithAck version first calls the Ackless version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestWithAck(const UpdateAck& ack) override; + +private: + // Returns a new'd pointer, meant to be owned by the caller. + envoy::api::v2::DiscoveryRequest* getNextRequestInternal(); + + void handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message); + void handleBadResponse(const EnvoyException& e, UpdateAck& ack); + + // The version_info value most recently received in a DiscoveryResponse that was accepted. + // Remains empty until one is accepted. + absl::optional last_good_version_info_; + + bool any_request_sent_yet_in_current_stream_{}; + // Starts true because we should send a request upon subscription start. + bool update_pending_{true}; + + absl::flat_hash_set names_tracked_; + + SotwSubscriptionState(const SotwSubscriptionState&) = delete; + SotwSubscriptionState& operator=(const SotwSubscriptionState&) = delete; +}; + +class SotwSubscriptionStateFactory : public SubscriptionStateFactory { +public: + SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + ~SotwSubscriptionStateFactory() override; + std::unique_ptr + makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + +private: + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index b85c8bbb1d1c..61aacb4b1b6f 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -1,8 +1,7 @@ #include "common/config/subscription_factory_impl.h" -#include "common/config/delta_subscription_impl.h" #include "common/config/filesystem_subscription_impl.h" -#include "common/config/grpc_mux_subscription_impl.h" +#include "common/config/grpc_mux_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" #include "common/config/type_to_endpoint.h" @@ -22,15 +21,14 @@ SubscriptionFactoryImpl::SubscriptionFactoryImpl( SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, SubscriptionCallbacks& callbacks) { - Config::Utility::checkLocalInfo(type_url, local_info_); + Utility::checkLocalInfo(type_url, local_info_); std::unique_ptr result; SubscriptionStats stats = Utility::generateStats(scope); switch (config.config_source_specifier_case()) { case envoy::api::v2::core::ConfigSource::kPath: { Utility::checkFilesystemSubscriptionBackingPath(config.path(), api_); - result = std::make_unique( - dispatcher_, config.path(), callbacks, stats, validation_visitor_, api_); - break; + return std::make_unique(dispatcher_, config.path(), callbacks, + stats, validation_visitor_, api_); } case envoy::api::v2::core::ConfigSource::kApiConfigSource: { const envoy::api::v2::core::ApiConfigSource& api_config_source = config.api_config_source(); @@ -42,50 +40,44 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( "Please specify an explicit supported api_type in the following config:\n" + config.DebugString()); case envoy::api::v2::core::ApiConfigSource::REST: - result = std::make_unique( + return std::make_unique( local_info_, cm_, api_config_source.cluster_names()[0], dispatcher_, random_, Utility::apiConfigSourceRefreshDelay(api_config_source), Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(type_url), callbacks, stats, Utility::configSourceInitialFetchTimeout(config), validation_visitor_); - break; case envoy::api::v2::core::ApiConfigSource::GRPC: - result = std::make_unique( - local_info_, - Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher_, random_, sotwGrpcMethod(type_url), type_url, callbacks, stats, scope, - Utility::parseRateLimitSettings(api_config_source), - Utility::configSourceInitialFetchTimeout(config), - api_config_source.set_node_on_first_message_only()); - break; - case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { - Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.clusters(), api_config_source); - result = std::make_unique( - local_info_, - Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), type_url, random_, scope, - Utility::parseRateLimitSettings(api_config_source), callbacks, stats, - Utility::configSourceInitialFetchTimeout(config)); - break; - } + return std::make_unique( + std::make_shared(Utility::factoryForGrpcApiConfigSource( + cm_.grpcAsyncClientManager(), api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), + local_info_, /*skip_subsequent_node=*/false), + type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), + /*is_aggregated=*/false); + case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: + return std::make_unique( + std::make_shared(Utility::factoryForGrpcApiConfigSource( + cm_.grpcAsyncClientManager(), api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), + local_info_, /*skip_subsequent_node=*/false), + type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), + /*is_aggregated=*/false); default: NOT_REACHED_GCOVR_EXCL_LINE; } - break; } case envoy::api::v2::core::ConfigSource::kAds: { - result = std::make_unique( - cm_.adsMux(), callbacks, stats, type_url, dispatcher_, - Utility::configSourceInitialFetchTimeout(config)); - break; + return std::make_unique(cm_.adsMux(), type_url, callbacks, stats, + Utility::configSourceInitialFetchTimeout(config), + /*is_aggregated=*/true); } default: throw EnvoyException("Missing config source specifier in envoy::api::v2::core::ConfigSource"); } - return result; + NOT_REACHED_GCOVR_EXCL_LINE; } } // namespace Config diff --git a/source/common/config/subscription_state.cc b/source/common/config/subscription_state.cc new file mode 100644 index 000000000000..c68b20c28d09 --- /dev/null +++ b/source/common/config/subscription_state.cc @@ -0,0 +1,36 @@ +#include "common/config/subscription_state.h" + +#include +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/pure.h" +#include "envoy/config/subscription.h" + +namespace Envoy { +namespace Config { + +SubscriptionState::SubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, bool skip_subsequent_node) + : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), + skip_subsequent_node_(skip_subsequent_node) { + if (init_fetch_timeout.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout); + } +} + +void SubscriptionState::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h index 7fe69ccf769c..de8147e2992b 100644 --- a/source/common/config/subscription_state.h +++ b/source/common/config/subscription_state.h @@ -6,7 +6,10 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/common/pure.h" #include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/local_info/local_info.h" +#include "common/config/update_ack.h" #include "common/protobuf/protobuf.h" #include "absl/strings/string_view.h" @@ -14,28 +17,31 @@ namespace Envoy { namespace Config { -struct UpdateAck { - UpdateAck(absl::string_view nonce, absl::string_view type_url) - : nonce_(nonce), type_url_(type_url) {} - std::string nonce_; - std::string type_url_; - ::google::rpc::Status error_detail_; -}; - -class SubscriptionState { +// Tracks the protocol state of an individual ongoing xDS-over-gRPC session, for a single type_url. +// There can be multiple DeltaSubscriptionStates active, one per type_url. They will all be +// blissfully unaware of each other's existence, even when their messages are being multiplexed +// together by ADS. +// This is the abstract parent class for both the delta and state-of-the-world xDS variants. +class SubscriptionState : public Logger::Loggable { public: + SubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, + bool skip_subsequent_node); virtual ~SubscriptionState() = default; // Update which resources we're interested in subscribing to. - virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; + virtual void updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) PURE; // Whether there was a change in our subscription interest we have yet to inform the server of. virtual bool subscriptionUpdatePending() const PURE; virtual void markStreamFresh() PURE; - // Argument should have been static_cast from GrpcStream's ResponseProto type. - virtual UpdateAck handleResponse(const Protobuf::Message& message) PURE; + // Implementations expect either a DeltaDiscoveryResponse or DiscoveryResponse. The caller is + // expected to know which it should be providing. + virtual UpdateAck handleResponse(const void* reponse_proto_ptr) PURE; virtual void handleEstablishmentFailure() PURE; @@ -43,21 +49,35 @@ class SubscriptionState { // understanding of the current protocol state, and new resources that Envoy wants to request. // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the // pointer actually is. - void* getNextRequestAckless(); + virtual void* getNextRequestAckless() PURE; // The WithAck version first calls the Ackless version, then adds in the passed-in ack. // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the // pointer actually is. - void* getNextRequestWithAck(const UpdateAck& ack); + virtual void* getNextRequestWithAck(const UpdateAck& ack) PURE; + +protected: + void disableInitFetchTimeoutTimer(); + + std::string type_url() const { return type_url_; } + const LocalInfo::LocalInfo& local_info() const { return local_info_; } + SubscriptionCallbacks& callbacks() const { return callbacks_; } + bool skip_subsequent_node() const { return skip_subsequent_node_; } + +private: + const std::string type_url_; + // callbacks_ is expected to be a WatchMap. + SubscriptionCallbacks& callbacks_; + const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; + Event::TimerPtr init_fetch_timeout_timer_; }; class SubscriptionStateFactory { public: virtual ~SubscriptionStateFactory() = default; - virtual SubscriptionState makeSubscriptionState(const std::string& type_url, - const std::set& resource_names, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - SubscriptionStats& stats) PURE; + virtual std::unique_ptr + makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; }; } // namespace Config diff --git a/source/common/config/subscription_state_sotw.cc b/source/common/config/subscription_state_sotw.cc deleted file mode 100644 index 0816001882c4..000000000000 --- a/source/common/config/subscription_state_sotw.cc +++ /dev/null @@ -1,146 +0,0 @@ -#include "common/common/assert.h" -#include "common/common/hash.h" -#include "common/config/sotw_subscription_state.h" - -namespace Envoy { -namespace Config { - -SubscriptionStateSotw::SubscriptionStateSotw(const std::string& type_url, - SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, - std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout) { - setInitFetchTimeout(dispatcher); -} - -void SubscriptionStateSotw::setInitFetchTimeout(Event::Dispatcher& dispatcher) { - if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { - init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } -} - -void SubscriptionStateSotw::pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; -} - -void SubscriptionStateSotw::resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; -} - -void SubscriptionStateSotw::updateSubscriptionInterest(const std::set& cur_added, - const std::set& cur_removed) { - for (const auto& a : cur_added) { - names_tracked_.insert(a); - } - for (const auto& r : cur_removed) { - names_tracked_.erase(r); - } - update_pending_ = true; -} - -// Not having sent any requests yet counts as an "update pending" since you're supposed to resend -// the entirety of your interest at the start of a stream, even if nothing has changed. -bool SubscriptionStateSotw::subscriptionUpdatePending() const { return update_pending_; } - -UpdateAck SubscriptionStateSotw::handleResponse(const envoy::api::v2::DiscoveryResponse& message) { - // We *always* copy the response's nonce into the next request, even if we're going to make that - // request a NACK by setting error_detail. - UpdateAck ack(message.nonce(), type_url_); - try { - handleGoodResponse(message); - } catch (const EnvoyException& e) { - handleBadResponse(e, ack); - } - return ack; -} - -void SubscriptionStateSotw::handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message) { - disableInitFetchTimeoutTimer(); - callbacks_.onConfigUpdate(message.resources(), message.version_info()); - // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. - last_good_version_info_ = message.version_info(); - ENVOY_LOG(debug, "Config update for {} accepted with {} resources", type_url_, - message.resources().size()); -} - -void SubscriptionStateSotw::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { - // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. - ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); - ack.error_detail_.set_message(e.what()); - disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "Config update for {} rejected: {}", type_url_, e.what()); - callbacks_.onConfigUpdateFailed(&e); -} - -void SubscriptionStateSotw::handleEstablishmentFailure() { - disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); - callbacks_.onConfigUpdateFailed(nullptr); -} - -envoy::api::v2::DiscoveryRequest SubscriptionStateSotw::getNextRequestAckless() { - envoy::api::v2::DiscoveryRequest request; - - std::copy(names_tracked_.begin(), names_tracked_.end(), - Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names())); - - request.set_type_url(type_url_); - request.mutable_node()->MergeFrom(local_info_.node()); - update_pending_ = false; - return request; -} - -envoy::api::v2::DiscoveryRequest -SubscriptionStateSotw::getNextRequestWithAck(const UpdateAck& ack) { - envoy::api::v2::DiscoveryRequest request = getNextRequestAckless(); - request.set_response_nonce(ack.nonce_); - if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { - // Don't needlessly make the field present-but-empty if status is ok. - request.mutable_error_detail()->CopyFrom(ack.error_detail_); - // TODO TODO? if last_good_version_info_ != nullopt - request.set_version_info(last_good_version_info_); - } else { - // TODO TODO in SotW, seems like previous response's version_info should accompany the - // response_nonce.... - // ........ hmmmm..... i think everything is fine if we can add a version_info field to - // UpdateAck (even though unusued for delta) - request.set_version_info(ack.version_info_) - } - return request; -} - -void SubscriptionStateSotw::disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } -} - -class SubscriptionStateSotw : public Logger::Loggable { - const std::string type_url_; - // callbacks_ is expected to be a WatchMap. - SubscriptionCallbacks& callbacks_; - const LocalInfo::LocalInfo& local_info_; - std::chrono::milliseconds init_fetch_timeout_; - Event::TimerPtr init_fetch_timeout_timer_; - std::string last_good_version_info_; // should this be absl::optional to distinguish none and ""? - - bool paused_{}; - bool update_pending_{true}; // Should send a request upon subscription start. - - SubscriptionStateSotw(const SubscriptionStateSotw&) = delete; - SubscriptionStateSotw& operator=(const SubscriptionStateSotw&) = delete; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/update_ack.h b/source/common/config/update_ack.h new file mode 100644 index 000000000000..193e86e436a0 --- /dev/null +++ b/source/common/config/update_ack.h @@ -0,0 +1,21 @@ +#pragma once + +#include + +#include "envoy/api/v2/discovery.pb.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Config { + +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 9c710540930f..ae77d119a584 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -221,8 +221,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either // by Server init or LDS init. - factory_context_.clusterManager().adsMux().pause( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } resume_rds = std::make_unique([this, &noop_init_manager, version_info] { // For new RDS subscriptions created after listener warming up, we don't wait for them to warm // up. @@ -234,8 +236,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // New RDS subscriptions should have been created, now lift the floodgate. // Note in the case of partial acceptance, accepted RDS subscriptions should be started // despite of any error. - factory_context_.clusterManager().adsMux().resume( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } }); } std::vector exception_msgs; diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 41f7ec4faca6..bf55ce5483a0 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -56,8 +56,12 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); - Cleanup eds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + std::unique_ptr maybe_eds_resume; + if (cm_.adsMux()) { + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + maybe_eds_resume = std::make_unique( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + } ENVOY_LOG(info, "cds: add {} cluster(s), remove {} cluster(s)", added_resources.size(), removed_resources.size()); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e5c06231c4ca..d84640b326d7 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -137,12 +137,13 @@ void ClusterManagerInitHelper::maybeFinishInitialize() { // If the first CDS response doesn't have any primary cluster, ClusterLoadAssignment // should be already paused by CdsApiImpl::onConfigUpdate(). Need to check that to // avoid double pause ClusterLoadAssignment. - if (cm_.adsMux().paused(Config::TypeUrl::get().ClusterLoadAssignment)) { + if (cm_.adsMux() == nullptr || + cm_.adsMux()->paused(Config::TypeUrl::get().ClusterLoadAssignment)) { initializeSecondaryClusters(); } else { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume( - [this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); initializeSecondaryClusters(); } } @@ -233,18 +234,30 @@ ClusterManagerImpl::ClusterManagerImpl( } // Now setup ADS if needed, this might rely on a primary cluster. + // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. + // After here, we just have a GrpcMux interface held in ads_mux_, which hides + // whether the backing implementation is delta or SotW. if (bootstrap.dynamic_resources().has_ads_config()) { - ads_mux_ = std::make_unique( - local_info, - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) - ->create(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(bootstrap.dynamic_resources().ads_config()), - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + auto& ads_config = bootstrap.dynamic_resources().ads_config(); + if (ads_config.api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, ads_config, stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), + random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + } else { + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, ads_config, stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + } } else { ads_mux_ = std::make_unique(); } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index cb45bb14aca7..9e70fe4e0ed7 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -226,7 +226,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + std::unique_ptr maybe_eds_resume; + if (cm_.adsMux()) { + cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + maybe_eds_resume = std::make_unique( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); + } bool any_applied = false; // We do all listener removals before adding the new listeners. This allows adding a new listener diff --git a/source/server/server.cc b/source/server/server.cc index 4994d53ccdf8..4598e7e83292 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -508,14 +508,18 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + } ENVOY_LOG(info, "all clusters initialized. initializing init manager"); init_manager.initialize(init_watcher_); // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); + } }); } diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 60101fa4aa22..2786dd2862b6 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -10,28 +10,28 @@ load( envoy_package() -envoy_cc_test( - name = "delta_subscription_impl_test", - srcs = ["delta_subscription_impl_test.cc"], - deps = [ - ":delta_subscription_test_harness", - "//source/common/config:delta_subscription_lib", - "//source/common/stats:isolated_store_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - ], -) +#envoy_cc_test( +# name = "delta_subscription_impl_test", +# srcs = ["delta_subscription_impl_test.cc"], +# deps = [ +# ":delta_subscription_test_harness", +# "//source/common/config:delta_subscription_lib", +# "//source/common/stats:isolated_store_lib", +# "//test/mocks:common_lib", +# "//test/mocks/config:config_mocks", +# "//test/mocks/event:event_mocks", +# "//test/mocks/grpc:grpc_mocks", +# "//test/mocks/local_info:local_info_mocks", +# "//test/mocks/runtime:runtime_mocks", +# "//test/test_common:logging_lib", +# ], +#) envoy_cc_test( name = "delta_subscription_state_test", srcs = ["delta_subscription_state_test.cc"], deps = [ - "//source/common/config:delta_subscription_lib", + "//source/common/config:delta_subscription_state_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", "//test/mocks/config:config_mocks", @@ -71,29 +71,29 @@ envoy_cc_test_library( ], ) -envoy_cc_test( - name = "grpc_mux_impl_test", - srcs = ["grpc_mux_impl_test.cc"], - deps = [ - "//source/common/config:grpc_mux_lib", - "//source/common/config:protobuf_link_hacks", - "//source/common/config:resources_lib", - "//source/common/protobuf", - "//source/common/stats:isolated_store_lib", - "//test/mocks:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - "//test/test_common:logging_lib", - "//test/test_common:simulated_time_system_lib", - "//test/test_common:utility_lib", - "@envoy_api//envoy/api/v2:discovery_cc", - "@envoy_api//envoy/api/v2:eds_cc", - "@envoy_api//envoy/service/discovery/v2:ads_cc", - ], -) +#envoy_cc_test( +# name = "grpc_mux_impl_test", +# srcs = ["grpc_mux_impl_test.cc"], +# deps = [ +# "//source/common/config:grpc_mux_lib", +# "//source/common/config:protobuf_link_hacks", +# "//source/common/config:resources_lib", +# "//source/common/protobuf", +# "//source/common/stats:isolated_store_lib", +# "//test/mocks:common_lib", +# "//test/mocks/config:config_mocks", +# "//test/mocks/event:event_mocks", +# "//test/mocks/grpc:grpc_mocks", +# "//test/mocks/local_info:local_info_mocks", +# "//test/mocks/runtime:runtime_mocks", +# "//test/test_common:logging_lib", +# "//test/test_common:simulated_time_system_lib", +# "//test/test_common:utility_lib", +# "@envoy_api//envoy/api/v2:discovery_cc", +# "@envoy_api//envoy/api/v2:eds_cc", +# "@envoy_api//envoy/service/discovery/v2:ads_cc", +# ], +#) envoy_cc_test( name = "grpc_stream_test", @@ -107,47 +107,47 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "grpc_subscription_impl_test", - srcs = ["grpc_subscription_impl_test.cc"], - deps = [ - ":grpc_subscription_test_harness", - "//source/common/buffer:zero_copy_input_stream_lib", - ], -) +#envoy_cc_test( +# name = "grpc_subscription_impl_test", +# srcs = ["grpc_subscription_impl_test.cc"], +# deps = [ +# ":grpc_subscription_test_harness", +# "//source/common/buffer:zero_copy_input_stream_lib", +# ], +#) -envoy_cc_test_library( - name = "grpc_subscription_test_harness", - hdrs = ["grpc_subscription_test_harness.h"], - deps = [ - ":subscription_test_harness", - "//source/common/common:hash_lib", - "//source/common/config:grpc_subscription_lib", - "//source/common/config:resources_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/upstream:upstream_mocks", - "//test/test_common:utility_lib", - "@envoy_api//envoy/api/v2:eds_cc", - ], -) +#envoy_cc_test_library( +# name = "grpc_subscription_test_harness", +# hdrs = ["grpc_subscription_test_harness.h"], +# deps = [ +# ":subscription_test_harness", +# "//source/common/common:hash_lib", +# "//source/common/config:grpc_subscription_lib", +# "//source/common/config:resources_lib", +# "//test/mocks/config:config_mocks", +# "//test/mocks/event:event_mocks", +# "//test/mocks/grpc:grpc_mocks", +# "//test/mocks/local_info:local_info_mocks", +# "//test/mocks/upstream:upstream_mocks", +# "//test/test_common:utility_lib", +# "@envoy_api//envoy/api/v2:eds_cc", +# ], +#) -envoy_cc_test_library( - name = "delta_subscription_test_harness", - hdrs = ["delta_subscription_test_harness.h"], - deps = [ - ":subscription_test_harness", - "//source/common/config:delta_subscription_lib", - "//source/common/grpc:common_lib", - "//test/mocks/config:config_mocks", - "//test/mocks/event:event_mocks", - "//test/mocks/grpc:grpc_mocks", - "//test/mocks/local_info:local_info_mocks", - "//test/mocks/runtime:runtime_mocks", - ], -) +#envoy_cc_test_library( +# name = "delta_subscription_test_harness", +# hdrs = ["delta_subscription_test_harness.h"], +# deps = [ +# ":subscription_test_harness", +# "//source/common/config:delta_subscription_lib", +# "//source/common/grpc:common_lib", +# "//test/mocks/config:config_mocks", +# "//test/mocks/event:event_mocks", +# "//test/mocks/grpc:grpc_mocks", +# "//test/mocks/local_info:local_info_mocks", +# "//test/mocks/runtime:runtime_mocks", +# ], +#) envoy_cc_test( name = "http_subscription_impl_test", @@ -198,17 +198,17 @@ envoy_cc_test( ], ) -envoy_cc_test( - name = "subscription_impl_test", - srcs = ["subscription_impl_test.cc"], - deps = [ - ":delta_subscription_test_harness", - ":filesystem_subscription_test_harness", - ":grpc_subscription_test_harness", - ":http_subscription_test_harness", - ":subscription_test_harness", - ], -) +#envoy_cc_test( +# name = "subscription_impl_test", +# srcs = ["subscription_impl_test.cc"], +# deps = [ +# ":delta_subscription_test_harness", +# ":filesystem_subscription_test_harness", +# ":grpc_subscription_test_harness", +# ":http_subscription_test_harness", +# ":subscription_test_harness", +# ], +#) envoy_cc_test_library( name = "subscription_test_harness", diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 2a96f2b0953f..cdbc35675b93 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -24,11 +24,16 @@ class DeltaSubscriptionStateTest : public testing::Test { DeltaSubscriptionStateTest() : state_(TypeUrl, callbacks_, local_info_, std::chrono::milliseconds(0U), dispatcher_) { state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name1", "name2", "name3")); } + std::unique_ptr getNextDeltaDiscoveryRequestAckless() { + auto* ptr = static_cast(state_.getNextRequestAckless()); + return std::unique_ptr(ptr); + } + UpdateAck deliverDiscoveryResponse( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, @@ -42,7 +47,7 @@ class DeltaSubscriptionStateTest : public testing::Test { message.set_nonce(nonce.value()); } EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).Times(expect_config_update_call ? 1 : 0); - return state_.handleResponse(message); + return state_.handleResponse(&message); } UpdateAck deliverBadDiscoveryResponse( @@ -55,7 +60,7 @@ class DeltaSubscriptionStateTest : public testing::Test { message.set_system_version_info(version_info); message.set_nonce(nonce); EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).WillOnce(Throw(EnvoyException("oh no"))); - return state_.handleResponse(message); + return state_.handleResponse(&message); } NiceMock> callbacks_; @@ -80,15 +85,15 @@ populateRepeatedResource(std::vector> items) TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { { state_.updateSubscriptionInterest({"name4"}, {"name1"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1")); } { state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1")); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name1")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); } } @@ -104,9 +109,9 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name3")); - EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name3")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); } // Due to how our implementation provides the required behavior tested in RemoveThenAdd, the @@ -120,9 +125,9 @@ TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { TEST_F(DeltaSubscriptionStateTest, AddThenRemove) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name4")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name4")); } // add/remove/add == add. @@ -130,9 +135,9 @@ TEST_F(DeltaSubscriptionStateTest, AddRemoveAdd) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); } // remove/add/remove == remove. @@ -140,9 +145,9 @@ TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); state_.updateSubscriptionInterest({}, {"name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_TRUE(cur_request.resource_names_subscribe().empty()); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name3")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3")); } // Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are @@ -151,18 +156,18 @@ TEST_F(DeltaSubscriptionStateTest, BothAddAndRemove) { state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2", "name3")); } TEST_F(DeltaSubscriptionStateTest, CumulativeUpdates) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({"name5"}, {}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); - EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); } // Verifies that a sequence of good and bad responses from the server all get the appropriate @@ -214,11 +219,11 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); deliverDiscoveryResponse(add1_2, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); - EXPECT_EQ(cur_request.initial_resource_versions().end(), - cur_request.initial_resource_versions().find("name3")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); + EXPECT_EQ(cur_request->initial_resource_versions().end(), + cur_request->initial_resource_versions().find("name3")); } { @@ -229,11 +234,11 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove2.Add() = "name2"; deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); - EXPECT_EQ(cur_request.initial_resource_versions().end(), - cur_request.initial_resource_versions().find("name2")); - EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1B", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ(cur_request->initial_resource_versions().end(), + cur_request->initial_resource_versions().find("name2")); + EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); } { @@ -243,17 +248,17 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name3"; deliverDiscoveryResponse({}, remove1_3, "debugversion3"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_TRUE(cur_request.initial_resource_versions().empty()); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->initial_resource_versions().empty()); } { // ...but our own map should remember our interest. In particular, losing interest in a // resource should cause its name to appear in the next request's resource_names_unsubscribe. state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); } } @@ -272,16 +277,16 @@ TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { state_.updateSubscriptionInterest({"name4"}, {"name1"}); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); // Regarding the resource_names_subscribe field: // name1: do not include: we lost interest. // name2: yes do include: we're interested and we have a version of it. // name3: yes do include: even though we don't have a version of it, we are interested. // name4: yes do include: we are newly interested. (If this wasn't a stream reconnect, only name4 // would belong in this subscribe field). - EXPECT_THAT(cur_request.resource_names_subscribe(), + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name2", "name3", "name4")); - EXPECT_TRUE(cur_request.resource_names_unsubscribe().empty()); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); } // initial_resource_versions should not be present on messages after the first in a stream. @@ -293,10 +298,10 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion1"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); - EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); + EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); } // Then, after updating the resources but not reconnecting the stream, verify that initial // versions are not sent. @@ -309,8 +314,8 @@ TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { {"name3", "version3B"}, {"name4", "version4A"}}); deliverDiscoveryResponse(add_all, {}, "debugversion2"); - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); - EXPECT_TRUE(cur_request.initial_resource_versions().empty()); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->initial_resource_versions().empty()); } } diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 49f102993ce2..c5cd9c4c771a 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -38,7 +38,7 @@ class SubscriptionFactoryTest : public testing::Test { return SubscriptionFactoryImpl(local_info_, dispatcher_, cm_, random_, validation_visitor_, *api_) .subscriptionFromConfigSource(config, Config::TypeUrl::get().ClusterLoadAssignment, - stats_store_, callbacks_, false); + stats_store_, callbacks_); } Upstream::MockClusterManager cm_; diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 1ea5f5fe67e8..d3013870ea94 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -122,7 +122,8 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht bool test_skipped_{true}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // 1) Envoy starts up with no static clusters (other than the CDS-over-gRPC server). // 2) Envoy is told of a cluster via CDS. diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 4a3e3099e0c1..234288262b03 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -25,21 +25,12 @@ MockSubscriptionFactory::MockSubscriptionFactory() { MockSubscriptionFactory::~MockSubscriptionFactory() = default; -MockGrpcMuxWatch::MockGrpcMuxWatch() = default; -MockGrpcMuxWatch::~MockGrpcMuxWatch() { cancel(); } - MockGrpcMux::MockGrpcMux() = default; MockGrpcMux::~MockGrpcMux() = default; MockGrpcStreamCallbacks::MockGrpcStreamCallbacks() = default; MockGrpcStreamCallbacks::~MockGrpcStreamCallbacks() = default; -GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) { - return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); -} - MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { ON_CALL(*this, resourceName(testing::_)) .WillByDefault(testing::Invoke(TestUtility::xdsResourceName)); diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index b579cb56b722..8bb0c8eb0098 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -4,7 +4,6 @@ #include "envoy/config/config_provider_manager.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" #include "common/config/config_provider_impl.h" #include "common/config/resources.h" @@ -45,7 +44,7 @@ template class MockSubscriptionCallbacks : public Subscript class MockSubscription : public Subscription { public: MOCK_METHOD1(start, void(const std::set& resources)); - MOCK_METHOD1(updateResources, void(const std::set& update_to_these_names)); + MOCK_METHOD1(updateResourceInterest, void(const std::set& update_to_these_names)); }; class MockSubscriptionFactory : public SubscriptionFactory { @@ -63,25 +62,17 @@ class MockSubscriptionFactory : public SubscriptionFactory { SubscriptionCallbacks* callbacks_{}; }; -class MockGrpcMuxWatch : public GrpcMuxWatch { -public: - MockGrpcMuxWatch(); - ~MockGrpcMuxWatch() override; - - MOCK_METHOD0(cancel, void()); -}; - class MockGrpcMux : public GrpcMux { public: MockGrpcMux(); ~MockGrpcMux() override; MOCK_METHOD0(start, void()); - MOCK_METHOD3(subscribe_, - GrpcMuxWatch*(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks)); - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks) override; + MOCK_METHOD5(addOrUpdateWatch, + Watch*(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout)); + MOCK_METHOD2(removeWatch, void(const std::string& type_url, Watch* watch)); MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 7d629b106108..8686be781d2f 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -139,7 +139,8 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ON_CALL(*this, adsMux()).WillByDefault(ReturnRef(ads_mux_)); + ads_mux_ = std::make_shared>(); + ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 1b7f0b07b237..49d31c2ff072 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -322,7 +322,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, Config::GrpcMux&()); + MOCK_METHOD0(adsMux, Config::GrpcMuxSharedPtr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); @@ -336,7 +336,7 @@ class MockClusterManager : public ClusterManager { NiceMock tcp_conn_pool_; NiceMock thread_local_cluster_; envoy::api::v2::core::BindConfig bind_config_; - NiceMock ads_mux_; + std::shared_ptr> ads_mux_; NiceMock async_client_manager_; std::string local_cluster_name_; NiceMock cluster_manager_factory_; From 0a8d087cca06ba0b5aa5fc81bd67bb9be72fa490 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 16 Sep 2019 15:40:16 -0400 Subject: [PATCH 03/39] much of ads_integration_test passes, support skip_subsequent_node Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_state.cc | 30 ++-- .../common/config/delta_subscription_state.h | 8 +- source/common/config/grpc_mux_impl.cc | 7 +- source/common/config/grpc_mux_impl.h | 52 +++++-- .../common/config/sotw_subscription_state.cc | 36 +++-- .../common/config/sotw_subscription_state.h | 10 +- source/common/config/subscription_state.cc | 6 +- source/common/config/subscription_state.h | 9 +- test/integration/cds_integration_test.cc | 10 +- test/integration/integration.cc | 133 +++++++++++------- test/integration/integration.h | 16 +-- 11 files changed, 180 insertions(+), 137 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 6a706b689e3c..ea753dc8cf9d 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -8,20 +8,14 @@ namespace Config { DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, - bool skip_subsequent_node) - : SubscriptionState(type_url, callbacks, local_info, init_fetch_timeout, dispatcher, - skip_subsequent_node) {} + Event::Dispatcher& dispatcher) + : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) {} DeltaSubscriptionState::~DeltaSubscriptionState() {} -DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, - bool skip_subsequent_node) - : dispatcher_(dispatcher), local_info_(local_info), - skip_subsequent_node_(skip_subsequent_node) {} +DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher) + : dispatcher_(dispatcher) {} DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() {} @@ -29,8 +23,8 @@ std::unique_ptr DeltaSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) { - return std::make_unique( - type_url, callbacks, local_info_, init_fetch_timeout, dispatcher_, skip_subsequent_node_); + return std::make_unique(type_url, callbacks, init_fetch_timeout, + dispatcher_); } void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, @@ -80,6 +74,11 @@ void DeltaSubscriptionState::handleGoodResponse( disableInitFetchTimeoutTimer(); absl::flat_hash_set names_added_removed; for (const auto& resource : message.resources()) { + if (resource.resource().type_url() != type_url()) { + throw EnvoyException(fmt::format("{} does not match {} type URL in DeltaDiscoveryResponse {}", + resource.resource().type_url(), type_url(), + message.DebugString())); + } if (!names_added_removed.insert(resource.name()).second) { throw EnvoyException( fmt::format("duplicate name {} found among added/updated resources", resource.name())); @@ -137,6 +136,8 @@ void DeltaSubscriptionState::handleEstablishmentFailure() { envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInternal() { auto* request = new envoy::api::v2::DeltaDiscoveryRequest; + request->set_type_url(type_url()); + if (!any_request_sent_yet_in_current_stream_) { any_request_sent_yet_in_current_stream_ = true; // initial_resource_versions "must be populated for first request in a stream". @@ -155,6 +156,7 @@ envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInt } names_removed_.clear(); } + std::copy(names_added_.begin(), names_added_.end(), Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_subscribe())); std::copy(names_removed_.begin(), names_removed_.end(), @@ -162,10 +164,6 @@ envoy::api::v2::DeltaDiscoveryRequest* DeltaSubscriptionState::getNextRequestInt names_added_.clear(); names_removed_.clear(); - request->set_type_url(type_url()); - if (!any_request_sent_yet_in_current_stream_ || !skip_subsequent_node()) { - request->mutable_node()->MergeFrom(local_info().node()); - } return request; } diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 0df0045369f6..3541a63a1481 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -16,9 +16,8 @@ namespace Config { class DeltaSubscriptionState : public SubscriptionState { public: DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, bool skip_subsequent_node); + Event::Dispatcher& dispatcher); ~DeltaSubscriptionState() override; // Update which resources we're interested in subscribing to. @@ -96,8 +95,7 @@ class DeltaSubscriptionState : public SubscriptionState { class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { public: - DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher); ~DeltaSubscriptionStateFactory() override; std::unique_ptr makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, @@ -105,8 +103,6 @@ class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { private: Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; }; } // namespace Config diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index a8943d631813..40a9137134b9 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -10,8 +10,10 @@ namespace Envoy { namespace Config { -GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory) - : subscription_state_factory_(std::move(subscription_state_factory)) {} +GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) + : subscription_state_factory_(std::move(subscription_state_factory)), + skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) {} Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, @@ -105,6 +107,7 @@ void GrpcMuxImpl::handleEstablishedStream() { for (auto& sub : subscriptions_) { sub.second->markStreamFresh(); } + set_any_request_sent_yet_in_current_stream(false); trySendDiscoveryRequests(); } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 042b9b8173f0..8fc4ee5d9a9d 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -25,7 +25,8 @@ namespace Config { // work done so far. Should be changed. class GrpcMuxImpl : public GrpcMux, Logger::Loggable { public: - GrpcMuxImpl(std::unique_ptr subscription_state_factory); + GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, @@ -57,6 +58,14 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { void handleStreamEstablishmentFailure(); void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); void trySendDiscoveryRequests(); + bool skip_subsequent_node() const { return skip_subsequent_node_; } + bool any_request_sent_yet_in_current_stream() const { + return any_request_sent_yet_in_current_stream_; + } + void set_any_request_sent_yet_in_current_stream(bool value) { + any_request_sent_yet_in_current_stream_ = value; + } + const LocalInfo::LocalInfo& local_info() const { return local_info_; } private: Watch* addWatch(const std::string& type_url, const std::set& resources, @@ -99,6 +108,19 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { // Determines the order of initial discovery requests. (Assumes that subscriptions are added // to this GrpcMux in the order of Envoy's dependency ordering). std::list subscription_ordering_; + + // Whether to enable the optimization of only including the node field in the very first + // discovery request in an xDS gRPC stream (really just one: *not* per-type_url). + const bool skip_subsequent_node_; + + // State to help with skip_subsequent_node's logic. + bool any_request_sent_yet_in_current_stream_{}; + + // Used to populate the [Delta]DiscoveryRequest's node field. That field is the same across + // all type_urls, and moreover, the 'skip_subsequent_node' logic needs to operate across all + // the type_urls. So, while the SubscriptionStates populate every other field of these messages, + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; }; class GrpcMuxDelta : public GrpcMuxImpl, @@ -108,8 +130,8 @@ class GrpcMuxDelta : public GrpcMuxImpl, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher, local_info, - skip_subsequent_node)), + : GrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} @@ -125,9 +147,13 @@ class GrpcMuxDelta : public GrpcMuxImpl, protected: void establishGrpcStream() override { grpc_stream_.establishNewStream(); } void sendGrpcMessage(void* msg_proto_ptr) override { - auto* typed_proto_ptr = static_cast(msg_proto_ptr); - grpc_stream_.sendMessage(*typed_proto_ptr); - delete typed_proto_ptr; + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); } void maybeUpdateQueueSizeStat(uint64_t size) override { grpc_stream_.maybeUpdateQueueSizeStat(size); @@ -146,8 +172,8 @@ class GrpcMuxSotw : public GrpcMuxImpl, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher, local_info, - skip_subsequent_node)), + : GrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} @@ -162,9 +188,13 @@ class GrpcMuxSotw : public GrpcMuxImpl, protected: void establishGrpcStream() override { grpc_stream_.establishNewStream(); } void sendGrpcMessage(void* msg_proto_ptr) override { - auto* typed_proto_ptr = static_cast(msg_proto_ptr); - grpc_stream_.sendMessage(*typed_proto_ptr); - delete typed_proto_ptr; + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); } void maybeUpdateQueueSizeStat(uint64_t size) override { grpc_stream_.maybeUpdateQueueSizeStat(size); diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index 94757847ac39..0725f8d549d2 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -8,20 +8,14 @@ namespace Config { SotwSubscriptionState::SotwSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, - bool skip_subsequent_node) - : SubscriptionState(type_url, callbacks, local_info, init_fetch_timeout, dispatcher, - skip_subsequent_node) {} + Event::Dispatcher& dispatcher) + : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) {} SotwSubscriptionState::~SotwSubscriptionState() {} -SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, - bool skip_subsequent_node) - : dispatcher_(dispatcher), local_info_(local_info), - skip_subsequent_node_(skip_subsequent_node) {} +SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher) + : dispatcher_(dispatcher) {} SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() {} @@ -29,8 +23,8 @@ std::unique_ptr SotwSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) { - return std::make_unique( - type_url, callbacks, local_info_, init_fetch_timeout, dispatcher_, skip_subsequent_node_); + return std::make_unique(type_url, callbacks, init_fetch_timeout, + dispatcher_); } void SotwSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, @@ -63,6 +57,12 @@ UpdateAck SotwSubscriptionState::handleResponse(const void* reponse_proto_ptr) { void SotwSubscriptionState::handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message) { disableInitFetchTimeoutTimer(); + for (const auto& resource : message.resources()) { + if (resource.type_url() != type_url()) { + throw EnvoyException(fmt::format("{} does not match {} type URL in DiscoveryResponse {}", + resource.type_url(), type_url(), message.DebugString())); + } + } callbacks().onConfigUpdate(message.resources(), message.version_info()); // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. last_good_version_info_ = message.version_info(); @@ -88,15 +88,14 @@ void SotwSubscriptionState::handleEstablishmentFailure() { envoy::api::v2::DiscoveryRequest* SotwSubscriptionState::getNextRequestInternal() { auto* request = new envoy::api::v2::DiscoveryRequest; + request->set_type_url(type_url()); std::copy(names_tracked_.begin(), names_tracked_.end(), Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names())); - - request->set_type_url(type_url()); - if (!any_request_sent_yet_in_current_stream_ || !skip_subsequent_node()) { - request->mutable_node()->MergeFrom(local_info().node()); + if (last_good_version_info_.has_value()) { + request->set_version_info(last_good_version_info_.value()); } - any_request_sent_yet_in_current_stream_ = true; + update_pending_ = false; return request; } @@ -106,9 +105,6 @@ void* SotwSubscriptionState::getNextRequestAckless() { return getNextRequestInte void* SotwSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { envoy::api::v2::DiscoveryRequest* request = getNextRequestInternal(); request->set_response_nonce(ack.nonce_); - if (last_good_version_info_.has_value()) { - request->set_version_info(last_good_version_info_.value()); - } if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { // Don't needlessly make the field present-but-empty if status is ok. request->mutable_error_detail()->CopyFrom(ack.error_detail_); diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h index c2b3814cbeb7..da9a5a18b1eb 100644 --- a/source/common/config/sotw_subscription_state.h +++ b/source/common/config/sotw_subscription_state.h @@ -16,9 +16,8 @@ namespace Config { class SotwSubscriptionState : public SubscriptionState { public: SotwSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, - std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, - bool skip_subsequent_node); + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher); ~SotwSubscriptionState() override; // Update which resources we're interested in subscribing to. @@ -66,8 +65,7 @@ class SotwSubscriptionState : public SubscriptionState { class SotwSubscriptionStateFactory : public SubscriptionStateFactory { public: - SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher); ~SotwSubscriptionStateFactory() override; std::unique_ptr makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, @@ -75,8 +73,6 @@ class SotwSubscriptionStateFactory : public SubscriptionStateFactory { private: Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; }; } // namespace Config diff --git a/source/common/config/subscription_state.cc b/source/common/config/subscription_state.cc index c68b20c28d09..5ba6f887e143 100644 --- a/source/common/config/subscription_state.cc +++ b/source/common/config/subscription_state.cc @@ -11,11 +11,9 @@ namespace Envoy { namespace Config { SubscriptionState::SubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, bool skip_subsequent_node) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - skip_subsequent_node_(skip_subsequent_node) { + Event::Dispatcher& dispatcher) + : type_url_(type_url), callbacks_(callbacks) { if (init_fetch_timeout.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { ENVOY_LOG(warn, "config: initial fetch timed out for {}", type_url_); diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h index de8147e2992b..67cc807f7b7e 100644 --- a/source/common/config/subscription_state.h +++ b/source/common/config/subscription_state.h @@ -7,7 +7,6 @@ #include "envoy/common/pure.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher.h" -#include "envoy/local_info/local_info.h" #include "common/config/update_ack.h" #include "common/protobuf/protobuf.h" @@ -25,9 +24,7 @@ namespace Config { class SubscriptionState : public Logger::Loggable { public: SubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, - const LocalInfo::LocalInfo& local_info, - std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, - bool skip_subsequent_node); + std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); virtual ~SubscriptionState() = default; // Update which resources we're interested in subscribing to. @@ -59,16 +56,12 @@ class SubscriptionState : public Logger::Loggable { void disableInitFetchTimeoutTimer(); std::string type_url() const { return type_url_; } - const LocalInfo::LocalInfo& local_info() const { return local_info_; } SubscriptionCallbacks& callbacks() const { return callbacks_; } - bool skip_subsequent_node() const { return skip_subsequent_node_; } private: const std::string type_url_; // callbacks_ is expected to be a WatchMap. SubscriptionCallbacks& callbacks_; - const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; Event::TimerPtr init_fetch_timeout_timer_; }; diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index d3013870ea94..2858dc02b421 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -137,7 +137,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) { testRouterHeaderOnlyRequestAndResponse(nullptr, UpstreamIndex1, "/cluster1"); // Tell Envoy that cluster_1 is gone. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of @@ -154,7 +154,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is back. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_}, {cluster1_}, {}, "413"); @@ -177,7 +177,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_2 is here. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_, cluster2_}, {cluster2_}, {}, "42"); // The '3' includes the fake CDS server. @@ -189,7 +189,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is gone. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster2_}, {}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of @@ -202,7 +202,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is back. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_, cluster2_}, {cluster1_}, {}, "413"); diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 595c78f1e24b..f476509a6ad7 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -554,21 +554,37 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, bool expect_node, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { return compareSotwDiscoveryRequest(expected_type_url, expected_version, expected_resource_names, - expect_node, expected_error_code, expected_error_message); + expect_node, expected_error_code, expected_error_substring); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, - expected_resource_names_removed, expected_error_code, - expected_error_message); + expected_resource_names_removed, expect_node, + expected_error_code, expected_error_substring); } } +AssertionResult compareSets(const std::set& set1, const std::set& set2, + absl::string_view name) { + if (set1 == set2) { + return AssertionSuccess(); + } + auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; + for (const auto& x : set1) { + failure << x << ", "; + } + failure << "}\nActual: {"; + for (const auto& x : set2) { + failure << x << ", "; + } + return failure << "}"; +} + AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, bool expect_node, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DiscoveryRequest discovery_request; VERIFY_ASSERTION(xds_stream_->waitForGrpcMessage(*dispatcher_, discovery_request)); @@ -584,27 +600,34 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( return AssertionFailure() << fmt::format("type_url {} does not match expected {}", discovery_request.type_url(), expected_type_url); } - if (!(expected_error_code == discovery_request.error_detail().code())) { - return AssertionFailure() << fmt::format("error_code {} does not match expected {}", - discovery_request.error_detail().code(), - expected_error_code); - } - EXPECT_TRUE( - IsSubstring("", "", expected_error_message, discovery_request.error_detail().message())); - const std::vector resource_names(discovery_request.resource_names().cbegin(), - discovery_request.resource_names().cend()); - if (expected_resource_names != resource_names) { - return AssertionFailure() << fmt::format( - "resources {} do not match expected {} in {}", - fmt::join(resource_names.begin(), resource_names.end(), ","), - fmt::join(expected_resource_names.begin(), expected_resource_names.end(), ","), - discovery_request.DebugString()); + + // Sort to ignore ordering. + std::set expected_sub{expected_resource_names.begin(), + expected_resource_names.end()}; + std::set actual_sub{discovery_request.resource_names().cbegin(), + discovery_request.resource_names().cend()}; + auto sub_result = compareSets(expected_sub, actual_sub, "resource_names"); + if (!sub_result) { + return sub_result; } if (expected_version != discovery_request.version_info()) { return AssertionFailure() << fmt::format("version {} does not match expected {} in {}", discovery_request.version_info(), expected_version, discovery_request.DebugString()); } + if (discovery_request.error_detail().code() != expected_error_code) { + return AssertionFailure() << fmt::format( + "error code {} does not match expected {}. (Error message is {}).", + discovery_request.error_detail().code(), expected_error_code, + discovery_request.error_detail().message()); + } + if (expected_error_code != Grpc::Status::GrpcStatus::Ok && + discovery_request.error_detail().message().find(expected_error_substring) == + std::string::npos) { + return AssertionFailure() << "\"" << expected_error_substring + << "\" is not a substring of actual error message \"" + << discovery_request.error_detail().message() << "\""; + } return AssertionSuccess(); } @@ -612,43 +635,53 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& xds_stream, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + bool expect_node, const Protobuf::int32 expected_error_code, + const std::string& expected_error_substring) { envoy::api::v2::DeltaDiscoveryRequest request; VERIFY_ASSERTION(xds_stream->waitForGrpcMessage(*dispatcher_, request)); - EXPECT_TRUE(request.has_node()); - EXPECT_FALSE(request.node().id().empty()); - EXPECT_FALSE(request.node().cluster().empty()); + if (expect_node) { + EXPECT_TRUE(request.has_node()); + EXPECT_FALSE(request.node().id().empty()); + EXPECT_FALSE(request.node().cluster().empty()); + } else { + EXPECT_FALSE(request.has_node()); + } - if (expected_type_url != request.type_url()) { - return AssertionFailure() << fmt::format("type_url {} does not match expected {}", + if (request.type_url() != expected_type_url) { + return AssertionFailure() << fmt::format("type_url {} does not match expected {}.", request.type_url(), expected_type_url); } - if (!(expected_error_code == request.error_detail().code())) { - return AssertionFailure() << fmt::format("error_code {} does not match expected {}", - request.error_detail().code(), expected_error_code); - } - EXPECT_TRUE(IsSubstring("", "", expected_error_message, request.error_detail().message())); - - const std::vector resource_subscriptions(request.resource_names_subscribe().cbegin(), - request.resource_names_subscribe().cend()); - if (expected_resource_subscriptions != resource_subscriptions) { - return AssertionFailure() << fmt::format( - "newly subscribed resources {} do not match expected {} in {}", - fmt::join(resource_subscriptions.begin(), resource_subscriptions.end(), ","), - fmt::join(expected_resource_subscriptions.begin(), - expected_resource_subscriptions.end(), ","), - request.DebugString()); - } - const std::vector resource_unsubscriptions( - request.resource_names_unsubscribe().cbegin(), request.resource_names_unsubscribe().cend()); - if (expected_resource_unsubscriptions != resource_unsubscriptions) { + // Sort to ignore ordering. + std::set expected_sub{expected_resource_subscriptions.begin(), + expected_resource_subscriptions.end()}; + std::set expected_unsub{expected_resource_unsubscriptions.begin(), + expected_resource_unsubscriptions.end()}; + std::set actual_sub{request.resource_names_subscribe().begin(), + request.resource_names_subscribe().end()}; + std::set actual_unsub{request.resource_names_unsubscribe().begin(), + request.resource_names_unsubscribe().end()}; + auto sub_result = compareSets(expected_sub, actual_sub, "resource_names_subscribe"); + if (!sub_result) { + return sub_result; + } + auto unsub_result = compareSets(expected_unsub, actual_unsub, "resource_names_unsubscribe"); + if (!unsub_result) { + return unsub_result; + } + // (We don't care about response_nonce or initial_resource_versions.) + + if (request.error_detail().code() != expected_error_code) { return AssertionFailure() << fmt::format( - "newly UNsubscribed resources {} do not match expected {} in {}", - fmt::join(resource_unsubscriptions.begin(), resource_unsubscriptions.end(), ","), - fmt::join(expected_resource_unsubscriptions.begin(), - expected_resource_unsubscriptions.end(), ","), - request.DebugString()); + "error code {} does not match expected {}. (Error message is {}).", + request.error_detail().code(), expected_error_code, + request.error_detail().message()); + } + if (expected_error_code != Grpc::Status::GrpcStatus::Ok && + request.error_detail().message().find(expected_error_substring) == std::string::npos) { + return AssertionFailure() << "\"" << expected_error_substring + << "\" is not a substring of actual error message \"" + << request.error_detail().message() << "\""; } return AssertionSuccess(); } diff --git a/test/integration/integration.h b/test/integration/integration.h index 7cfa3b726435..118071c5fbdb 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -227,7 +227,7 @@ class BaseIntegrationTest : Logger::Loggable { const std::vector& expected_resource_names_removed, bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, - const std::string& expected_error_message = ""); + const std::string& expected_error_substring = ""); template void sendDiscoveryResponse(const std::string& type_url, const std::vector& state_of_the_world, const std::vector& added_or_updated, @@ -242,25 +242,25 @@ class BaseIntegrationTest : Logger::Loggable { AssertionResult compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, - const std::vector& expected_resource_unsubscriptions, + const std::vector& expected_resource_unsubscriptions, bool expect_node, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, - const std::string& expected_error_message = "") { + const std::string& expected_error_substring = "") { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_subscriptions, - expected_resource_unsubscriptions, xds_stream_, - expected_error_code, expected_error_message); + expected_resource_unsubscriptions, xds_stream_, expect_node, + expected_error_code, expected_error_substring); } AssertionResult compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& stream, - const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, - const std::string& expected_error_message = ""); + bool expect_node, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, + const std::string& expected_error_substring = ""); AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, - const std::string& expected_error_message = ""); + const std::string& expected_error_substring = ""); template void sendSotwDiscoveryResponse(const std::string& type_url, const std::vector& messages, From 2b5109c97a37eb1c15619a0ad1389fbeb399e0da Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 16 Sep 2019 16:24:04 -0400 Subject: [PATCH 04/39] incorporate PR 7427 init timeout disable Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 2 ++ .../common/config/delta_subscription_state.cc | 2 +- source/common/config/grpc_mux_impl.cc | 6 ++++++ source/common/config/grpc_mux_impl.h | 2 ++ .../common/config/grpc_subscription_impl.cc | 21 ++++++++++++++----- source/common/config/grpc_subscription_impl.h | 2 +- .../common/config/sotw_subscription_state.cc | 4 ++-- source/common/config/subscription_state.h | 2 +- test/mocks/config/mocks.h | 1 + 9 files changed, 32 insertions(+), 10 deletions(-) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index d6ccdfe6e3ef..976993b6a997 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -101,6 +101,8 @@ class GrpcMux { * @return bool whether the API is paused. */ virtual bool paused(const std::string& type_url) const PURE; + + virtual void disableInitFetchTimeoutTimer() PURE; }; using GrpcMuxPtr = std::unique_ptr; diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index ea753dc8cf9d..c155274dee0c 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -129,7 +129,7 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc void DeltaSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url()); + ENVOY_LOG(debug, "gRPC update for {} failed: couldn't connect", type_url()); callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 40a9137134b9..19c4727a731d 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -45,6 +45,12 @@ bool GrpcMuxImpl::paused(const std::string& type_url) const { void GrpcMuxImpl::start() { establishGrpcStream(); } +void GrpcMuxImpl::disableInitFetchTimeoutTimer() { + for (auto& sub : subscriptions_) { + sub.second->disableInitFetchTimeoutTimer(); + } +} + Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) { diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 8fc4ee5d9a9d..ec0fc4e13b4b 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -37,6 +37,7 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; void start() override; + void disableInitFetchTimeoutTimer() override; protected: // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes @@ -220,6 +221,7 @@ class NullGrpcMuxImpl : public GrpcMux { void removeWatch(const std::string&, Watch*) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } + void disableInitFetchTimeoutTimer() override {} }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 49c03a2aba32..94edc03ed7b5 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -48,6 +48,7 @@ void GrpcSubscriptionImpl::onConfigUpdate( stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } + void GrpcSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, @@ -57,18 +58,28 @@ void GrpcSubscriptionImpl::onConfigUpdate( stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } + void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) { - stats_.update_attempt_.inc(); - if (reason == ConfigUpdateFailureReason::FetchTimedout) { - stats_.init_fetch_timeout_.inc(); - } else if (e == nullptr) { + switch (reason) { + case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: stats_.update_failure_.inc(); - } else { + break; + case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + stats_.init_fetch_timeout_.inc(); + context_->disableInitFetchTimeoutTimer(); + break; + case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: + // We expect Envoy exception to be thrown when update is rejected. + ASSERT(e != nullptr); + context_->disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); + break; } + stats_.update_attempt_.inc(); callbacks_.onConfigUpdateFailed(reason, e); } + std::string GrpcSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { return callbacks_.resourceName(resource); } diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 598a25577a93..8dba696fb018 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -54,7 +54,7 @@ class GrpcSubscriptionImpl : public Subscription, public SubscriptionCallbacks { GrpcMuxSharedPtr getContextForTest() { return context_; } private: - GrpcMuxSharedPtr context_; + GrpcMuxSharedPtr const context_; const std::string type_url_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index 0725f8d549d2..38804327e72f 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -75,13 +75,13 @@ void SotwSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "Config update for {} rejected: {}", type_url(), e.what()); + ENVOY_LOG(warn, "gRPC state-of-the-world config for {} rejected: {}", type_url(), e.what()); callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } void SotwSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url()); + ENVOY_LOG(debug, "gRPC update for {} failed: couldn't connect", type_url()); callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h index 67cc807f7b7e..5dec01f79f92 100644 --- a/source/common/config/subscription_state.h +++ b/source/common/config/subscription_state.h @@ -52,9 +52,9 @@ class SubscriptionState : public Logger::Loggable { // pointer actually is. virtual void* getNextRequestWithAck(const UpdateAck& ack) PURE; -protected: void disableInitFetchTimeoutTimer(); +protected: std::string type_url() const { return type_url_; } SubscriptionCallbacks& callbacks() const { return callbacks_; } diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 8bb0c8eb0098..bc62ec56e7c2 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -76,6 +76,7 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); + MOCK_METHOD0(disableInitFetchTimeoutTimer, void()); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { From 282d3469a7589b47ef6612e165b9fbbc0bab728a Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 24 Sep 2019 16:07:40 -0400 Subject: [PATCH 05/39] WatchMap changes from PR 8350 Signed-off-by: Fred Douglas --- source/common/config/watch_map.cc | 30 ++++++++--- source/common/config/watch_map.h | 4 ++ test/common/config/watch_map_test.cc | 75 ++++++++++++++++++++-------- 3 files changed, 83 insertions(+), 26 deletions(-) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index f351dd962a2f..6fa1c07924d6 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -54,8 +54,17 @@ absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& res void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { if (watches_.empty()) { - ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); - return; + if (resources.empty()) { + // We have no watches, and the update contained no resources. This can happen when Envoy + // unregisters from a resource that's removed from the server as well. For example, + // a deleted cluster triggers un-watching the ClusterLoadAssignment watch, and at the + // same time the xDS server sends an empty list of ClusterLoadAssignment resources. + return; + } else { + // We have no watches, but the update contained resources. This should not happen. + ENVOY_LOG(warn, "Rejecting non-empty update for unwatched type URL"); + throw EnvoyException("Rejecting non-empty update for unwatched type URL"); + } } SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; @@ -75,12 +84,21 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField for (auto& watch : watches_) { const auto this_watch_updates = per_watch_updates.find(watch); if (this_watch_updates == per_watch_updates.end()) { - // This update included no resources this watch cares about - so we do an empty - // onConfigUpdate(), to notify the watch that its resources - if they existed before this - - // were dropped. - watch->callbacks_.onConfigUpdate({}, version_info); + // This update included no resources this watch cares about. + // 1) If there is only a single, wildcard watch (i.e. Cluster or Listener), always call + // its onConfigUpdate even if just a no-op, to properly maintain state-of-the-world + // semantics and the update_empty stat. + // 2) If this watch previously had some resources, it means this update is removing all + // of this watch's resources, so the watch must be informed with an onConfigUpdate. + // 3) Otherwise, we can skip onConfigUpdate for this watch. + if ((watches_.size() == 1 && wildcard_watches_.size() == 1) || + !watch->state_of_the_world_empty_) { + watch->callbacks_.onConfigUpdate({}, version_info); + } + watch->state_of_the_world_empty_ = true; } else { watch->callbacks_.onConfigUpdate(this_watch_updates->second, version_info); + watch->state_of_the_world_empty_ = false; } } } diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 5e75e5e88dd7..9df852c7f712 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -26,6 +26,10 @@ struct Watch { Watch(SubscriptionCallbacks& callbacks) : callbacks_(callbacks) {} SubscriptionCallbacks& callbacks_; std::set resource_names_; // must be sorted set, for set_difference. + // Needed only for state-of-the-world. + // Whether the most recent update contained any resources this watch cares about. + // If true, a new update that also contains no resources can skip this watch. + bool state_of_the_world_empty_{true}; }; // NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc index 543298557fab..b41b1ec8bcb2 100644 --- a/test/common/config/watch_map_test.cc +++ b/test/common/config/watch_map_test.cc @@ -67,13 +67,17 @@ void expectDeltaAndSotwUpdate( })); } -// Sometimes we want to verify that a delta onConfigUpdate simply doesn't happen. However, for SotW, -// every update triggers all onConfigUpdate()s, so we should still expect empty calls for that. -void expectNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { +void expectNoUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)).Times(0); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); +} + +void expectEmptySotwNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { EXPECT_CALL(callbacks, onConfigUpdate(_, version)) - .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& gotten_resources, - const std::string&) { EXPECT_EQ(0, gotten_resources.size()); })); - EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)).Times(0); + .WillOnce(Invoke( + [](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { EXPECT_EQ(gotten_resources.size(), 0); })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); } Protobuf::RepeatedPtrField @@ -105,7 +109,7 @@ void doDeltaAndSotwUpdate(SubscriptionCallbacks& watch_map, for (const auto& n : removed_names) { *removed_names_proto.Add() = n; } - watch_map.onConfigUpdate(delta_resources, removed_names_proto, "version1"); + watch_map.onConfigUpdate(delta_resources, removed_names_proto, version); } // Tests the simple case of a single watch. Checks that the watch will not be told of updates to @@ -195,9 +199,9 @@ TEST(WatchMapTest, Overlap) { EXPECT_TRUE(added_removed.removed_.empty()); watch_map.updateWatchInterest(watch2, {"dummy"}); - // First watch receives update. + // *Only* first watch receives update. expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); - expectNoDeltaUpdate(callbacks2, "version1"); + expectNoUpdate(callbacks2, "version1"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); } // Second watch becomes interested. @@ -218,9 +222,13 @@ TEST(WatchMapTest, Overlap) { EXPECT_TRUE(added_removed.added_.empty()); // nothing happens EXPECT_TRUE(added_removed.removed_.empty()); - // *Only* second watch receives update. - expectNoDeltaUpdate(callbacks1, "version3"); + // Both watches receive the update. For watch2, this is obviously desired. expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version3"); + // For watch1, it's more subtle: the WatchMap sees that this update has no + // resources watch1 cares about, but also knows that watch1 previously had + // some resources. So, it must inform watch1 that it now has no resources. + // (SotW only: delta's explicit removals avoid the need for this guessing.) + expectEmptySotwNoDeltaUpdate(callbacks1, "version3"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version3"); } // Second watch loses interest. @@ -257,9 +265,9 @@ TEST(WatchMapTest, AddRemoveAdd) { EXPECT_TRUE(added_removed.removed_.empty()); watch_map.updateWatchInterest(watch2, {"dummy"}); - // First watch receives update. + // *Only* first watch receives update. expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); - expectNoDeltaUpdate(callbacks2, "version1"); + expectNoUpdate(callbacks2, "version1"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); } // First watch loses interest. @@ -278,9 +286,13 @@ TEST(WatchMapTest, AddRemoveAdd) { EXPECT_EQ(std::set({"alice"}), added_removed.added_); // add to subscription EXPECT_TRUE(added_removed.removed_.empty()); - // *Only* second watch receives update. - expectNoDeltaUpdate(callbacks1, "version2"); + // Both watches receive the update. For watch2, this is obviously desired. expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + // For watch1, it's more subtle: the WatchMap sees that this update has no + // resources watch1 cares about, but also knows that watch1 previously had + // some resources. So, it must inform watch1 that it now has no resources. + // (SotW only: delta's explicit removals avoid the need for this guessing.) + expectEmptySotwNoDeltaUpdate(callbacks1, "version2"); doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); } } @@ -302,21 +314,31 @@ TEST(WatchMapTest, UninterestingUpdate) { bob.set_cluster_name("bob"); bob_update.Add()->PackFrom(bob); - expectNoDeltaUpdate(callbacks, "version1"); + + // We are watching for alice, and an update for just bob arrives. It should be ignored. + expectNoUpdate(callbacks, "version1"); doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version1"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); + // The server sends an update adding alice and removing bob. We pay attention only to alice. expectDeltaAndSotwUpdate(callbacks, {alice}, {}, "version2"); doDeltaAndSotwUpdate(watch_map, alice_update, {}, "version2"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); - expectNoDeltaUpdate(callbacks, "version3"); - doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version3"); + // The server sends an update removing alice and adding bob. We pay attention only to alice. + expectDeltaAndSotwUpdate(callbacks, {}, {"alice"}, "version3"); + doDeltaAndSotwUpdate(watch_map, bob_update, {"alice"}, "version3"); + ::testing::Mock::VerifyAndClearExpectations(&callbacks); // Clean removal of the watch: first update to "interested in nothing", then remove. watch_map.updateWatchInterest(watch, {}); watch_map.removeWatch(watch); - // Finally, test that calling onConfigUpdate on a map with no watches doesn't break. - doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"); + // Finally, test that calling onConfigUpdate on a map with no watches (which should not + // happen) throws an exception. + EXPECT_THROW_WITH_MESSAGE( + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"), + EnvoyException, "Rejecting non-empty update for unwatched type URL"); } // Tests that a watch that specifies no particular resource interest is treated as interested in @@ -366,6 +388,19 @@ TEST(WatchMapTest, DeltaOnConfigUpdate) { watch_map.updateWatchInterest(watch2, {"updated", "removed"}); watch_map.updateWatchInterest(watch3, {"removed"}); + // First, create the "removed" resource. We want to test SotW being handed an empty + // onConfigUpdate. But, if SotW holds no resources, then an update with nothing it cares about + // will just not trigger any onConfigUpdate at all. + { + Protobuf::RepeatedPtrField prepare_removed; + envoy::api::v2::ClusterLoadAssignment will_be_removed_later; + will_be_removed_later.set_cluster_name("removed"); + prepare_removed.Add()->PackFrom(will_be_removed_later); + expectDeltaAndSotwUpdate(callbacks2, {will_be_removed_later}, {}, "version0"); + expectDeltaAndSotwUpdate(callbacks3, {will_be_removed_later}, {}, "version0"); + doDeltaAndSotwUpdate(watch_map, prepare_removed, {}, "version0"); + } + Protobuf::RepeatedPtrField update; envoy::api::v2::ClusterLoadAssignment updated; updated.set_cluster_name("updated"); From bce3da060e449b654a20faa244fa5a6924f5f960 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 24 Sep 2019 16:17:57 -0400 Subject: [PATCH 06/39] PR 8334 Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 7 ++++--- api/envoy/api/v3alpha/discovery.proto | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 2d04a24817f9..a8423f5f904f 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -44,8 +44,9 @@ message DiscoveryRequest { // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above // discussion on version_info and the DiscoveryResponse nonce comment. This - // may be empty if no nonce is available, e.g. at startup or for non-stream - // xDS implementations. + // may be empty only if 1) this is a non-persistent-stream xDS such as HTTP, + // or 2) the client has not yet accepted an update in this xDS stream (unlike + // delta, where it is populated only for new explicit ACKs). string response_nonce = 5; // This is populated when the previous :ref:`DiscoveryResponse ` @@ -176,7 +177,7 @@ message DeltaDiscoveryRequest { // When the DeltaDiscoveryRequest is a ACK or NACK message in response // to a previous DeltaDiscoveryResponse, the response_nonce must be the // nonce in the DeltaDiscoveryResponse. - // Otherwise response_nonce must be omitted. + // Otherwise (unlike in DiscoveryRequest) response_nonce must be omitted. string response_nonce = 6; // This is populated when the previous :ref:`DiscoveryResponse ` diff --git a/api/envoy/api/v3alpha/discovery.proto b/api/envoy/api/v3alpha/discovery.proto index 1547b030062f..c03bb57cebd5 100644 --- a/api/envoy/api/v3alpha/discovery.proto +++ b/api/envoy/api/v3alpha/discovery.proto @@ -44,8 +44,9 @@ message DiscoveryRequest { // nonce corresponding to DiscoveryResponse being ACK/NACKed. See above // discussion on version_info and the DiscoveryResponse nonce comment. This - // may be empty if no nonce is available, e.g. at startup or for non-stream - // xDS implementations. + // may be empty only if 1) this is a non-persistent-stream xDS such as HTTP, + // or 2) the client has not yet accepted an update in this xDS stream (unlike + // delta, where it is populated only for new explicit ACKs). string response_nonce = 5; // This is populated when the previous :ref:`DiscoveryResponse ` @@ -176,7 +177,7 @@ message DeltaDiscoveryRequest { // When the DeltaDiscoveryRequest is a ACK or NACK message in response // to a previous DeltaDiscoveryResponse, the response_nonce must be the // nonce in the DeltaDiscoveryResponse. - // Otherwise response_nonce must be omitted. + // Otherwise (unlike in DiscoveryRequest) response_nonce must be omitted. string response_nonce = 6; // This is populated when the previous :ref:`DiscoveryResponse ` From df64fc904f2409e82d7e13b58873cec7bb1829ab Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 24 Sep 2019 16:19:14 -0400 Subject: [PATCH 07/39] add to the mock gRPC version of the protobuf matchers Signed-off-by: Fred Douglas --- test/mocks/grpc/mocks.h | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/test/mocks/grpc/mocks.h b/test/mocks/grpc/mocks.h index a2393b32444a..db94dc44fbff 100644 --- a/test/mocks/grpc/mocks.h +++ b/test/mocks/grpc/mocks.h @@ -124,14 +124,35 @@ MATCHER_P2(ProtoBufferEqIgnoringField, expected, ignored_field, "") { *result_listener << "\nParse of buffer failed\n"; return false; } - auto equal = ::Envoy::TestUtility::protoEqualIgnoringField(proto, expected, ignored_field); + const bool equal = ::Envoy::TestUtility::protoEqualIgnoringField(proto, expected, ignored_field); if (!equal) { std::string but_ignoring = absl::StrCat("(but ignoring ", ignored_field, ")"); + *result_listener << "\n" + << TestUtility::addLeftAndRightPadding("Expected proto:") << "\n" + << TestUtility::addLeftAndRightPadding(but_ignoring) << "\n" + << expected.DebugString() + << TestUtility::addLeftAndRightPadding("is not equal to actual proto:") << "\n" + << proto.DebugString() + << TestUtility::addLeftAndRightPadding("") // line full of padding + << "\n"; + } + return equal; +} + +MATCHER_P(ProtoBufferEqIgnoreRepeatedFieldOrdering, expected, "") { + typename std::remove_const::type proto; + if (!proto.ParseFromArray(static_cast(arg->linearize(arg->length())), arg->length())) { + *result_listener << "\nParse of buffer failed\n"; + return false; + } + const bool equal = + ::Envoy::TestUtility::protoEqual(proto, expected, /*ignore_repeated_field_ordering=*/true); + if (!equal) { *result_listener << "\n" << "=======================Expected proto:===========================\n" - << expected.DebugString() << " " << but_ignoring + << expected.DebugString() << "------------------is not equal to actual proto:------------------\n" - << proto.DebugString() << " " << but_ignoring + << proto.DebugString() << "=================================================================\n"; } return equal; From 9d79ab871acaf19f6ec3293e4921d98cd87db5b0 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 24 Sep 2019 17:41:52 -0400 Subject: [PATCH 08/39] all config unit tests pass, found some moderate differences from old SotW behavior Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_state.cc | 11 +- source/common/config/grpc_mux_impl.cc | 4 +- source/common/config/grpc_mux_impl.h | 5 + .../common/config/grpc_subscription_impl.cc | 6 +- .../common/config/sotw_subscription_state.cc | 25 +- .../common/config/sotw_subscription_state.h | 9 +- .../common/upstream/cluster_manager_impl.cc | 4 +- test/common/config/BUILD | 179 +++++----- .../config/delta_subscription_impl_test.cc | 20 +- .../config/delta_subscription_state_test.cc | 3 +- .../config/delta_subscription_test_harness.h | 16 +- test/common/config/grpc_mux_impl_test.cc | 315 ++++++++++-------- .../config/grpc_subscription_impl_test.cc | 20 +- .../config/grpc_subscription_test_harness.h | 43 +-- .../config/subscription_factory_impl_test.cc | 1 - test/common/config/subscription_impl_test.cc | 6 +- test/common/config/watch_map_test.cc | 14 +- .../upstream/cluster_manager_impl_test.cc | 2 +- test/integration/ads_integration.h | 8 +- test/integration/ads_integration_test.cc | 39 ++- test/integration/integration.h | 5 +- test/integration/rtds_integration_test.cc | 3 +- .../scoped_rds_integration_test.cc | 8 +- 23 files changed, 403 insertions(+), 343 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index c155274dee0c..14a43026e587 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -10,7 +10,9 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) {} + : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) { + std::cerr << "hi! constructing DELTA SubscriptionState" << std::endl; +} DeltaSubscriptionState::~DeltaSubscriptionState() {} @@ -74,11 +76,6 @@ void DeltaSubscriptionState::handleGoodResponse( disableInitFetchTimeoutTimer(); absl::flat_hash_set names_added_removed; for (const auto& resource : message.resources()) { - if (resource.resource().type_url() != type_url()) { - throw EnvoyException(fmt::format("{} does not match {} type URL in DeltaDiscoveryResponse {}", - resource.resource().type_url(), type_url(), - message.DebugString())); - } if (!names_added_removed.insert(resource.name()).second) { throw EnvoyException( fmt::format("duplicate name {} found among added/updated resources", resource.name())); @@ -128,8 +125,6 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc } void DeltaSubscriptionState::handleEstablishmentFailure() { - disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed: couldn't connect", type_url()); callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 19c4727a731d..2b874384b7e3 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -13,7 +13,9 @@ namespace Config { GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) : subscription_state_factory_(std::move(subscription_state_factory)), - skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) {} + skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) { + Config::Utility::checkLocalInfo("ads", local_info); +} Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index ec0fc4e13b4b..e201753e168c 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -185,6 +185,10 @@ class GrpcMuxSotw : public GrpcMuxImpl, void onDiscoveryResponse(std::unique_ptr&& message) override { genericHandleResponse(message->type_url(), message.get()); } + GrpcStream& + grpcStreamForTest() { + return grpc_stream_; + } protected: void establishGrpcStream() override { grpc_stream_.establishNewStream(); } @@ -194,6 +198,7 @@ class GrpcMuxSotw : public GrpcMuxImpl, if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { typed_proto->mutable_node()->MergeFrom(local_info().node()); } + grpc_stream_.sendMessage(*typed_proto); set_any_request_sent_yet_in_current_stream(true); } diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 94edc03ed7b5..862262b2c73d 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -63,21 +63,25 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason const EnvoyException* e) { switch (reason) { case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: + // This is a gRPC-stream-level establishment failure, not an xDS-protocol-level failure. + // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. + // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. stats_.update_failure_.inc(); break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: stats_.init_fetch_timeout_.inc(); context_->disableInitFetchTimeoutTimer(); + callbacks_.onConfigUpdateFailed(reason, e); break; case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: // We expect Envoy exception to be thrown when update is rejected. ASSERT(e != nullptr); context_->disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); + callbacks_.onConfigUpdateFailed(reason, e); break; } stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(reason, e); } std::string GrpcSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index 38804327e72f..0bf43a232562 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -10,7 +10,9 @@ SotwSubscriptionState::SotwSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) {} + : SubscriptionState(type_url, callbacks, init_fetch_timeout, dispatcher) { + std::cerr << "hi! constructing SOTW SubscriptionState" << std::endl; +} SotwSubscriptionState::~SotwSubscriptionState() {} @@ -35,13 +37,21 @@ void SotwSubscriptionState::updateSubscriptionInterest(const std::set(reponse_proto_ptr); // We *always* copy the response's nonce into the next request, even if we're going to make that @@ -59,13 +69,15 @@ void SotwSubscriptionState::handleGoodResponse(const envoy::api::v2::DiscoveryRe disableInitFetchTimeoutTimer(); for (const auto& resource : message.resources()) { if (resource.type_url() != type_url()) { - throw EnvoyException(fmt::format("{} does not match {} type URL in DiscoveryResponse {}", + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DiscoveryResponse {}", resource.type_url(), type_url(), message.DebugString())); } } callbacks().onConfigUpdate(message.resources(), message.version_info()); // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. last_good_version_info_ = message.version_info(); + last_good_nonce_ = message.nonce(); ENVOY_LOG(debug, "Config update for {} accepted with {} resources", type_url(), message.resources().size()); } @@ -80,8 +92,6 @@ void SotwSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck } void SotwSubscriptionState::handleEstablishmentFailure() { - disableInitFetchTimeoutTimer(); - ENVOY_LOG(debug, "gRPC update for {} failed: couldn't connect", type_url()); callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } @@ -95,6 +105,11 @@ envoy::api::v2::DiscoveryRequest* SotwSubscriptionState::getNextRequestInternal( if (last_good_version_info_.has_value()) { request->set_version_info(last_good_version_info_.value()); } + // Default response_nonce to the last known good one. If we are being called by + // getNextRequestWithAck(), this value will be overwritten. + if (last_good_nonce_.has_value()) { + request->set_response_nonce(last_good_nonce_.value()); + } update_pending_ = false; return request; diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h index da9a5a18b1eb..7fd1b0e2f99b 100644 --- a/source/common/config/sotw_subscription_state.h +++ b/source/common/config/sotw_subscription_state.h @@ -27,7 +27,7 @@ class SotwSubscriptionState : public SubscriptionState { // Whether there was a change in our subscription interest we have yet to inform the server of. bool subscriptionUpdatePending() const override; - void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } + void markStreamFresh() override; // message is expected to be a envoy::api::v2::DiscoveryResponse. UpdateAck handleResponse(const void* reponse_proto_ptr) override; @@ -49,11 +49,14 @@ class SotwSubscriptionState : public SubscriptionState { void handleGoodResponse(const envoy::api::v2::DiscoveryResponse& message); void handleBadResponse(const EnvoyException& e, UpdateAck& ack); - // The version_info value most recently received in a DiscoveryResponse that was accepted. + // The version_info carried by the last accepted DiscoveryResponse. // Remains empty until one is accepted. absl::optional last_good_version_info_; + // The nonce carried by the last accepted DiscoveryResponse. + // Remains empty until one is accepted. + // Used when it's time to make a spontaneous (i.e. not primarily meant as an ACK) request. + absl::optional last_good_nonce_; - bool any_request_sent_yet_in_current_stream_{}; // Starts true because we should send a request upon subscription start. bool update_pending_{true}; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index d84640b326d7..83cbe17f1616 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -247,7 +247,7 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + ads_config.set_node_on_first_message_only()); } else { ads_mux_ = std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, ads_config, stats) @@ -256,7 +256,7 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), random_, stats_, Envoy::Config::Utility::parseRateLimitSettings(ads_config), local_info, - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + ads_config.set_node_on_first_message_only()); } } else { ads_mux_ = std::make_unique(); diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 2786dd2862b6..046ed95d2f86 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -10,22 +10,22 @@ load( envoy_package() -#envoy_cc_test( -# name = "delta_subscription_impl_test", -# srcs = ["delta_subscription_impl_test.cc"], -# deps = [ -# ":delta_subscription_test_harness", -# "//source/common/config:delta_subscription_lib", -# "//source/common/stats:isolated_store_lib", -# "//test/mocks:common_lib", -# "//test/mocks/config:config_mocks", -# "//test/mocks/event:event_mocks", -# "//test/mocks/grpc:grpc_mocks", -# "//test/mocks/local_info:local_info_mocks", -# "//test/mocks/runtime:runtime_mocks", -# "//test/test_common:logging_lib", -# ], -#) +envoy_cc_test( + name = "delta_subscription_impl_test", + srcs = ["delta_subscription_impl_test.cc"], + deps = [ + ":delta_subscription_test_harness", + "//source/common/config:grpc_subscription_lib", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + ], +) envoy_cc_test( name = "delta_subscription_state_test", @@ -71,29 +71,30 @@ envoy_cc_test_library( ], ) -#envoy_cc_test( -# name = "grpc_mux_impl_test", -# srcs = ["grpc_mux_impl_test.cc"], -# deps = [ -# "//source/common/config:grpc_mux_lib", -# "//source/common/config:protobuf_link_hacks", -# "//source/common/config:resources_lib", -# "//source/common/protobuf", -# "//source/common/stats:isolated_store_lib", -# "//test/mocks:common_lib", -# "//test/mocks/config:config_mocks", -# "//test/mocks/event:event_mocks", -# "//test/mocks/grpc:grpc_mocks", -# "//test/mocks/local_info:local_info_mocks", -# "//test/mocks/runtime:runtime_mocks", -# "//test/test_common:logging_lib", -# "//test/test_common:simulated_time_system_lib", -# "//test/test_common:utility_lib", -# "@envoy_api//envoy/api/v2:discovery_cc", -# "@envoy_api//envoy/api/v2:eds_cc", -# "@envoy_api//envoy/service/discovery/v2:ads_cc", -# ], -#) +envoy_cc_test( + name = "grpc_mux_impl_test", + srcs = ["grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:grpc_mux_lib", + "//source/common/config:grpc_subscription_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resources_lib", + "//source/common/protobuf", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/api/v2:eds_cc", + "@envoy_api//envoy/service/discovery/v2:ads_cc", + ], +) envoy_cc_test( name = "grpc_stream_test", @@ -107,47 +108,47 @@ envoy_cc_test( ], ) -#envoy_cc_test( -# name = "grpc_subscription_impl_test", -# srcs = ["grpc_subscription_impl_test.cc"], -# deps = [ -# ":grpc_subscription_test_harness", -# "//source/common/buffer:zero_copy_input_stream_lib", -# ], -#) +envoy_cc_test( + name = "grpc_subscription_impl_test", + srcs = ["grpc_subscription_impl_test.cc"], + deps = [ + ":grpc_subscription_test_harness", + "//source/common/buffer:zero_copy_input_stream_lib", + ], +) -#envoy_cc_test_library( -# name = "grpc_subscription_test_harness", -# hdrs = ["grpc_subscription_test_harness.h"], -# deps = [ -# ":subscription_test_harness", -# "//source/common/common:hash_lib", -# "//source/common/config:grpc_subscription_lib", -# "//source/common/config:resources_lib", -# "//test/mocks/config:config_mocks", -# "//test/mocks/event:event_mocks", -# "//test/mocks/grpc:grpc_mocks", -# "//test/mocks/local_info:local_info_mocks", -# "//test/mocks/upstream:upstream_mocks", -# "//test/test_common:utility_lib", -# "@envoy_api//envoy/api/v2:eds_cc", -# ], -#) +envoy_cc_test_library( + name = "grpc_subscription_test_harness", + hdrs = ["grpc_subscription_test_harness.h"], + deps = [ + ":subscription_test_harness", + "//source/common/common:hash_lib", + "//source/common/config:grpc_subscription_lib", + "//source/common/config:resources_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/upstream:upstream_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:eds_cc", + ], +) -#envoy_cc_test_library( -# name = "delta_subscription_test_harness", -# hdrs = ["delta_subscription_test_harness.h"], -# deps = [ -# ":subscription_test_harness", -# "//source/common/config:delta_subscription_lib", -# "//source/common/grpc:common_lib", -# "//test/mocks/config:config_mocks", -# "//test/mocks/event:event_mocks", -# "//test/mocks/grpc:grpc_mocks", -# "//test/mocks/local_info:local_info_mocks", -# "//test/mocks/runtime:runtime_mocks", -# ], -#) +envoy_cc_test_library( + name = "delta_subscription_test_harness", + hdrs = ["delta_subscription_test_harness.h"], + deps = [ + ":subscription_test_harness", + "//source/common/config:grpc_subscription_lib", + "//source/common/grpc:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + ], +) envoy_cc_test( name = "http_subscription_impl_test", @@ -198,17 +199,17 @@ envoy_cc_test( ], ) -#envoy_cc_test( -# name = "subscription_impl_test", -# srcs = ["subscription_impl_test.cc"], -# deps = [ -# ":delta_subscription_test_harness", -# ":filesystem_subscription_test_harness", -# ":grpc_subscription_test_harness", -# ":http_subscription_test_harness", -# ":subscription_test_harness", -# ], -#) +envoy_cc_test( + name = "subscription_impl_test", + srcs = ["subscription_impl_test.cc"], + deps = [ + ":delta_subscription_test_harness", + ":filesystem_subscription_test_harness", + ":grpc_subscription_test_harness", + ":http_subscription_test_harness", + ":subscription_test_harness", + ], +) envoy_cc_test_library( name = "subscription_test_harness", diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index f8127f93cfdf..992394ab11be 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -11,7 +11,7 @@ class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public te DeltaSubscriptionImplTest() = default; // We need to destroy the subscription before the test's destruction, because the subscription's - // destructor removes its watch from the NewGrpcMuxImpl, and that removal process involves + // destructor removes its watch from the GrpcMuxDelta, and that removal process involves // some things held by the test fixture. void TearDown() override { doSubscriptionTearDown(); } }; @@ -71,8 +71,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) - ->onDiscoveryResponse(std::move(message)); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). @@ -85,8 +85,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) - ->onDiscoveryResponse(std::move(message)); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). @@ -99,8 +99,8 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) - ->onDiscoveryResponse(std::move(message)); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)) @@ -135,11 +135,11 @@ TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { const Protobuf::MethodDescriptor* method_descriptor = Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); - std::shared_ptr xds_context = std::make_shared( + std::shared_ptr xds_context = std::make_shared( std::unique_ptr(async_client), dispatcher, *method_descriptor, random, - stats_store, rate_limit_settings, local_info); + stats_store, rate_limit_settings, local_info, true); - std::unique_ptr subscription = std::make_unique( + auto subscription = std::make_unique( xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, stats, std::chrono::milliseconds(12345), false); diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index cdbc35675b93..0614eecb0229 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -22,7 +22,7 @@ const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; class DeltaSubscriptionStateTest : public testing::Test { protected: DeltaSubscriptionStateTest() - : state_(TypeUrl, callbacks_, local_info_, std::chrono::milliseconds(0U), dispatcher_) { + : state_(TypeUrl, callbacks_, std::chrono::milliseconds(0U), dispatcher_) { state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); auto cur_request = getNextDeltaDiscoveryRequestAckless(); EXPECT_THAT(cur_request->resource_names_subscribe(), @@ -64,7 +64,6 @@ class DeltaSubscriptionStateTest : public testing::Test { } NiceMock> callbacks_; - NiceMock local_info_; NiceMock dispatcher_; // We start out interested in three resources: name1, name2, and name3. DeltaSubscriptionState state_; diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 35e8a4bbef06..3c0b88d51066 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -2,7 +2,7 @@ #include -#include "common/config/delta_subscription_impl.h" +#include "common/config/grpc_subscription_impl.h" #include "common/grpc/common.h" #include "test/common/config/subscription_test_harness.h" @@ -34,10 +34,10 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); - xds_context_ = std::make_shared( + xds_context_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, - random_, stats_store_, rate_limit_settings_, local_info_); - subscription_ = std::make_unique( + random_, stats_store_, rate_limit_settings_, local_info_, false); + subscription_ = std::make_unique( xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout, false); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); @@ -148,8 +148,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - static_cast(subscription_->getContextForTest().get()) - ->onDiscoveryResponse(std::move(response)); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } @@ -189,8 +189,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; - std::shared_ptr xds_context_; - std::unique_ptr subscription_; + std::shared_ptr xds_context_; + std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; Envoy::Config::RateLimitSettings rate_limit_settings_; diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 1d1df3f57931..bffeece69ee5 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -48,19 +48,19 @@ class GrpcMuxImplTestBase : public testing::Test { stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)) {} void setup() { - grpc_mux_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, true); + random_, stats_, rate_limit_settings_, local_info_, true); } void setup(const RateLimitSettings& custom_rate_limit_settings) { - grpc_mux_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, custom_rate_limit_settings, true); + random_, stats_, custom_rate_limit_settings, local_info_, true); } void expectSendMessage(const std::string& type_url, @@ -85,15 +85,49 @@ class GrpcMuxImplTestBase : public testing::Test { error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); + EXPECT_CALL( + async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); + } + + // These tests were written around GrpcMuxWatch, an RAII type returned by the old subscribe(). + // To preserve these tests for the new code, we need an RAII watch handler. That is + // GrpcSubscriptionImpl, but to keep things simple, we'll fake it. (What we really care about + // is the destructor, which is identical to the real one). + class FakeGrpcSubscription { + public: + FakeGrpcSubscription(GrpcMux* grpc_mux, const std::string& type_url, Watch* watch) + : grpc_mux_(grpc_mux), type_url_(type_url), watch_(watch) {} + ~FakeGrpcSubscription() { grpc_mux_->removeWatch(type_url_, watch_); } + + private: + GrpcMux* const grpc_mux_; + std::string type_url_; + Watch* const watch_; + }; + + FakeGrpcSubscription makeWatch(const std::string& type_url, + const std::set& resources) { + return FakeGrpcSubscription(grpc_mux_.get(), type_url, + grpc_mux_->addOrUpdateWatch(type_url, nullptr, resources, + callbacks_, + std::chrono::milliseconds(0))); + } + + FakeGrpcSubscription + makeWatch(const std::string& type_url, const std::set& resources, + NiceMock>& callbacks) { + return FakeGrpcSubscription(grpc_mux_.get(), type_url, + grpc_mux_->addOrUpdateWatch(type_url, nullptr, resources, callbacks, + std::chrono::milliseconds(0))); } NiceMock dispatcher_; NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; - std::unique_ptr grpc_mux_; - NiceMock callbacks_; + std::unique_ptr grpc_mux_; + NiceMock> callbacks_; NiceMock local_info_; Stats::IsolatedStoreImpl stats_; Envoy::Config::RateLimitSettings rate_limit_settings_; @@ -105,25 +139,25 @@ class GrpcMuxImplTest : public GrpcMuxImplTestBase { Event::SimulatedTimeSystem time_system_; }; -// Validate behavior when multiple type URL watches are maintained, watches are created/destroyed -// (via RAII). +// Validate behavior when multiple type URL watches are maintained, watches are created/destroyed. TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { setup(); InSequence s; - auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); - auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); + + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, "", true); - expectSendMessage("bar", {}, ""); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); grpc_mux_->start(); EXPECT_EQ(1, control_plane_connected_state_.value()); - expectSendMessage("bar", {"z"}, ""); - auto bar_z_sub = grpc_mux_->subscribe("bar", {"z"}, callbacks_); - expectSendMessage("bar", {"zz", "z"}, ""); - auto bar_zz_sub = grpc_mux_->subscribe("bar", {"zz"}, callbacks_); - expectSendMessage("bar", {"z"}, ""); - expectSendMessage("bar", {}, ""); - expectSendMessage("foo", {}, ""); + expectSendMessage("type_url_bar", {"z"}, ""); + FakeGrpcSubscription bar_z_sub = makeWatch("type_url_bar", {"z"}); + expectSendMessage("type_url_bar", {"zz", "z"}, ""); + FakeGrpcSubscription bar_zz_sub = makeWatch("type_url_bar", {"zz"}); + expectSendMessage("type_url_bar", {"z"}, ""); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_foo", {}, ""); } // Validate behavior when multiple type URL watches are maintained and the stream is reset. @@ -140,13 +174,13 @@ TEST_F(GrpcMuxImplTest, ResetStream) { })); setup(); - auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); - auto bar_sub = grpc_mux_->subscribe("bar", {}, callbacks_); - auto baz_sub = grpc_mux_->subscribe("baz", {"z"}, callbacks_); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); + FakeGrpcSubscription baz_sub = makeWatch("type_url_baz", {"z"}); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, "", true); - expectSendMessage("bar", {}, ""); - expectSendMessage("baz", {"z"}, ""); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_baz", {"z"}, ""); grpc_mux_->start(); EXPECT_CALL(callbacks_, @@ -158,87 +192,87 @@ TEST_F(GrpcMuxImplTest, ResetStream) { grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, control_plane_connected_state_.value()); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, "", true); - expectSendMessage("bar", {}, ""); - expectSendMessage("baz", {"z"}, ""); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_baz", {"z"}, ""); timer_cb(); - expectSendMessage("baz", {}, ""); - expectSendMessage("foo", {}, ""); + expectSendMessage("type_url_baz", {}, ""); + expectSendMessage("type_url_foo", {}, ""); } // Validate pause-resume behavior. TEST_F(GrpcMuxImplTest, PauseResume) { setup(); InSequence s; - auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); - grpc_mux_->pause("foo"); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + grpc_mux_->pause("type_url_foo"); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); grpc_mux_->start(); - expectSendMessage("foo", {"x", "y"}, "", true); - grpc_mux_->resume("foo"); - grpc_mux_->pause("bar"); - expectSendMessage("foo", {"z", "x", "y"}, ""); - auto foo_z_sub = grpc_mux_->subscribe("foo", {"z"}, callbacks_); - grpc_mux_->resume("bar"); - grpc_mux_->pause("foo"); - auto foo_zz_sub = grpc_mux_->subscribe("foo", {"zz"}, callbacks_); - expectSendMessage("foo", {"zz", "z", "x", "y"}, ""); - grpc_mux_->resume("foo"); - grpc_mux_->pause("foo"); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + grpc_mux_->resume("type_url_foo"); + grpc_mux_->pause("type_url_bar"); + expectSendMessage("type_url_foo", {"z", "x", "y"}, ""); + FakeGrpcSubscription foo_z_sub = makeWatch("type_url_foo", {"z"}); + grpc_mux_->resume("type_url_bar"); + grpc_mux_->pause("type_url_foo"); + FakeGrpcSubscription foo_zz_sub = makeWatch("type_url_foo", {"zz"}); + expectSendMessage("type_url_foo", {"zz", "z", "x", "y"}, ""); + grpc_mux_->resume("type_url_foo"); + grpc_mux_->pause("type_url_foo"); } // Validate behavior when type URL mismatches occur. TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { setup(); - std::unique_ptr invalid_response( - new envoy::api::v2::DiscoveryResponse()); - InSequence s; - auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); + auto invalid_response = std::make_unique(); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - expectSendMessage("foo", {"x", "y"}, "", true); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); grpc_mux_->start(); { - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); - response->set_type_url("bar"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + auto response = std::make_unique(); + response->set_type_url("type_url_bar"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } { - invalid_response->set_type_url("foo"); - invalid_response->mutable_resources()->Add()->set_type_url("bar"); + invalid_response->set_type_url("type_url_foo"); + invalid_response->mutable_resources()->Add()->set_type_url("type_url_bar"); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) .WillOnce(Invoke([](Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) { - EXPECT_TRUE(IsSubstring("", "", "bar does not match foo type URL in DiscoveryResponse", - e->what())); + EXPECT_TRUE( + IsSubstring("", "", + "type URL type_url_bar embedded in an individual Any does not match the " + "message-wide type URL type_url_foo in DiscoveryResponse", + e->what())); })); - expectSendMessage("foo", {"x", "y"}, "", false, "", Grpc::Status::GrpcStatus::Internal, - fmt::format("bar does not match foo type URL in DiscoveryResponse {}", - invalid_response->DebugString())); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(invalid_response)); + expectSendMessage( + "type_url_foo", {"x", "y"}, "", false, "", Grpc::Status::GrpcStatus::Internal, + fmt::format("type URL type_url_bar embedded in an individual Any does not match the " + "message-wide type URL type_url_foo in DiscoveryResponse {}", + invalid_response->DebugString())); + grpc_mux_->onDiscoveryResponse(std::move(invalid_response)); } - expectSendMessage("foo", {}, ""); + expectSendMessage("type_url_foo", {}, ""); } // Validate behavior when watches has an unknown resource name. TEST_F(GrpcMuxImplTest, WildcardWatch) { setup(); - InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - auto foo_sub = grpc_mux_->subscribe(type_url, {}, callbacks_); + FakeGrpcSubscription foo_sub = makeWatch(type_url, {}); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); { - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_type_url(type_url); response->set_version_info("1"); envoy::api::v2::ClusterLoadAssignment load_assignment; @@ -254,27 +288,28 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); })); expectSendMessage(type_url, {}, "1"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + grpc_mux_->onDiscoveryResponse(std::move(response)); } } // Validate behavior when watches specify resources (potentially overlapping). TEST_F(GrpcMuxImplTest, WatchDemux) { setup(); - InSequence s; + // We will not require InSequence here: an update that causes multiple onConfigUpdates + // causes them in an indeterminate order, based on the whims of the hash map. const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - NiceMock foo_callbacks; - auto foo_sub = grpc_mux_->subscribe(type_url, {"x", "y"}, foo_callbacks); - NiceMock bar_callbacks; - auto bar_sub = grpc_mux_->subscribe(type_url, {"y", "z"}, bar_callbacks); + NiceMock> foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks); + NiceMock> bar_callbacks; + FakeGrpcSubscription bar_sub = makeWatch(type_url, {"y", "z"}, bar_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); // Should dedupe the "x" resource. expectSendMessage(type_url, {"y", "z", "x"}, "", true); grpc_mux_->start(); + // Send just x; only foo_callbacks should receive an onConfigUpdate(). { - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_type_url(type_url); response->set_version_info("1"); envoy::api::v2::ClusterLoadAssignment load_assignment; @@ -291,12 +326,13 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); })); expectSendMessage(type_url, {"y", "z", "x"}, "1"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + grpc_mux_->onDiscoveryResponse(std::move(response)); } + // Send x y and z; foo_ and bar_callbacks should both receive onConfigUpdate()s, carrying {x,y} + // and {y,z} respectively. { - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_type_url(type_url); response->set_version_info("2"); envoy::api::v2::ClusterLoadAssignment load_assignment_x; @@ -331,7 +367,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment_y)); })); expectSendMessage(type_url, {"y", "z", "x"}, "2"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + grpc_mux_->onDiscoveryResponse(std::move(response)); } expectSendMessage(type_url, {"x", "y"}, "2"); @@ -343,21 +379,20 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { setup(); InSequence s; const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - NiceMock foo_callbacks; - auto foo_sub = grpc_mux_->subscribe(type_url, {"x", "y"}, foo_callbacks); + NiceMock> foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {"x", "y"}, "", true); grpc_mux_->start(); - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_type_url(type_url); response->set_version_info("1"); EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")).Times(0); expectSendMessage(type_url, {"x", "y"}, "1"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + grpc_mux_->onDiscoveryResponse(std::move(response)); expectSendMessage(type_url, {}, "1"); } @@ -366,15 +401,14 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { setup(); const std::string& type_url = Config::TypeUrl::get().Cluster; - NiceMock foo_callbacks; - auto foo_sub = grpc_mux_->subscribe(type_url, {}, foo_callbacks); + NiceMock> foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {}, foo_callbacks); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_type_url(type_url); response->set_version_info("1"); // Validate that onConfigUpdate is called with empty resources. @@ -382,7 +416,7 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& resources, const std::string&) { EXPECT_TRUE(resources.empty()); })); expectSendMessage(type_url, {}, "1"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + grpc_mux_->onDiscoveryResponse(std::move(response)); } // Exactly one test requires a mock time system to provoke behavior that cannot @@ -416,15 +450,15 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("baz"); - response->set_nonce("bar"); - response->set_type_url("foo"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } }; - auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, "", true); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); grpc_mux_->start(); // Exhausts the limit. @@ -469,23 +503,24 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("baz"); - response->set_nonce("bar"); - response->set_type_url("foo"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } }; - auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, "", true); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. - EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)); - onReceiveMessage(99); - EXPECT_EQ(1, stats_.counter("control_plane.rate_limit_enforced").value()); - EXPECT_EQ( - 1, + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)) + .Times(AtLeast(1)); + onReceiveMessage(110); + EXPECT_LE(10, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_LE( + 10, stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::Accumulate).value()); } @@ -525,15 +560,15 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { for (uint64_t i = 0; i < burst; i++) { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); - response->set_version_info("baz"); - response->set_nonce("bar"); - response->set_type_url("foo"); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } }; - auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); - expectSendMessage("foo", {"x"}, "", true); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); grpc_mux_->start(); // Validate that rate limit is not enforced for 100 requests. @@ -544,10 +579,10 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500), _)) .Times(AtLeast(1)); onReceiveMessage(160); - EXPECT_EQ(12, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_LE(10, stats_.counter("control_plane.rate_limit_enforced").value()); Stats::Gauge& pending_requests = stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::Accumulate); - EXPECT_EQ(12, pending_requests.value()); + EXPECT_LE(10, pending_requests.value()); // Validate that drain requests call when there are multiple requests in queue. time_system_.setMonotonicTime(std::chrono::seconds(10)); @@ -557,7 +592,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { EXPECT_EQ(0, pending_requests.value()); } -// Verifies that a message with no resources is accepted. +// Verifies that a message with no resources is accepted. TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { setup(); @@ -569,25 +604,26 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { { // subscribe and unsubscribe to simulate a cluster added and removed expectSendMessage(type_url, {"y"}, "", true); - auto temp_sub = grpc_mux_->subscribe(type_url, {"y"}, callbacks_); + FakeGrpcSubscription temp_sub = makeWatch(type_url, {"y"}); expectSendMessage(type_url, {}, ""); } // simulate the server sending empty CLA message to notify envoy that the CLA was removed. - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_nonce("bar"); response->set_version_info("1"); response->set_type_url(type_url); - // This contains zero resources. No discovery request should be sent. - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + // Although the update will change nothing for us, we will "accept" it, and so according + // to the spec we should ACK it. + expectSendMessage(type_url, {}, "1", false, "bar"); + grpc_mux_->onDiscoveryResponse(std::move(response)); - // when we add the new subscription version should be 1 and nonce should be bar + // When we become interested in "x", we should send a request indicating that interest. expectSendMessage(type_url, {"x"}, "1", false, "bar"); + FakeGrpcSubscription sub = makeWatch(type_url, {"x"}); - // simulate a new cluster x is added. add CLA subscription for it. - auto sub = grpc_mux_->subscribe(type_url, {"x"}, callbacks_); + // Watch destroyed -> interest gone -> unsubscribe request. expectSendMessage(type_url, {}, "1", false, "bar"); } @@ -600,15 +636,16 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; grpc_mux_->start(); - // subscribe and unsubscribe (by not keeping the return watch) so that the type is known to envoy - expectSendMessage(type_url, {"y"}, "", true); - expectSendMessage(type_url, {}, ""); - grpc_mux_->subscribe(type_url, {"y"}, callbacks_); + // subscribe and unsubscribe so that the type is known to envoy + { + expectSendMessage(type_url, {"y"}, "", true); + expectSendMessage(type_url, {}, ""); + FakeGrpcSubscription delete_immediately = makeWatch(type_url, {"y"}); + } // simulate the server sending CLA message to notify envoy that the CLA was added, // even though envoy doesn't expect it. Envoy should reject this update. - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_nonce("bar"); response->set_version_info("1"); response->set_type_url(type_url); @@ -618,19 +655,19 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { response->add_resources()->PackFrom(load_assignment); // The message should be rejected. - expectSendMessage(type_url, {}, "", false, "bar"); - EXPECT_LOG_CONTAINS("warning", "Ignoring unwatched type URL " + type_url, - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response))); + expectSendMessage(type_url, {}, "", false, "bar", Grpc::Status::GrpcStatus::Internal, + "Rejecting non-empty update for unwatched type URL"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_THROW_WITH_MESSAGE( - GrpcMuxImpl( - local_info_, std::unique_ptr(async_client_), dispatcher_, + GrpcMuxSotw( + std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, true), + random_, stats_, rate_limit_settings_, local_info_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); @@ -639,11 +676,11 @@ TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); EXPECT_THROW_WITH_MESSAGE( - GrpcMuxImpl( - local_info_, std::unique_ptr(async_client_), dispatcher_, + GrpcMuxSotw( + std::unique_ptr(async_client_), dispatcher_, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, rate_limit_settings_, true), + random_, stats_, rate_limit_settings_, local_info_, true), EnvoyException, "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 8a33cf7a894e..cecf8aabfafa 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -15,12 +15,14 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { InSequence s; EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(nullptr)); + // onConfigUpdateFailed() should not be called for gRPC stream connection failure EXPECT_CALL(callbacks_, - onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(0); EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_, _)); subscription_->start({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0)); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. subscription_->updateResourceInterest({"cluster2"}); @@ -30,7 +32,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { expectSendMessage({"cluster2"}, "", true); timer_cb_(); - EXPECT_TRUE(statsAre(3, 0, 0, 1, 0, 0)); + EXPECT_TRUE(statsAre(3, 0, 0, 0, 0, 0)); verifyControlPlaneStats(1); } @@ -38,12 +40,16 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); + // onConfigUpdateFailed() should not be called for gRPC stream connection failure EXPECT_CALL(callbacks_, - onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(0); EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); - subscription_->grpcMux()->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, - ""); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get()) + ->grpcStreamForTest() + .onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); verifyControlPlaneStats(0); @@ -54,7 +60,7 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0)); } -// Validate that When the management server gets multiple requests for the same version, it can +// Validate that when the management server gets multiple requests for the same version, it can // ignore later ones. This allows the nonce to be used. TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { InSequence s; diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 788aa7293055..b7a6256e4c7c 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -36,16 +36,22 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { : method_descriptor_(Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints")), async_client_(new NiceMock()), timer_(new Event::MockTimer()) { - node_.set_id("fo0"); - EXPECT_CALL(local_info_, node()).WillOnce(testing::ReturnRef(node_)); + node_.set_id("node_name"); + node_.set_cluster("cluster_name"); + node_.mutable_locality()->set_zone("zone_name"); + EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)).WillOnce(Invoke([this](Event::TimerCb timer_cb) { timer_cb_ = timer_cb; return timer_; })); + subscription_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, random_, - *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, - stats_store_, rate_limit_settings_, init_fetch_timeout, true); + std::make_shared(std::unique_ptr(async_client_), + dispatcher_, *method_descriptor_, random_, stats_store_, + rate_limit_settings_, local_info_, + /*skip_subsequent_node=*/true), + Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout, + /*is_aggregated=*/false); } ~GrpcSubscriptionTestHarness() override { EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); } @@ -76,7 +82,9 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL(async_stream_, sendMessageRaw_(Grpc::ProtoBufferEq(expected_request), false)); + EXPECT_CALL( + async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); } void startSubscription(const std::set& cluster_names) override { @@ -114,36 +122,19 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::GrpcStatus::Internal, "bad config"); } - subscription_->grpcMux()->onDiscoveryResponse(std::move(response)); + auto shared_mux = subscription_->getContextForTest(); + static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } void updateResourceInterest(const std::set& cluster_names) override { - // The "watch" mechanism means that updates that lose interest in a resource - // will first generate a request for [still watched resources, i.e. without newly unwatched - // ones] before generating the request for all of cluster_names. - // TODO(fredlas) this unnecessary second request will stop happening once the watch mechanism is - // no longer internally used by GrpcSubscriptionImpl. - // TODO TODO i think this should now be resolved - std::set both; - for (const auto& n : cluster_names) { - if (last_cluster_names_.find(n) != last_cluster_names_.end()) { - both.insert(n); - } - } - expectSendMessage(both, version_); expectSendMessage(cluster_names, version_); subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)) - .WillOnce([this](ConfigUpdateFailureReason reason, const EnvoyException*) { - if (reason == ConfigUpdateFailureReason::FetchTimedout) { - stats_.init_fetch_timeout_.inc(); - } - }); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index c5cd9c4c771a..c387cd23dc1c 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -302,7 +302,6 @@ TEST_F(SubscriptionFactoryTest, GrpcSubscription) { })); EXPECT_CALL(random_, random()); EXPECT_CALL(dispatcher_, createTimer_(_)).Times(2); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); subscriptionFromConfigSource(config)->start({"static_cluster"}); } diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 6beb74812412..0781a7a63883 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -150,18 +150,19 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { if (GetParam() == SubscriptionType::Filesystem) { return; // initial_fetch_timeout not implemented for filesystem. } - InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); expectConfigUpdateFailed(); + if (GetParam() != SubscriptionType::Http) { + expectDisableInitFetchTimeoutTimer(); + } callInitFetchTimeoutCb(); EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } // Validate that initial fetch timer is disabled on config update TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { - InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); @@ -171,7 +172,6 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { // Validate that initial fetch timer is disabled on config update failed TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { - InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc index b41b1ec8bcb2..3410986ca88d 100644 --- a/test/common/config/watch_map_test.cc +++ b/test/common/config/watch_map_test.cc @@ -72,11 +72,11 @@ void expectNoUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); } -void expectEmptySotwNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { +void expectEmptySotwNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, + const std::string& version) { EXPECT_CALL(callbacks, onConfigUpdate(_, version)) - .WillOnce(Invoke( - [](const Protobuf::RepeatedPtrField& gotten_resources, - const std::string&) { EXPECT_EQ(gotten_resources.size(), 0); })); + .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { EXPECT_EQ(gotten_resources.size(), 0); })); EXPECT_CALL(callbacks, onConfigUpdate(_, _, version)).Times(0); } @@ -314,7 +314,6 @@ TEST(WatchMapTest, UninterestingUpdate) { bob.set_cluster_name("bob"); bob_update.Add()->PackFrom(bob); - // We are watching for alice, and an update for just bob arrives. It should be ignored. expectNoUpdate(callbacks, "version1"); doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version1"); @@ -336,9 +335,8 @@ TEST(WatchMapTest, UninterestingUpdate) { // Finally, test that calling onConfigUpdate on a map with no watches (which should not // happen) throws an exception. - EXPECT_THROW_WITH_MESSAGE( - doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"), - EnvoyException, "Rejecting non-empty update for unwatched type URL"); + EXPECT_THROW_WITH_MESSAGE(doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"), + EnvoyException, "Rejecting non-empty update for unwatched type URL"); } // Tests that a watch that specifies no particular resource interest is treated as interested in diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index da72909457d6..74de0b2e60d0 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -101,7 +101,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { return std::make_pair(result.first, ThreadAwareLoadBalancerPtr(result.second)); } - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, bool, ClusterManager&) override { + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, ClusterManager&) override { return CdsApiPtr{createCds_()}; } diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 69aa472c2f9b..c806cd8107e7 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -13,9 +13,9 @@ #include "test/integration/http_integration.h" namespace Envoy { -static const std::string& AdsIntegrationConfig(const std::string& api_type) { - CONSTRUCT_ON_FIRST_USE(std::string, fmt::format( - R"EOF( +static std::string AdsIntegrationConfig(const std::string& api_type) { + // Note: do not use CONSTRUCT_ON_FIRST_USE here! + return fmt::format(R"EOF( dynamic_resources: lds_config: ads: {{}} @@ -43,7 +43,7 @@ static const std::string& AdsIntegrationConfig(const std::string& api_type) { address: 127.0.0.1 port_value: 0 )EOF", - api_type)); + api_type); } class AdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 91e916ed451f..c271a57a39cc 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -56,8 +56,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest( Config::TypeUrl::get().Cluster, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().ClusterLoadAssignment, - Config::TypeUrl::get().Cluster))); + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Cluster))); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -69,11 +68,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildCluster("cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"cluster_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Cluster, - Config::TypeUrl::get().ClusterLoadAssignment))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {}, {}, false, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().ClusterLoadAssignment))); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); @@ -86,8 +85,7 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest( Config::TypeUrl::get().Listener, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().RouteConfiguration, - Config::TypeUrl::get().Listener))); + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Listener))); sendDiscoveryResponse( Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); @@ -99,11 +97,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildListener("route_config_0", "cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Listener, - Config::TypeUrl::get().RouteConfiguration))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {}, {}, false, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().RouteConfiguration))); sendDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); @@ -211,7 +209,7 @@ TEST_P(AdsIntegrationTest, RedisClusterRemoval) { {buildRedisCluster("redis_cluster")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", - {"redis_cluster"}, {}, {})); + {"redis_cluster"}, {"redis_cluster"}, {})); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("redis_cluster")}, {buildClusterLoadAssignment("redis_cluster")}, {}, "1"); @@ -364,6 +362,10 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. + // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't + // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 + // ACK goes out, they're both acknowledging version 3. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); @@ -433,7 +435,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response. - test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + /* TODO TODO HANGS?*/ // test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); // Disconnect and reconnect the stream. xds_stream_->finishGrpcStream(Grpc::Status::Internal); @@ -444,7 +446,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response even after disconnect and reconnect. - test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + /* TODO TODO HANGS?*/ // test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); // Finish warming the second cluster. sendDiscoveryResponse( @@ -763,7 +765,8 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { // Remove listener. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); + sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, + {"listener_0"}, "2"); test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); } diff --git a/test/integration/integration.h b/test/integration/integration.h index 118071c5fbdb..c374ebae0163 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -242,7 +242,7 @@ class BaseIntegrationTest : Logger::Loggable { AssertionResult compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, - const std::vector& expected_resource_unsubscriptions, bool expect_node, + const std::vector& expected_resource_unsubscriptions, bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_substring = "") { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_subscriptions, @@ -254,7 +254,8 @@ class BaseIntegrationTest : Logger::Loggable { const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& stream, - bool expect_node, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, + bool expect_node = false, + const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_substring = ""); AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 02283b3927c2..3ffcc5cf54a5 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -110,7 +110,8 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H uint32_t initial_keys_{}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); TEST_P(RtdsIntegrationTest, RtdsReload) { initialize(); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 5f4bb27395e4..484c07239964 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -13,7 +13,7 @@ namespace Envoy { namespace { class ScopedRdsIntegrationTest : public HttpIntegrationTest, - public Grpc::DeltaSotwGrpcClientIntegrationParamTest { + public Grpc::DeltaSotwIntegrationParamTest { protected: struct FakeUpstreamInfo { FakeHttpConnectionPtr connection_; @@ -82,7 +82,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, scoped_routes->mutable_scoped_rds() ->mutable_scoped_rds_config_source() ->mutable_api_config_source(); - if (isDelta()) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Delta) { srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); } else { srds_api_config_source->set_api_type(envoy::api::v2::core::ApiConfigSource::GRPC); @@ -169,7 +169,7 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, const std::vector& to_add_list, const std::vector& to_delete_list, const std::string& version) { - if (isDelta()) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Delta) { sendDeltaScopedRdsResponse(to_add_list, to_delete_list, version); } else { sendSotwScopedRdsResponse(sotw_list, version); @@ -283,7 +283,7 @@ route_configuration_name: {} } test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_attempt", // update_attempt only increase after a response - isDelta() ? 1 : 2); + sotwOrDelta() == Grpc::SotwOrDelta::Delta ? 1 : 2); test_server_->waitForCounterGe("http.config_test.scoped_rds.foo-scoped-routes.update_success", 1); // The version gauge should be set to xxHash64("1"). test_server_->waitForGaugeEq("http.config_test.scoped_rds.foo-scoped-routes.version", From 4cf6dabee7286a28331535412cd5f45f54721c09 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 26 Sep 2019 10:40:28 -0400 Subject: [PATCH 09/39] passes scoped_rds and vhds with stat commented out Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 1 + source/common/config/grpc_mux_impl.h | 11 +- .../config/grpc_mux_subscription_impl.cc | 107 ------------------ source/common/config/grpc_stream.h | 16 ++- .../common/config/grpc_subscription_impl.cc | 9 +- .../config/subscription_factory_impl.cc | 3 + source/common/config/watch_map.cc | 18 +-- source/common/router/scoped_rds.cc | 3 +- source/common/router/scoped_rds.h | 8 +- source/common/stats/isolated_store_impl.h | 2 + test/common/config/watch_map_test.cc | 6 +- test/integration/ads_integration_test.cc | 4 +- test/integration/integration.cc | 4 - test/integration/rtds_integration_test.cc | 2 +- .../scoped_rds_integration_test.cc | 3 + test/integration/server.h | 4 + 16 files changed, 58 insertions(+), 143 deletions(-) delete mode 100644 source/common/config/grpc_mux_subscription_impl.cc diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 2b874384b7e3..c45c16851c12 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -181,6 +181,7 @@ void GrpcMuxImpl::trySendDiscoveryRequests() { sendGrpcMessage(sub->getNextRequestAckless()); } } + std::cerr << ++vinny_the_variable_ << std::endl; maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index e201753e168c..401469643ba5 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -37,6 +37,7 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; void start() override; + // void shutdown() override; void disableInitFetchTimeoutTimer() override; protected: @@ -122,6 +123,10 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { // the type_urls. So, while the SubscriptionStates populate every other field of these messages, // this one is up to GrpcMux. const LocalInfo::LocalInfo& local_info_; + + // Demonstrates that when the maybeUpdateQueueSizeStat() segfault happens, this object is + // still valid. + int vinny_the_variable_{}; }; class GrpcMuxDelta : public GrpcMuxImpl, @@ -176,8 +181,10 @@ class GrpcMuxSotw : public GrpcMuxImpl, : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - + rate_limit_settings) { + std::cerr << "creating STOW mux " << this << std::endl; + } + ~GrpcMuxSotw() { std::cerr << "destroying STOW mux " << this << std::endl; } // TODO REMOVE // GrpcStreamCallbacks void onStreamEstablished() override { handleEstablishedStream(); } void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc deleted file mode 100644 index dffab9f0caea..000000000000 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ /dev/null @@ -1,107 +0,0 @@ -#include "common/config/grpc_mux_subscription_impl.h" - -#include "common/common/assert.h" -#include "common/common/logger.h" -#include "common/grpc/common.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, - SubscriptionCallbacks& callbacks, - SubscriptionStats stats, - absl::string_view type_url, - Event::Dispatcher& dispatcher, - std::chrono::milliseconds init_fetch_timeout) - : grpc_mux_(grpc_mux), callbacks_(callbacks), stats_(stats), type_url_(type_url), - dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} - -// Config::Subscription -void GrpcMuxSubscriptionImpl::start(const std::set& resources) { - if (init_fetch_timeout_.count() > 0) { - init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { - callbacks_.onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::FetchTimedout, - nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } - - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); - // The attempt stat here is maintained for the purposes of having consistency between ADS and - // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an - // "attempt" for a given xDS API combined by ADS is not really that meaningful. - stats_.update_attempt_.inc(); -} - -void GrpcMuxSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - // First destroy the watch, so that this subscribe doesn't send a request for both the - // previously watched resources and the new ones (we may have lost interest in some of the - // previously watched ones). - watch_.reset(); - watch_ = grpc_mux_.subscribe(type_url_, update_to_these_names, *this); - stats_.update_attempt_.inc(); -} - -// Config::GrpcMuxCallbacks -void GrpcMuxSubscriptionImpl::onConfigUpdate( - const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { - disableInitFetchTimeoutTimer(); - // TODO(mattklein123): In the future if we start tracking per-resource versions, we need to - // supply those versions to onConfigUpdate() along with the xDS response ("system") - // version_info. This way, both types of versions can be tracked and exposed for debugging by - // the configuration update targets. - callbacks_.onConfigUpdate(resources, version_info); - stats_.update_success_.inc(); - stats_.update_attempt_.inc(); - stats_.version_.set(HashUtil::xxHash64(version_info)); - ENVOY_LOG(debug, "gRPC config for {} accepted with {} resources with version {}", type_url_, - resources.size(), version_info); -} - -void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, - const EnvoyException* e) { - switch (reason) { - case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: - stats_.update_failure_.inc(); - ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); - break; - case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: - stats_.init_fetch_timeout_.inc(); - disableInitFetchTimeoutTimer(); - ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); - break; - case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: - // We expect Envoy exception to be thrown when update is rejected. - ASSERT(e != nullptr); - disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); - ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); - break; - } - - stats_.update_attempt_.inc(); - if (reason == Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure) { - // New gRPC stream will be established and send requests again. - // If init_fetch_timeout is non-zero, server will continue startup after it timeout - return; - } - - callbacks_.onConfigUpdateFailed(reason, e); -} - -std::string GrpcMuxSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { - return callbacks_.resourceName(resource); -} - -void GrpcMuxSubscriptionImpl::disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 1be922742930..6f15ee697460 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -37,7 +37,9 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } backoff_strategy_ = std::make_unique(RETRY_INITIAL_DELAY_MS, RETRY_MAX_DELAY_MS, random_); + std::cerr << "created GrpcStream " << this << " with scope " << &scope << std::endl; } + ~GrpcStream() { std::cerr << "destroying GrpcStream " << this << std::endl; } // TODO REMOVE void establishNewStream() { ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); @@ -91,7 +93,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, setRetryTimer(); } - void maybeUpdateQueueSizeStat(uint64_t size) { + void maybeUpdateQueueSizeStat(uint64_t /*size*/) { // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter // because we always attempt this drain immediately after the push). Basically, a change in @@ -99,9 +101,11 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // if(>0 || used) to keep this stat from being wrongly marked interesting by a pointless set(0) // and needlessly taking up space. The first time we set(123), used becomes true, and so we will // subsequently always do the set (including set(0)). - if (size > 0 || control_plane_stats_.pending_requests_.used()) { - control_plane_stats_.pending_requests_.set(size); - } + // TODO TODO segfault here in test teardown + // if (size > 0 || control_plane_stats_.pending_requests_.used()) { + // control_plane_stats_.pending_requests_.set(size); + // } + std::cerr << ++vinny_the_variable_ << std::endl; } bool checkRateLimitAllowsDrain() { @@ -147,6 +151,10 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; + + // Demonstrates that when the maybeUpdateQueueSizeStat() segfault (would) happen, this object is + // still valid. + int vinny_the_variable_{}; }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 862262b2c73d..01d5f4c15637 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -9,10 +9,17 @@ GrpcSubscriptionImpl::GrpcSubscriptionImpl(GrpcMuxSharedPtr context, absl::strin std::chrono::milliseconds init_fetch_timeout, bool is_aggregated) : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), - init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) { + if (!context_) { + std::cerr << "HEY you gave me a null context_!!!!!!!" << std::endl; + } + std::cerr << "GrpcSubscriptionImpl CTOR " << this << std::endl; +} GrpcSubscriptionImpl::~GrpcSubscriptionImpl() { + std::cerr << "~GrpcSubscriptionImpl " << this << std::endl; if (watch_) { + std::cerr << "~GrpcSubscriptionImpl removing wathc" << std::endl; context_->removeWatch(type_url_, watch_); } } diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 61aacb4b1b6f..a8e678b01837 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -46,6 +46,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(type_url), callbacks, stats, Utility::configSourceInitialFetchTimeout(config), validation_visitor_); case envoy::api::v2::core::ApiConfigSource::GRPC: + std::cerr << "subscriptionFromConfigSource single SOTW" << std::endl; return std::make_unique( std::make_shared(Utility::factoryForGrpcApiConfigSource( cm_.grpcAsyncClientManager(), api_config_source, scope) @@ -56,6 +57,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false); case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: + std::cerr << "subscriptionFromConfigSource single DELTA" << std::endl; return std::make_unique( std::make_shared(Utility::factoryForGrpcApiConfigSource( cm_.grpcAsyncClientManager(), api_config_source, scope) @@ -70,6 +72,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( } } case envoy::api::v2::core::ConfigSource::kAds: { + std::cerr << "subscriptionFromConfigSource ADS" << std::endl; return std::make_unique(cm_.adsMux(), type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/true); diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index 6fa1c07924d6..adc99f145f55 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -54,17 +54,7 @@ absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& res void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { if (watches_.empty()) { - if (resources.empty()) { - // We have no watches, and the update contained no resources. This can happen when Envoy - // unregisters from a resource that's removed from the server as well. For example, - // a deleted cluster triggers un-watching the ClusterLoadAssignment watch, and at the - // same time the xDS server sends an empty list of ClusterLoadAssignment resources. - return; - } else { - // We have no watches, but the update contained resources. This should not happen. - ENVOY_LOG(warn, "Rejecting non-empty update for unwatched type URL"); - throw EnvoyException("Rejecting non-empty update for unwatched type URL"); - } + return; } SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; @@ -80,6 +70,7 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField } } + const bool map_is_single_wildcard = (watches_.size() == 1 && wildcard_watches_.size() == 1); // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (auto& watch : watches_) { const auto this_watch_updates = per_watch_updates.find(watch); @@ -91,11 +82,10 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField // 2) If this watch previously had some resources, it means this update is removing all // of this watch's resources, so the watch must be informed with an onConfigUpdate. // 3) Otherwise, we can skip onConfigUpdate for this watch. - if ((watches_.size() == 1 && wildcard_watches_.size() == 1) || - !watch->state_of_the_world_empty_) { + if (map_is_single_wildcard || !watch->state_of_the_world_empty_) { watch->callbacks_.onConfigUpdate({}, version_info); + watch->state_of_the_world_empty_ = true; } - watch->state_of_the_world_empty_ = true; } else { watch->callbacks_.onConfigUpdate(this_watch_updates->second, version_info); watch->state_of_the_world_empty_ = false; diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index ae77d119a584..f9ad7d2cb7af 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -87,8 +87,8 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), + factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), rds_config_source_(std::move(rds_config_source)), validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), @@ -99,6 +99,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), *scope_, *this); + std::cerr << "made SRDS with scope " << scope_.get() << std::endl; initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { return std::make_shared( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 422551449acd..0b6f1a391bf1 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -98,7 +98,9 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); - ~ScopedRdsConfigSubscription() override = default; + ~ScopedRdsConfigSubscription() override { + std::cerr << "destroying SRDS with scope " << scope_.get() << std::endl; + } // = default; const std::string& name() const { return name_; } @@ -160,6 +162,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio void onRdsConfigUpdate(const std::string& scope_name, RdsRouteConfigSubscription& rds_subscription); + Stats::ScopePtr scope_; // ScopedRouteInfo by scope name. ScopedRouteMap scoped_route_map_; // RdsRouteConfigProvider by scope name. @@ -170,15 +173,14 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio // For creating RDS subscriptions. Server::Configuration::FactoryContext& factory_context_; const std::string name_; - std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder_; - Stats::ScopePtr scope_; ScopedRdsStats stats_; const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; const std::string stat_prefix_; RouteConfigProviderManager& route_config_provider_manager_; + std::unique_ptr subscription_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index cce741211df1..4d1a31f7c74d 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -2,6 +2,7 @@ #include #include +#include // TODO REMOVE #include #include "envoy/stats/stats.h" @@ -83,6 +84,7 @@ class IsolatedStoreImpl : public StoreImpl { public: IsolatedStoreImpl(); explicit IsolatedStoreImpl(SymbolTable& symbol_table); + ~IsolatedStoreImpl() { std::cerr << "~IsolatedStoreImpl " << this << std::endl; } // TODO remove // Stats::Scope Counter& counterFromStatName(StatName name) override { return counters_.get(name); } diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc index 3410986ca88d..77003b2a61c5 100644 --- a/test/common/config/watch_map_test.cc +++ b/test/common/config/watch_map_test.cc @@ -333,10 +333,8 @@ TEST(WatchMapTest, UninterestingUpdate) { watch_map.updateWatchInterest(watch, {}); watch_map.removeWatch(watch); - // Finally, test that calling onConfigUpdate on a map with no watches (which should not - // happen) throws an exception. - EXPECT_THROW_WITH_MESSAGE(doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"), - EnvoyException, "Rejecting non-empty update for unwatched type URL"); + // Finally, test that calling onConfigUpdate on a map with no watches doesn't break. + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"); } // Tests that a watch that specifies no particular resource interest is treated as interested in diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index c271a57a39cc..c364618ec3af 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -435,7 +435,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response. - /* TODO TODO HANGS?*/ // test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); // Disconnect and reconnect the stream. xds_stream_->finishGrpcStream(Grpc::Status::Internal); @@ -446,7 +446,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response even after disconnect and reconnect. - /* TODO TODO HANGS?*/ // test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); // Finish warming the second cluster. sendDiscoveryResponse( diff --git a/test/integration/integration.cc b/test/integration/integration.cc index f476509a6ad7..945a05450e11 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -592,8 +592,6 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( EXPECT_TRUE(discovery_request.has_node()); EXPECT_FALSE(discovery_request.node().id().empty()); EXPECT_FALSE(discovery_request.node().cluster().empty()); - } else { - EXPECT_FALSE(discovery_request.has_node()); } if (expected_type_url != discovery_request.type_url()) { @@ -644,8 +642,6 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( EXPECT_TRUE(request.has_node()); EXPECT_FALSE(request.node().id().empty()); EXPECT_FALSE(request.node().cluster().empty()); - } else { - EXPECT_FALSE(request.has_node()); } if (request.type_url() != expected_type_url) { diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 3ffcc5cf54a5..10b2cb1fab01 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -36,7 +36,7 @@ std::string tdsBootstrapConfig(absl::string_view api_type) { grpc_services: envoy_grpc: cluster_name: rtds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false - name: some_admin_layer admin_layer: {{}} admin: diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 484c07239964..a380d84d3d72 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -25,8 +25,11 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion(), realTime()) {} ~ScopedRdsIntegrationTest() override { + std::cerr << "hmm1" << std::endl; resetConnections(); + std::cerr << "hmm2" << std::endl; cleanupUpstreamAndDownstream(); + std::cerr << "hmm3" << std::endl; } void initialize() override { diff --git a/test/integration/server.h b/test/integration/server.h index 4489c436df19..2dd1e676ab69 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -69,6 +69,7 @@ class TestScopeWrapper : public Scope { public: TestScopeWrapper(Thread::MutexBasicLockable& lock, ScopePtr wrapped_scope) : lock_(lock), wrapped_scope_(std::move(wrapped_scope)) {} + ~TestScopeWrapper() { std::cerr << "destroying TestScopeWrapper" << std::endl; } // TODO REMOVE ScopePtr createScope(const std::string& name) override { Thread::LockGuard lock(lock_); @@ -140,6 +141,9 @@ class TestScopeWrapper : public Scope { */ class TestIsolatedStoreImpl : public StoreRoot { public: + ~TestIsolatedStoreImpl() override { + std::cerr << "destroying TestIsolatedStoreImpl" << std::endl; + } // TODO REMOVE // Stats::Scope Counter& counterFromStatName(StatName name) override { Thread::LockGuard lock(lock_); From 676d46756f411a73b5bafa0546ddb7717ec1b3df Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 27 Sep 2019 17:23:42 -0400 Subject: [PATCH 10/39] reenable segfaulting stat access, reshuffle onConfigUpdate update_attempt Signed-off-by: Fred Douglas --- source/common/config/grpc_stream.h | 8 ++++---- source/common/config/grpc_subscription_impl.cc | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 6f15ee697460..ef5e1d4bb1fa 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -93,7 +93,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, setRetryTimer(); } - void maybeUpdateQueueSizeStat(uint64_t /*size*/) { + void maybeUpdateQueueSizeStat(uint64_t size) { // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter // because we always attempt this drain immediately after the push). Basically, a change in @@ -102,9 +102,9 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // and needlessly taking up space. The first time we set(123), used becomes true, and so we will // subsequently always do the set (including set(0)). // TODO TODO segfault here in test teardown - // if (size > 0 || control_plane_stats_.pending_requests_.used()) { - // control_plane_stats_.pending_requests_.set(size); - // } + if (size > 0 || control_plane_stats_.pending_requests_.used()) { + control_plane_stats_.pending_requests_.set(size); + } std::cerr << ++vinny_the_variable_ << std::endl; } diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 01d5f4c15637..374a67cd1903 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -50,9 +50,9 @@ void GrpcSubscriptionImpl::updateResourceInterest( void GrpcSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { + stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(resources, version_info); stats_.update_success_.inc(); - stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } @@ -60,9 +60,9 @@ void GrpcSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { + stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); stats_.update_success_.inc(); - stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } @@ -74,11 +74,17 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. stats_.update_failure_.inc(); + // TODO remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: stats_.init_fetch_timeout_.inc(); context_->disableInitFetchTimeoutTimer(); callbacks_.onConfigUpdateFailed(reason, e); + // TODO remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); break; case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: // We expect Envoy exception to be thrown when update is rejected. @@ -88,7 +94,6 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason callbacks_.onConfigUpdateFailed(reason, e); break; } - stats_.update_attempt_.inc(); } std::string GrpcSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { From 7398569f187fb46c4266b2671cb910c2218956e2 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 27 Sep 2019 18:37:26 -0400 Subject: [PATCH 11/39] make the update_attempt_ TODOs actual TODOs, since it should be its own PR Signed-off-by: Fred Douglas --- source/common/config/grpc_subscription_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 1ca8d4e23ac7..dd1b4483e030 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -78,7 +78,7 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. stats_.update_failure_.inc(); - // TODO remove; it only makes sense to count start() and updateResourceInterest() + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() // as attempts. stats_.update_attempt_.inc(); break; @@ -86,7 +86,7 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason stats_.init_fetch_timeout_.inc(); context_->disableInitFetchTimeoutTimer(); callbacks_.onConfigUpdateFailed(reason, e); - // TODO remove; it only makes sense to count start() and updateResourceInterest() + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() // as attempts. stats_.update_attempt_.inc(); break; From 4f7b1e78d6cae0e9077dc83f304b458fb8561b70 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 2 Oct 2019 12:00:54 -0400 Subject: [PATCH 12/39] get github to diff grpc_mux_impl against new_grpc_mux impl, part 1 Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 368 ++++++++++++---------- source/common/config/grpc_mux_impl.h | 292 ++++++----------- source/common/config/new_grpc_mux_impl.cc | 228 ++++++++++++++ source/common/config/new_grpc_mux_impl.h | 117 +++++++ 4 files changed, 645 insertions(+), 360 deletions(-) create mode 100644 source/common/config/new_grpc_mux_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.h diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 1a566126f15b..fdb3fa603758 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -1,226 +1,248 @@ #include "common/config/grpc_mux_impl.h" -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/token_bucket_impl.h" +#include + #include "common/config/utility.h" #include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" namespace Envoy { namespace Config { -GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) - : subscription_state_factory_(std::move(subscription_state_factory)), - skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) { +GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, + Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node) + : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings), + local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), + first_stream_request_(true) { Config::Utility::checkLocalInfo("ads", local_info); } -Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - if (watch == nullptr) { - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } else { - updateWatch(type_url, watch, resources); - return watch; +GrpcMuxImpl::~GrpcMuxImpl() { + for (const auto& api_state : api_state_) { + for (auto watch : api_state.second.watches_) { + watch->clear(); + } } } -void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { - updateWatch(type_url, watch, {}); - watchMapFor(type_url).removeWatch(watch); -} - -void GrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } +void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } -void GrpcMuxImpl::resume(const std::string& type_url) { - pausable_ack_queue_.resume(type_url); - trySendDiscoveryRequests(); -} +void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); + return; // Drop this request; the reconnect will enqueue a new one. + } -bool GrpcMuxImpl::paused(const std::string& type_url) const { - return pausable_ack_queue_.paused(type_url); -} + ApiState& api_state = api_state_[type_url]; + if (api_state.paused_) { + ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest(), setting pending.", type_url); + api_state.pending_ = true; + return; // Drop this request; the unpause will enqueue a new one. + } -void GrpcMuxImpl::start() { establishGrpcStream(); } + auto& request = api_state.request_; + request.mutable_resource_names()->Clear(); -void GrpcMuxImpl::disableInitFetchTimeoutTimer() { - for (auto& sub : subscriptions_) { - sub.second->disableInitFetchTimeoutTimer(); + // Maintain a set to avoid dupes. + std::unordered_set resources; + for (const auto* watch : api_state.watches_) { + for (const std::string& resource : watch->resources_) { + if (resources.count(resource) == 0) { + resources.emplace(resource); + request.add_resource_names(resource); + } + } } -} -Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto watch_map = watch_maps_.find(type_url); - if (watch_map == watch_maps_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); + if (skip_subsequent_node_ && !first_stream_request_) { + request.clear_node(); } + ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); + grpc_stream_.sendMessage(request); + first_stream_request_ = false; - Watch* watch = watch_map->second->addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch, resources); - return watch; + // clear error_detail after the request is sent if it exists. + if (api_state_[type_url].request_.has_error_detail()) { + api_state_[type_url].request_.clear_error_detail(); + } } -// Updates the list of resource names watched by the given watch. If an added name is new across -// the whole subscription, or if a removed name has no other watch interested in it, then the -// subscription will enqueue and attempt to send an appropriate discovery request. -void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); - SubscriptionState* sub = subscriptionStateFor(type_url); - WatchMap& watch_map = watchMapFor(type_url); - - auto added_removed = watch_map.updateWatchInterest(watch, resources); - sub->updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - - // Tell the server about our change in interest, if any. - if (sub->subscriptionUpdatePending()) { - trySendDiscoveryRequests(); +GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) { + auto watch = + std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); + ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); + + // Lazily kick off the requests based on first subscription. This has the + // convenient side-effect that we order messages on the channel based on + // Envoy's internal dependency ordering. + // TODO(gsagula): move TokenBucketImpl params to a config. + if (!api_state_[type_url].subscribed_) { + api_state_[type_url].request_.set_type_url(type_url); + api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); + api_state_[type_url].subscribed_ = true; + subscriptions_.emplace_back(type_url); } -} -void GrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { - watch_maps_.emplace(type_url, std::make_unique()); - subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( - type_url, *watch_maps_[type_url], init_fetch_timeout)); - subscription_ordering_.emplace_back(type_url); -} + // This will send an updated request on each subscription. + // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. + // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we + // only send a single RDS/EDS update after the CDS/LDS update. + queueDiscoveryRequest(type_url); -SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { - auto sub = subscriptions_.find(type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", type_url)); - return sub->second.get(); + return watch; } -WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { - auto watch_map = watch_maps_.find(type_url); - RELEASE_ASSERT(watch_map != watch_maps_.end(), fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); - return *watch_map->second; +void GrpcMuxImpl::pause(const std::string& type_url) { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(!api_state.paused_); + ASSERT(!api_state.pending_); + api_state.paused_ = true; } -void GrpcMuxImpl::handleEstablishedStream() { - for (auto& sub : subscriptions_) { - sub.second->markStreamFresh(); +void GrpcMuxImpl::resume(const std::string& type_url) { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(api_state.paused_); + api_state.paused_ = false; + + if (api_state.pending_) { + ASSERT(api_state.subscribed_); + queueDiscoveryRequest(type_url); + api_state.pending_ = false; } - set_any_request_sent_yet_in_current_stream(false); - trySendDiscoveryRequests(); } -void GrpcMuxImpl::handleStreamEstablishmentFailure() { - // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately - // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a - // crash, the iteration needs to dance around a little: collect pointers to all - // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are - // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; - do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = sub.second.get(); - } - for (auto& sub : all_subscribed) { - if (already_called.insert(sub).second) { // insert succeeded ==> not already called - sub.second->handleEstablishmentFailure(); - } - } - } while (all_subscribed.size() != subscriptions_.size()); +bool GrpcMuxImpl::paused(const std::string& type_url) const { + auto entry = api_state_.find(type_url); + if (entry == api_state_.end()) { + return false; + } + return entry->second.paused_; } -void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, - const void* response_proto_ptr) { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "The server sent an xDS response proto with type_url {}, which we have " - "not subscribed to. Ignoring.", +void GrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + const std::string& type_url = message->type_url(); + ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); + if (api_state_.count(type_url) == 0) { + ENVOY_LOG(warn, "Ignoring the message for type URL {} as it has no current subscribers.", type_url); + // TODO(yuval-k): This should never happen. consider dropping the stream as this is a + // protocol violation return; } - pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); - trySendDiscoveryRequests(); -} - -void GrpcMuxImpl::trySendDiscoveryRequests() { - while (true) { - // Do any of our subscriptions even want to send a request? - absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); - if (!maybe_request_type.has_value()) { - break; + if (api_state_[type_url].watches_.empty()) { + // update the nonce as we are processing this response. + api_state_[type_url].request_.set_response_nonce(message->nonce()); + if (message->resources().empty()) { + // No watches and no resources. This can happen when envoy unregisters from a + // resource that's removed from the server as well. For example, a deleted cluster + // triggers un-watching the ClusterLoadAssignment watch, and at the same time the + // xDS server sends an empty list of ClusterLoadAssignment resources. we'll accept + // this update. no need to send a discovery request, as we don't watch for anything. + api_state_[type_url].request_.set_version_info(message->version_info()); + } else { + // No watches and we have resources - this should not happen. send a NACK (by not + // updating the version). + ENVOY_LOG(warn, "Ignoring unwatched type URL {}", type_url); + queueDiscoveryRequest(type_url); + } + return; + } + try { + // To avoid O(n^2) explosion (e.g. when we have 1000s of EDS watches), we + // build a map here from resource name to resource and then walk watches_. + // We have to walk all watches (and need an efficient map as a result) to + // ensure we deliver empty config updates when a resource is dropped. + std::unordered_map resources; + GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; + for (const auto& resource : message->resources()) { + if (type_url != resource.type_url()) { + throw EnvoyException( + fmt::format("{} does not match the message-wide type URL {} in DiscoveryResponse {}", + resource.type_url(), type_url, message->DebugString())); + } + const std::string resource_name = callbacks.resourceName(resource); + resources.emplace(resource_name, resource); } - // If so, which one (by type_url)? - std::string next_request_type_url = maybe_request_type.value(); - // If we don't have a subscription object for this request's type_url, drop the request. - SubscriptionState* sub = subscriptionStateFor(next_request_type_url); - // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url)) { - break; + for (auto watch : api_state_[type_url].watches_) { + // onConfigUpdate should be called in all cases for single watch xDS (Cluster and + // Listener) even if the message does not have resources so that update_empty stat + // is properly incremented and state-of-the-world semantics are maintained. + if (watch->resources_.empty()) { + watch->callbacks_.onConfigUpdate(message->resources(), message->version_info()); + continue; + } + Protobuf::RepeatedPtrField found_resources; + for (const auto& watched_resource_name : watch->resources_) { + auto it = resources.find(watched_resource_name); + if (it != resources.end()) { + found_resources.Add()->MergeFrom(it->second); + } + } + // onConfigUpdate should be called only on watches(clusters/routes) that have + // updates in the message for EDS/RDS. + if (!found_resources.empty()) { + watch->callbacks_.onConfigUpdate(found_resources, message->version_info()); + } } - // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. - if (!pausable_ack_queue_.empty()) { - // Because ACKs take precedence over plain requests, if there is anything in the queue, it's - // safe to assume it's of the type_url that we're wanting to send. - UpdateAck ack = pausable_ack_queue_.front(); - pausable_ack_queue_.pop(); - // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub->getNextRequestWithAck(ack)); - } else { - // getNextRequestAckless() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub->getNextRequestAckless()); + // TODO(mattklein123): In the future if we start tracking per-resource versions, we + // would do that tracking here. + api_state_[type_url].request_.set_version_info(message->version_info()); + } catch (const EnvoyException& e) { + for (auto watch : api_state_[type_url].watches_) { + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); } + ::google::rpc::Status* error_detail = api_state_[type_url].request_.mutable_error_detail(); + error_detail->set_code(Grpc::Status::GrpcStatus::Internal); + error_detail->set_message(e.what()); } - std::cerr << ++vinny_the_variable_ << std::endl; - maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); + api_state_[type_url].request_.set_response_nonce(message->nonce()); + queueDiscoveryRequest(type_url); } -// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check -// whether we *want* to send a DeltaDiscoveryRequest). -bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { - RELEASE_ASSERT( - !pausable_ack_queue_.paused(type_url), - fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " - "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", - type_url)); - - if (!grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); - return false; - } else if (!rateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); - return false; +void GrpcMuxImpl::onWriteable() { drainRequests(); } + +void GrpcMuxImpl::onStreamEstablished() { + first_stream_request_ = true; + for (const auto& type_url : subscriptions_) { + queueDiscoveryRequest(type_url); } - return true; } -// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or -// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). -// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). -// First, prioritizes ACKs over non-ACK subscription interest updates. -// Then, prioritizes non-ACK updates in the order the various types -// of subscriptions were activated. -absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { - // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose - // type_url from pausable_ack_queue_ if possible, before looking at pending updates. - if (!pausable_ack_queue_.empty()) { - return pausable_ack_queue_.front().type_url_; - } - // If we're looking to send multiple non-ACK requests, send them in the order that their - // subscriptions were initiated. - for (const auto& sub_type : subscription_ordering_) { - SubscriptionState* sub = subscriptionStateFor(sub_type); - if (sub->subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { - return sub_type; +void GrpcMuxImpl::onEstablishmentFailure() { + for (const auto& api_state : api_state_) { + for (auto watch : api_state.second.watches_) { + watch->callbacks_.onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); } } - return absl::nullopt; +} + +void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { + request_queue_.push(queue_item); + drainRequests(); +} + +void GrpcMuxImpl::clearRequestQueue() { + grpc_stream_.maybeUpdateQueueSizeStat(0); + request_queue_ = {}; +} + +void GrpcMuxImpl::drainRequests() { + while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { + // Process the request, if rate limiting is not enabled at all or if it is under rate limit. + sendDiscoveryRequest(request_queue_.front()); + request_queue_.pop(); + } + grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); } } // namespace Config diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index bf9d38fed8d0..83821a91504a 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -1,224 +1,138 @@ #pragma once -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" +#include +#include + +#include "envoy/common/time.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/grpc/status.h" +#include "envoy/upstream/cluster_manager.h" +#include "common/common/cleanup.h" #include "common/common/logger.h" -#include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" -#include "common/config/pausable_ack_queue.h" -#include "common/config/sotw_subscription_state.h" -#include "common/config/watch_map.h" -#include "common/grpc/common.h" +#include "common/config/utility.h" namespace Envoy { namespace Config { -// Manages subscriptions to one or more type of resource. The logical protocol -// state of those subscription(s) is handled by SubscriptionState. -// This class owns the GrpcStream used to talk to the server, maintains queuing -// logic to properly order the subscription(s)' various messages, and allows -// starting/stopping/pausing of the subscriptions. -class GrpcMuxImpl : public GrpcMux, Logger::Loggable { +/** + * ADS API implementation that fetches via gRPC. + */ +class GrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + public Logger::Loggable { public: - GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); + GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, + Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); + ~GrpcMuxImpl() override; - Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; - void removeWatch(const std::string& type_url, Watch* watch) override; + void start() override; + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, + GrpcMuxCallbacks& callbacks) override; + // GrpcMux void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; - void start() override; - void disableInitFetchTimeoutTimer() override; - -protected: - // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes - // that underlie it) are templated on protobufs. That means that a single implementation that - // supports different types of protobufs cannot use polymorphism to share code. The workaround: - // the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is - // seen here in the base class as calls to abstract functions, to be provided by those derived - // classes. - virtual void establishGrpcStream() PURE; - // Deletes msg_proto_ptr. - virtual void sendGrpcMessage(void* msg_proto_ptr) PURE; - virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; - virtual bool grpcStreamAvailable() const PURE; - virtual bool rateLimitAllowsDrain() PURE; - - SubscriptionState* subscriptionStateFor(const std::string& type_url); - WatchMap& watchMapFor(const std::string& type_url); - void handleEstablishedStream(); - void handleStreamEstablishmentFailure(); - void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); - void trySendDiscoveryRequests(); - bool skip_subsequent_node() const { return skip_subsequent_node_; } - bool any_request_sent_yet_in_current_stream() const { - return any_request_sent_yet_in_current_stream_; - } - void set_any_request_sent_yet_in_current_stream(bool value) { - any_request_sent_yet_in_current_stream_ = value; - } - const LocalInfo::LocalInfo& local_info() const { return local_info_; } - -private: - Watch* addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); - - // Updates the list of resource names watched by the given watch. If an added name is new across - // the whole subscription, or if a removed name has no other watch interested in it, then the - // subscription will enqueue and attempt to send an appropriate discovery request. - void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources); - - void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(const std::string& type_url); - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). - // First, prioritizes ACKs over non-ACK subscription interest updates. - // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated (as tracked by subscription_ordering_). - absl::optional whoWantsToSendDiscoveryRequest(); - - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All - // of our different resource types' ACKs are mixed together in this queue. See class for - // description of how it interacts with pause() and resume(). - PausableAckQueue pausable_ack_queue_; - - // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing - // delta or SotW xDS is determined by whether this is a {Delta,Sotw}SubscriptionStateFactory. - std::unique_ptr subscription_state_factory_; - - // Map key is type_url. - // Only addWatch() should insert into these maps. - absl::flat_hash_map> subscriptions_; - absl::flat_hash_map> watch_maps_; - - // Determines the order of initial discovery requests. (Assumes that subscriptions are added - // to this GrpcMux in the order of Envoy's dependency ordering). - std::list subscription_ordering_; - - // Whether to enable the optimization of only including the node field in the very first - // discovery request in an xDS gRPC stream (really just one: *not* per-type_url). - const bool skip_subsequent_node_; - - // State to help with skip_subsequent_node's logic. - bool any_request_sent_yet_in_current_stream_{}; - // Used to populate the [Delta]DiscoveryRequest's node field. That field is the same across - // all type_urls, and moreover, the 'skip_subsequent_node' logic needs to operate across all - // the type_urls. So, while the SubscriptionStates populate every other field of these messages, - // this one is up to GrpcMux. - const LocalInfo::LocalInfo& local_info_; - - // Demonstrates that when the maybeUpdateQueueSizeStat() segfault happens, this object is - // still valid. - int vinny_the_variable_{}; -}; - -class GrpcMuxDelta : public GrpcMuxImpl, - public GrpcStreamCallbacks { -public: - GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - - // GrpcStreamCallbacks - void onStreamEstablished() override { handleEstablishedStream(); } - void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } - void onWriteable() override { trySendDiscoveryRequests(); } - void - onDiscoveryResponse(std::unique_ptr&& message) override { - genericHandleResponse(message->type_url(), message.get()); + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } -protected: - void establishGrpcStream() override { grpc_stream_.establishNewStream(); } - void sendGrpcMessage(void* msg_proto_ptr) override { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); - } - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); - } - void maybeUpdateQueueSizeStat(uint64_t size) override { - grpc_stream_.maybeUpdateQueueSizeStat(size); - } - bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } - bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + void handleDiscoveryResponse(std::unique_ptr&& message); - GrpcStream - grpc_stream_; -}; + void sendDiscoveryRequest(const std::string& type_url); + + // Config::GrpcStreamCallbacks + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onDiscoveryResponse(std::unique_ptr&& message) override; + void onWriteable() override; -class GrpcMuxSotw : public GrpcMuxImpl, - public GrpcStreamCallbacks { -public: - GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) { - std::cerr << "creating STOW mux " << this << std::endl; - } - ~GrpcMuxSotw() { std::cerr << "destroying STOW mux " << this << std::endl; } // TODO REMOVE - // GrpcStreamCallbacks - void onStreamEstablished() override { handleEstablishedStream(); } - void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } - void onWriteable() override { trySendDiscoveryRequests(); } - void onDiscoveryResponse(std::unique_ptr&& message) override { - genericHandleResponse(message->type_url(), message.get()); - } GrpcStream& grpcStreamForTest() { return grpc_stream_; } -protected: - void establishGrpcStream() override { grpc_stream_.establishNewStream(); } - void sendGrpcMessage(void* msg_proto_ptr) override { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); +private: + void drainRequests(); + void setRetryTimer(); + + struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { + GrpcMuxWatchImpl(const std::set& resources, GrpcMuxCallbacks& callbacks, + const std::string& type_url, GrpcMuxImpl& parent) + : RaiiListElement(parent.api_state_[type_url].watches_, this), + resources_(resources), callbacks_(callbacks), type_url_(type_url), parent_(parent), + inserted_(true) {} + ~GrpcMuxWatchImpl() override { + if (inserted_) { + erase(); + if (!resources_.empty()) { + parent_.sendDiscoveryRequest(type_url_); + } + } } - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); - } - void maybeUpdateQueueSizeStat(uint64_t size) override { - grpc_stream_.maybeUpdateQueueSizeStat(size); - } - bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } - bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + void clear() { + inserted_ = false; + cancel(); + } + + std::set resources_; + GrpcMuxCallbacks& callbacks_; + const std::string type_url_; + GrpcMuxImpl& parent_; + + private: + bool inserted_; + }; + + // Per muxed API state. + struct ApiState { + // Watches on the returned resources for the API; + std::list watches_; + // Current DiscoveryRequest for API. + envoy::api::v2::DiscoveryRequest request_; + // Paused via pause()? + bool paused_{}; + // Was a DiscoveryRequest elided during a pause? + bool pending_{}; + // Has this API been tracked in subscriptions_? + bool subscribed_{}; + }; + + // Request queue management logic. + void queueDiscoveryRequest(const std::string& queue_item); + void clearRequestQueue(); GrpcStream grpc_stream_; + const LocalInfo::LocalInfo& local_info_; + const bool skip_subsequent_node_; + bool first_stream_request_; + std::unordered_map api_state_; + // Envoy's dependency ordering. + std::list subscriptions_; + + // A queue to store requests while rate limited. Note that when requests cannot be sent due to the + // gRPC stream being down, this queue does not store them; rather, they are simply dropped. + // This string is a type URL. + std::queue request_queue_; }; -class NullGrpcMuxImpl : public GrpcMux { +class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: void start() override {} - + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } void pause(const std::string&) override {} void resume(const std::string&) override {} bool paused(const std::string&) const override { return false; } @@ -230,7 +144,11 @@ class NullGrpcMuxImpl : public GrpcMux { void removeWatch(const std::string&, Watch*) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } - void disableInitFetchTimeoutTimer() override {} + + void onWriteable() override {} + void onStreamEstablished() override {} + void onEstablishmentFailure() override {} + void onDiscoveryResponse(std::unique_ptr&&) override {} }; } // namespace Config diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc new file mode 100644 index 000000000000..d22593d0bf99 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.cc @@ -0,0 +1,228 @@ +#include "common/config/new_grpc_mux_impl.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch, resources); + return watch; + } +} + +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + RELEASE_ASSERT(entry != subscriptions_.end(), + fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); + entry->second->watch_map_.removeWatch(watch); +} + +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } + +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} + +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + kickOffAck(sub->second->sub_state_.handleResponse(*message)); +} + +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); + } + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } + +void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { + pausable_ack_queue_.push(std::move(ack)); + trySendDiscoveryRequests(); +} + +// TODO(fredlas) to be removed from the GrpcMux interface very soon. +GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} + +void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } + +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + Watch* watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Watch of {} has no subscription to update.", type_url)); + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our change in interest, if any. + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to send discovery request for non-existent subscription {}.", + next_request_type_url)); + + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); + } + } + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); + + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } + } + return absl::nullopt; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h new file mode 100644 index 000000000000..1bd3b5a1f174 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.h @@ -0,0 +1,117 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" + +#include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" + +namespace Envoy { +namespace Config { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { +public: + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info); + + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; + + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; + + void onStreamEstablished() override; + + void onEstablishmentFailure() override; + + void onWriteable() override; + + void kickOffAck(UpdateAck ack); + + // TODO(fredlas) remove these two from the GrpcMux interface. + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override; + void start() override; + +private: + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); + + void trySendDiscoveryRequests(); + + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMap watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + GrpcStream + grpc_stream_; +}; + +using NewGrpcMuxImplSharedPtr = std::shared_ptr; + +} // namespace Config +} // namespace Envoy From 52c798e2c4d8b65e3d9d655668b4602bf3a603a3 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 2 Oct 2019 12:01:45 -0400 Subject: [PATCH 13/39] get github to diff grpc_mux_impl against new_grpc_mux impl, part 2 Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 381 ++++++++++------------ source/common/config/grpc_mux_impl.h | 206 +++++------- source/common/config/new_grpc_mux_impl.cc | 228 ------------- source/common/config/new_grpc_mux_impl.h | 117 ------- 4 files changed, 264 insertions(+), 668 deletions(-) delete mode 100644 source/common/config/new_grpc_mux_impl.cc delete mode 100644 source/common/config/new_grpc_mux_impl.h diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index fdb3fa603758..d22593d0bf99 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -1,248 +1,227 @@ -#include "common/config/grpc_mux_impl.h" - -#include +#include "common/config/new_grpc_mux_impl.h" +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" #include "common/config/utility.h" #include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" namespace Envoy { namespace Config { -GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, - Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node) - : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings), - local_info_(local_info), skip_subsequent_node_(skip_subsequent_node), - first_stream_request_(true) { - Config::Utility::checkLocalInfo("ads", local_info); -} - -GrpcMuxImpl::~GrpcMuxImpl() { - for (const auto& api_state : api_state_) { - for (auto watch : api_state.second.watches_) { - watch->clear(); - } +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch, resources); + return watch; } } -void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - -void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { - if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); - return; // Drop this request; the reconnect will enqueue a new one. - } +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + RELEASE_ASSERT(entry != subscriptions_.end(), + fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); + entry->second->watch_map_.removeWatch(watch); +} - ApiState& api_state = api_state_[type_url]; - if (api_state.paused_) { - ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest(), setting pending.", type_url); - api_state.pending_ = true; - return; // Drop this request; the unpause will enqueue a new one. - } +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } - auto& request = api_state.request_; - request.mutable_resource_names()->Clear(); +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} - // Maintain a set to avoid dupes. - std::unordered_set resources; - for (const auto* watch : api_state.watches_) { - for (const std::string& resource : watch->resources_) { - if (resources.count(resource) == 0) { - resources.emplace(resource); - request.add_resource_names(resource); - } - } - } +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} - if (skip_subsequent_node_ && !first_stream_request_) { - request.clear_node(); +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; } - ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); - grpc_stream_.sendMessage(request); - first_stream_request_ = false; + kickOffAck(sub->second->sub_state_.handleResponse(*message)); +} - // clear error_detail after the request is sent if it exists. - if (api_state_[type_url].request_.has_error_detail()) { - api_state_[type_url].request_.clear_error_detail(); +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); } + trySendDiscoveryRequests(); } -GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) { - auto watch = - std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); - ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); - - // Lazily kick off the requests based on first subscription. This has the - // convenient side-effect that we order messages on the channel based on - // Envoy's internal dependency ordering. - // TODO(gsagula): move TokenBucketImpl params to a config. - if (!api_state_[type_url].subscribed_) { - api_state_[type_url].request_.set_type_url(type_url); - api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); - api_state_[type_url].subscribed_ = true; - subscriptions_.emplace_back(type_url); - } +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} - // This will send an updated request on each subscription. - // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. - // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we - // only send a single RDS/EDS update after the CDS/LDS update. - queueDiscoveryRequest(type_url); +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } - return watch; +void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { + pausable_ack_queue_.push(std::move(ack)); + trySendDiscoveryRequests(); } -void GrpcMuxImpl::pause(const std::string& type_url) { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(!api_state.paused_); - ASSERT(!api_state.pending_); - api_state.paused_ = true; +// TODO(fredlas) to be removed from the GrpcMux interface very soon. +GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } -void GrpcMuxImpl::resume(const std::string& type_url) { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(api_state.paused_); - api_state.paused_ = false; +void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - if (api_state.pending_) { - ASSERT(api_state.subscribed_); - queueDiscoveryRequest(type_url); - api_state.pending_ = false; +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); } + + Watch* watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; } -bool GrpcMuxImpl::paused(const std::string& type_url) const { - auto entry = api_state_.find(type_url); - if (entry == api_state_.end()) { - return false; +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Watch of {} has no subscription to update.", type_url)); + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our change in interest, if any. + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); } - return entry->second.paused_; } -void GrpcMuxImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - const std::string& type_url = message->type_url(); - ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); - if (api_state_.count(type_url) == 0) { - ENVOY_LOG(warn, "Ignoring the message for type URL {} as it has no current subscribers.", - type_url); - // TODO(yuval-k): This should never happen. consider dropping the stream as this is a - // protocol violation - return; - } - if (api_state_[type_url].watches_.empty()) { - // update the nonce as we are processing this response. - api_state_[type_url].request_.set_response_nonce(message->nonce()); - if (message->resources().empty()) { - // No watches and no resources. This can happen when envoy unregisters from a - // resource that's removed from the server as well. For example, a deleted cluster - // triggers un-watching the ClusterLoadAssignment watch, and at the same time the - // xDS server sends an empty list of ClusterLoadAssignment resources. we'll accept - // this update. no need to send a discovery request, as we don't watch for anything. - api_state_[type_url].request_.set_version_info(message->version_info()); - } else { - // No watches and we have resources - this should not happen. send a NACK (by not - // updating the version). - ENVOY_LOG(warn, "Ignoring unwatched type URL {}", type_url); - queueDiscoveryRequest(type_url); - } - return; - } - try { - // To avoid O(n^2) explosion (e.g. when we have 1000s of EDS watches), we - // build a map here from resource name to resource and then walk watches_. - // We have to walk all watches (and need an efficient map as a result) to - // ensure we deliver empty config updates when a resource is dropped. - std::unordered_map resources; - GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; - for (const auto& resource : message->resources()) { - if (type_url != resource.type_url()) { - throw EnvoyException( - fmt::format("{} does not match the message-wide type URL {} in DiscoveryResponse {}", - resource.type_url(), type_url, message->DebugString())); - } - const std::string resource_name = callbacks.resourceName(resource); - resources.emplace(resource_name, resource); +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; } - for (auto watch : api_state_[type_url].watches_) { - // onConfigUpdate should be called in all cases for single watch xDS (Cluster and - // Listener) even if the message does not have resources so that update_empty stat - // is properly incremented and state-of-the-world semantics are maintained. - if (watch->resources_.empty()) { - watch->callbacks_.onConfigUpdate(message->resources(), message->version_info()); - continue; - } - Protobuf::RepeatedPtrField found_resources; - for (const auto& watched_resource_name : watch->resources_) { - auto it = resources.find(watched_resource_name); - if (it != resources.end()) { - found_resources.Add()->MergeFrom(it->second); - } - } - // onConfigUpdate should be called only on watches(clusters/routes) that have - // updates in the message for EDS/RDS. - if (!found_resources.empty()) { - watch->callbacks_.onConfigUpdate(found_resources, message->version_info()); - } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to send discovery request for non-existent subscription {}.", + next_request_type_url)); + + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; } - // TODO(mattklein123): In the future if we start tracking per-resource versions, we - // would do that tracking here. - api_state_[type_url].request_.set_version_info(message->version_info()); - } catch (const EnvoyException& e) { - for (auto watch : api_state_[type_url].watches_) { - watch->callbacks_.onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } - ::google::rpc::Status* error_detail = api_state_[type_url].request_.mutable_error_detail(); - error_detail->set_code(Grpc::Status::GrpcStatus::Internal); - error_detail->set_message(e.what()); } - api_state_[type_url].request_.set_response_nonce(message->nonce()); - queueDiscoveryRequest(type_url); + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } -void GrpcMuxImpl::onWriteable() { drainRequests(); } +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); -void GrpcMuxImpl::onStreamEstablished() { - first_stream_request_ = true; - for (const auto& type_url : subscriptions_) { - queueDiscoveryRequest(type_url); + if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; } + return true; } -void GrpcMuxImpl::onEstablishmentFailure() { - for (const auto& api_state : api_state_) { - for (auto watch : api_state.second.watches_) { - watch->callbacks_.onConfigUpdateFailed( - Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, nullptr); - } +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; } -} - -void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { - request_queue_.push(queue_item); - drainRequests(); -} - -void GrpcMuxImpl::clearRequestQueue() { - grpc_stream_.maybeUpdateQueueSizeStat(0); - request_queue_ = {}; -} - -void GrpcMuxImpl::drainRequests() { - while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { - // Process the request, if rate limiting is not enabled at all or if it is under rate limit. - sendDiscoveryRequest(request_queue_.front()); - request_queue_.pop(); + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } } - grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); + return absl::nullopt; } } // namespace Config diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 83821a91504a..1bd3b5a1f174 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -1,155 +1,117 @@ #pragma once -#include -#include - -#include "envoy/common/time.h" +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/grpc/status.h" -#include "envoy/upstream/cluster_manager.h" -#include "common/common/cleanup.h" #include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" -#include "common/config/utility.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" namespace Envoy { namespace Config { -/** - * ADS API implementation that fetches via gRPC. - */ -class GrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - public Logger::Loggable { +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { public: - GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); - ~GrpcMuxImpl() override; + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info); - void start() override; - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks) override; + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; - // GrpcMux void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; - Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - - void handleDiscoveryResponse(std::unique_ptr&& message); - - void sendDiscoveryRequest(const std::string& type_url); - - // Config::GrpcStreamCallbacks void onStreamEstablished() override; + void onEstablishmentFailure() override; - void onDiscoveryResponse(std::unique_ptr&& message) override; + void onWriteable() override; - GrpcStream& - grpcStreamForTest() { - return grpc_stream_; - } + void kickOffAck(UpdateAck ack); + + // TODO(fredlas) remove these two from the GrpcMux interface. + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override; + void start() override; private: - void drainRequests(); - void setRetryTimer(); - - struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { - GrpcMuxWatchImpl(const std::set& resources, GrpcMuxCallbacks& callbacks, - const std::string& type_url, GrpcMuxImpl& parent) - : RaiiListElement(parent.api_state_[type_url].watches_, this), - resources_(resources), callbacks_(callbacks), type_url_(type_url), parent_(parent), - inserted_(true) {} - ~GrpcMuxWatchImpl() override { - if (inserted_) { - erase(); - if (!resources_.empty()) { - parent_.sendDiscoveryRequest(type_url_); - } - } - } - - void clear() { - inserted_ = false; - cancel(); - } - - std::set resources_; - GrpcMuxCallbacks& callbacks_; - const std::string type_url_; - GrpcMuxImpl& parent_; - - private: - bool inserted_; - }; + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); - // Per muxed API state. - struct ApiState { - // Watches on the returned resources for the API; - std::list watches_; - // Current DiscoveryRequest for API. - envoy::api::v2::DiscoveryRequest request_; - // Paused via pause()? - bool paused_{}; - // Was a DiscoveryRequest elided during a pause? - bool pending_{}; - // Has this API been tracked in subscriptions_? - bool subscribed_{}; - }; + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - // Request queue management logic. - void queueDiscoveryRequest(const std::string& queue_item); - void clearRequestQueue(); + void trySendDiscoveryRequests(); - GrpcStream grpc_stream_; + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; - bool first_stream_request_; - std::unordered_map api_state_; - // Envoy's dependency ordering. - std::list subscriptions_; - - // A queue to store requests while rate limited. Note that when requests cannot be sent due to the - // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - // This string is a type URL. - std::queue request_queue_; -}; -class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { -public: - void start() override {} - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - void pause(const std::string&) override {} - void resume(const std::string&) override {} - bool paused(const std::string&) const override { return false; } - - Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - void removeWatch(const std::string&, Watch*) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - - void onWriteable() override {} - void onStreamEstablished() override {} - void onEstablishmentFailure() override {} - void onDiscoveryResponse(std::unique_ptr&&) override {} + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMap watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + GrpcStream + grpc_stream_; }; +using NewGrpcMuxImplSharedPtr = std::shared_ptr; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc deleted file mode 100644 index d22593d0bf99..000000000000 --- a/source/common/config/new_grpc_mux_impl.cc +++ /dev/null @@ -1,228 +0,0 @@ -#include "common/config/new_grpc_mux_impl.h" - -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/token_bucket_impl.h" -#include "common/config/utility.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, - Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - -Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - if (watch == nullptr) { - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } else { - updateWatch(type_url, watch, resources); - return watch; - } -} - -void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { - updateWatch(type_url, watch, {}); - auto entry = subscriptions_.find(type_url); - RELEASE_ASSERT(entry != subscriptions_.end(), - fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); - entry->second->watch_map_.removeWatch(watch); -} - -void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } - -void NewGrpcMuxImpl::resume(const std::string& type_url) { - pausable_ack_queue_.resume(type_url); - trySendDiscoveryRequests(); -} - -bool NewGrpcMuxImpl::paused(const std::string& type_url) const { - return pausable_ack_queue_.paused(type_url); -} - -void NewGrpcMuxImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), - message->system_version_info()); - auto sub = subscriptions_.find(message->type_url()); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " - "subscription {}.", - message->system_version_info(), message->type_url()); - return; - } - kickOffAck(sub->second->sub_state_.handleResponse(*message)); -} - -void NewGrpcMuxImpl::onStreamEstablished() { - for (auto& sub : subscriptions_) { - sub.second->sub_state_.markStreamFresh(); - } - trySendDiscoveryRequests(); -} - -void NewGrpcMuxImpl::onEstablishmentFailure() { - // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately - // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a - // crash, the iteration needs to dance around a little: collect pointers to all - // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are - // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; - do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = &sub.second->sub_state_; - } - for (auto& sub : all_subscribed) { - if (already_called.insert(sub).second) { // insert succeeded ==> not already called - sub.second->handleEstablishmentFailure(); - } - } - } while (all_subscribed.size() != subscriptions_.size()); -} - -void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } - -void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { - pausable_ack_queue_.push(std::move(ack)); - trySendDiscoveryRequests(); -} - -// TODO(fredlas) to be removed from the GrpcMux interface very soon. -GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; -} - -void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - -Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } - - Watch* watch = entry->second->watch_map_.addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch, resources); - return watch; -} - -// Updates the list of resource names watched by the given watch. If an added name is new across -// the whole subscription, or if a removed name has no other watch interested in it, then the -// subscription will enqueue and attempt to send an appropriate discovery request. -void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); - auto sub = subscriptions_.find(type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), - fmt::format("Watch of {} has no subscription to update.", type_url)); - auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); - sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - // Tell the server about our change in interest, if any. - if (sub->second->sub_state_.subscriptionUpdatePending()) { - trySendDiscoveryRequests(); - } -} - -void NewGrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { - subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, - dispatcher_, local_info_)); - subscription_ordering_.emplace_back(type_url); -} - -void NewGrpcMuxImpl::trySendDiscoveryRequests() { - while (true) { - // Do any of our subscriptions even want to send a request? - absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); - if (!maybe_request_type.has_value()) { - break; - } - // If so, which one (by type_url)? - std::string next_request_type_url = maybe_request_type.value(); - // If we don't have a subscription object for this request's type_url, drop the request. - auto sub = subscriptions_.find(next_request_type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), - fmt::format("Tried to send discovery request for non-existent subscription {}.", - next_request_type_url)); - - // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url)) { - break; - } - // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. - if (!pausable_ack_queue_.empty()) { - // Because ACKs take precedence over plain requests, if there is anything in the queue, it's - // safe to assume it's of the type_url that we're wanting to send. - grpc_stream_.sendMessage( - sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); - pausable_ack_queue_.pop(); - } else { - grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); - } - } - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); -} - -// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check -// whether we *want* to send a DeltaDiscoveryRequest). -bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { - RELEASE_ASSERT( - !pausable_ack_queue_.paused(type_url), - fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " - "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", - type_url)); - - if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); - return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); - return false; - } - return true; -} - -// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or -// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). -// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). -// First, prioritizes ACKs over non-ACK subscription interest updates. -// Then, prioritizes non-ACK updates in the order the various types -// of subscriptions were activated. -absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { - // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose - // type_url from pausable_ack_queue_ if possible, before looking at pending updates. - if (!pausable_ack_queue_.empty()) { - return pausable_ack_queue_.front().type_url_; - } - // If we're looking to send multiple non-ACK requests, send them in the order that their - // subscriptions were initiated. - for (const auto& sub_type : subscription_ordering_) { - auto sub = subscriptions_.find(sub_type); - if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && - !pausable_ack_queue_.paused(sub_type)) { - return sub->first; - } - } - return absl::nullopt; -} - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h deleted file mode 100644 index 1bd3b5a1f174..000000000000 --- a/source/common/config/new_grpc_mux_impl.h +++ /dev/null @@ -1,117 +0,0 @@ -#pragma once - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" -#include "envoy/config/grpc_mux.h" -#include "envoy/config/subscription.h" - -#include "common/common/logger.h" -#include "common/config/delta_subscription_state.h" -#include "common/config/grpc_stream.h" -#include "common/config/pausable_ack_queue.h" -#include "common/config/watch_map.h" -#include "common/grpc/common.h" - -namespace Envoy { -namespace Config { - -// Manages subscriptions to one or more type of resource. The logical protocol -// state of those subscription(s) is handled by DeltaSubscriptionState. -// This class owns the GrpcStream used to talk to the server, maintains queuing -// logic to properly order the subscription(s)' various messages, and allows -// starting/stopping/pausing of the subscriptions. -class NewGrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - Logger::Loggable { -public: - NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info); - - Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; - void removeWatch(const std::string& type_url, Watch* watch) override; - - void pause(const std::string& type_url) override; - void resume(const std::string& type_url) override; - bool paused(const std::string& type_url) const override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; - - void onStreamEstablished() override; - - void onEstablishmentFailure() override; - - void onWriteable() override; - - void kickOffAck(UpdateAck ack); - - // TODO(fredlas) remove these two from the GrpcMux interface. - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override; - void start() override; - -private: - Watch* addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); - - // Updates the list of resource names watched by the given watch. If an added name is new across - // the whole subscription, or if a removed name has no other watch interested in it, then the - // subscription will enqueue and attempt to send an appropriate discovery request. - void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources); - - void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - - void trySendDiscoveryRequests(); - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(const std::string& type_url); - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). - // First, prioritizes ACKs over non-ACK subscription interest updates. - // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated. - absl::optional whoWantsToSendDiscoveryRequest(); - - Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All - // of our different resource types' ACKs are mixed together in this queue. See class for - // description of how it interacts with pause() and resume(). - PausableAckQueue pausable_ack_queue_; - - struct SubscriptionStuff { - SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} - - WatchMap watch_map_; - DeltaSubscriptionState sub_state_; - const std::chrono::milliseconds init_fetch_timeout_; - - SubscriptionStuff(const SubscriptionStuff&) = delete; - SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; - }; - // Map key is type_url. - absl::flat_hash_map> subscriptions_; - - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in - // the order of Envoy's dependency ordering). - std::list subscription_ordering_; - - GrpcStream - grpc_stream_; -}; - -using NewGrpcMuxImplSharedPtr = std::shared_ptr; - -} // namespace Config -} // namespace Envoy From 96ce9f0cc4af7cec54de1bfba06ee43bd3f693ee Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 2 Oct 2019 12:02:41 -0400 Subject: [PATCH 14/39] get github to diff grpc_mux_impl against new_grpc_mux impl, part 3 Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 219 +++++++++++++------------- source/common/config/grpc_mux_impl.h | 208 ++++++++++++++++++------ 2 files changed, 273 insertions(+), 154 deletions(-) diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index d22593d0bf99..1a566126f15b 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "common/config/new_grpc_mux_impl.h" +#include "common/config/grpc_mux_impl.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" @@ -10,20 +10,17 @@ namespace Envoy { namespace Config { -NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, - Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - -Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { +GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) + : subscription_state_factory_(std::move(subscription_state_factory)), + skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) { + Config::Utility::checkLocalInfo("ads", local_info); +} + +Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { if (watch == nullptr) { return addWatch(type_url, resources, callbacks, init_fetch_timeout); } else { @@ -32,58 +29,103 @@ Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watc } } -void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { +void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { updateWatch(type_url, watch, {}); - auto entry = subscriptions_.find(type_url); - RELEASE_ASSERT(entry != subscriptions_.end(), - fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); - entry->second->watch_map_.removeWatch(watch); + watchMapFor(type_url).removeWatch(watch); } -void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } +void GrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } -void NewGrpcMuxImpl::resume(const std::string& type_url) { +void GrpcMuxImpl::resume(const std::string& type_url) { pausable_ack_queue_.resume(type_url); trySendDiscoveryRequests(); } -bool NewGrpcMuxImpl::paused(const std::string& type_url) const { +bool GrpcMuxImpl::paused(const std::string& type_url) const { return pausable_ack_queue_.paused(type_url); } -void NewGrpcMuxImpl::onDiscoveryResponse( - std::unique_ptr&& message) { - ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), - message->system_version_info()); - auto sub = subscriptions_.find(message->type_url()); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " - "subscription {}.", - message->system_version_info(), message->type_url()); - return; +void GrpcMuxImpl::start() { establishGrpcStream(); } + +void GrpcMuxImpl::disableInitFetchTimeoutTimer() { + for (auto& sub : subscriptions_) { + sub.second->disableInitFetchTimeoutTimer(); + } +} + +Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto watch_map = watch_maps_.find(type_url); + if (watch_map == watch_maps_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); } - kickOffAck(sub->second->sub_state_.handleResponse(*message)); + + Watch* watch = watch_map->second->addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + SubscriptionState* sub = subscriptionStateFor(type_url); + WatchMap& watch_map = watchMapFor(type_url); + + auto added_removed = watch_map.updateWatchInterest(watch, resources); + sub->updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + + // Tell the server about our change in interest, if any. + if (sub->subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void GrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + watch_maps_.emplace(type_url, std::make_unique()); + subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( + type_url, *watch_maps_[type_url], init_fetch_timeout)); + subscription_ordering_.emplace_back(type_url); +} + +SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", type_url)); + return sub->second.get(); } -void NewGrpcMuxImpl::onStreamEstablished() { +WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { + auto watch_map = watch_maps_.find(type_url); + RELEASE_ASSERT(watch_map != watch_maps_.end(), fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); + return *watch_map->second; +} + +void GrpcMuxImpl::handleEstablishedStream() { for (auto& sub : subscriptions_) { - sub.second->sub_state_.markStreamFresh(); + sub.second->markStreamFresh(); } + set_any_request_sent_yet_in_current_stream(false); trySendDiscoveryRequests(); } -void NewGrpcMuxImpl::onEstablishmentFailure() { +void GrpcMuxImpl::handleStreamEstablishmentFailure() { // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a // crash, the iteration needs to dance around a little: collect pointers to all // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; do { for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = &sub.second->sub_state_; + all_subscribed[sub.first] = sub.second.get(); } for (auto& sub : all_subscribed) { if (already_called.insert(sub).second) { // insert succeeded ==> not already called @@ -93,62 +135,21 @@ void NewGrpcMuxImpl::onEstablishmentFailure() { } while (all_subscribed.size() != subscriptions_.size()); } -void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } - -void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { - pausable_ack_queue_.push(std::move(ack)); - trySendDiscoveryRequests(); -} - -// TODO(fredlas) to be removed from the GrpcMux interface very soon. -GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; -} - -void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - -Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } - - Watch* watch = entry->second->watch_map_.addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch, resources); - return watch; -} - -// Updates the list of resource names watched by the given watch. If an added name is new across -// the whole subscription, or if a removed name has no other watch interested in it, then the -// subscription will enqueue and attempt to send an appropriate discovery request. -void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); +void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, + const void* response_proto_ptr) { auto sub = subscriptions_.find(type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), - fmt::format("Watch of {} has no subscription to update.", type_url)); - auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); - sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - // Tell the server about our change in interest, if any. - if (sub->second->sub_state_.subscriptionUpdatePending()) { - trySendDiscoveryRequests(); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "The server sent an xDS response proto with type_url {}, which we have " + "not subscribed to. Ignoring.", + type_url); + return; } + pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); + trySendDiscoveryRequests(); } -void NewGrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { - subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, - dispatcher_, local_info_)); - subscription_ordering_.emplace_back(type_url); -} - -void NewGrpcMuxImpl::trySendDiscoveryRequests() { +void GrpcMuxImpl::trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); @@ -158,11 +159,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { // If so, which one (by type_url)? std::string next_request_type_url = maybe_request_type.value(); // If we don't have a subscription object for this request's type_url, drop the request. - auto sub = subscriptions_.find(next_request_type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), - fmt::format("Tried to send discovery request for non-existent subscription {}.", - next_request_type_url)); - + SubscriptionState* sub = subscriptionStateFor(next_request_type_url); // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { break; @@ -171,29 +168,32 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { if (!pausable_ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's of the type_url that we're wanting to send. - grpc_stream_.sendMessage( - sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + UpdateAck ack = pausable_ack_queue_.front(); pausable_ack_queue_.pop(); + // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. + sendGrpcMessage(sub->getNextRequestWithAck(ack)); } else { - grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); + // getNextRequestAckless() returns a raw unowned pointer, which sendGrpcMessage deletes. + sendGrpcMessage(sub->getNextRequestAckless()); } } - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); + std::cerr << ++vinny_the_variable_ << std::endl; + maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). -bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { +bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { RELEASE_ASSERT( !pausable_ack_queue_.paused(type_url), fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", type_url)); - if (!grpc_stream_.grpcStreamAvailable()) { + if (!grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + } else if (!rateLimitAllowsDrain()) { ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); return false; } @@ -206,7 +206,7 @@ bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { // First, prioritizes ACKs over non-ACK subscription interest updates. // Then, prioritizes non-ACK updates in the order the various types // of subscriptions were activated. -absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { +absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from pausable_ack_queue_ if possible, before looking at pending updates. if (!pausable_ack_queue_.empty()) { @@ -215,10 +215,9 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // If we're looking to send multiple non-ACK requests, send them in the order that their // subscriptions were initiated. for (const auto& sub_type : subscription_ordering_) { - auto sub = subscriptions_.find(sub_type); - if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && - !pausable_ack_queue_.paused(sub_type)) { - return sub->first; + SubscriptionState* sub = subscriptionStateFor(sub_type); + if (sub->subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { + return sub_type; } } return absl::nullopt; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 1bd3b5a1f174..bf9d38fed8d0 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -9,6 +9,7 @@ #include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" #include "common/config/pausable_ack_queue.h" +#include "common/config/sotw_subscription_state.h" #include "common/config/watch_map.h" #include "common/grpc/common.h" @@ -16,18 +17,14 @@ namespace Envoy { namespace Config { // Manages subscriptions to one or more type of resource. The logical protocol -// state of those subscription(s) is handled by DeltaSubscriptionState. +// state of those subscription(s) is handled by SubscriptionState. // This class owns the GrpcStream used to talk to the server, maintains queuing // logic to properly order the subscription(s)' various messages, and allows // starting/stopping/pausing of the subscriptions. -class NewGrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - Logger::Loggable { +class GrpcMuxImpl : public GrpcMux, Logger::Loggable { public: - NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info); + GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, @@ -37,21 +34,37 @@ class NewGrpcMuxImpl : public GrpcMux, void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; - void - onDiscoveryResponse(std::unique_ptr&& message) override; - - void onStreamEstablished() override; - - void onEstablishmentFailure() override; - - void onWriteable() override; - - void kickOffAck(UpdateAck ack); - - // TODO(fredlas) remove these two from the GrpcMux interface. - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override; void start() override; + void disableInitFetchTimeoutTimer() override; + +protected: + // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes + // that underlie it) are templated on protobufs. That means that a single implementation that + // supports different types of protobufs cannot use polymorphism to share code. The workaround: + // the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is + // seen here in the base class as calls to abstract functions, to be provided by those derived + // classes. + virtual void establishGrpcStream() PURE; + // Deletes msg_proto_ptr. + virtual void sendGrpcMessage(void* msg_proto_ptr) PURE; + virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; + virtual bool grpcStreamAvailable() const PURE; + virtual bool rateLimitAllowsDrain() PURE; + + SubscriptionState* subscriptionStateFor(const std::string& type_url); + WatchMap& watchMapFor(const std::string& type_url); + void handleEstablishedStream(); + void handleStreamEstablishmentFailure(); + void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); + void trySendDiscoveryRequests(); + bool skip_subsequent_node() const { return skip_subsequent_node_; } + bool any_request_sent_yet_in_current_stream() const { + return any_request_sent_yet_in_current_stream_; + } + void set_any_request_sent_yet_in_current_stream(bool value) { + any_request_sent_yet_in_current_stream_ = value; + } + const LocalInfo::LocalInfo& local_info() const { return local_info_; } private: Watch* addWatch(const std::string& type_url, const std::set& resources, @@ -65,8 +78,6 @@ class NewGrpcMuxImpl : public GrpcMux, void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - void trySendDiscoveryRequests(); - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). bool canSendDiscoveryRequest(const std::string& type_url); @@ -76,42 +87,151 @@ class NewGrpcMuxImpl : public GrpcMux, // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). // First, prioritizes ACKs over non-ACK subscription interest updates. // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated. + // of subscriptions were activated (as tracked by subscription_ordering_). absl::optional whoWantsToSendDiscoveryRequest(); - Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All // of our different resource types' ACKs are mixed together in this queue. See class for // description of how it interacts with pause() and resume(). PausableAckQueue pausable_ack_queue_; - struct SubscriptionStuff { - SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} - - WatchMap watch_map_; - DeltaSubscriptionState sub_state_; - const std::chrono::milliseconds init_fetch_timeout_; + // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing + // delta or SotW xDS is determined by whether this is a {Delta,Sotw}SubscriptionStateFactory. + std::unique_ptr subscription_state_factory_; - SubscriptionStuff(const SubscriptionStuff&) = delete; - SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; - }; // Map key is type_url. - absl::flat_hash_map> subscriptions_; + // Only addWatch() should insert into these maps. + absl::flat_hash_map> subscriptions_; + absl::flat_hash_map> watch_maps_; - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in - // the order of Envoy's dependency ordering). + // Determines the order of initial discovery requests. (Assumes that subscriptions are added + // to this GrpcMux in the order of Envoy's dependency ordering). std::list subscription_ordering_; + // Whether to enable the optimization of only including the node field in the very first + // discovery request in an xDS gRPC stream (really just one: *not* per-type_url). + const bool skip_subsequent_node_; + + // State to help with skip_subsequent_node's logic. + bool any_request_sent_yet_in_current_stream_{}; + + // Used to populate the [Delta]DiscoveryRequest's node field. That field is the same across + // all type_urls, and moreover, the 'skip_subsequent_node' logic needs to operate across all + // the type_urls. So, while the SubscriptionStates populate every other field of these messages, + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; + + // Demonstrates that when the maybeUpdateQueueSizeStat() segfault happens, this object is + // still valid. + int vinny_the_variable_{}; +}; + +class GrpcMuxDelta : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + + // GrpcStreamCallbacks + void onStreamEstablished() override { handleEstablishedStream(); } + void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } + void onWriteable() override { trySendDiscoveryRequests(); } + void + onDiscoveryResponse(std::unique_ptr&& message) override { + genericHandleResponse(message->type_url(), message.get()); + } + +protected: + void establishGrpcStream() override { grpc_stream_.establishNewStream(); } + void sendGrpcMessage(void* msg_proto_ptr) override { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); + } + void maybeUpdateQueueSizeStat(uint64_t size) override { + grpc_stream_.maybeUpdateQueueSizeStat(size); + } + bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } + bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + GrpcStream grpc_stream_; }; -using NewGrpcMuxImplSharedPtr = std::shared_ptr; +class GrpcMuxSotw : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) { + std::cerr << "creating STOW mux " << this << std::endl; + } + ~GrpcMuxSotw() { std::cerr << "destroying STOW mux " << this << std::endl; } // TODO REMOVE + // GrpcStreamCallbacks + void onStreamEstablished() override { handleEstablishedStream(); } + void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } + void onWriteable() override { trySendDiscoveryRequests(); } + void onDiscoveryResponse(std::unique_ptr&& message) override { + genericHandleResponse(message->type_url(), message.get()); + } + GrpcStream& + grpcStreamForTest() { + return grpc_stream_; + } + +protected: + void establishGrpcStream() override { grpc_stream_.establishNewStream(); } + void sendGrpcMessage(void* msg_proto_ptr) override { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); + } + void maybeUpdateQueueSizeStat(uint64_t size) override { + grpc_stream_.maybeUpdateQueueSizeStat(size); + } + bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } + bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + + GrpcStream grpc_stream_; +}; + +class NullGrpcMuxImpl : public GrpcMux { +public: + void start() override {} + + void pause(const std::string&) override {} + void resume(const std::string&) override {} + bool paused(const std::string&) const override { return false; } + + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void removeWatch(const std::string&, Watch*) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void disableInitFetchTimeoutTimer() override {} +}; } // namespace Config } // namespace Envoy From 7d0904eb01e9363cce7111b32716a962e199acab Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 2 Oct 2019 16:42:37 -0400 Subject: [PATCH 15/39] is_delta removed from everywhere, new sotw_sub_state_test passes Signed-off-by: Fred Douglas --- include/envoy/config/subscription_factory.h | 2 +- .../router/route_config_provider_manager.h | 2 +- include/envoy/server/listener_manager.h | 6 +- include/envoy/upstream/cluster_manager.h | 4 +- source/common/config/grpc_mux_impl.cc | 8 +- .../common/config/grpc_subscription_impl.cc | 2 +- .../common/config/sotw_subscription_state.cc | 3 +- .../config/subscription_factory_impl.cc | 26 +- source/common/router/rds_impl.cc | 12 +- source/common/router/rds_impl.h | 8 +- source/common/router/vhds.cc | 2 +- source/common/runtime/runtime_impl.cc | 2 +- source/common/secret/sds_api.cc | 2 +- source/common/upstream/cds_api_impl.cc | 13 +- source/common/upstream/cds_api_impl.h | 9 +- source/common/upstream/cluster_manager_impl.h | 6 +- source/common/upstream/eds.cc | 7 +- source/common/upstream/eds.h | 2 +- .../network/http_connection_manager/config.cc | 3 +- .../config_validation/cluster_manager.cc | 4 +- .../config_validation/cluster_manager.h | 2 +- source/server/config_validation/server.h | 9 +- source/server/lds_api.cc | 4 +- source/server/lds_api.h | 2 +- source/server/listener_manager_impl.h | 10 +- source/server/server.cc | 8 +- .../config/delta_subscription_impl_test.cc | 6 +- .../config/delta_subscription_test_harness.h | 8 +- .../config/grpc_subscription_impl_test.cc | 2 +- .../config/grpc_subscription_test_harness.h | 5 +- .../config/sotw_subscription_state_test.cc | 287 +++--------------- test/common/config/subscription_impl_test.cc | 14 +- test/common/upstream/cds_api_impl_test.cc | 4 +- test/integration/ads_integration_test.cc | 2 +- test/mocks/config/mocks.cc | 18 +- test/mocks/config/mocks.h | 4 +- test/mocks/router/mocks.h | 4 +- test/mocks/server/mocks.h | 11 +- test/mocks/upstream/mocks.h | 5 +- 39 files changed, 156 insertions(+), 372 deletions(-) diff --git a/include/envoy/config/subscription_factory.h b/include/envoy/config/subscription_factory.h index 5268ac7f2abe..b1733afda6c4 100644 --- a/include/envoy/config/subscription_factory.h +++ b/include/envoy/config/subscription_factory.h @@ -24,7 +24,7 @@ class SubscriptionFactory { virtual SubscriptionPtr subscriptionFromConfigSource(const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, - SubscriptionCallbacks& callbacks, bool is_delta) PURE; + SubscriptionCallbacks& callbacks) PURE; }; } // namespace Config diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index accb99c85f24..9912aa460356 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -39,7 +39,7 @@ class RouteConfigProviderManager { virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager, bool is_delta) PURE; + Init::Manager& init_manager) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/include/envoy/server/listener_manager.h b/include/envoy/server/listener_manager.h index 54d4b4a9bbcb..ce2a015decf3 100644 --- a/include/envoy/server/listener_manager.h +++ b/include/envoy/server/listener_manager.h @@ -39,8 +39,7 @@ class ListenerComponentFactory { * @return an LDS API provider. * @param lds_config supplies the management server configuration. */ - virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, - bool is_delta) PURE; + virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; /** * Creates a socket. @@ -129,8 +128,7 @@ class ListenerManager { * pieces of the server existing. * @param lds_config supplies the management server configuration. */ - virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, - bool is_delta) PURE; + virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; /** * @return std::vector> a list of the currently diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index fd3255fc2dde..29d20890f233 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -223,8 +223,6 @@ class ClusterManager { virtual Config::SubscriptionFactory& subscriptionFactory() PURE; virtual std::size_t warmingClusterCount() const PURE; - - virtual bool xdsIsDelta() const PURE; }; using ClusterManagerPtr = std::unique_ptr; @@ -297,7 +295,7 @@ class ClusterManagerFactory { /** * Create a CDS API provider from configuration proto. */ - virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm) PURE; /** diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 1a566126f15b..158ac178dddf 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -97,13 +97,17 @@ void GrpcMuxImpl::addSubscription(const std::string& type_url, SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { auto sub = subscriptions_.find(type_url); - RELEASE_ASSERT(sub != subscriptions_.end(), fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", type_url)); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", + type_url)); return sub->second.get(); } WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { auto watch_map = watch_maps_.find(type_url); - RELEASE_ASSERT(watch_map != watch_maps_.end(), fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); + RELEASE_ASSERT( + watch_map != watch_maps_.end(), + fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); return *watch_map->second; } diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 20a84948bc66..4103f50de555 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -42,7 +42,7 @@ void GrpcSubscriptionImpl::start(const std::set& resources) { void GrpcSubscriptionImpl::updateResourceInterest( const std::set& update_to_these_names) { watch_ = grpc_mux_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, - init_fetch_timeout_); + init_fetch_timeout_); stats_.update_attempt_.inc(); } diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index b8724078e64c..0a18d92c448d 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -6,8 +6,7 @@ namespace Envoy { namespace Config { -SotwSubscriptionState::SotwSubscriptionState(std::string type_url, - SubscriptionCallbacks& callbacks, +SotwSubscriptionState::SotwSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) { diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index acf66910ea72..769b6104ca8c 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -48,23 +48,25 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( case envoy::api::v2::core::ApiConfigSource::GRPC: std::cerr << "subscriptionFromConfigSource single SOTW" << std::endl; return std::make_unique( - std::make_shared(Utility::factoryForGrpcApiConfigSource( - cm_.grpcAsyncClientManager(), api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), - local_info_, /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false + std::make_shared( + Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), local_info_, + /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false); case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: std::cerr << "subscriptionFromConfigSource single DELTA" << std::endl; return std::make_unique( - std::make_shared(Utility::factoryForGrpcApiConfigSource( - cm_.grpcAsyncClientManager(), api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), - local_info_, /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false + std::make_shared( + Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), local_info_, + /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false); default: diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index af64085ddd19..c9fd1b300255 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -23,7 +23,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager, bool is_delta) { + RouteConfigProviderManager& route_config_provider_manager) { switch (config.route_specifier_case()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kRouteConfig: @@ -31,7 +31,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: return route_config_provider_manager.createRdsRouteConfigProvider( - config.rds(), factory_context, stat_prefix, factory_context.initManager(), is_delta); + config.rds(), factory_context, stat_prefix, factory_context.initManager()); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -56,7 +56,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager, bool is_delta) + Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager) : route_config_name_(rds.route_config_name()), factory_context_(factory_context), init_target_(fmt::format("RdsRouteConfigSubscription {}", route_config_name_), [this]() { subscription_->start({route_config_name_}); }), @@ -69,7 +69,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( rds.config_source(), Grpc::Common::typeUrl(envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name()), - *scope_, *this, is_delta); + *scope_, *this); config_update_info_ = std::make_unique( factory_context.timeSource(), factory_context.messageValidationVisitor()); @@ -225,7 +225,7 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager, bool is_delta) { + Init::Manager& init_manager) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -237,7 +237,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // around it. However, since this is not a performance critical path we err on the side // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, - stat_prefix, *this, is_delta)); + stat_prefix, *this)); init_manager.add(subscription->init_target_); route_config_subscriptions_.insert({manager_identifier, subscription}); } else { diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index caf2340f084e..418512bb6e50 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -48,7 +48,7 @@ class RouteConfigProviderUtil { create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager, bool is_delta); + RouteConfigProviderManager& route_config_provider_manager); }; class RouteConfigProviderManagerImpl; @@ -132,8 +132,8 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, RouteConfigProviderManagerImpl& route_config_provider_manager, - bool is_delta); + const std::string& stat_prefix, + RouteConfigProviderManagerImpl& route_config_provider_manager); bool validateUpdateSize(int num_resources); @@ -207,7 +207,7 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Init::Manager& init_manager, bool is_delta) override; + Init::Manager& init_manager) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index de67108a9e26..8e715aef04dc 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -42,7 +42,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, factory_context.clusterManager().subscriptionFactory().subscriptionFromConfigSource( config_update_info_->routeConfiguration().vhds().config_source(), Grpc::Common::typeUrl(envoy::api::v2::route::VirtualHost().GetDescriptor()->full_name()), - *scope_, *this, /*is_delta=*/true); + *scope_, *this); } void VhdsSubscription::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason reason, diff --git a/source/common/runtime/runtime_impl.cc b/source/common/runtime/runtime_impl.cc index 22a0508bc56a..3d0a721c6aa1 100644 --- a/source/common/runtime/runtime_impl.cc +++ b/source/common/runtime/runtime_impl.cc @@ -568,7 +568,7 @@ void RtdsSubscription::start() { subscription_ = parent_.cm_->subscriptionFactory().subscriptionFromConfigSource( config_source_, Grpc::Common::typeUrl(envoy::service::discovery::v2::Runtime().GetDescriptor()->full_name()), - store_, *this, /*is_delta=*/false); + store_, *this); subscription_->start({resource_name_}); } diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 3e3478f0beb7..4514f5ce0792 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -79,7 +79,7 @@ void SdsApi::initialize() { subscription_ = subscription_factory_.subscriptionFromConfigSource( sds_config_, Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), stats_, - *this, /*is_delta=*/false); + *this); subscription_->start({sds_config_name_}); } diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 3269d58a2c3f..82521e3e0891 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -15,22 +15,19 @@ namespace Envoy { namespace Upstream { -// TODO(fredlas) the is_delta argument can be removed upon delta+SotW ADS Envoy code unification. It -// is only actually needed to choose the grpc_method, which is irrelevant if ADS is used. -CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, +CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor) { - return CdsApiPtr{new CdsApiImpl(cds_config, is_delta, cm, scope, validation_visitor)}; + return CdsApiPtr{new CdsApiImpl(cds_config, cm, scope, validation_visitor)}; } -CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, - ClusterManager& cm, Stats::Scope& scope, - ProtobufMessage::ValidationVisitor& validation_visitor) +CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, + Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor) : cm_(cm), scope_(scope.createScope("cluster_manager.cds.")), validation_visitor_(validation_visitor) { subscription_ = cm_.subscriptionFactory().subscriptionFromConfigSource( cds_config, Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), - *scope_, *this, is_delta); + *scope_, *this); } void CdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index 66bb7e4c3df8..b17d4bbc9989 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -22,8 +22,8 @@ class CdsApiImpl : public CdsApi, Config::SubscriptionCallbacks, Logger::Loggable { public: - static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, - ClusterManager& cm, Stats::Scope& scope, + static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, + Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor); // Upstream::CdsApi @@ -46,9 +46,8 @@ class CdsApiImpl : public CdsApi, return MessageUtil::anyConvert(resource).name(); } - CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, - ClusterManager& cm, Stats::Scope& scope, - ProtobufMessage::ValidationVisitor& validation_visitor); + CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, + Stats::Scope& scope, ProtobufMessage::ValidationVisitor& validation_visitor); void runInitializeCallbackIfAny(); ClusterManager& cm_; diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 7c969984ca7c..d1d7b204c709 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -68,7 +68,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { std::pair clusterFromProto(const envoy::api::v2::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) override; - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm) override; Secret::SecretManager& secretManager() override { return secret_manager_; } @@ -238,9 +238,6 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggablefull_name()), - info_->statsScope(), *this, is_delta); + info_->statsScope(), *this); } void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}); } @@ -269,8 +269,7 @@ EdsClusterFactory::createClusterImpl( return std::make_pair( std::make_unique(cluster, context.runtime(), socket_factory_context, - std::move(stats_scope), context.addedViaApi(), - context.clusterManager().xdsIsDelta()), + std::move(stats_scope), context.addedViaApi()), nullptr); } diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index 3f0a74b8dfca..0df5f4c84473 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -24,7 +24,7 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba public: EdsClusterImpl(const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& factory_context, - Stats::ScopePtr&& stats_scope, bool added_via_api, bool is_delta); + Stats::ScopePtr&& stats_scope, bool added_via_api); // Upstream::Cluster InitializePhase initializePhase() const override { return InitializePhase::Secondary; } diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index b18316ad5217..998dce6709ab 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -183,8 +183,7 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kRouteConfig: route_config_provider_ = Router::RouteConfigProviderUtil::create( - config, context_, stats_prefix_, route_config_provider_manager_, - context_.clusterManager().xdsIsDelta()); + config, context_, stats_prefix_, route_config_provider_manager_); break; case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kScopedRoutes: diff --git a/source/server/config_validation/cluster_manager.cc b/source/server/config_validation/cluster_manager.cc index 0f014a1741ea..cf20a6c0e221 100644 --- a/source/server/config_validation/cluster_manager.cc +++ b/source/server/config_validation/cluster_manager.cc @@ -14,9 +14,9 @@ ClusterManagerPtr ValidationClusterManagerFactory::clusterManagerFromProto( CdsApiPtr ValidationClusterManagerFactory::createCds(const envoy::api::v2::core::ConfigSource& cds_config, - bool is_delta, ClusterManager& cm) { + ClusterManager& cm) { // Create the CdsApiImpl... - ProdClusterManagerFactory::createCds(cds_config, is_delta, cm); + ProdClusterManagerFactory::createCds(cds_config, cm); // ... and then throw it away, so that we don't actually connect to it. return nullptr; } diff --git a/source/server/config_validation/cluster_manager.h b/source/server/config_validation/cluster_manager.h index b62387c3341d..0fe33294d190 100644 --- a/source/server/config_validation/cluster_manager.h +++ b/source/server/config_validation/cluster_manager.h @@ -38,7 +38,7 @@ class ValidationClusterManagerFactory : public ProdClusterManagerFactory { // Delegates to ProdClusterManagerFactory::createCds, but discards the result and returns nullptr // unconditionally. - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm) override; private: diff --git a/source/server/config_validation/server.h b/source/server/config_validation/server.h index 58ad9affadc3..d447c5822940 100644 --- a/source/server/config_validation/server.h +++ b/source/server/config_validation/server.h @@ -109,11 +109,10 @@ class ValidationInstance : Logger::Loggable, } // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, - bool is_delta) override { - return std::make_unique( - lds_config, clusterManager(), initManager(), stats(), listenerManager(), - messageValidationContext().dynamicValidationVisitor(), is_delta); + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { + return std::make_unique(lds_config, clusterManager(), initManager(), stats(), + listenerManager(), + messageValidationContext().dynamicValidationVisitor()); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index ca5cb3ed187d..5254fdbac3dc 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -17,13 +17,13 @@ namespace Server { LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Init::Manager& init_manager, Stats::Scope& scope, ListenerManager& lm, - ProtobufMessage::ValidationVisitor& validation_visitor, bool is_delta) + ProtobufMessage::ValidationVisitor& validation_visitor) : listener_manager_(lm), scope_(scope.createScope("listener_manager.lds.")), cm_(cm), init_target_("LDS", [this]() { subscription_->start({}); }), validation_visitor_(validation_visitor) { subscription_ = cm.subscriptionFactory().subscriptionFromConfigSource( lds_config, Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), - *scope_, *this, is_delta); + *scope_, *this); init_manager.add(init_target_); } diff --git a/source/server/lds_api.h b/source/server/lds_api.h index 2ef9209542b2..24ca66cfe73f 100644 --- a/source/server/lds_api.h +++ b/source/server/lds_api.h @@ -24,7 +24,7 @@ class LdsApiImpl : public LdsApi, public: LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Init::Manager& init_manager, Stats::Scope& scope, ListenerManager& lm, - ProtobufMessage::ValidationVisitor& validation_visitor, bool is_delta); + ProtobufMessage::ValidationVisitor& validation_visitor); // Server::LdsApi std::string versionInfo() const override { return system_version_info_; } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index ca00bc11082e..599f471ea7fe 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -59,12 +59,10 @@ class ProdListenerComponentFactory : public ListenerComponentFactory, Configuration::ListenerFactoryContext& context); // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, - bool is_delta) override { + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { return std::make_unique( lds_config, server_.clusterManager(), server_.initManager(), server_.stats(), - server_.listenerManager(), server_.messageValidationContext().dynamicValidationVisitor(), - is_delta); + server_.listenerManager(), server_.messageValidationContext().dynamicValidationVisitor()); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, @@ -129,9 +127,9 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable> listeners() override; uint64_t numConnections() override; diff --git a/source/server/server.cc b/source/server/server.cc index 1afeff023cfe..3b4d7e3d6ac1 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -402,13 +402,7 @@ void InstanceImpl::initialize(const Options& options, // Instruct the listener manager to create the LDS provider if needed. This must be done later // because various items do not yet exist when the listener manager is created. if (bootstrap_.dynamic_resources().has_lds_config()) { - const bool is_delta = - bootstrap_.dynamic_resources().lds_config().api_config_source().api_type() == - envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || - (bootstrap_.dynamic_resources().has_ads_config() && - bootstrap_.dynamic_resources().ads_config().api_type() == - envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); - listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config(), is_delta); + listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config()); } // We have to defer RTDS initialization until after the cluster manager is diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 992394ab11be..9586788bfe8e 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -71,7 +71,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. @@ -85,7 +85,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. @@ -99,7 +99,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get())->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 17377f52ded5..40fd021febf5 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -34,11 +34,11 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); - xds_context_ = std::make_shared( TODO TODO // TODO TODO rename this and other contexts to grpc_mux_ + grpc_mux_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, random_, stats_store_, rate_limit_settings_, local_info_, false); subscription_ = std::make_unique( - xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + grpc_mux_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout, false); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); } @@ -148,7 +148,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } @@ -189,7 +189,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; - std::shared_ptr xds_context_; + std::shared_ptr grpc_mux_; std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index cecf8aabfafa..82a7c06a0e2e 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -46,7 +46,7 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { .Times(0); EXPECT_CALL(*timer_, enableTimer(_, _)); EXPECT_CALL(random_, random()); - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get()) ->grpcStreamForTest() .onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index b7a6256e4c7c..642c647577d6 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -96,8 +96,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) override { - std::unique_ptr response( - new envoy::api::v2::DiscoveryResponse()); + auto response = std::make_unique(); response->set_version_info(version); last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); response->set_nonce(last_response_nonce_); @@ -122,7 +121,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, false, Grpc::Status::GrpcStatus::Internal, "bad config"); } - auto shared_mux = subscription_->getContextForTest(); + auto shared_mux = subscription_->getGrpcMuxForTest(); static_cast(shared_mux.get())->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc index edf367de3b92..6ddb3d35ef08 100644 --- a/test/common/config/sotw_subscription_state_test.cc +++ b/test/common/config/sotw_subscription_state_test.cc @@ -17,156 +17,113 @@ namespace Envoy { namespace Config { namespace { -const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; +const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"; class SotwSubscriptionStateTest : public testing::Test { protected: SotwSubscriptionStateTest() : state_(TypeUrl, callbacks_, std::chrono::milliseconds(0U), dispatcher_) { state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), - UnorderedElementsAre("name1", "name2", "name3")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); } - std::unique_ptr getNextDeltaDiscoveryRequestAckless() { + std::unique_ptr getNextDiscoveryRequestAckless() { auto* ptr = static_cast(state_.getNextRequestAckless()); return std::unique_ptr(ptr); } - UpdateAck deliverDiscoveryResponse( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& version_info, absl::optional nonce = absl::nullopt, - bool expect_config_update_call = true) { - envoy::api::v2::DiscoveryResponse message; - *message.mutable_resources() = added_resources; - *message.mutable_removed_resources() = removed_resources; - message.set_system_version_info(version_info); - if (nonce.has_value()) { - message.set_nonce(nonce.value()); + UpdateAck deliverDiscoveryResponse(const std::vector& resource_names, + const std::string& version_info, std::string nonce) { + envoy::api::v2::DiscoveryResponse response; + response.set_version_info(version_info); + response.set_nonce(nonce); + Protobuf::RepeatedPtrField typed_resources; + for (const auto& resource_name : resource_names) { + envoy::api::v2::ClusterLoadAssignment* load_assignment = typed_resources.Add(); + load_assignment->set_cluster_name(resource_name); + response.add_resources()->PackFrom(*load_assignment); } - EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).Times(expect_config_update_call ? 1 : 0); - return state_.handleResponse(&message); + EXPECT_CALL(callbacks_, onConfigUpdate(_, version_info)); + return state_.handleResponse(&response); } - UpdateAck deliverBadDiscoveryResponse( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& version_info, std::string nonce) { + UpdateAck deliverBadDiscoveryResponse(const std::string& version_info, std::string nonce) { envoy::api::v2::DiscoveryResponse message; - *message.mutable_resources() = added_resources; - *message.mutable_removed_resources() = removed_resources; - message.set_system_version_info(version_info); + message.set_version_info(version_info); message.set_nonce(nonce); - EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).WillOnce(Throw(EnvoyException("oh no"))); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _)).WillOnce(Throw(EnvoyException("oh no"))); return state_.handleResponse(&message); } - NiceMock> callbacks_; + NiceMock> callbacks_; NiceMock dispatcher_; // We start out interested in three resources: name1, name2, and name3. SotwSubscriptionState state_; }; -Protobuf::RepeatedPtrField -populateRepeatedResource(std::vector> items) { - Protobuf::RepeatedPtrField add_to; - for (const auto& item : items) { - auto* resource = add_to.Add(); - resource->set_name(item.first); - resource->set_version(item.second); - } - return add_to; -} - // Basic gaining/losing interest in resources should lead to (un)subscriptions. TEST_F(SotwSubscriptionStateTest, SubscribeAndUnsubscribe) { { state_.updateSubscriptionInterest({"name4"}, {"name1"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name2", "name3", "name4")); } { state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name1")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); } } -// Delta xDS reliably queues up and sends all discovery requests, even in situations where it isn't -// strictly necessary. E.g.: if you subscribe but then unsubscribe to a given resource, all before a -// request was able to be sent, two requests will be sent. The following tests demonstrate this. -// -// If Envoy decided it wasn't interested in a resource and then (before a request was sent) decided -// it was again, for all we know, it dropped that resource in between and needs to retrieve it -// again. So, we *should* send a request "re-"subscribing. This means that the server needs to -// interpret the resource_names_subscribe field as "send these resources even if you think Envoy -// already has them". +// Unlike delta, if SotW gets multiple interest updates before being able to send a request, they +// all collapse to a single update. However, even if the updates all cancel each other out, there +// still will be a request generated. All of the following tests explore different such cases. TEST_F(SotwSubscriptionStateTest, RemoveThenAdd) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name3")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); } -// Due to how our implementation provides the required behavior tested in RemoveThenAdd, the -// add-then-remove case *also* causes the resource to be referred to in the request (as an -// unsubscribe). -// Unlike the remove-then-add case, this one really is unnecessary, and ideally we would have -// the request simply not include any mention of the resource. Oh well. -// This test is just here to illustrate that this behavior exists, not to enforce that it -// should be like this. What *is* important: the server must happily and cleanly ignore -// "unsubscribe from [resource name I have never before referred to]" requests. TEST_F(SotwSubscriptionStateTest, AddThenRemove) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name4")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); } -// add/remove/add == add. TEST_F(SotwSubscriptionStateTest, AddRemoveAdd) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), + UnorderedElementsAre("name1", "name2", "name3", "name4")); } -// remove/add/remove == remove. TEST_F(SotwSubscriptionStateTest, RemoveAddRemove) { state_.updateSubscriptionInterest({}, {"name3"}); state_.updateSubscriptionInterest({"name3"}, {}); state_.updateSubscriptionInterest({}, {"name3"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); } -// Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are -// removed/added/removed. End result should be 4 added and 1,2,3 removed. TEST_F(SotwSubscriptionStateTest, BothAddAndRemove) { state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), - UnorderedElementsAre("name1", "name2", "name3")); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name4")); } TEST_F(SotwSubscriptionStateTest, CumulativeUpdates) { state_.updateSubscriptionInterest({"name4"}, {}); state_.updateSubscriptionInterest({"name5"}, {}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), + UnorderedElementsAre("name1", "name2", "name3", "name4", "name5")); } // Verifies that a sequence of good and bad responses from the server all get the appropriate @@ -174,150 +131,31 @@ TEST_F(SotwSubscriptionStateTest, CumulativeUpdates) { TEST_F(SotwSubscriptionStateTest, AckGenerated) { // The xDS server's first response includes items for name1 and 2, but not 3. { - Protobuf::RepeatedPtrField added_resources = - populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug1", "nonce1"); + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2"}, "version1", "nonce1"); EXPECT_EQ("nonce1", ack.nonce_); EXPECT_EQ(Grpc::Status::GrpcStatus::Ok, ack.error_detail_.code()); + std::cerr << "the error is: " << ack.error_detail_.message() << std::endl; } // The next response updates 1 and 2, and adds 3. { - Protobuf::RepeatedPtrField added_resources = populateRepeatedResource( - {{"name1", "version1B"}, {"name2", "version2B"}, {"name3", "version3A"}}); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug2", "nonce2"); + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version2", "nonce2"); EXPECT_EQ("nonce2", ack.nonce_); EXPECT_EQ(Grpc::Status::GrpcStatus::Ok, ack.error_detail_.code()); } // The next response tries but fails to update all 3, and so should produce a NACK. { - Protobuf::RepeatedPtrField added_resources = populateRepeatedResource( - {{"name1", "version1C"}, {"name2", "version2C"}, {"name3", "version3B"}}); - UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug3", "nonce3"); + UpdateAck ack = deliverBadDiscoveryResponse("version3", "nonce3"); EXPECT_EQ("nonce3", ack.nonce_); EXPECT_NE(Grpc::Status::GrpcStatus::Ok, ack.error_detail_.code()); } // The last response successfully updates all 3. { - Protobuf::RepeatedPtrField added_resources = populateRepeatedResource( - {{"name1", "version1D"}, {"name2", "version2D"}, {"name3", "version3C"}}); - UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug4", "nonce4"); + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version4", "nonce4"); EXPECT_EQ("nonce4", ack.nonce_); EXPECT_EQ(Grpc::Status::GrpcStatus::Ok, ack.error_detail_.code()); } } -// Tests population of the initial_resource_versions map in the first request of a new stream. -// Tests that -// 1) resources we have a version of are present in the map, -// 2) resources we are interested in but don't have are not present, and -// 3) resources we have lost interest in are not present. -TEST_F(SotwSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { - { - // The xDS server's first update includes items for name1 and 2, but not 3. - Protobuf::RepeatedPtrField add1_2 = - populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); - deliverDiscoveryResponse(add1_2, {}, "debugversion1"); - state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); - EXPECT_EQ(cur_request->initial_resource_versions().end(), - cur_request->initial_resource_versions().find("name3")); - } - - { - // The next update updates 1, removes 2, and adds 3. The map should then have 1 and 3. - Protobuf::RepeatedPtrField add1_3 = - populateRepeatedResource({{"name1", "version1B"}, {"name3", "version3A"}}); - Protobuf::RepeatedPtrField remove2; - *remove2.Add() = "name2"; - deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); - state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1B", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ(cur_request->initial_resource_versions().end(), - cur_request->initial_resource_versions().find("name2")); - EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); - } - - { - // The next update removes 1 and 3. The map we send the server should be empty... - Protobuf::RepeatedPtrField remove1_3; - *remove1_3.Add() = "name1"; - *remove1_3.Add() = "name3"; - deliverDiscoveryResponse({}, remove1_3, "debugversion3"); - state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->initial_resource_versions().empty()); - } - - { - // ...but our own map should remember our interest. In particular, losing interest in a - // resource should cause its name to appear in the next request's resource_names_unsubscribe. - state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); - } -} - -// Upon a reconnection, the server is supposed to assume a blank slate for the Envoy's state -// (hence the need for initial_resource_versions). The resource_names_subscribe of the first -// message must therefore be every resource the Envoy is interested in. -// -// resource_names_unsubscribe, on the other hand, is always blank in the first request - even if, -// in between the last request of the last stream and the first request of the new stream, Envoy -// lost interest in a resource. The unsubscription implicitly takes effect by simply saying -// nothing about the resource in the newly reconnected stream. -TEST_F(SotwSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { - Protobuf::RepeatedPtrField add1_2 = - populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); - deliverDiscoveryResponse(add1_2, {}, "debugversion1"); - - state_.updateSubscriptionInterest({"name4"}, {"name1"}); - state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - // Regarding the resource_names_subscribe field: - // name1: do not include: we lost interest. - // name2: yes do include: we're interested and we have a version of it. - // name3: yes do include: even though we don't have a version of it, we are interested. - // name4: yes do include: we are newly interested. (If this wasn't a stream reconnect, only name4 - // would belong in this subscribe field). - EXPECT_THAT(cur_request->resource_names_subscribe(), - UnorderedElementsAre("name2", "name3", "name4")); - EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); -} - -// initial_resource_versions should not be present on messages after the first in a stream. -TEST_F(SotwSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { - // First, verify that the first message of a new stream sends initial versions. - { - // The xDS server's first update gives us all three resources. - Protobuf::RepeatedPtrField add_all = populateRepeatedResource( - {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); - deliverDiscoveryResponse(add_all, {}, "debugversion1"); - state_.markStreamFresh(); // simulate a stream reconnection - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); - EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); - } - // Then, after updating the resources but not reconnecting the stream, verify that initial - // versions are not sent. - { - state_.updateSubscriptionInterest({"name4"}, {}); - // The xDS server updates our resources, and gives us our newly requested one too. - Protobuf::RepeatedPtrField add_all = - populateRepeatedResource({{"name1", "version1B"}, - {"name2", "version2B"}, - {"name3", "version3B"}, - {"name4", "version4A"}}); - deliverDiscoveryResponse(add_all, {}, "debugversion2"); - auto cur_request = getNextDeltaDiscoveryRequestAckless(); - EXPECT_TRUE(cur_request->initial_resource_versions().empty()); - } -} - TEST_F(SotwSubscriptionStateTest, CheckUpdatePending) { // Note that the test fixture ctor causes the first request to be "sent", so we start in the // middle of a stream, with our initially interested resources having been requested already. @@ -332,40 +170,9 @@ TEST_F(SotwSubscriptionStateTest, CheckUpdatePending) { EXPECT_TRUE(state_.subscriptionUpdatePending()); } -// The next three tests test that duplicate resource names (whether additions or removals) cause -// DeltaSubscriptionState to reject the update without even trying to hand it to the consuming API's -// onConfigUpdate(). -TEST_F(SotwSubscriptionStateTest, DuplicatedAdd) { - Protobuf::RepeatedPtrField additions = - populateRepeatedResource({{"name1", "version1A"}, {"name1", "sdfsdfsdfds"}}); - UpdateAck ack = deliverDiscoveryResponse(additions, {}, "debugversion1", absl::nullopt, false); - EXPECT_EQ("duplicate name name1 found among added/updated resources", - ack.error_detail_.message()); -} - -TEST_F(SotwSubscriptionStateTest, DuplicatedRemove) { - Protobuf::RepeatedPtrField removals; - *removals.Add() = "name1"; - *removals.Add() = "name1"; - UpdateAck ack = deliverDiscoveryResponse({}, removals, "debugversion1", absl::nullopt, false); - EXPECT_EQ("duplicate name name1 found in the union of added+removed resources", - ack.error_detail_.message()); -} - -TEST_F(SotwSubscriptionStateTest, AddedAndRemoved) { - Protobuf::RepeatedPtrField additions = - populateRepeatedResource({{"name1", "version1A"}}); - Protobuf::RepeatedPtrField removals; - *removals.Add() = "name1"; - UpdateAck ack = - deliverDiscoveryResponse(additions, removals, "debugversion1", absl::nullopt, false); - EXPECT_EQ("duplicate name name1 found in the union of added+removed resources", - ack.error_detail_.message()); -} - TEST_F(SotwSubscriptionStateTest, handleEstablishmentFailure) { // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the - // ultimate actual subscription callbacks, DeltaSubscriptionState's callbacks are actually + // ultimate actual subscription callbacks, SotwSubscriptionState's callbacks are actually // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index f154a3cc3310..f22254db7561 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -150,11 +150,11 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { if (GetParam() == SubscriptionType::Filesystem) { return; // initial_fetch_timeout not implemented for filesystem. } -// TODO BEGIN HEAD -// expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); -// startSubscription({"cluster0", "cluster1"}); -// statsAre(1, 0, 0, 0, 0, 0); -// TODO BEGIN upstream master + // TODO BEGIN HEAD + // expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); + // startSubscription({"cluster0", "cluster1"}); + // statsAre(1, 0, 0, 0, 0, 0); + // TODO BEGIN upstream master InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); @@ -162,9 +162,9 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { if (GetParam() == SubscriptionType::Http) { expectDisableInitFetchTimeoutTimer(); } -// TODO END upstream master + // TODO END upstream master expectConfigUpdateFailed(); -// TODO THIS LINE WAS IN HEAD expectDisableInitFetchTimeoutTimer(); + // TODO THIS LINE WAS IN HEAD expectDisableInitFetchTimeoutTimer(); callInitFetchTimeoutCb(); EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 6c0138cbaf04..228aaeba9d79 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -32,9 +32,9 @@ MATCHER_P(WithName, expectedName, "") { return arg.name() == expectedName; } class CdsApiImplTest : public testing::Test { protected: - void setup(bool is_delta = false) { + void setup() { envoy::api::v2::core::ConfigSource cds_config; - cds_ = CdsApiImpl::create(cds_config, is_delta, cm_, store_, validation_visitor_); + cds_ = CdsApiImpl::create(cds_config, cm_, store_, validation_visitor_); cds_->setInitializedCb([this]() -> void { initialized_.ready(); }); EXPECT_CALL(*cm_.subscription_factory_.subscription_, start(_)); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index b59cd3750fab..184dd0f9073f 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -367,7 +367,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 // ACK goes out, they're both acknowledging version 3. // TODO ONLY ON UPSTREAM if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); // TODO ONLY ON UPSTREAM } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index dfc2a157fddc..19b4a3ec0d98 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -10,15 +10,15 @@ namespace Envoy { namespace Config { MockSubscriptionFactory::MockSubscriptionFactory() { - ON_CALL(*this, subscriptionFromConfigSource(_, _, _, _, _)) - .WillByDefault(testing::Invoke( - [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - SubscriptionCallbacks& callbacks, bool) -> SubscriptionPtr { - auto ret = std::make_unique>(); - subscription_ = ret.get(); - callbacks_ = &callbacks; - return ret; - })); + ON_CALL(*this, subscriptionFromConfigSource(_, _, _, _)) + .WillByDefault(testing::Invoke([this](const envoy::api::v2::core::ConfigSource&, + absl::string_view, Stats::Scope&, + SubscriptionCallbacks& callbacks) -> SubscriptionPtr { + auto ret = std::make_unique>(); + subscription_ = ret.get(); + callbacks_ = &callbacks; + return ret; + })); ON_CALL(*this, messageValidationVisitor()) .WillByDefault(testing::ReturnRef(ProtobufMessage::getStrictValidationVisitor())); } diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 75f21af0d5b9..ee4bae564f76 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -52,10 +52,10 @@ class MockSubscriptionFactory : public SubscriptionFactory { MockSubscriptionFactory(); ~MockSubscriptionFactory() override; - MOCK_METHOD5(subscriptionFromConfigSource, + MOCK_METHOD4(subscriptionFromConfigSource, SubscriptionPtr(const envoy::api::v2::core::ConfigSource& config, absl::string_view type_url, Stats::Scope& scope, - SubscriptionCallbacks& callbacks, bool is_delta)); + SubscriptionCallbacks& callbacks)); MOCK_METHOD0(messageValidationVisitor, ProtobufMessage::ValidationVisitor&()); MockSubscription* subscription_{}; diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index ba07f3c6f97c..e1416736308f 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -406,11 +406,11 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager() override; - MOCK_METHOD5(createRdsRouteConfigProvider, + MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, Init::Manager& init_manager, bool is_delta)); + const std::string& stat_prefix, Init::Manager& init_manager)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 1487bc0b8852..bcbc232bd240 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -234,13 +234,11 @@ class MockListenerComponentFactory : public ListenerComponentFactory { DrainManagerPtr createDrainManager(envoy::api::v2::Listener::DrainType drain_type) override { return DrainManagerPtr{createDrainManager_(drain_type)}; } - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, - bool is_delta) override { - return LdsApiPtr{createLdsApi_(lds_config, is_delta)}; + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { + return LdsApiPtr{createLdsApi_(lds_config)}; } - MOCK_METHOD2(createLdsApi_, - LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); + MOCK_METHOD1(createLdsApi_, LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config)); MOCK_METHOD2(createNetworkFilterFactoryList, std::vector( const Protobuf::RepeatedPtrField& filters, @@ -271,8 +269,7 @@ class MockListenerManager : public ListenerManager { MOCK_METHOD3(addOrUpdateListener, bool(const envoy::api::v2::Listener& config, const std::string& version_info, bool modifiable)); - MOCK_METHOD2(createLdsApi, - void(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); + MOCK_METHOD1(createLdsApi, void(const envoy::api::v2::core::ConfigSource& lds_config)); MOCK_METHOD0(listeners, std::vector>()); MOCK_METHOD0(numConnections, uint64_t()); MOCK_METHOD1(removeListener, bool(const std::string& listener_name)); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 39e2be2bb5a2..4be83f9d8fc2 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -267,8 +267,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { const envoy::api::v2::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api)); - MOCK_METHOD3(createCds, CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, - bool is_delta, ClusterManager& cm)); + MOCK_METHOD2(createCds, + CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm)); private: NiceMock secret_manager_; @@ -326,7 +326,6 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(addThreadLocalClusterUpdateCallbacks_, ClusterUpdateCallbacksHandle*(ClusterUpdateCallbacks& callbacks)); MOCK_CONST_METHOD0(warmingClusterCount, std::size_t()); - MOCK_CONST_METHOD0(xdsIsDelta, bool()); MOCK_METHOD0(subscriptionFactory, Config::SubscriptionFactory&()); NiceMock conn_pool_; From 0e9a90d63398955dda2e457addf3af4cba684c4c Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 12:59:42 -0400 Subject: [PATCH 16/39] everything passes except integration framework lifetimes bug Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 1 - source/common/config/grpc_mux_impl.h | 9 +- source/common/config/grpc_stream.h | 7 -- .../common/config/grpc_subscription_impl.cc | 9 +- .../common/config/sotw_subscription_state.cc | 4 +- source/common/stats/isolated_store_impl.h | 1 - .../common/upstream/cluster_manager_impl.cc | 31 +++--- test/common/config/grpc_mux_impl_test.cc | 104 ++++++++---------- .../config/sotw_subscription_state_test.cc | 1 - .../config/subscription_factory_impl_test.cc | 2 +- test/common/config/subscription_impl_test.cc | 14 +-- test/integration/ads_integration.h | 2 - test/integration/integration.cc | 16 --- 13 files changed, 70 insertions(+), 131 deletions(-) diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 158ac178dddf..1a874989a14f 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -181,7 +181,6 @@ void GrpcMuxImpl::trySendDiscoveryRequests() { sendGrpcMessage(sub->getNextRequestAckless()); } } - std::cerr << ++vinny_the_variable_ << std::endl; maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index bf9d38fed8d0..f5228051e01b 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -120,10 +120,6 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { // the type_urls. So, while the SubscriptionStates populate every other field of these messages, // this one is up to GrpcMux. const LocalInfo::LocalInfo& local_info_; - - // Demonstrates that when the maybeUpdateQueueSizeStat() segfault happens, this object is - // still valid. - int vinny_the_variable_{}; }; class GrpcMuxDelta : public GrpcMuxImpl, @@ -178,10 +174,7 @@ class GrpcMuxSotw : public GrpcMuxImpl, : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) { - std::cerr << "creating STOW mux " << this << std::endl; - } - ~GrpcMuxSotw() { std::cerr << "destroying STOW mux " << this << std::endl; } // TODO REMOVE + rate_limit_settings) {} // GrpcStreamCallbacks void onStreamEstablished() override { handleEstablishedStream(); } void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index ef5e1d4bb1fa..eef60e699eab 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -37,9 +37,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } backoff_strategy_ = std::make_unique(RETRY_INITIAL_DELAY_MS, RETRY_MAX_DELAY_MS, random_); - std::cerr << "created GrpcStream " << this << " with scope " << &scope << std::endl; } - ~GrpcStream() { std::cerr << "destroying GrpcStream " << this << std::endl; } // TODO REMOVE void establishNewStream() { ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); @@ -105,7 +103,6 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, if (size > 0 || control_plane_stats_.pending_requests_.used()) { control_plane_stats_.pending_requests_.set(size); } - std::cerr << ++vinny_the_variable_ << std::endl; } bool checkRateLimitAllowsDrain() { @@ -151,10 +148,6 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; - - // Demonstrates that when the maybeUpdateQueueSizeStat() segfault (would) happen, this object is - // still valid. - int vinny_the_variable_{}; }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 4103f50de555..8f3ea08d55cc 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -9,17 +9,10 @@ GrpcSubscriptionImpl::GrpcSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, absl::stri std::chrono::milliseconds init_fetch_timeout, bool is_aggregated) : grpc_mux_(std::move(grpc_mux)), type_url_(type_url), callbacks_(callbacks), stats_(stats), - init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) { - if (!grpc_mux_) { - std::cerr << "HEY you gave me a null grpc_mux_!!!!!!!" << std::endl; - } - std::cerr << "GrpcSubscriptionImpl CTOR " << this << std::endl; -} + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} GrpcSubscriptionImpl::~GrpcSubscriptionImpl() { - std::cerr << "~GrpcSubscriptionImpl " << this << std::endl; if (watch_) { - std::cerr << "~GrpcSubscriptionImpl removing wathc" << std::endl; grpc_mux_->removeWatch(type_url_, watch_); } } diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index 0a18d92c448d..ee922829d46d 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -9,9 +9,7 @@ namespace Config { SotwSubscriptionState::SotwSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) { - std::cerr << "hi! constructing SOTW SubscriptionState" << std::endl; -} + : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} SotwSubscriptionState::~SotwSubscriptionState() {} diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 4d1a31f7c74d..8360109ee734 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -84,7 +84,6 @@ class IsolatedStoreImpl : public StoreImpl { public: IsolatedStoreImpl(); explicit IsolatedStoreImpl(SymbolTable& symbol_table); - ~IsolatedStoreImpl() { std::cerr << "~IsolatedStoreImpl " << this << std::endl; } // TODO remove // Stats::Scope Counter& counterFromStatName(StatName name) override { return counters_.get(name); } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 158c4b7101b1..e865070fd4b6 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -752,9 +752,9 @@ ClusterManagerImpl::httpConnPoolForCluster(const std::string& cluster, ResourceP return entry->second->connPool(priority, protocol, context); } -Tcp::ConnectionPool::Instance* ClusterManagerImpl::tcpConnPoolForCluster( - const std::string& cluster, ResourcePriority priority, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { +Tcp::ConnectionPool::Instance* +ClusterManagerImpl::tcpConnPoolForCluster(const std::string& cluster, ResourcePriority priority, + LoadBalancerContext* context) { ThreadLocalClusterManagerImpl& cluster_manager = tls_->getTyped(); auto entry = cluster_manager.thread_local_clusters_.find(cluster); @@ -763,7 +763,7 @@ Tcp::ConnectionPool::Instance* ClusterManagerImpl::tcpConnPoolForCluster( } // Select a host and create a connection pool for it if it does not already exist. - return entry->second->tcpConnPool(priority, context, transport_socket_options); + return entry->second->tcpConnPool(priority, context); } void ClusterManagerImpl::postThreadLocalDrainConnections(const Cluster& cluster, @@ -793,9 +793,8 @@ void ClusterManagerImpl::postThreadLocalHealthFailure(const HostSharedPtr& host) [this, host] { ThreadLocalClusterManagerImpl::onHostHealthFailure(host, *tls_); }); } -Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster( - const std::string& cluster, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { +Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster(const std::string& cluster, + LoadBalancerContext* context) { ThreadLocalClusterManagerImpl& cluster_manager = tls_->getTyped(); auto entry = cluster_manager.thread_local_clusters_.find(cluster); @@ -805,8 +804,9 @@ Host::CreateConnectionData ClusterManagerImpl::tcpConnForCluster( HostConstSharedPtr logical_host = entry->second->lb_->chooseHost(context); if (logical_host) { - auto conn_info = logical_host->createConnection(cluster_manager.thread_local_dispatcher_, - nullptr, transport_socket_options); + auto conn_info = logical_host->createConnection( + cluster_manager.thread_local_dispatcher_, nullptr, + context == nullptr ? nullptr : context->upstreamTransportSocketOptions()); if ((entry->second->cluster_info_->features() & ClusterInfo::Features::CLOSE_CONNECTIONS_ON_HOST_HEALTH_FAILURE) && conn_info.connection_ != nullptr) { @@ -1240,8 +1240,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::connPool( Tcp::ConnectionPool::Instance* ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( - ResourcePriority priority, LoadBalancerContext* context, - Network::TransportSocketOptionsSharedPtr transport_socket_options) { + ResourcePriority priority, LoadBalancerContext* context) { HostConstSharedPtr host = lb_->chooseHost(context); if (!host) { ENVOY_LOG(debug, "no healthy host for TCP connection pool"); @@ -1256,7 +1255,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( // This allows socket options to control connection pooling so that connections with // different options are not pooled together. bool have_options = false; - if (context && context->downstreamConnection()) { + if (context != nullptr && context->downstreamConnection()) { const Network::ConnectionSocket::OptionsSharedPtr& options = context->downstreamConnection()->socketOptions(); if (options) { @@ -1267,8 +1266,10 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( } } - if (transport_socket_options != nullptr) { - transport_socket_options->hashKey(hash_key); + bool have_transport_socket_options = false; + if (context != nullptr && context->upstreamTransportSocketOptions() != nullptr) { + have_transport_socket_options = true; + context->upstreamTransportSocketOptions()->hashKey(hash_key); } TcpConnPoolsContainer& container = parent_.host_tcp_conn_pool_map_[host]; @@ -1276,7 +1277,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::tcpConnPool( container.pools_[hash_key] = parent_.parent_.factory_.allocateTcpConnPool( parent_.thread_local_dispatcher_, host, priority, have_options ? context->downstreamConnection()->socketOptions() : nullptr, - transport_socket_options); + have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr); } return container.pools_[hash_key].get(); diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 98739c9125a2..1356aa158c01 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -271,25 +271,23 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); - { - auto response = std::make_unique(); - response->set_type_url(type_url); - response->set_version_info("1"); - envoy::api::v2::ClusterLoadAssignment load_assignment; - load_assignment.set_cluster_name("x"); - response->add_resources()->PackFrom(load_assignment); - EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) - .WillOnce( - Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, - const std::string&) { - EXPECT_EQ(1, resources.size()); - envoy::api::v2::ClusterLoadAssignment expected_assignment; - resources[0].UnpackTo(&expected_assignment); - EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); - })); - expectSendMessage(type_url, {}, "1"); - grpc_mux_->onDiscoveryResponse(std::move(response)); - } + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce( + Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + resources[0].UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } // Validate behavior when watches specify resources (potentially overlapping). @@ -627,36 +625,32 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { expectSendMessage(type_url, {}, "1", false, "bar"); } -// Verifies that a messsage with some resources is rejected when there are no watches. -TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { +// Verifies that a messsage with some resources is accepted even when there are no watches. +// Rationale: SotW gRPC xDS has always been willing to accept updates that include +// uninteresting resources. It should not matter whether those uninteresting resources +// are accompanied by interesting ones. +// Note: this was previously "rejects", not "accepts". See +// https://github.com/envoyproxy/envoy/pull/8350#discussion_r328218220 for discussion. +TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { setup(); - EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); - const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - grpc_mux_->start(); + // subscribe and unsubscribe so that the type is known to envoy { expectSendMessage(type_url, {"y"}, "", true); expectSendMessage(type_url, {}, ""); FakeGrpcSubscription delete_immediately = makeWatch(type_url, {"y"}); } - - // simulate the server sending CLA message to notify envoy that the CLA was added, - // even though envoy doesn't expect it. Envoy should reject this update. auto response = std::make_unique(); - response->set_nonce("bar"); - response->set_version_info("1"); response->set_type_url(type_url); - envoy::api::v2::ClusterLoadAssignment load_assignment; load_assignment.set_cluster_name("x"); response->add_resources()->PackFrom(load_assignment); + response->set_version_info("1"); - // The message should be rejected. - expectSendMessage(type_url, {}, "", false, "bar", Grpc::Status::GrpcStatus::Internal, - "Rejecting non-empty update for unwatched type URL"); + expectSendMessage(type_url, {}, "1"); grpc_mux_->onDiscoveryResponse(std::move(response)); } @@ -694,34 +688,32 @@ TEST_F(GrpcMuxImplTest, DiscoveryResponseNonexistentSub) { grpc_mux_->addOrUpdateWatch(type_url, nullptr, {}, callbacks_, std::chrono::milliseconds(0)); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {}, "", true); grpc_mux_->start(); - { - auto unexpected_response = std::make_unique(); - unexpected_response->set_type_url(type_url); - unexpected_response->set_system_version_info("0"); + auto unexpected_response = std::make_unique(); + unexpected_response->set_type_url("unexpected_type_url"); + unexpected_response->set_version_info("0"); EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); grpc_mux_->onDiscoveryResponse(std::move(unexpected_response)); } - { - auto response = std::make_unique(); - response->set_type_url(type_url); - response->set_system_version_info("1"); - envoy::api::v2::ClusterLoadAssignment load_assignment; - load_assignment.set_cluster_name("x"); - response->add_resources()->mutable_resource()->PackFrom(load_assignment); - EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "1")) - .WillOnce( - Invoke([&load_assignment]( - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField&, const std::string&) { - EXPECT_EQ(1, added_resources.size()); - envoy::api::v2::ClusterLoadAssignment expected_assignment; - added_resources[0].resource().UnpackTo(&expected_assignment); - EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); - })); - grpc_mux_->onDiscoveryResponse(std::move(response)); - } + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce( + Invoke([&load_assignment](const Protobuf::RepeatedPtrField& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + resources[0].UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response)); } } // namespace diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc index 6ddb3d35ef08..417dbdada5f7 100644 --- a/test/common/config/sotw_subscription_state_test.cc +++ b/test/common/config/sotw_subscription_state_test.cc @@ -134,7 +134,6 @@ TEST_F(SotwSubscriptionStateTest, AckGenerated) { UpdateAck ack = deliverDiscoveryResponse({"name1", "name2"}, "version1", "nonce1"); EXPECT_EQ("nonce1", ack.nonce_); EXPECT_EQ(Grpc::Status::GrpcStatus::Ok, ack.error_detail_.code()); - std::cerr << "the error is: " << ack.error_detail_.message() << std::endl; } // The next response updates 1 and 2, and adds 3. { diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 2426c10d8395..403ad4d5f91f 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -38,7 +38,7 @@ class SubscriptionFactoryTest : public testing::Test { return SubscriptionFactoryImpl(local_info_, dispatcher_, cm_, random_, validation_visitor_, *api_) .subscriptionFromConfigSource(config, Config::TypeUrl::get().ClusterLoadAssignment, - stats_store_, callbacks_, false); + stats_store_, callbacks_); } Upstream::MockClusterManager cm_; diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index f22254db7561..66c37b7c1769 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -150,21 +150,11 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { if (GetParam() == SubscriptionType::Filesystem) { return; // initial_fetch_timeout not implemented for filesystem. } - // TODO BEGIN HEAD - // expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); - // startSubscription({"cluster0", "cluster1"}); - // statsAre(1, 0, 0, 0, 0, 0); - // TODO BEGIN upstream master - InSequence s; expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); startSubscription({"cluster0", "cluster1"}); - EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0)); - if (GetParam() == SubscriptionType::Http) { - expectDisableInitFetchTimeoutTimer(); - } - // TODO END upstream master + statsAre(1, 0, 0, 0, 0, 0); expectConfigUpdateFailed(); - // TODO THIS LINE WAS IN HEAD expectDisableInitFetchTimeoutTimer(); + expectDisableInitFetchTimeoutTimer(); callInitFetchTimeoutCb(); EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0)); } diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 9c524d505b05..c806cd8107e7 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -12,8 +12,6 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" -// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification -// work restores it here. namespace Envoy { static std::string AdsIntegrationConfig(const std::string& api_type) { // Note: do not use CONSTRUCT_ON_FIRST_USE here! diff --git a/test/integration/integration.cc b/test/integration/integration.cc index d0cd85c00fd6..945a05450e11 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -629,22 +629,6 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( return AssertionSuccess(); } -AssertionResult compareSets(const std::set& set1, const std::set& set2, - absl::string_view name) { - if (set1 == set2) { - return AssertionSuccess(); - } - auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; - for (const auto& x : set1) { - failure << x << ", "; - } - failure << "}\nActual: {"; - for (const auto& x : set2) { - failure << x << ", "; - } - return failure << "}"; -} - AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, From c8d159f56b3f1d153bedacb23f61d41c250b347e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 13:07:09 -0400 Subject: [PATCH 17/39] GrpcMuxImpl back to NewGrpcMuxImpl for cleaner diff Signed-off-by: Fred Douglas --- source/common/config/BUILD | 4 ++-- source/common/config/grpc_subscription_impl.h | 2 +- .../common/config/{grpc_mux_impl.cc => new_grpc_mux_impl.cc} | 2 +- source/common/config/{grpc_mux_impl.h => new_grpc_mux_impl.h} | 0 source/common/config/subscription_factory_impl.cc | 2 +- test/common/config/grpc_mux_impl_test.cc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename source/common/config/{grpc_mux_impl.cc => new_grpc_mux_impl.cc} (99%) rename source/common/config/{grpc_mux_impl.h => new_grpc_mux_impl.h} (100%) diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 004492fc04fa..3bcff4cdb102 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -194,8 +194,8 @@ envoy_cc_library( envoy_cc_library( name = "grpc_mux_lib", - srcs = ["grpc_mux_impl.cc"], - hdrs = ["grpc_mux_impl.h"], + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], deps = [ ":delta_subscription_state_lib", ":grpc_stream_lib", diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 594cec7d4372..7f6997a3d6da 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -2,7 +2,7 @@ #include "envoy/config/subscription.h" -#include "common/config/grpc_mux_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/utility.h" namespace Envoy { diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc similarity index 99% rename from source/common/config/grpc_mux_impl.cc rename to source/common/config/new_grpc_mux_impl.cc index 1a874989a14f..d9ddbaae98af 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -1,4 +1,4 @@ -#include "common/config/grpc_mux_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h similarity index 100% rename from source/common/config/grpc_mux_impl.h rename to source/common/config/new_grpc_mux_impl.h diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 769b6104ca8c..87993d5b5c4b 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -1,9 +1,9 @@ #include "common/config/subscription_factory_impl.h" #include "common/config/filesystem_subscription_impl.h" -#include "common/config/grpc_mux_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/type_to_endpoint.h" #include "common/config/utility.h" #include "common/protobuf/protobuf.h" diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 1356aa158c01..6fc9d0f0e795 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -4,7 +4,7 @@ #include "envoy/api/v2/eds.pb.h" #include "common/common/empty_string.h" -#include "common/config/grpc_mux_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/protobuf_link_hacks.h" #include "common/config/resources.h" #include "common/config/utility.h" From fdb501c426cc31a64ac8d7ccfade4284a5d8619e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 13:13:00 -0400 Subject: [PATCH 18/39] rename class GrpcMuxImpl to NewGrpcMuxImpl Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 56 +++++++++++------------ source/common/config/new_grpc_mux_impl.h | 18 ++++---- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index d9ddbaae98af..f24e7793585f 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -10,17 +10,17 @@ namespace Envoy { namespace Config { -GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) +NewGrpcMuxImpl::NewGrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info) : subscription_state_factory_(std::move(subscription_state_factory)), skip_subsequent_node_(skip_subsequent_node), local_info_(local_info) { Config::Utility::checkLocalInfo("ads", local_info); } -Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { if (watch == nullptr) { return addWatch(type_url, resources, callbacks, init_fetch_timeout); } else { @@ -29,33 +29,33 @@ Watch* GrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, } } -void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { updateWatch(type_url, watch, {}); watchMapFor(type_url).removeWatch(watch); } -void GrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } -void GrpcMuxImpl::resume(const std::string& type_url) { +void NewGrpcMuxImpl::resume(const std::string& type_url) { pausable_ack_queue_.resume(type_url); trySendDiscoveryRequests(); } -bool GrpcMuxImpl::paused(const std::string& type_url) const { +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { return pausable_ack_queue_.paused(type_url); } -void GrpcMuxImpl::start() { establishGrpcStream(); } +void NewGrpcMuxImpl::start() { establishGrpcStream(); } -void GrpcMuxImpl::disableInitFetchTimeoutTimer() { +void NewGrpcMuxImpl::disableInitFetchTimeoutTimer() { for (auto& sub : subscriptions_) { sub.second->disableInitFetchTimeoutTimer(); } } -Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { auto watch_map = watch_maps_.find(type_url); if (watch_map == watch_maps_.end()) { // We don't yet have a subscription for type_url! Make one! @@ -72,8 +72,8 @@ Watch* GrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources) { +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { ASSERT(watch != nullptr); SubscriptionState* sub = subscriptionStateFor(type_url); WatchMap& watch_map = watchMapFor(type_url); @@ -87,15 +87,15 @@ void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, } } -void GrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { watch_maps_.emplace(type_url, std::make_unique()); subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( type_url, *watch_maps_[type_url], init_fetch_timeout)); subscription_ordering_.emplace_back(type_url); } -SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { +SubscriptionState* NewGrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { auto sub = subscriptions_.find(type_url); RELEASE_ASSERT(sub != subscriptions_.end(), fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", @@ -103,7 +103,7 @@ SubscriptionState* GrpcMuxImpl::subscriptionStateFor(const std::string& type_url return sub->second.get(); } -WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { +WatchMap& NewGrpcMuxImpl::watchMapFor(const std::string& type_url) { auto watch_map = watch_maps_.find(type_url); RELEASE_ASSERT( watch_map != watch_maps_.end(), @@ -111,7 +111,7 @@ WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { return *watch_map->second; } -void GrpcMuxImpl::handleEstablishedStream() { +void NewGrpcMuxImpl::handleEstablishedStream() { for (auto& sub : subscriptions_) { sub.second->markStreamFresh(); } @@ -119,7 +119,7 @@ void GrpcMuxImpl::handleEstablishedStream() { trySendDiscoveryRequests(); } -void GrpcMuxImpl::handleStreamEstablishmentFailure() { +void NewGrpcMuxImpl::handleStreamEstablishmentFailure() { // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a // crash, the iteration needs to dance around a little: collect pointers to all @@ -139,8 +139,8 @@ void GrpcMuxImpl::handleStreamEstablishmentFailure() { } while (all_subscribed.size() != subscriptions_.size()); } -void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, - const void* response_proto_ptr) { +void NewGrpcMuxImpl::genericHandleResponse(const std::string& type_url, + const void* response_proto_ptr) { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, @@ -153,7 +153,7 @@ void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, trySendDiscoveryRequests(); } -void GrpcMuxImpl::trySendDiscoveryRequests() { +void NewGrpcMuxImpl::trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); @@ -186,7 +186,7 @@ void GrpcMuxImpl::trySendDiscoveryRequests() { // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). -bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { RELEASE_ASSERT( !pausable_ack_queue_.paused(type_url), fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " @@ -209,7 +209,7 @@ bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { // First, prioritizes ACKs over non-ACK subscription interest updates. // Then, prioritizes non-ACK updates in the order the various types // of subscriptions were activated. -absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from pausable_ack_queue_ if possible, before looking at pending updates. if (!pausable_ack_queue_.empty()) { diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index f5228051e01b..6197264399de 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -21,10 +21,10 @@ namespace Config { // This class owns the GrpcStream used to talk to the server, maintains queuing // logic to properly order the subscription(s)' various messages, and allows // starting/stopping/pausing of the subscriptions. -class GrpcMuxImpl : public GrpcMux, Logger::Loggable { +class NewGrpcMuxImpl : public GrpcMux, Logger::Loggable { public: - GrpcMuxImpl(std::unique_ptr subscription_state_factory, - bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); + NewGrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info); Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, @@ -122,15 +122,15 @@ class GrpcMuxImpl : public GrpcMux, Logger::Loggable { const LocalInfo::LocalInfo& local_info_; }; -class GrpcMuxDelta : public GrpcMuxImpl, +class GrpcMuxDelta : public NewGrpcMuxImpl, public GrpcStreamCallbacks { public: GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), + : NewGrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} @@ -164,15 +164,15 @@ class GrpcMuxDelta : public GrpcMuxImpl, grpc_stream_; }; -class GrpcMuxSotw : public GrpcMuxImpl, +class GrpcMuxSotw : public NewGrpcMuxImpl, public GrpcStreamCallbacks { public: GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : GrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), + : NewGrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} // GrpcStreamCallbacks From b9a6b0bd824397d756a196d1af973b6ea4af5381 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 13:32:46 -0400 Subject: [PATCH 19/39] reorder NewGrpcMuxImpl and split Delta and Sotw out of header Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 160 ++++++++++++++++------ source/common/config/new_grpc_mux_impl.h | 76 ++++------ 2 files changed, 141 insertions(+), 95 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index f24e7793585f..d092de4a265a 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -45,14 +45,56 @@ bool NewGrpcMuxImpl::paused(const std::string& type_url) const { return pausable_ack_queue_.paused(type_url); } +void NewGrpcMuxImpl::genericHandleResponse(const std::string& type_url, + const void* response_proto_ptr) { + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "The server sent an xDS response proto with type_url {}, which we have " + "not subscribed to. Ignoring.", + type_url); + return; + } + pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); + trySendDiscoveryRequests(); +} + void NewGrpcMuxImpl::start() { establishGrpcStream(); } +void NewGrpcMuxImpl::handleEstablishedStream() { + for (auto& sub : subscriptions_) { + sub.second->markStreamFresh(); + } + set_any_request_sent_yet_in_current_stream(false); + trySendDiscoveryRequests(); +} + void NewGrpcMuxImpl::disableInitFetchTimeoutTimer() { for (auto& sub : subscriptions_) { sub.second->disableInitFetchTimeoutTimer(); } } +void NewGrpcMuxImpl::handleStreamEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = sub.second.get(); + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) { @@ -111,48 +153,6 @@ WatchMap& NewGrpcMuxImpl::watchMapFor(const std::string& type_url) { return *watch_map->second; } -void NewGrpcMuxImpl::handleEstablishedStream() { - for (auto& sub : subscriptions_) { - sub.second->markStreamFresh(); - } - set_any_request_sent_yet_in_current_stream(false); - trySendDiscoveryRequests(); -} - -void NewGrpcMuxImpl::handleStreamEstablishmentFailure() { - // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately - // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a - // crash, the iteration needs to dance around a little: collect pointers to all - // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are - // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; - do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = sub.second.get(); - } - for (auto& sub : all_subscribed) { - if (already_called.insert(sub).second) { // insert succeeded ==> not already called - sub.second->handleEstablishmentFailure(); - } - } - } while (all_subscribed.size() != subscriptions_.size()); -} - -void NewGrpcMuxImpl::genericHandleResponse(const std::string& type_url, - const void* response_proto_ptr) { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "The server sent an xDS response proto with type_url {}, which we have " - "not subscribed to. Ignoring.", - type_url); - return; - } - pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); - trySendDiscoveryRequests(); -} - void NewGrpcMuxImpl::trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? @@ -226,5 +226,81 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { return absl::nullopt; } +// Delta- and SotW-specific concrete subclasses: +// GrpcMuxDelta +GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : NewGrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +// GrpcStreamCallbacks for GrpcMuxDelta +void GrpcMuxDelta::onStreamEstablished() { handleEstablishedStream(); } +void GrpcMuxDelta::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } +void GrpcMuxDelta::onWriteable() { trySendDiscoveryRequests(); } +void GrpcMuxDelta::onDiscoveryResponse( + std::unique_ptr&& message) { + genericHandleResponse(message->type_url(), message.get()); +} + +void GrpcMuxDelta::establishGrpcStream() { grpc_stream_.establishNewStream(); } +void GrpcMuxDelta::sendGrpcMessage(void* msg_proto_ptr) { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); +} +void GrpcMuxDelta::maybeUpdateQueueSizeStat(uint64_t size) { + grpc_stream_.maybeUpdateQueueSizeStat(size); +} +bool GrpcMuxDelta::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } +bool GrpcMuxDelta::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } + +// GrpcMuxSotw +GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : NewGrpcMuxImpl(std::make_unique(dispatcher), + skip_subsequent_node, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +// GrpcStreamCallbacks for GrpcMuxSotw +void GrpcMuxSotw::onStreamEstablished() { handleEstablishedStream(); } +void GrpcMuxSotw::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } +void GrpcMuxSotw::onWriteable() { trySendDiscoveryRequests(); } +void GrpcMuxSotw::onDiscoveryResponse( + std::unique_ptr&& message) { + genericHandleResponse(message->type_url(), message.get()); +} + +void GrpcMuxSotw::establishGrpcStream() { grpc_stream_.establishNewStream(); } + +void GrpcMuxSotw::sendGrpcMessage(void* msg_proto_ptr) { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); +} + +void GrpcMuxSotw::maybeUpdateQueueSizeStat(uint64_t size) { + grpc_stream_.maybeUpdateQueueSizeStat(size); +} + +bool GrpcMuxSotw::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } +bool GrpcMuxSotw::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } + } // namespace Config } // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index 6197264399de..e22ba540bfbf 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -128,38 +128,23 @@ class GrpcMuxDelta : public NewGrpcMuxImpl, GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : NewGrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); // GrpcStreamCallbacks - void onStreamEstablished() override { handleEstablishedStream(); } - void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } - void onWriteable() override { trySendDiscoveryRequests(); } + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onWriteable() override; void - onDiscoveryResponse(std::unique_ptr&& message) override { - genericHandleResponse(message->type_url(), message.get()); - } + onDiscoveryResponse(std::unique_ptr&& message) override; protected: - void establishGrpcStream() override { grpc_stream_.establishNewStream(); } - void sendGrpcMessage(void* msg_proto_ptr) override { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); - } - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); - } - void maybeUpdateQueueSizeStat(uint64_t size) override { - grpc_stream_.maybeUpdateQueueSizeStat(size); - } - bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } - bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + void establishGrpcStream() override; + void sendGrpcMessage(void* msg_proto_ptr) override; + void maybeUpdateQueueSizeStat(uint64_t size) override; + bool grpcStreamAvailable() const override; + bool rateLimitAllowsDrain() override; +private: GrpcStream grpc_stream_; }; @@ -170,41 +155,26 @@ class GrpcMuxSotw : public NewGrpcMuxImpl, GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) - : NewGrpcMuxImpl(std::make_unique(dispatcher), - skip_subsequent_node, local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node); + // GrpcStreamCallbacks - void onStreamEstablished() override { handleEstablishedStream(); } - void onEstablishmentFailure() override { handleStreamEstablishmentFailure(); } - void onWriteable() override { trySendDiscoveryRequests(); } - void onDiscoveryResponse(std::unique_ptr&& message) override { - genericHandleResponse(message->type_url(), message.get()); - } + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onWriteable() override; + void onDiscoveryResponse(std::unique_ptr&& message) override; GrpcStream& grpcStreamForTest() { return grpc_stream_; } protected: - void establishGrpcStream() override { grpc_stream_.establishNewStream(); } - void sendGrpcMessage(void* msg_proto_ptr) override { - std::unique_ptr typed_proto( - static_cast(msg_proto_ptr)); - if (!any_request_sent_yet_in_current_stream() || !skip_subsequent_node()) { - typed_proto->mutable_node()->MergeFrom(local_info().node()); - } - - grpc_stream_.sendMessage(*typed_proto); - set_any_request_sent_yet_in_current_stream(true); - } - void maybeUpdateQueueSizeStat(uint64_t size) override { - grpc_stream_.maybeUpdateQueueSizeStat(size); - } - bool grpcStreamAvailable() const override { return grpc_stream_.grpcStreamAvailable(); } - bool rateLimitAllowsDrain() override { return grpc_stream_.checkRateLimitAllowsDrain(); } + void establishGrpcStream() override; + void sendGrpcMessage(void* msg_proto_ptr) override; + void maybeUpdateQueueSizeStat(uint64_t size) override; + bool grpcStreamAvailable() const override; + bool rateLimitAllowsDrain() override; +private: GrpcStream grpc_stream_; }; From 1c1f447f9b63a9a5ac1c390034302c0068398dd3 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 14:03:03 -0400 Subject: [PATCH 20/39] clean up remaining delta-specific cruft Signed-off-by: Fred Douglas --- source/common/upstream/cluster_manager_impl.h | 2 +- test/common/router/rds_impl_test.cc | 14 +++--- test/common/router/scoped_rds_test.cc | 8 ++-- test/common/upstream/cds_api_impl_test.cc | 48 ------------------- .../upstream/cluster_manager_impl_test.cc | 2 +- test/common/upstream/eds_test.cc | 4 +- 6 files changed, 15 insertions(+), 63 deletions(-) diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index d1d7b204c709..19ee8ebfe412 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -21,7 +21,7 @@ #include "envoy/upstream/cluster_manager.h" #include "common/common/cleanup.h" -#include "common/config/grpc_mux_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/subscription_factory_impl.h" #include "common/http/async_client_impl.h" #include "common/upstream/load_stats_reporter.h" diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index 9be73fbefe8c..11424515eb32 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -92,9 +92,9 @@ class RdsImplTest : public RdsTestBase { )EOF"; EXPECT_CALL(factory_context_.init_manager_, add(_)); - rds_ = RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), - factory_context_, "foo.", - *route_config_provider_manager_, false); + rds_ = + RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), + factory_context_, "foo.", *route_config_provider_manager_); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; EXPECT_CALL(*factory_context_.cluster_manager_.subscription_factory_.subscription_, start(_)); factory_context_.init_manager_.initialize(init_watcher_); @@ -126,7 +126,7 @@ TEST_F(RdsImplTest, RdsAndStatic) { EXPECT_THROW(RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), factory_context_, "foo.", - *route_config_provider_manager_, false), + *route_config_provider_manager_), EnvoyException); } @@ -264,7 +264,7 @@ class RouteConfigProviderManagerImplTest : public RdsTestBase { rds_.set_route_config_name("foo_route_config"); rds_.mutable_config_source()->set_path("foo_path"); provider_ = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix.", factory_context_.initManager(), false); + rds_, factory_context_, "foo_prefix.", factory_context_.initManager()); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; } @@ -416,7 +416,7 @@ name: foo_route_config "1"); RouteConfigProviderPtr provider2 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds_, factory_context_, "foo_prefix", factory_context_.initManager(), false); + rds_, factory_context_, "foo_prefix", factory_context_.initManager()); // provider2 should have route config immediately after create EXPECT_TRUE(provider2->configInfo().has_value()); @@ -430,7 +430,7 @@ name: foo_route_config rds2.set_route_config_name("foo_route_config"); rds2.mutable_config_source()->set_path("bar_path"); RouteConfigProviderPtr provider3 = route_config_provider_manager_->createRdsRouteConfigProvider( - rds2, factory_context_, "foo_prefix", factory_context_.initManager(), false); + rds2, factory_context_, "foo_prefix", factory_context_.initManager()); EXPECT_NE(provider3, provider_); factory_context_.cluster_manager_.subscription_factory_.callbacks_->onConfigUpdate(route_configs, "provider3"); diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index d8a3fcd5ff23..2279ce2546b7 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -6,7 +6,7 @@ #include "envoy/init/manager.h" #include "envoy/stats/scope.h" -#include "common/config/grpc_mux_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/router/scoped_rds.h" #include "test/mocks/config/mocks.h" @@ -104,18 +104,18 @@ class ScopedRdsTest : public ScopedRoutesTestBase { // To build the map from RDS route_config_name to the RDS subscription, we need to get the // route_config_name by mocking start() on the Config::Subscription. EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, - subscriptionFromConfigSource(_, _, _, _, _)) + subscriptionFromConfigSource(_, _, _, _)) .Times(AnyNumber()); EXPECT_CALL(factory_context_.cluster_manager_.subscription_factory_, subscriptionFromConfigSource( _, Eq(Grpc::Common::typeUrl( envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name())), - _, _, _)) + _, _)) .Times(AnyNumber()) .WillRepeatedly(Invoke([this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - Envoy::Config::SubscriptionCallbacks& callbacks, bool) { + Envoy::Config::SubscriptionCallbacks& callbacks) { auto ret = std::make_unique>(); rds_subscription_by_config_subscription_[ret.get()] = &callbacks; EXPECT_CALL(*ret, start(_)) diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 228aaeba9d79..b7f0214339be 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -177,54 +177,6 @@ TEST_F(CdsApiImplTest, ConfigUpdateWith2ValidClusters) { cds_callbacks_->onConfigUpdate(clusters, ""); } -TEST_F(CdsApiImplTest, DeltaConfigUpdate) { - { - InSequence s; - setup(true); - } - EXPECT_CALL(initialized_, ready()); - - { - Protobuf::RepeatedPtrField resources; - { - envoy::api::v2::Cluster cluster; - cluster.set_name("cluster_1"); - expectAdd("cluster_1", "v1"); - auto* resource = resources.Add(); - resource->mutable_resource()->PackFrom(cluster); - resource->set_name("cluster_1"); - resource->set_version("v1"); - } - { - envoy::api::v2::Cluster cluster; - cluster.set_name("cluster_2"); - expectAdd("cluster_2", "v1"); - auto* resource = resources.Add(); - resource->mutable_resource()->PackFrom(cluster); - resource->set_name("cluster_2"); - resource->set_version("v1"); - } - cds_callbacks_->onConfigUpdate(resources, {}, "v1"); - } - - { - Protobuf::RepeatedPtrField resources; - { - envoy::api::v2::Cluster cluster; - cluster.set_name("cluster_3"); - expectAdd("cluster_3", "v2"); - auto* resource = resources.Add(); - resource->mutable_resource()->PackFrom(cluster); - resource->set_name("cluster_3"); - resource->set_version("v2"); - } - Protobuf::RepeatedPtrField removed; - *removed.Add() = "cluster_1"; - EXPECT_CALL(cm_, removeCluster(StrEq("cluster_1"))).WillOnce(Return(true)); - cds_callbacks_->onConfigUpdate(resources, removed, "v2"); - } -} - TEST_F(CdsApiImplTest, ConfigUpdateAddsSecondClusterEvenIfFirstThrows) { { InSequence s; diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 4ea0c9080cd1..98e9fdf9b1ce 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -101,7 +101,7 @@ class TestClusterManagerFactory : public ClusterManagerFactory { return std::make_pair(result.first, ThreadAwareLoadBalancerPtr(result.second)); } - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, bool, ClusterManager&) override { + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource&, ClusterManager&) override { return CdsApiPtr{createCds_()}; } diff --git a/test/common/upstream/eds_test.cc b/test/common/upstream/eds_test.cc index 1e520b815163..9247c9d84eb8 100644 --- a/test/common/upstream/eds_test.cc +++ b/test/common/upstream/eds_test.cc @@ -75,8 +75,8 @@ class EdsTest : public testing::Test { Envoy::Server::Configuration::TransportSocketFactoryContextImpl factory_context( admin_, ssl_context_manager_, *scope, cm_, local_info_, dispatcher_, random_, stats_, singleton_manager_, tls_, validation_visitor_, *api_); - cluster_.reset(new EdsClusterImpl(eds_cluster_, runtime_, factory_context, std::move(scope), - false, false)); + cluster_.reset( + new EdsClusterImpl(eds_cluster_, runtime_, factory_context, std::move(scope), false)); EXPECT_EQ(Cluster::InitializePhase::Secondary, cluster_->initializePhase()); eds_callbacks_ = cm_.subscription_factory_.callbacks_; } From f83d1e6eed8a8dc2efb8e8078d8352dc588f1874 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 14:22:34 -0400 Subject: [PATCH 21/39] fully cleaned up, only lifetimes problem left Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_state.cc | 4 +-- source/common/config/grpc_stream.h | 1 - .../common/config/grpc_subscription_impl.cc | 8 ++--- .../config/subscription_factory_impl.cc | 31 +++++++++---------- source/common/router/scoped_rds.cc | 1 - source/common/router/scoped_rds.h | 4 +-- source/common/stats/isolated_store_impl.h | 1 - test/integration/ads_integration_test.cc | 2 -- test/integration/cds_integration_test.cc | 10 +++--- .../scoped_rds_integration_test.cc | 3 -- test/integration/server.h | 4 --- 11 files changed, 25 insertions(+), 44 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index d4a612c04b52..51e5e934d10a 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -10,9 +10,7 @@ DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) { - std::cerr << "hi! constructing DELTA SubscriptionState" << std::endl; -} + : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} DeltaSubscriptionState::~DeltaSubscriptionState() {} diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index eef60e699eab..1be922742930 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -99,7 +99,6 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, // if(>0 || used) to keep this stat from being wrongly marked interesting by a pointless set(0) // and needlessly taking up space. The first time we set(123), used becomes true, and so we will // subsequently always do the set (including set(0)). - // TODO TODO segfault here in test teardown if (size > 0 || control_plane_stats_.pending_requests_.used()) { control_plane_stats_.pending_requests_.set(size); } diff --git a/source/common/config/grpc_subscription_impl.cc b/source/common/config/grpc_subscription_impl.cc index 8f3ea08d55cc..77f003b061b5 100644 --- a/source/common/config/grpc_subscription_impl.cc +++ b/source/common/config/grpc_subscription_impl.cc @@ -45,7 +45,7 @@ void GrpcSubscriptionImpl::onConfigUpdate( const std::string& version_info) { stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(resources, version_info); - grpc_mux_->disableInitFetchTimeoutTimer(); // TODO keep this here? + grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_success_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } @@ -56,7 +56,7 @@ void GrpcSubscriptionImpl::onConfigUpdate( const std::string& system_version_info) { stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); - grpc_mux_->disableInitFetchTimeoutTimer(); // TODO keep this here? + grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_success_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } @@ -75,7 +75,7 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: stats_.init_fetch_timeout_.inc(); - grpc_mux_->disableInitFetchTimeoutTimer(); // TODO keep this here? + grpc_mux_->disableInitFetchTimeoutTimer(); callbacks_.onConfigUpdateFailed(reason, e); // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() // as attempts. @@ -84,7 +84,7 @@ void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: // We expect Envoy exception to be thrown when update is rejected. ASSERT(e != nullptr); - grpc_mux_->disableInitFetchTimeoutTimer(); // TODO keep this here? + grpc_mux_->disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); callbacks_.onConfigUpdateFailed(reason, e); break; diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 87993d5b5c4b..887e81f708fc 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -46,27 +46,25 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::apiConfigSourceRequestTimeout(api_config_source), restMethod(type_url), callbacks, stats, Utility::configSourceInitialFetchTimeout(config), validation_visitor_); case envoy::api::v2::core::ApiConfigSource::GRPC: - std::cerr << "subscriptionFromConfigSource single SOTW" << std::endl; return std::make_unique( - std::make_shared( - Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false + std::make_shared(Utility::factoryForGrpcApiConfigSource( + cm_.grpcAsyncClientManager(), api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), + local_info_, + api_config_source.set_node_on_first_message_only()), type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false); case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: - std::cerr << "subscriptionFromConfigSource single DELTA" << std::endl; return std::make_unique( - std::make_shared( - Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher_, deltaGrpcMethod(type_url), random_, scope, - Utility::parseRateLimitSettings(api_config_source), local_info_, - /*skip_subsequent_node=*/false), // TODO verify it's ok for this to be hardcoded false + std::make_shared(Utility::factoryForGrpcApiConfigSource( + cm_.grpcAsyncClientManager(), api_config_source, scope) + ->create(), + dispatcher_, deltaGrpcMethod(type_url), random_, scope, + Utility::parseRateLimitSettings(api_config_source), + local_info_, + api_config_source.set_node_on_first_message_only()), type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/false); default: @@ -74,7 +72,6 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( } } case envoy::api::v2::core::ConfigSource::kAds: { - std::cerr << "subscriptionFromConfigSource ADS" << std::endl; return std::make_unique(cm_.adsMux(), type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), /*is_aggregated=*/true); diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index f9ad7d2cb7af..479c65b231c7 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -99,7 +99,6 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), *scope_, *this); - std::cerr << "made SRDS with scope " << scope_.get() << std::endl; initialize([scope_key_builder]() -> Envoy::Config::ConfigProvider::ConfigConstSharedPtr { return std::make_shared( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 0b6f1a391bf1..4baf96cc34d8 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -98,9 +98,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio RouteConfigProviderManager& route_config_provider_manager, ScopedRoutesConfigProviderManager& config_provider_manager); - ~ScopedRdsConfigSubscription() override { - std::cerr << "destroying SRDS with scope " << scope_.get() << std::endl; - } // = default; + ~ScopedRdsConfigSubscription() override = default; const std::string& name() const { return name_; } diff --git a/source/common/stats/isolated_store_impl.h b/source/common/stats/isolated_store_impl.h index 8360109ee734..cce741211df1 100644 --- a/source/common/stats/isolated_store_impl.h +++ b/source/common/stats/isolated_store_impl.h @@ -2,7 +2,6 @@ #include #include -#include // TODO REMOVE #include #include "envoy/stats/stats.h" diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 184dd0f9073f..0ce4f3e07f00 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -366,9 +366,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 // ACK goes out, they're both acknowledging version 3. - // TODO ONLY ON UPSTREAM if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); - // TODO ONLY ON UPSTREAM } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 2858dc02b421..d3013870ea94 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -137,7 +137,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) { testRouterHeaderOnlyRequestAndResponse(nullptr, UpstreamIndex1, "/cluster1"); // Tell Envoy that cluster_1 is gone. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}, true)); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {}, {}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of @@ -154,7 +154,7 @@ TEST_P(CdsIntegrationTest, CdsClusterUpDownUp) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is back. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_}, {cluster1_}, {}, "413"); @@ -177,7 +177,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_2 is here. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {}, true)); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "55", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_, cluster2_}, {cluster2_}, {}, "42"); // The '3' includes the fake CDS server. @@ -189,7 +189,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is gone. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster2_}, {}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of @@ -202,7 +202,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { codec_client_->waitForDisconnect(); // Tell Envoy that cluster_1 is back. - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {}, true)); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "42", {}, {}, {})); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {cluster1_, cluster2_}, {cluster1_}, {}, "413"); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 99b2f119a92f..b43eb790306a 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -25,11 +25,8 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, : HttpIntegrationTest(Http::CodecClient::Type::HTTP1, ipVersion(), realTime()) {} ~ScopedRdsIntegrationTest() override { - std::cerr << "hmm1" << std::endl; resetConnections(); - std::cerr << "hmm2" << std::endl; cleanupUpstreamAndDownstream(); - std::cerr << "hmm3" << std::endl; } void initialize() override { diff --git a/test/integration/server.h b/test/integration/server.h index 2dd1e676ab69..4489c436df19 100644 --- a/test/integration/server.h +++ b/test/integration/server.h @@ -69,7 +69,6 @@ class TestScopeWrapper : public Scope { public: TestScopeWrapper(Thread::MutexBasicLockable& lock, ScopePtr wrapped_scope) : lock_(lock), wrapped_scope_(std::move(wrapped_scope)) {} - ~TestScopeWrapper() { std::cerr << "destroying TestScopeWrapper" << std::endl; } // TODO REMOVE ScopePtr createScope(const std::string& name) override { Thread::LockGuard lock(lock_); @@ -141,9 +140,6 @@ class TestScopeWrapper : public Scope { */ class TestIsolatedStoreImpl : public StoreRoot { public: - ~TestIsolatedStoreImpl() override { - std::cerr << "destroying TestIsolatedStoreImpl" << std::endl; - } // TODO REMOVE // Stats::Scope Counter& counterFromStatName(StatName name) override { Thread::LockGuard lock(lock_); From c779600b01f45187837fdd5cc3e5cf53ff2c96f0 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 14:35:22 -0400 Subject: [PATCH 22/39] simplify queue size stat setting Signed-off-by: Fred Douglas --- source/common/config/grpc_stream.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 1be922742930..f3f579b78d56 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -92,15 +92,10 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, } void maybeUpdateQueueSizeStat(uint64_t size) { - // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently - // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter - // because we always attempt this drain immediately after the push). Basically, a change in - // queue length is not "meaningful" until it has persisted until here. We need the - // if(>0 || used) to keep this stat from being wrongly marked interesting by a pointless set(0) - // and needlessly taking up space. The first time we set(123), used becomes true, and so we will - // subsequently always do the set (including set(0)). - if (size > 0 || control_plane_stats_.pending_requests_.used()) { + // A change in queue length is not "meaningful" until it has persisted until here. + if (size != last_queue_size_) { control_plane_stats_.pending_requests_.set(size); + last_queue_size_ = size; } } @@ -147,6 +142,7 @@ class GrpcStream : public Grpc::AsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; + uint64_t last_queue_size_{}; }; } // namespace Config From a7c3bd3e6c192a233573174751aaf609609e209f Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 15:32:52 -0400 Subject: [PATCH 23/39] all tests passing Signed-off-by: Fred Douglas --- test/common/runtime/runtime_impl_test.cc | 4 ++-- test/server/lds_api_test.cc | 2 +- test/server/listener_manager_impl_test.cc | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index 182b92d1ef5d..ef5428e19ea9 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -808,10 +808,10 @@ class RtdsLoaderImplTest : public LoaderImplTest { EXPECT_CALL(init_manager_, add(_)).WillRepeatedly(Invoke([this](const Init::Target& target) { init_target_handles_.emplace_back(target.createHandle("test")); })); - ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _, _)) + ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _)) .WillByDefault(testing::Invoke( [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - Config::SubscriptionCallbacks& callbacks, bool) -> Config::SubscriptionPtr { + Config::SubscriptionCallbacks& callbacks) -> Config::SubscriptionPtr { auto ret = std::make_unique>(); rtds_subscriptions_.push_back(ret.get()); rtds_callbacks_.push_back(&callbacks); diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index 72aee52328b5..8ce4b67f2791 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -36,7 +36,7 @@ class LdsApiTest : public testing::Test { envoy::api::v2::core::ConfigSource lds_config; EXPECT_CALL(init_manager_, add(_)); lds_ = std::make_unique(lds_config, cluster_manager_, init_manager_, store_, - listener_manager_, validation_visitor_, false); + listener_manager_, validation_visitor_); EXPECT_CALL(*cluster_manager_.subscription_factory_.subscription_, start(_)); init_target_handle_->initialize(init_watcher_); lds_callbacks_ = cluster_manager_.subscription_factory_.callbacks_; diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index 91882a64f73f..12c436b8ad60 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -590,9 +590,9 @@ TEST_F(ListenerManagerImplTest, AddOrUpdateListener) { InSequence s; auto* lds_api = new MockLdsApi(); - EXPECT_CALL(listener_factory_, createLdsApi_(_, _)).WillOnce(Return(lds_api)); + EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); envoy::api::v2::core::ConfigSource lds_config; - manager_->createLdsApi(lds_config, false); + manager_->createLdsApi(lds_config); EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("")); checkConfigDump(R"EOF( From 008ca2de6a8e6b24f595f38d07921597e26da107 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 15:59:20 -0400 Subject: [PATCH 24/39] remove unneeded extra line Signed-off-by: Fred Douglas --- source/common/router/scoped_rds.cc | 2 +- source/common/router/scoped_rds.h | 4 ++-- source/common/upstream/cluster_manager_impl.cc | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 479c65b231c7..ae77d119a584 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -87,8 +87,8 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( ScopedRoutesConfigProviderManager& config_provider_manager) : DeltaConfigSubscriptionInstance("SRDS", manager_identifier, config_provider_manager, factory_context), - scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), factory_context_(factory_context), name_(name), scope_key_builder_(scope_key_builder), + scope_(factory_context.scope().createScope(stat_prefix + "scoped_rds." + name + ".")), stats_({ALL_SCOPED_RDS_STATS(POOL_COUNTER(*scope_))}), rds_config_source_(std::move(rds_config_source)), validation_visitor_(factory_context.messageValidationVisitor()), stat_prefix_(stat_prefix), diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index 4baf96cc34d8..422551449acd 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -160,7 +160,6 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio void onRdsConfigUpdate(const std::string& scope_name, RdsRouteConfigSubscription& rds_subscription); - Stats::ScopePtr scope_; // ScopedRouteInfo by scope name. ScopedRouteMap scoped_route_map_; // RdsRouteConfigProvider by scope name. @@ -171,14 +170,15 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio // For creating RDS subscriptions. Server::Configuration::FactoryContext& factory_context_; const std::string name_; + std::unique_ptr subscription_; const envoy::config::filter::network::http_connection_manager::v2::ScopedRoutes::ScopeKeyBuilder scope_key_builder_; + Stats::ScopePtr scope_; ScopedRdsStats stats_; const envoy::api::v2::core::ConfigSource rds_config_source_; ProtobufMessage::ValidationVisitor& validation_visitor_; const std::string stat_prefix_; RouteConfigProviderManager& route_config_provider_manager_; - std::unique_ptr subscription_; }; using ScopedRdsConfigSubscriptionSharedPtr = std::shared_ptr; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index e865070fd4b6..3eeeb973c2bf 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -147,7 +147,6 @@ void ClusterManagerInitHelper::maybeFinishInitialize() { initializeSecondaryClusters(); } } - return; } From 075b3628b9b73b469ad654b9a5eb86ad4eb52b6d Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 16:27:58 -0400 Subject: [PATCH 25/39] fix typos Signed-off-by: Fred Douglas --- source/common/config/README.md | 2 +- source/common/config/delta_subscription_state.cc | 4 ++-- source/common/config/delta_subscription_state.h | 2 +- source/common/config/sotw_subscription_state.cc | 4 ++-- source/common/config/sotw_subscription_state.h | 2 +- source/common/config/subscription_state.h | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/common/config/README.md b/source/common/config/README.md index c3e2be84bc65..cfcecd7678a1 100644 --- a/source/common/config/README.md +++ b/source/common/config/README.md @@ -48,7 +48,7 @@ spontaneous updates. ACKs are not shown in this diagram; they are also carried b What does GrpcMux even do in this diagram? Just own things and pass through function calls? Answer: 1) it sequences the requests and ACKs that the various type_urls send, 2) it handles the -protobuf vs polymorphism impedence mismatch, allowing all delta-vs-SotW-agnostic code +protobuf vs polymorphism impedance mismatch, allowing all delta-vs-SotW-agnostic code to be reused. Note that there can be multiple active gRPC subscriptions for a single resource diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 51e5e934d10a..09a405ea708b 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -56,8 +56,8 @@ bool DeltaSubscriptionState::subscriptionUpdatePending() const { !any_request_sent_yet_in_current_stream_; } -UpdateAck DeltaSubscriptionState::handleResponse(const void* reponse_proto_ptr) { - auto* response = static_cast(reponse_proto_ptr); +UpdateAck DeltaSubscriptionState::handleResponse(const void* response_proto_ptr) { + auto* response = static_cast(response_proto_ptr); // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. UpdateAck ack(response->nonce(), type_url()); diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index dfc244ff54df..da994692c5be 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -30,7 +30,7 @@ class DeltaSubscriptionState : public SubscriptionState { void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } // message is expected to be a envoy::api::v2::DeltaDiscoveryResponse. - UpdateAck handleResponse(const void* reponse_proto_ptr) override; + UpdateAck handleResponse(const void* response_proto_ptr) override; void handleEstablishmentFailure() override; diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index ee922829d46d..90652933247a 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -49,8 +49,8 @@ void SotwSubscriptionState::markStreamFresh() { update_pending_ = true; } -UpdateAck SotwSubscriptionState::handleResponse(const void* reponse_proto_ptr) { - auto* response = static_cast(reponse_proto_ptr); +UpdateAck SotwSubscriptionState::handleResponse(const void* response_proto_ptr) { + auto* response = static_cast(response_proto_ptr); // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. UpdateAck ack(response->nonce(), type_url()); diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h index 763e1e4b2ca4..e417147b35e8 100644 --- a/source/common/config/sotw_subscription_state.h +++ b/source/common/config/sotw_subscription_state.h @@ -30,7 +30,7 @@ class SotwSubscriptionState : public SubscriptionState { void markStreamFresh() override; // message is expected to be a envoy::api::v2::DiscoveryResponse. - UpdateAck handleResponse(const void* reponse_proto_ptr) override; + UpdateAck handleResponse(const void* response_proto_ptr) override; void handleEstablishmentFailure() override; diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h index 7619d04b8cee..c69f2240c443 100644 --- a/source/common/config/subscription_state.h +++ b/source/common/config/subscription_state.h @@ -38,7 +38,7 @@ class SubscriptionState : public Logger::Loggable { // Implementations expect either a DeltaDiscoveryResponse or DiscoveryResponse. The caller is // expected to know which it should be providing. - virtual UpdateAck handleResponse(const void* reponse_proto_ptr) PURE; + virtual UpdateAck handleResponse(const void* response_proto_ptr) PURE; virtual void handleEstablishmentFailure() PURE; From 0040f369b799538d433b54f2c58406b740a5c42f Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 18:18:31 -0400 Subject: [PATCH 26/39] format and tidy fixes Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_state.cc | 4 ++-- source/common/config/sotw_subscription_state.cc | 4 ++-- source/common/config/sotw_subscription_state.h | 6 +++--- test/common/config/grpc_mux_impl_test.cc | 4 ++-- test/common/config/sotw_subscription_state_test.cc | 4 ++-- tools/spelling_dictionary.txt | 3 +++ 6 files changed, 14 insertions(+), 11 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 09a405ea708b..bfdf63885874 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -12,12 +12,12 @@ DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, Event::Dispatcher& dispatcher) : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} -DeltaSubscriptionState::~DeltaSubscriptionState() {} +DeltaSubscriptionState::~DeltaSubscriptionState() = default; DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} -DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() {} +DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() = default; std::unique_ptr DeltaSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, diff --git a/source/common/config/sotw_subscription_state.cc b/source/common/config/sotw_subscription_state.cc index 90652933247a..97afca688572 100644 --- a/source/common/config/sotw_subscription_state.cc +++ b/source/common/config/sotw_subscription_state.cc @@ -11,12 +11,12 @@ SotwSubscriptionState::SotwSubscriptionState(std::string type_url, SubscriptionC Event::Dispatcher& dispatcher) : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} -SotwSubscriptionState::~SotwSubscriptionState() {} +SotwSubscriptionState::~SotwSubscriptionState() = default; SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher) : dispatcher_(dispatcher) {} -SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() {} +SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() = default; std::unique_ptr SotwSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h index e417147b35e8..37a89e510891 100644 --- a/source/common/config/sotw_subscription_state.h +++ b/source/common/config/sotw_subscription_state.h @@ -42,6 +42,9 @@ class SotwSubscriptionState : public SubscriptionState { // Returns a new'd pointer, meant to be owned by the caller. void* getNextRequestWithAck(const UpdateAck& ack) override; + SotwSubscriptionState(const SotwSubscriptionState&) = delete; + SotwSubscriptionState& operator=(const SotwSubscriptionState&) = delete; + private: // Returns a new'd pointer, meant to be owned by the caller. envoy::api::v2::DiscoveryRequest* getNextRequestInternal(); @@ -61,9 +64,6 @@ class SotwSubscriptionState : public SubscriptionState { bool update_pending_{true}; absl::flat_hash_set names_tracked_; - - SotwSubscriptionState(const SotwSubscriptionState&) = delete; - SotwSubscriptionState& operator=(const SotwSubscriptionState&) = delete; }; class SotwSubscriptionStateFactory : public SubscriptionStateFactory { diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 6fc9d0f0e795..e611b3422ce7 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -96,8 +96,8 @@ class GrpcMuxImplTestBase : public testing::Test { // is the destructor, which is identical to the real one). class FakeGrpcSubscription { public: - FakeGrpcSubscription(GrpcMux* grpc_mux, const std::string& type_url, Watch* watch) - : grpc_mux_(grpc_mux), type_url_(type_url), watch_(watch) {} + FakeGrpcSubscription(GrpcMux* grpc_mux, std::string type_url, Watch* watch) + : grpc_mux_(grpc_mux), type_url_(std::move(type_url)), watch_(watch) {} ~FakeGrpcSubscription() { grpc_mux_->removeWatch(type_url_, watch_); } private: diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc index 417dbdada5f7..d361c46e5261 100644 --- a/test/common/config/sotw_subscription_state_test.cc +++ b/test/common/config/sotw_subscription_state_test.cc @@ -34,7 +34,7 @@ class SotwSubscriptionStateTest : public testing::Test { } UpdateAck deliverDiscoveryResponse(const std::vector& resource_names, - const std::string& version_info, std::string nonce) { + const std::string& version_info, const std::string& nonce) { envoy::api::v2::DiscoveryResponse response; response.set_version_info(version_info); response.set_nonce(nonce); @@ -48,7 +48,7 @@ class SotwSubscriptionStateTest : public testing::Test { return state_.handleResponse(&response); } - UpdateAck deliverBadDiscoveryResponse(const std::string& version_info, std::string nonce) { + UpdateAck deliverBadDiscoveryResponse(const std::string& version_info, const std::string& nonce) { envoy::api::v2::DiscoveryResponse message; message.set_version_info(version_info); message.set_nonce(nonce); diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index ab5842d2a194..7a6d6742bee5 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -517,6 +517,7 @@ goog google goto gregs +GrpcMuxSotw gzip hacky handshaker @@ -652,6 +653,7 @@ pclose pfctl pkey pointee +polymorphism popen pos posix @@ -740,6 +742,7 @@ sizeof smatch sockaddr sockfd +SotW spanid spdlog splitter From 675f3a88a65d2fd5b0ab08140435d5fb51f33af4 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 3 Oct 2019 19:07:21 -0400 Subject: [PATCH 27/39] additional things the pedantic spell check thing disliked without actually pointing to Signed-off-by: Fred Douglas --- tools/spelling_dictionary.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 7a6d6742bee5..4780b1dd4be1 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -743,6 +743,7 @@ smatch sockaddr sockfd SotW +SotwSubscriptionState spanid spdlog splitter @@ -763,6 +764,7 @@ struct structs subexpr subdirs +SubscriptionStateFactory symlink symlinks symlinked From 31650749aa571c292b3a88580a023d182233fc5b Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 8 Oct 2019 18:54:12 -0400 Subject: [PATCH 28/39] maybe it needs the punctuation too, if this doesnt work i will disable it on these files Signed-off-by: Fred Douglas --- tools/spelling_dictionary.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 4780b1dd4be1..d83121e2ebab 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -744,6 +744,8 @@ sockaddr sockfd SotW SotwSubscriptionState +SotwSubscriptionState's +{Delta,Sotw}SubscriptionStateFactory spanid spdlog splitter From 0961ce89e9614f8640885a4d93d6ca2693274972 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 8 Oct 2019 19:15:08 -0400 Subject: [PATCH 29/39] add grpc_mux.h comments Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 61817cbe206a..c07b0bab1646 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -45,7 +45,7 @@ class GrpcMux { * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery * requests may later be resumed with resume(). * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster. + * type.googleapis.com/envoy.api.v2.Cluster */ virtual void pause(const std::string& type_url) PURE; @@ -56,10 +56,30 @@ class GrpcMux { */ virtual void resume(const std::string& type_url) PURE; + /** + * Registers a GrpcSubscription with the GrpcMux. 'watch' may be null (meaning this is an add), + * or it may be the Watch* previously returned by this function (which makes it an update). + * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster + * @param watch the Watch* to be updated, or nullptr to add one. + * @param resources the set of resource names for 'watch' to start out interested in. If empty, + * 'watch' is treated as interested in *all* resources (of type type_url). + * @param callbacks the callbacks that receive updates for 'resources' when they arrive. + * @param init_fetch_timeout how long to wait for this new subscription's first update. Ignored + * unless the addOrUpdateWatch() call is the first for 'type_url'. + * @return Watch* the opaque watch token added or updated, to be used in future addOrUpdateWatch + * calls. + */ virtual Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, const std::set& resources, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) PURE; + + /** + * Cleanup of a Watch* added by addOrUpdateWatch(). Receiving a Watch* from addOrUpdateWatch() + * makes you responsible for eventually invoking this cleanup. + * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster + * @param watch the watch to be cleaned up. + */ virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; /** @@ -70,6 +90,10 @@ class GrpcMux { */ virtual bool paused(const std::string& type_url) const PURE; + /** + * Passes through to all multiplexed SubscriptionStates. To be called when something + * definitive happens with the initial fetch: either an update is successfully received, + * or some sort of error happened.*/ virtual void disableInitFetchTimeoutTimer() PURE; }; From 4350f2d14b348a01492dc21ba8815379588159cc Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 9 Oct 2019 11:23:42 -0400 Subject: [PATCH 30/39] clean up some TODOs Signed-off-by: Fred Douglas --- api/envoy/api/v2/core/config_source.proto | 11 ++++------- api/envoy/api/v3alpha/core/config_source.proto | 11 ++++------- source/common/config/watch_map.h | 3 +-- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/api/envoy/api/v2/core/config_source.proto b/api/envoy/api/v2/core/config_source.proto index 240d37b81ee2..a430e9d5a480 100644 --- a/api/envoy/api/v2/core/config_source.proto +++ b/api/envoy/api/v2/core/config_source.proto @@ -29,15 +29,12 @@ message ApiConfigSource { // the v2 protos is used. REST = 1; - // gRPC v2 API. + // "State of the world" gRPC v2 API, using Discovery{Request,Response} protos. GRPC = 2; - // Using the delta xDS gRPC service, i.e. DeltaDiscovery{Request,Response} - // rather than Discovery{Request,Response}. Rather than sending Envoy the entire state - // with every update, the xDS server only sends what has changed since the last update. - // - // DELTA_GRPC is not yet entirely implemented! Initially, only CDS is available. - // Do not use for other xDSes. TODO(fredlas) update/remove this warning when appropriate. + // "Delta" gRPC v2 API, using DeltaDiscovery{Request,Response} protos. + // Rather than sending Envoy the entire state with every update, the xDS server + // only sends what has changed since the last update. DELTA_GRPC = 3; } diff --git a/api/envoy/api/v3alpha/core/config_source.proto b/api/envoy/api/v3alpha/core/config_source.proto index 7f80ff523267..04509cad8a18 100644 --- a/api/envoy/api/v3alpha/core/config_source.proto +++ b/api/envoy/api/v3alpha/core/config_source.proto @@ -29,15 +29,12 @@ message ApiConfigSource { // the v2 protos is used. REST = 1; - // gRPC v2 API. + // "State of the world" gRPC v2 API, using Discovery{Request,Response} protos. GRPC = 2; - // Using the delta xDS gRPC service, i.e. DeltaDiscovery{Request,Response} - // rather than Discovery{Request,Response}. Rather than sending Envoy the entire state - // with every update, the xDS server only sends what has changed since the last update. - // - // DELTA_GRPC is not yet entirely implemented! Initially, only CDS is available. - // Do not use for other xDSes. TODO(fredlas) update/remove this warning when appropriate. + // "Delta" gRPC v2 API, using DeltaDiscovery{Request,Response} protos. + // Rather than sending Envoy the entire state with every update, the xDS server + // only sends what has changed since the last update. DELTA_GRPC = 3; } diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 9df852c7f712..e98c8af17122 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -33,8 +33,7 @@ struct Watch { }; // NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned -// by addWatch(). We don't expect there to be new users of this class beyond -// NewGrpcMuxImpl and DeltaSubscriptionImpl (TODO(fredlas) to be renamed). +// by addWatch(). However, we don't expect there to be new users of this class. // // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. From 1f682e73ac6ffef6224d2835df2b87ff801711a7 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 9 Oct 2019 12:20:57 -0400 Subject: [PATCH 31/39] fix compilation, BUILD proto dependencies have a new format Signed-off-by: Fred Douglas --- source/common/config/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 253ad52c4eb3..8d5816029da2 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -92,7 +92,7 @@ envoy_cc_library( ":subscription_state_lib", "//source/common/grpc:common_lib", "//source/common/protobuf", - "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/service/discovery/v2:pkg_cc_proto", ], ) @@ -104,7 +104,7 @@ envoy_cc_library( ":subscription_state_lib", "//source/common/grpc:common_lib", "//source/common/protobuf", - "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/service/discovery/v2:pkg_cc_proto", ], ) @@ -358,7 +358,7 @@ envoy_cc_library( name = "update_ack_lib", hdrs = ["update_ack.h"], deps = [ - "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/service/discovery/v2:pkg_cc_proto", ], ) From 0ff6538ddff39562c5325a1d7c78b683995d82ef Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 9 Oct 2019 12:46:37 -0400 Subject: [PATCH 32/39] check_spelling_pedantic appears to crash when i add the punctuated words it fails on to its dictionary Signed-off-by: Fred Douglas --- tools/spelling_dictionary.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 9cb8bfb039c4..8b6530e5b3b4 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -745,8 +745,6 @@ sockaddr sockfd SotW SotwSubscriptionState -SotwSubscriptionState's -{Delta,Sotw}SubscriptionStateFactory spanid spdlog splitter From b8137150943408bb937db019369a00c47b85ebbd Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 10 Oct 2019 18:13:59 -0400 Subject: [PATCH 33/39] review comments Signed-off-by: Fred Douglas --- source/common/config/grpc_subscription_impl.h | 4 +- source/common/config/new_grpc_mux_impl.cc | 38 ++++++++----------- source/common/config/new_grpc_mux_impl.h | 4 +- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 7f6997a3d6da..2a9e5f941acd 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -58,8 +58,8 @@ class GrpcSubscriptionImpl : public Subscription, public SubscriptionCallbacks { const std::string type_url_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; - // NOTE: if another subscription of the same type_url has already been started, this value will be - // ignored in favor of the other subscription's. + // NOTE: if another subscription of the same type_url has already been started, the value of + // init_fetch_timeout_ will be ignored in favor of the other subscription's. const std::chrono::milliseconds init_fetch_timeout_; Watch* watch_{}; const bool is_aggregated_; diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index d092de4a265a..19bad39e9f6d 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -101,8 +101,10 @@ Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set()).first; + subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( + type_url, *watch_maps_[type_url], init_fetch_timeout)); + subscription_ordering_.emplace_back(type_url); } Watch* watch = watch_map->second->addWatch(callbacks); @@ -117,32 +119,24 @@ Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources) { ASSERT(watch != nullptr); - SubscriptionState* sub = subscriptionStateFor(type_url); + SubscriptionState& sub = subscriptionStateFor(type_url); WatchMap& watch_map = watchMapFor(type_url); auto added_removed = watch_map.updateWatchInterest(watch, resources); - sub->updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + sub.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); // Tell the server about our change in interest, if any. - if (sub->subscriptionUpdatePending()) { + if (sub.subscriptionUpdatePending()) { trySendDiscoveryRequests(); } } -void NewGrpcMuxImpl::addSubscription(const std::string& type_url, - std::chrono::milliseconds init_fetch_timeout) { - watch_maps_.emplace(type_url, std::make_unique()); - subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( - type_url, *watch_maps_[type_url], init_fetch_timeout)); - subscription_ordering_.emplace_back(type_url); -} - -SubscriptionState* NewGrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { +SubscriptionState& NewGrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { auto sub = subscriptions_.find(type_url); RELEASE_ASSERT(sub != subscriptions_.end(), fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", type_url)); - return sub->second.get(); + return *sub->second; } WatchMap& NewGrpcMuxImpl::watchMapFor(const std::string& type_url) { @@ -163,7 +157,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { // If so, which one (by type_url)? std::string next_request_type_url = maybe_request_type.value(); // If we don't have a subscription object for this request's type_url, drop the request. - SubscriptionState* sub = subscriptionStateFor(next_request_type_url); + SubscriptionState& sub = subscriptionStateFor(next_request_type_url); // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { break; @@ -172,13 +166,13 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { if (!pausable_ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's of the type_url that we're wanting to send. - UpdateAck ack = pausable_ack_queue_.front(); - pausable_ack_queue_.pop(); + // // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub->getNextRequestWithAck(ack)); + sendGrpcMessage(sub.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); } else { // getNextRequestAckless() returns a raw unowned pointer, which sendGrpcMessage deletes. - sendGrpcMessage(sub->getNextRequestAckless()); + sendGrpcMessage(sub.getNextRequestAckless()); } } maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); @@ -218,8 +212,8 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // If we're looking to send multiple non-ACK requests, send them in the order that their // subscriptions were initiated. for (const auto& sub_type : subscription_ordering_) { - SubscriptionState* sub = subscriptionStateFor(sub_type); - if (sub->subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { + SubscriptionState& sub = subscriptionStateFor(sub_type); + if (sub.subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { return sub_type; } } diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index e22ba540bfbf..3edcb5e9fe16 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -51,7 +51,7 @@ class NewGrpcMuxImpl : public GrpcMux, Logger::Loggable { virtual bool grpcStreamAvailable() const PURE; virtual bool rateLimitAllowsDrain() PURE; - SubscriptionState* subscriptionStateFor(const std::string& type_url); + SubscriptionState& subscriptionStateFor(const std::string& type_url); WatchMap& watchMapFor(const std::string& type_url); void handleEstablishedStream(); void handleStreamEstablishmentFailure(); @@ -76,8 +76,6 @@ class NewGrpcMuxImpl : public GrpcMux, Logger::Loggable { void updateWatch(const std::string& type_url, Watch* watch, const std::set& resources); - void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). bool canSendDiscoveryRequest(const std::string& type_url); From a7693c3e37a5a2321df1ac24edce64789c75432a Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 11 Oct 2019 11:41:01 -0400 Subject: [PATCH 34/39] remove outdated comment Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index 19bad39e9f6d..5d20be80f1d2 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -156,7 +156,6 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { } // If so, which one (by type_url)? std::string next_request_type_url = maybe_request_type.value(); - // If we don't have a subscription object for this request's type_url, drop the request. SubscriptionState& sub = subscriptionStateFor(next_request_type_url); // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { From b893c174093783f3b3a82ee7494f4545198eb7cf Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 11 Oct 2019 11:45:33 -0400 Subject: [PATCH 35/39] change comments to work around broken pedantic spell checker Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.h | 2 +- test/common/config/sotw_subscription_state_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index 3edcb5e9fe16..67b68c789abb 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -94,7 +94,7 @@ class NewGrpcMuxImpl : public GrpcMux, Logger::Loggable { PausableAckQueue pausable_ack_queue_; // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing - // delta or SotW xDS is determined by whether this is a {Delta,Sotw}SubscriptionStateFactory. + // delta or state of the world xDS is determined by which concrete subclass this variable gets. std::unique_ptr subscription_state_factory_; // Map key is type_url. diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc index d361c46e5261..dd19413c1f96 100644 --- a/test/common/config/sotw_subscription_state_test.cc +++ b/test/common/config/sotw_subscription_state_test.cc @@ -171,7 +171,7 @@ TEST_F(SotwSubscriptionStateTest, CheckUpdatePending) { TEST_F(SotwSubscriptionStateTest, handleEstablishmentFailure) { // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the - // ultimate actual subscription callbacks, SotwSubscriptionState's callbacks are actually + // ultimate actual subscription callbacks, the callbacks reference held is actually to // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); From fbc907da5712a58c4d82f477fa508b7ab9f50d85 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 11 Oct 2019 16:05:37 -0400 Subject: [PATCH 36/39] undo merge accident yaml json Signed-off-by: Fred Douglas --- test/common/router/rds_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/common/router/rds_impl_test.cc b/test/common/router/rds_impl_test.cc index e1044018defd..564b1c01531a 100644 --- a/test/common/router/rds_impl_test.cc +++ b/test/common/router/rds_impl_test.cc @@ -90,7 +90,7 @@ stat_prefix: foo EXPECT_CALL(factory_context_.init_manager_, add(_)); rds_ = - RouteConfigProviderUtil::create(parseHttpConnectionManagerFromJson(config_json), + RouteConfigProviderUtil::create(parseHttpConnectionManagerFromYaml(config_yaml), factory_context_, "foo.", *route_config_provider_manager_); rds_callbacks_ = factory_context_.cluster_manager_.subscription_factory_.callbacks_; EXPECT_CALL(*factory_context_.cluster_manager_.subscription_factory_.subscription_, start(_)); From 61da213c12485fff74e428d3d384961982795b50 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 16 Oct 2019 11:39:19 -0400 Subject: [PATCH 37/39] add comments about callbacks being a WatchMap Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_state.h | 1 + source/common/config/sotw_subscription_state.h | 1 + source/common/config/subscription_state.h | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index da994692c5be..31c34c0322d2 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -15,6 +15,7 @@ namespace Config { // Tracks the state of a delta xDS-over-gRPC protocol session. class DeltaSubscriptionState : public SubscriptionState { public: + // Note that, outside of tests, we expect callbacks to always be a WatchMap. DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); diff --git a/source/common/config/sotw_subscription_state.h b/source/common/config/sotw_subscription_state.h index 37a89e510891..1133341e4539 100644 --- a/source/common/config/sotw_subscription_state.h +++ b/source/common/config/sotw_subscription_state.h @@ -15,6 +15,7 @@ namespace Config { // Tracks the state of a "state-of-the-world" (i.e. not delta) xDS-over-gRPC protocol session. class SotwSubscriptionState : public SubscriptionState { public: + // Note that, outside of tests, we expect callbacks to always be a WatchMap. SotwSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); diff --git a/source/common/config/subscription_state.h b/source/common/config/subscription_state.h index c69f2240c443..85bd56222def 100644 --- a/source/common/config/subscription_state.h +++ b/source/common/config/subscription_state.h @@ -23,6 +23,7 @@ namespace Config { // This is the abstract parent class for both the delta and state-of-the-world xDS variants. class SubscriptionState : public Logger::Loggable { public: + // Note that, outside of tests, we expect callbacks to always be a WatchMap. SubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); virtual ~SubscriptionState() = default; @@ -60,7 +61,7 @@ class SubscriptionState : public Logger::Loggable { private: const std::string type_url_; - // callbacks_ is expected to be a WatchMap. + // callbacks_ is expected (outside of tests) to be a WatchMap. SubscriptionCallbacks& callbacks_; Event::TimerPtr init_fetch_timeout_timer_; }; @@ -68,6 +69,7 @@ class SubscriptionState : public Logger::Loggable { class SubscriptionStateFactory { public: virtual ~SubscriptionStateFactory() = default; + // Note that, outside of tests, we expect callbacks to always be a WatchMap. virtual std::unique_ptr makeSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout) PURE; From 74033fa3a1dab9acc4ce9f9c563e69856d035dac Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 22 Oct 2019 13:15:56 -0400 Subject: [PATCH 38/39] remove isDelta Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.h | 159 ------------------ source/common/config/new_grpc_mux_impl.h | 3 - test/common/config/new_grpc_mux_impl_test.cc | 6 - .../scoped_rds_integration_test.cc | 2 - 4 files changed, 170 deletions(-) delete mode 100644 source/common/config/grpc_mux_impl.h diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h deleted file mode 100644 index 3f55178f4e1e..000000000000 --- a/source/common/config/grpc_mux_impl.h +++ /dev/null @@ -1,159 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/common/time.h" -#include "envoy/config/grpc_mux.h" -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/grpc/status.h" -#include "envoy/upstream/cluster_manager.h" - -#include "common/common/cleanup.h" -#include "common/common/logger.h" -#include "common/config/grpc_stream.h" -#include "common/config/utility.h" - -namespace Envoy { -namespace Config { - -/** - * ADS API implementation that fetches via gRPC. - */ -class GrpcMuxImpl : public GrpcMux, - public GrpcStreamCallbacks, - public Logger::Loggable { -public: - GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::RawAsyncClientPtr async_client, - Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, bool skip_subsequent_node); - ~GrpcMuxImpl() override; - - void start() override; - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks) override; - - // GrpcMux - // TODO(fredlas) PR #8478 will remove this. - bool isDelta() const override { return false; } - void pause(const std::string& type_url) override; - void resume(const std::string& type_url) override; - bool paused(const std::string& type_url) const override; - - Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - - void handleDiscoveryResponse(std::unique_ptr&& message); - - void sendDiscoveryRequest(const std::string& type_url); - - // Config::GrpcStreamCallbacks - void onStreamEstablished() override; - void onEstablishmentFailure() override; - void onDiscoveryResponse(std::unique_ptr&& message) override; - void onWriteable() override; - - GrpcStream& - grpcStreamForTest() { - return grpc_stream_; - } - -private: - void drainRequests(); - void setRetryTimer(); - - struct GrpcMuxWatchImpl : public GrpcMuxWatch, RaiiListElement { - GrpcMuxWatchImpl(const std::set& resources, GrpcMuxCallbacks& callbacks, - const std::string& type_url, GrpcMuxImpl& parent) - : RaiiListElement(parent.api_state_[type_url].watches_, this), - resources_(resources), callbacks_(callbacks), type_url_(type_url), parent_(parent), - inserted_(true) {} - ~GrpcMuxWatchImpl() override { - if (inserted_) { - erase(); - if (!resources_.empty()) { - parent_.sendDiscoveryRequest(type_url_); - } - } - } - - void clear() { - inserted_ = false; - cancel(); - } - - std::set resources_; - GrpcMuxCallbacks& callbacks_; - const std::string type_url_; - GrpcMuxImpl& parent_; - - private: - bool inserted_; - }; - - // Per muxed API state. - struct ApiState { - // Watches on the returned resources for the API; - std::list watches_; - // Current DiscoveryRequest for API. - envoy::api::v2::DiscoveryRequest request_; - // Paused via pause()? - bool paused_{}; - // Was a DiscoveryRequest elided during a pause? - bool pending_{}; - // Has this API been tracked in subscriptions_? - bool subscribed_{}; - }; - - // Request queue management logic. - void queueDiscoveryRequest(const std::string& queue_item); - void clearRequestQueue(); - - GrpcStream grpc_stream_; - const LocalInfo::LocalInfo& local_info_; - const bool skip_subsequent_node_; - bool first_stream_request_; - std::unordered_map api_state_; - // Envoy's dependency ordering. - std::list subscriptions_; - - // A queue to store requests while rate limited. Note that when requests cannot be sent due to the - // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - // This string is a type URL. - std::queue request_queue_; -}; - -class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { -public: - void start() override {} - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - // TODO(fredlas) PR #8478 will remove this. - bool isDelta() const override { return false; } - void pause(const std::string&) override {} - void resume(const std::string&) override {} - bool paused(const std::string&) const override { return false; } - - Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - void removeWatch(const std::string&, Watch*) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - - void onWriteable() override {} - void onStreamEstablished() override {} - void onEstablishmentFailure() override {} - void onDiscoveryResponse(std::unique_ptr&&) override {} -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index 2b519431b56a..67b68c789abb 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -31,9 +31,6 @@ class NewGrpcMuxImpl : public GrpcMux, Logger::Loggable { std::chrono::milliseconds init_fetch_timeout) override; void removeWatch(const std::string& type_url, Watch* watch) override; - // TODO(fredlas) PR #8478 will remove this. - bool isDelta() const override { return true; } - void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc index 57b1373dab22..68047881d54b 100644 --- a/test/common/config/new_grpc_mux_impl_test.cc +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -68,12 +68,6 @@ class NewGrpcMuxImplTest : public NewGrpcMuxImplTestBase { Event::SimulatedTimeSystem time_system_; }; -// TODO(fredlas) #8478 will delete this. -TEST_F(NewGrpcMuxImplTest, JustForCoverageTodoDelete) { - setup(); - EXPECT_TRUE(grpc_mux_->isDelta()); -} - // Test that we simply ignore a message for an unknown type_url, with no ill effects. TEST_F(NewGrpcMuxImplTest, DiscoveryResponseNonexistentSub) { setup(); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index b43eb790306a..484c07239964 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -217,8 +217,6 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, response); } - bool isDelta() { return sotwOrDelta() == Grpc::SotwOrDelta::Delta; } - const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; FakeUpstreamInfo rds_upstream_info_; From 1e4ada5029a10c89b1830de591c8317162d72dc0 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 22 Oct 2019 13:33:55 -0400 Subject: [PATCH 39/39] spellcheck Signed-off-by: Fred Douglas --- test/common/config/grpc_mux_impl_test.cc | 2 +- tools/spelling_dictionary.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index f42356c93556..54a12bc5b783 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -625,7 +625,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { expectSendMessage(type_url, {}, "1", false, "bar"); } -// Verifies that a messsage with some resources is accepted even when there are no watches. +// Verifies that a message with some resources is accepted even when there are no watches. // Rationale: SotW gRPC xDS has always been willing to accept updates that include // uninteresting resources. It should not matter whether those uninteresting resources // are accompanied by interesting ones. diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 6fc0c1a7337a..6c047ea93b2f 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -311,6 +311,7 @@ absl accesslog accessor accessors +ackless acks acls addr @@ -551,6 +552,7 @@ genrule getaddrinfo getaffinity getifaddrs +getNextRequestAckless getpeername getsockname getsockopt