diff --git a/envoy/http/BUILD b/envoy/http/BUILD index db1fc1968979..bda7b5972454 100644 --- a/envoy/http/BUILD +++ b/envoy/http/BUILD @@ -203,3 +203,8 @@ envoy_cc_library( "//envoy/upstream:upstream_interface", ], ) + +envoy_cc_library( + name = "persistent_quic_info_interface", + hdrs = ["persistent_quic_info.h"], +) diff --git a/envoy/http/persistent_quic_info.h b/envoy/http/persistent_quic_info.h new file mode 100644 index 000000000000..f5cd952fc176 --- /dev/null +++ b/envoy/http/persistent_quic_info.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace Envoy { +namespace Http { + +// Store quic helpers which can be shared between connections and must live beyond the lifetime of +// individual connections. When used in HTTP/3 upstream, it should be owned by cluster and shared +// across its HTTP/3 connection pools. This an opaque placeholder is needed so that an +// implementation can be passed around while the QUICHE members which are behind ENVOY_ENABLE_QUIC +// preprocessor in the actual implementation can be hidden from the Envoy intefaces. +struct PersistentQuicInfo { + virtual ~PersistentQuicInfo() = default; +}; + +using PersistentQuicInfoPtr = std::unique_ptr; + +} // namespace Http +} // namespace Envoy diff --git a/envoy/upstream/BUILD b/envoy/upstream/BUILD index 6189ddbc47f2..abf275887838 100644 --- a/envoy/upstream/BUILD +++ b/envoy/upstream/BUILD @@ -26,6 +26,7 @@ envoy_cc_library( "//envoy/grpc:async_client_manager_interface", "//envoy/http:async_client_interface", "//envoy/http:conn_pool_interface", + "//envoy/http:persistent_quic_info_interface", "//envoy/local_info:local_info_interface", "//envoy/runtime:runtime_interface", "//envoy/secret:secret_manager_interface", diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index 8643f623febc..2b4b0b4030ed 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -17,6 +17,7 @@ #include "envoy/config/subscription_factory.h" #include "envoy/grpc/async_client_manager.h" #include "envoy/http/conn_pool.h" +#include "envoy/http/persistent_quic_info.h" #include "envoy/local_info/local_info.h" #include "envoy/runtime/runtime.h" #include "envoy/secret/secret_manager.h" @@ -467,7 +468,8 @@ class ClusterManagerFactory { alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& time_source, ClusterConnectivityState& state) PURE; + TimeSource& time_source, ClusterConnectivityState& state, + Http::PersistentQuicInfoPtr& quic_info) PURE; /** * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and diff --git a/source/common/http/conn_pool_grid.cc b/source/common/http/conn_pool_grid.cc index a4f66261ba33..04bf8093691d 100644 --- a/source/common/http/conn_pool_grid.cc +++ b/source/common/http/conn_pool_grid.cc @@ -198,12 +198,13 @@ ConnectivityGrid::ConnectivityGrid( const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, Upstream::ClusterConnectivityState& state, TimeSource& time_source, AlternateProtocolsCacheSharedPtr alternate_protocols, ConnectivityOptions connectivity_options, - Quic::QuicStatNames& quic_stat_names, Stats::Scope& scope) + Quic::QuicStatNames& quic_stat_names, Stats::Scope& scope, Http::PersistentQuicInfo& quic_info) : dispatcher_(dispatcher), random_generator_(random_generator), host_(host), priority_(priority), options_(options), transport_socket_options_(transport_socket_options), state_(state), next_attempt_duration_(std::chrono::milliseconds(kDefaultTimeoutMs)), time_source_(time_source), http3_status_tracker_(dispatcher_), - alternate_protocols_(alternate_protocols), quic_stat_names_(quic_stat_names), scope_(scope) { + alternate_protocols_(alternate_protocols), quic_stat_names_(quic_stat_names), scope_(scope), + quic_info_(quic_info) { // ProdClusterManagerFactory::allocateConnPool verifies the protocols are HTTP/1, HTTP/2 and // HTTP/3. AlternateProtocolsCache::Origin origin("https", host_->hostname(), @@ -245,10 +246,10 @@ absl::optional ConnectivityGrid::createNextPool( // HTTP/3 is hard-coded as higher priority, H2 as secondary. ConnectionPool::InstancePtr pool; if (pools_.empty()) { - pool = Http3::allocateConnPool(dispatcher_, random_generator_, host_, priority_, options_, - transport_socket_options_, state_, quic_stat_names_, - *alternate_protocols_, scope_, - makeOptRefFromPtr(this)); + pool = Http3::allocateConnPool( + dispatcher_, random_generator_, host_, priority_, options_, transport_socket_options_, + state_, quic_stat_names_, *alternate_protocols_, scope_, + makeOptRefFromPtr(this), quic_info_); } else { pool = std::make_unique(dispatcher_, random_generator_, host_, priority_, options_, transport_socket_options_, state_); diff --git a/source/common/http/conn_pool_grid.h b/source/common/http/conn_pool_grid.h index 61b97d4b498d..571ac6f38cb8 100644 --- a/source/common/http/conn_pool_grid.h +++ b/source/common/http/conn_pool_grid.h @@ -139,7 +139,7 @@ class ConnectivityGrid : public ConnectionPool::Instance, Upstream::ClusterConnectivityState& state, TimeSource& time_source, AlternateProtocolsCacheSharedPtr alternate_protocols, ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names, - Stats::Scope& scope); + Stats::Scope& scope, Http::PersistentQuicInfo& quic_info); ~ConnectivityGrid() override; // Event::DeferredDeletable @@ -233,6 +233,7 @@ class ConnectivityGrid : public ConnectionPool::Instance, Quic::QuicStatNames& quic_stat_names_; Stats::Scope& scope_; + Http::PersistentQuicInfo& quic_info_; }; } // namespace Http diff --git a/source/common/http/http3/BUILD b/source/common/http/http3/BUILD index 92a28b20c1d5..96e9bd43b9f9 100644 --- a/source/common/http/http3/BUILD +++ b/source/common/http/http3/BUILD @@ -13,8 +13,8 @@ envoy_cc_library( srcs = ["conn_pool.cc"], hdrs = ["conn_pool.h"], deps = [ - ":quic_client_connection_factory_lib", "//envoy/event:dispatcher_interface", + "//envoy/http:persistent_quic_info_interface", "//envoy/upstream:upstream_interface", "//source/common/http:codec_client_lib", "//source/common/http:conn_pool_base_lib", @@ -22,11 +22,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "quic_client_connection_factory_lib", - hdrs = ["quic_client_connection_factory.h"], -) - envoy_cc_library( name = "codec_stats_lib", hdrs = ["codec_stats.h"], diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index 5574b8fa468e..4040e33dc404 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -12,8 +12,6 @@ #include "source/common/network/utility.h" #include "source/common/runtime/runtime_features.h" -#include "quiche/quic/core/crypto/quic_client_session_cache.h" - namespace Envoy { namespace Http { namespace Http3 { @@ -76,11 +74,11 @@ Http3ConnPoolImpl::Http3ConnPoolImpl( const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector protocol, - OptRef connect_callback) + OptRef connect_callback, Http::PersistentQuicInfo& quic_info) : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options, random_generator, state, client_fn, codec_fn, protocol), - quic_info_(Quic::createPersistentQuicInfoForCluster(dispatcher, host->cluster())), - server_id_(getConfig(host->transportSocketFactory()).serverNameIndication(), + quic_info_(dynamic_cast(quic_info)), + server_id_(getConfig(host_->transportSocketFactory()).serverNameIndication(), static_cast(host_->address()->ip()->port()), false), connect_callback_(connect_callback) {} @@ -93,31 +91,13 @@ void Http3ConnPoolImpl::onConnected(Envoy::ConnectionPool::ActiveClient&) { // Make sure all connections are torn down before quic_info_ is deleted. Http3ConnPoolImpl::~Http3ConnPoolImpl() { destructAllConnections(); } -std::shared_ptr Http3ConnPoolImpl::cryptoConfig() { - Envoy::Ssl::ClientContextSharedPtr context = - dynamic_cast(host_->transportSocketFactory()) - .sslCtx(); - // If the secrets haven't been loaded, there is no crypto config. - if (context == nullptr) { - return nullptr; - } - - // If the secret has been updated, update the proof source. - if (context.get() != client_context_.get()) { - client_context_ = context; - crypto_config_ = std::make_shared( - std::make_unique(std::move(context)), - std::make_unique()); - } - // Return the latest client config. - return crypto_config_; -} - std::unique_ptr Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope) { - std::shared_ptr crypto_config = cryptoConfig(); + std::shared_ptr crypto_config = + dynamic_cast(host_->transportSocketFactory()) + .getCryptoConfig(); if (crypto_config == nullptr) { return nullptr; // no secrets available yet. } @@ -127,7 +107,7 @@ Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names, source_address = Network::Utility::getLocalAddress(host_address->ip()->version()); } - return Quic::createQuicNetworkConnection(*quic_info_, std::move(crypto_config), server_id_, + return Quic::createQuicNetworkConnection(quic_info_, std::move(crypto_config), server_id_, dispatcher(), host()->address(), source_address, quic_stat_names, rtt_cache, scope); } @@ -139,7 +119,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope, - OptRef connect_callback) { + OptRef connect_callback, + Http::PersistentQuicInfo& quic_info) { return std::make_unique( host, priority, dispatcher, options, transport_socket_options, random_generator, state, [&quic_stat_names, rtt_cache, @@ -192,7 +173,7 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ pool->randomGenerator()); return codec; }, - std::vector{Protocol::Http3}, connect_callback); + std::vector{Protocol::Http3}, connect_callback, quic_info); } } // namespace Http3 diff --git a/source/common/http/http3/conn_pool.h b/source/common/http/http3/conn_pool.h index 1cf5c356f8f3..6219081797af 100644 --- a/source/common/http/http3/conn_pool.h +++ b/source/common/http/http3/conn_pool.h @@ -4,6 +4,7 @@ #include #include "envoy/common/optref.h" +#include "envoy/http/persistent_quic_info.h" #include "envoy/upstream/upstream.h" #include "source/common/http/codec_client.h" @@ -126,7 +127,8 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector protocol, - OptRef connect_callback); + OptRef connect_callback, + Http::PersistentQuicInfo& quic_info); ~Http3ConnPoolImpl() override; ConnectionPool::Cancellable* newStream(Http::ResponseDecoder& response_decoder, @@ -147,20 +149,11 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { private: friend class Http3ConnPoolImplPeer; - // Returns the most recent crypto config from host_; - std::shared_ptr cryptoConfig(); - - // Store quic helpers which can be shared between connections and must live - // beyond the lifetime of individual connections. - std::unique_ptr quic_info_; + // Latches Quic helpers shared across the cluster + Quic::PersistentQuicInfoImpl& quic_info_; // server-id can change over the lifetime of Envoy but will be consistent for a // given connection pool. quic::QuicServerId server_id_; - // Latch the latest crypto config, to determine if it has updated since last - // checked. - Envoy::Ssl::ClientContextSharedPtr client_context_; - // If client_context_ changes, client config will be updated as well. - std::shared_ptr crypto_config_; // If not nullopt, called when the handshake state changes. OptRef connect_callback_; }; @@ -172,7 +165,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope, - OptRef connect_callback); + OptRef connect_callback, + Http::PersistentQuicInfo& quic_info); } // namespace Http3 } // namespace Http diff --git a/source/common/http/http3/quic_client_connection_factory.h b/source/common/http/http3/quic_client_connection_factory.h deleted file mode 100644 index bfc639d47cfb..000000000000 --- a/source/common/http/http3/quic_client_connection_factory.h +++ /dev/null @@ -1,13 +0,0 @@ -#pragma once - -namespace Envoy { -namespace Http { - -// Store quic helpers which can be shared between connections and must live -// beyond the lifetime of individual connections. -struct PersistentQuicInfo { - virtual ~PersistentQuicInfo() = default; -}; - -} // namespace Http -} // namespace Envoy diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index 7c0ce2e2ae95..c238f90a9058 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -164,8 +164,8 @@ envoy_cc_library( ":envoy_quic_server_session_lib", ":envoy_quic_utils_lib", "//envoy/http:codec_interface", + "//envoy/http:persistent_quic_info_interface", "//envoy/registry", - "//source/common/http/http3:quic_client_connection_factory_lib", "//source/extensions/quic/crypto_stream:envoy_quic_crypto_client_stream_lib", "//source/extensions/transport_sockets/tls:ssl_socket_lib", "@com_github_google_quiche//:quic_core_http_spdy_session_lib", @@ -405,6 +405,7 @@ envoy_cc_library( hdrs = ["quic_transport_socket_factory.h"], tags = ["nofips"], deps = [ + ":envoy_quic_proof_verifier_lib", "//envoy/network:transport_socket_interface", "//envoy/server:transport_socket_config_interface", "//envoy/ssl:context_config_interface", @@ -412,6 +413,7 @@ envoy_cc_library( "//source/common/network:transport_socket_options_lib", "//source/extensions/transport_sockets/tls:context_config_lib", "//source/extensions/transport_sockets/tls:ssl_socket_lib", + "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib", "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto", ], ) diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 94a2e70db808..6e4278c9d375 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -1,9 +1,5 @@ #include "source/common/quic/client_connection_factory_impl.h" -#include "source/common/quic/quic_transport_socket_factory.h" - -#include "quiche/quic/core/crypto/quic_client_session_cache.h" - namespace Envoy { namespace Quic { @@ -46,21 +42,22 @@ std::unique_ptr createQuicNetworkConnection( quic::QuicUtils::CreateRandomConnectionId(), server_addr, info_impl->conn_helper_, info_impl->alarm_factory_, quic_versions, local_addr, dispatcher, nullptr); + // TODO (danzh) move this temporary config and initial RTT configuration to h3 pool. + quic::QuicConfig config = info_impl->quic_config_; // Update config with latest srtt, if available. if (rtt_cache.has_value()) { Http::AlternateProtocolsCache::Origin origin("https", server_id.host(), server_id.port()); std::chrono::microseconds rtt = rtt_cache.value().get().getSrtt(origin); if (rtt.count() != 0) { - info_impl->quic_config_.SetInitialRoundTripTimeUsToSend(rtt.count()); + config.SetInitialRoundTripTimeUsToSend(rtt.count()); } } // QUICHE client session always use the 1st version to start handshake. return std::make_unique( - info_impl->quic_config_, quic_versions, std::move(connection), server_id, - std::move(crypto_config), &info_impl->push_promise_index_, dispatcher, - info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, - scope); + config, quic_versions, std::move(connection), server_id, std::move(crypto_config), + &info_impl->push_promise_index_, dispatcher, info_impl->buffer_limit_, + info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, scope); } } // namespace Quic diff --git a/source/common/quic/client_connection_factory_impl.h b/source/common/quic/client_connection_factory_impl.h index 887bc6aca8e5..953177e3b7f1 100644 --- a/source/common/quic/client_connection_factory_impl.h +++ b/source/common/quic/client_connection_factory_impl.h @@ -1,10 +1,13 @@ #pragma once -#include "source/common/http/http3/quic_client_connection_factory.h" +#include + +#include "envoy/http/persistent_quic_info.h" +#include "envoy/upstream/upstream.h" + #include "source/common/quic/envoy_quic_alarm_factory.h" #include "source/common/quic/envoy_quic_client_session.h" #include "source/common/quic/envoy_quic_connection_helper.h" -#include "source/common/quic/envoy_quic_proof_verifier.h" #include "source/common/quic/envoy_quic_utils.h" #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_client_stream.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" @@ -16,6 +19,8 @@ namespace Envoy { namespace Quic { // Information which can be shared across connections, though not across threads. +// TODO(danzh) considering exposing these QUICHE interfaces via base class virtual methods, so that +// down casting can be avoided while passing around this object. struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo { PersistentQuicInfoImpl(Event::Dispatcher& dispatcher, uint32_t buffer_limit); diff --git a/source/common/quic/quic_transport_socket_factory.cc b/source/common/quic/quic_transport_socket_factory.cc index 452b4f45d7c6..60b70c6942f9 100644 --- a/source/common/quic/quic_transport_socket_factory.cc +++ b/source/common/quic/quic_transport_socket_factory.cc @@ -1,9 +1,14 @@ #include "source/common/quic/quic_transport_socket_factory.h" +#include + #include "envoy/extensions/transport_sockets/quic/v3/quic_transport.pb.validate.h" +#include "source/common/quic/envoy_quic_proof_verifier.h" #include "source/extensions/transport_sockets/tls/context_config_impl.h" +#include "quiche/quic/core/crypto/quic_client_session_cache.h" + namespace Envoy { namespace Quic { @@ -53,6 +58,26 @@ ProtobufTypes::MessagePtr QuicClientTransportSocketConfigFactory::createEmptyCon return std::make_unique(); } +std::shared_ptr QuicClientTransportSocketFactory::getCryptoConfig() { + Envoy::Ssl::ClientContextSharedPtr context = sslCtx(); + // If the secrets haven't been loaded, there is no crypto config. + if (context == nullptr) { + ENVOY_LOG(warn, "SDS hasn't finished updating Ssl context config yet."); + stats_.upstream_context_secrets_not_ready_.inc(); + return nullptr; + } + + if (client_context_ != context) { + // If the context has been updated, update the crypto config. + client_context_ = context; + crypto_config_ = std::make_shared( + std::make_unique(std::move(context)), + std::make_unique()); + } + // Return the latest crypto config. + return crypto_config_; +} + REGISTER_FACTORY(QuicServerTransportSocketConfigFactory, Server::Configuration::DownstreamTransportSocketConfigFactory); diff --git a/source/common/quic/quic_transport_socket_factory.h b/source/common/quic/quic_transport_socket_factory.h index ffc334480df5..ba3026663a1e 100644 --- a/source/common/quic/quic_transport_socket_factory.h +++ b/source/common/quic/quic_transport_socket_factory.h @@ -9,6 +9,8 @@ #include "source/common/network/transport_socket_options_impl.h" #include "source/extensions/transport_sockets/tls/ssl_socket.h" +#include "quiche/quic/core/crypto/quic_crypto_client_config.h" + namespace Envoy { namespace Quic { @@ -114,6 +116,10 @@ class QuicClientTransportSocketFactory : public QuicTransportSocketFactoryBase { return fallback_factory_->config(); } + // Returns a crypto config generated from the up-to-date client context config. Once the passed in + // context config gets updated, a new crypto config object will be returned by this method. + std::shared_ptr getCryptoConfig(); + protected: // fallback_factory_ will update the context. void onSecretUpdated() override {} @@ -121,6 +127,11 @@ class QuicClientTransportSocketFactory : public QuicTransportSocketFactoryBase { private: // The QUIC client transport socket can create TLS sockets for fallback to TCP. std::unique_ptr fallback_factory_; + // Latch the latest client context, to determine if it has updated since last + // checked. + Envoy::Ssl::ClientContextSharedPtr client_context_; + // If client_context_ changes, client config will be updated as well. + std::shared_ptr crypto_config_; }; // Base class to create above QuicTransportSocketFactory for server and client diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index a875a817dfdd..7b380f6bc10a 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -49,6 +49,7 @@ #ifdef ENVOY_ENABLE_QUIC #include "source/common/http/conn_pool_grid.h" #include "source/common/http/http3/conn_pool.h" +#include "source/common/quic/client_connection_factory_impl.h" #endif namespace Envoy { @@ -1685,7 +1686,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::httpConnPoolImp parent_.thread_local_dispatcher_, host, priority, upstream_protocols, alternate_protocol_options, !upstream_options->empty() ? upstream_options : nullptr, have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr, - parent_.parent_.time_source_, parent_.cluster_manager_state_); + parent_.parent_.time_source_, parent_.cluster_manager_state_, quic_info_); pool->addIdleCallback([&parent = parent_, host, priority, hash_key]() { parent.httpConnPoolIsIdle(host, priority, hash_key); @@ -1831,7 +1832,7 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& source, ClusterConnectivityState& state) { + TimeSource& source, ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& quic_info) { if (protocols.size() == 3 && context_.runtime().snapshot().featureEnabled("upstream.use_http3", 100)) { ASSERT(contains(protocols, @@ -1842,11 +1843,15 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( alternate_protocols_cache_manager_->getCache(alternate_protocol_options.value(), dispatcher); Envoy::Http::ConnectivityGrid::ConnectivityOptions coptions{protocols}; + if (quic_info == nullptr) { + quic_info = Quic::createPersistentQuicInfoForCluster(dispatcher, host->cluster()); + } return std::make_unique( dispatcher, context_.api().randomGenerator(), host, priority, options, transport_socket_options, state, source, alternate_protocols_cache, coptions, - quic_stat_names_, stats_); + quic_stat_names_, stats_, *quic_info); #else + (void)quic_info; // Should be blocked by configuration checking at an earlier point. PANIC("unexpected"); #endif @@ -1865,9 +1870,12 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( if (protocols.size() == 1 && protocols[0] == Http::Protocol::Http3 && context_.runtime().snapshot().featureEnabled("upstream.use_http3", 100)) { #ifdef ENVOY_ENABLE_QUIC + if (quic_info == nullptr) { + quic_info = Quic::createPersistentQuicInfoForCluster(dispatcher, host->cluster()); + } return Http::Http3::allocateConnPool(dispatcher, context_.api().randomGenerator(), host, priority, options, transport_socket_options, state, - quic_stat_names_, {}, stats_, {}); + quic_stat_names_, {}, stats_, {}, *quic_info); #else UNREFERENCED_PARAMETER(source); // Should be blocked by configuration checking at an earlier point. diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 1acaa9147028..1332a3e5c586 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -81,7 +81,8 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& time_source, ClusterConnectivityState& state) override; + TimeSource& time_source, ClusterConnectivityState& state, + Http::PersistentQuicInfoPtr& quic_info) override; Tcp::ConnectionPool::InstancePtr allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority, @@ -532,6 +533,9 @@ class ClusterManagerImpl : public ClusterManager, LoadBalancerPtr lb_; ClusterInfoConstSharedPtr cluster_info_; Http::AsyncClientPtr lazy_http_async_client_; + // Stores QUICHE specific objects which live through out the life time of the cluster and can + // be shared across its hosts. + Http::PersistentQuicInfoPtr quic_info_; }; using ClusterEntryPtr = std::unique_ptr; diff --git a/test/common/http/conn_pool_grid_test.cc b/test/common/http/conn_pool_grid_test.cc index 945279c94fa8..cdf0171b225f 100644 --- a/test/common/http/conn_pool_grid_test.cc +++ b/test/common/http/conn_pool_grid_test.cc @@ -1,3 +1,5 @@ +#include + #include "envoy/http/alternate_protocols_cache.h" #include "source/common/http/alternate_protocols_cache_impl.h" @@ -110,11 +112,19 @@ class ConnectivityGridTest : public Event::TestUsingSimulatedTime, public testin quic_stat_names_(store_.symbolTable()) {} void initialize() { + quic_connection_persistent_info_ = +#ifdef ENVOY_ENABLE_QUIC + std::make_unique(dispatcher_, 0); +#else + std::make_unique(); +#endif + grid_ = std::make_unique( dispatcher_, random_, Upstream::makeTestHost(cluster_, "hostname", "tcp://127.0.0.1:9000", simTime()), Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, - simTime(), alternate_protocols_, options_, quic_stat_names_, store_); + simTime(), alternate_protocols_, options_, quic_stat_names_, store_, + *quic_connection_persistent_info_); host_ = grid_->host(); grid_->info_ = &info_; grid_->encoder_ = &encoder_; @@ -149,6 +159,7 @@ class ConnectivityGridTest : public Event::TestUsingSimulatedTime, public testin AlternateProtocolsCacheSharedPtr alternate_protocols_; Stats::IsolatedStoreImpl store_; Quic::QuicStatNames quic_stat_names_; + PersistentQuicInfoPtr quic_connection_persistent_info_; std::unique_ptr grid_; Upstream::HostDescriptionConstSharedPtr host_; @@ -798,10 +809,11 @@ TEST_F(ConnectivityGridTest, RealGrid) { .WillRepeatedly( Return(Upstream::TransportSocketMatcher::MatchData(*factory, matcher.stats_, "test"))); - ConnectivityGrid grid( - dispatcher_, random_, Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), - Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, - simTime(), alternate_protocols_, options_, quic_stat_names_, store_); + ConnectivityGrid grid(dispatcher_, random_, + Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), + Upstream::ResourcePriority::Default, socket_options_, + transport_socket_options_, state_, simTime(), alternate_protocols_, + options_, quic_stat_names_, store_, *quic_connection_persistent_info_); // Create the HTTP/3 pool. auto optional_it1 = ConnectivityGridForTest::forceCreateNextPool(grid); @@ -842,10 +854,11 @@ TEST_F(ConnectivityGridTest, ConnectionCloseDuringCreation) { .WillRepeatedly( Return(Upstream::TransportSocketMatcher::MatchData(*factory, matcher.stats_, "test"))); - ConnectivityGrid grid( - dispatcher_, random_, Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), - Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, - simTime(), alternate_protocols_, options_, quic_stat_names_, store_); + ConnectivityGrid grid(dispatcher_, random_, + Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), + Upstream::ResourcePriority::Default, socket_options_, + transport_socket_options_, state_, simTime(), alternate_protocols_, + options_, quic_stat_names_, store_, *quic_connection_persistent_info_); // Create the HTTP/3 pool. auto optional_it1 = ConnectivityGridForTest::forceCreateNextPool(grid); @@ -913,10 +926,11 @@ TEST_F(ConnectivityGridTest, ConnectionCloseDuringAysnConnect) { .WillRepeatedly( Return(Upstream::TransportSocketMatcher::MatchData(*factory, matcher.stats_, "test"))); - ConnectivityGrid grid( - dispatcher_, random_, Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), - Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, - simTime(), alternate_protocols_, options_, quic_stat_names_, store_); + ConnectivityGrid grid(dispatcher_, random_, + Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), + Upstream::ResourcePriority::Default, socket_options_, + transport_socket_options_, state_, simTime(), alternate_protocols_, + options_, quic_stat_names_, store_, *quic_connection_persistent_info_); // Create the HTTP/3 pool. auto optional_it1 = ConnectivityGridForTest::forceCreateNextPool(grid); diff --git a/test/common/http/http3/conn_pool_test.cc b/test/common/http/http3/conn_pool_test.cc index 7f36ec8f9c41..f545d61222f9 100644 --- a/test/common/http/http3/conn_pool_test.cc +++ b/test/common/http/http3/conn_pool_test.cc @@ -57,13 +57,15 @@ class Http3ConnPoolImplTest : public Event::TestUsingSimulatedTime, public testi Network::TransportSocketOptionsConstSharedPtr transport_options; pool_ = allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, state_, quic_stat_names_, {}, store_, - makeOptRef(connect_result_callback_)); + makeOptRef(connect_result_callback_), + quic_info_); EXPECT_EQ(3000, Http3ConnPoolImplPeer::getServerId(*pool_).port()); } Upstream::MockHost& mockHost() { return static_cast(*host_); } NiceMock dispatcher_; + Quic::PersistentQuicInfoImpl quic_info_{dispatcher_, 45}; Upstream::HostSharedPtr host_{new NiceMock}; NiceMock random_; Upstream::ClusterConnectivityState state_; @@ -105,7 +107,7 @@ TEST_F(Http3ConnPoolImplTest, FastFailWithoutSecretsLoaded) { ConnectionPool::InstancePtr pool = allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, state_, quic_stat_names_, {}, store_, - makeOptRef(connect_result_callback_)); + makeOptRef(connect_result_callback_), quic_info_); EXPECT_EQ(static_cast(pool.get())->instantiateActiveClient(), nullptr); } @@ -129,10 +131,11 @@ TEST_F(Http3ConnPoolImplTest, FailWithSecretsBecomeEmpty) { ConnectionPool::InstancePtr pool = allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, state_, quic_stat_names_, {}, store_, - makeOptRef(connect_result_callback_)); + makeOptRef(connect_result_callback_), quic_info_); MockResponseDecoder decoder; ConnPoolCallbacks callbacks; + EXPECT_CALL(context_.store_.counter_, inc()); EXPECT_CALL(callbacks.pool_failure_, ready()); EXPECT_EQ(pool->newStream(decoder, callbacks, {/*can_send_early_data_=*/false, @@ -141,7 +144,6 @@ TEST_F(Http3ConnPoolImplTest, FailWithSecretsBecomeEmpty) { } TEST_F(Http3ConnPoolImplTest, CreationAndNewStream) { - EXPECT_CALL(mockHost().cluster_, perConnectionBufferLimitBytes); initialize(); MockResponseDecoder decoder; @@ -175,7 +177,6 @@ TEST_F(Http3ConnPoolImplTest, NewAndCancelStreamBeforeConnect) { return; } - EXPECT_CALL(mockHost().cluster_, perConnectionBufferLimitBytes); initialize(); MockResponseDecoder decoder; @@ -206,7 +207,6 @@ TEST_F(Http3ConnPoolImplTest, NewAndDrainClientBeforeConnect) { "envoy.reloadable_features.postpone_h3_client_connect_to_next_loop")) { return; } - EXPECT_CALL(mockHost().cluster_, perConnectionBufferLimitBytes); initialize(); MockResponseDecoder decoder; diff --git a/test/common/quic/BUILD b/test/common/quic/BUILD index 3074476f09af..abd4f636efb4 100644 --- a/test/common/quic/BUILD +++ b/test/common/quic/BUILD @@ -341,3 +341,15 @@ envoy_cc_test_library( "@com_github_google_quiche//:quic_test_tools_qpack_qpack_encoder_test_utils_lib", ], ) + +envoy_cc_test( + name = "quic_transport_socket_factory_test", + srcs = ["quic_transport_socket_factory_test.cc"], + tags = ["nofips"], + deps = [ + "//source/common/quic:quic_transport_socket_factory_lib", + "//source/extensions/transport_sockets/tls:context_config_lib", + "//test/mocks/server:transport_socket_factory_context_mocks", + "//test/mocks/ssl:ssl_mocks", + ], +) diff --git a/test/common/quic/client_connection_factory_impl_test.cc b/test/common/quic/client_connection_factory_impl_test.cc index a647b2750e03..4b2abca47d3b 100644 --- a/test/common/quic/client_connection_factory_impl_test.cc +++ b/test/common/quic/client_connection_factory_impl_test.cc @@ -52,9 +52,7 @@ class QuicNetworkConnectionTest : public Event::TestUsingSimulatedTime, std::unique_ptr( new NiceMock), context_); - crypto_config_ = std::make_shared( - std::make_unique(factory_->sslCtx()), - std::make_unique()); + crypto_config_ = factory_->getCryptoConfig(); } uint32_t highWatermark(EnvoyQuicClientSession* session) { @@ -108,9 +106,9 @@ TEST_P(QuicNetworkConnectionTest, Srtt) { quic::QuicServerId{factory_->clientContextConfig().serverNameIndication(), port, false}, dispatcher_, test_address_, test_address_, quic_stat_names_, rtt_cache, store_); - EXPECT_EQ(info.quic_config_.GetInitialRoundTripTimeUsToSend(), 5); - EnvoyQuicClientSession* session = static_cast(client_connection.get()); + + EXPECT_EQ(session->config()->GetInitialRoundTripTimeUsToSend(), 5); session->Initialize(); client_connection->connect(); EXPECT_TRUE(client_connection->connecting()); diff --git a/test/common/quic/quic_transport_socket_factory_test.cc b/test/common/quic/quic_transport_socket_factory_test.cc new file mode 100644 index 000000000000..661e84da646b --- /dev/null +++ b/test/common/quic/quic_transport_socket_factory_test.cc @@ -0,0 +1,49 @@ +#include "source/common/quic/quic_transport_socket_factory.h" + +#include "test/mocks/server/transport_socket_factory_context.h" +#include "test/mocks/ssl/mocks.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Quic { + +class QuicClientTransportSocketFactoryTest : public testing::Test { +public: + QuicClientTransportSocketFactoryTest() { + EXPECT_CALL(context_.context_manager_, createSslClientContext(_, _)).WillOnce(Return(nullptr)); + EXPECT_CALL(*context_config_, setSecretUpdateCallback(_)) + .WillOnce(testing::SaveArg<0>(&update_callback_)); + factory_.emplace(std::unique_ptr(context_config_), context_); + factory_->initialize(); + } + + NiceMock context_; + absl::optional factory_; + // Will be owned by factory_. + NiceMock* context_config_{ + new NiceMock}; + std::function update_callback_; +}; + +TEST_F(QuicClientTransportSocketFactoryTest, GetCryptoConfig) { + EXPECT_EQ(nullptr, factory_->getCryptoConfig()); + + Ssl::ClientContextSharedPtr ssl_context1{new Ssl::MockClientContext()}; + EXPECT_CALL(context_.context_manager_, createSslClientContext(_, _)) + .WillOnce(Return(ssl_context1)); + update_callback_(); + std::shared_ptr crypto_config1 = factory_->getCryptoConfig(); + EXPECT_NE(nullptr, crypto_config1); + + Ssl::ClientContextSharedPtr ssl_context2{new Ssl::MockClientContext()}; + EXPECT_CALL(context_.context_manager_, createSslClientContext(_, _)) + .WillOnce(Return(ssl_context2)); + update_callback_(); + std::shared_ptr crypto_config2 = factory_->getCryptoConfig(); + EXPECT_NE(crypto_config2, crypto_config1); +} + +} // namespace Quic +} // namespace Envoy diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index 536417cf44b6..36fc57a88a15 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -76,14 +76,13 @@ class TestClusterManagerFactory : public ClusterManagerFactory { })); } - Http::ConnectionPool::InstancePtr - allocateConnPool(Event::Dispatcher&, HostConstSharedPtr host, ResourcePriority, - std::vector&, - const absl::optional& - alternate_protocol_options, - const Network::ConnectionSocket::OptionsSharedPtr& options, - const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource&, ClusterConnectivityState& state) override { + Http::ConnectionPool::InstancePtr allocateConnPool( + Event::Dispatcher&, HostConstSharedPtr host, ResourcePriority, std::vector&, + const absl::optional& + alternate_protocol_options, + const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, TimeSource&, + ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& /*quic_info*/) override { return Http::ConnectionPool::InstancePtr{allocateConnPool_( host, alternate_protocol_options, options, transport_socket_options, state)}; } diff --git a/test/integration/BUILD b/test/integration/BUILD index 9f9bf0fa2e4a..63b9444cd01e 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -862,7 +862,6 @@ envoy_cc_test_library( "//source/common/common:thread_lib", "//source/common/common:utility_lib", "//source/common/http:codec_client_lib", - "//source/common/http/http3:quic_client_connection_factory_lib", "//source/common/json:json_loader_lib", "//source/common/network:utility_lib", "//source/common/quic:quic_stat_names_lib", diff --git a/test/integration/http_integration.cc b/test/integration/http_integration.cc index 2a5098b19ade..016ad1a0a7ef 100644 --- a/test/integration/http_integration.cc +++ b/test/integration/http_integration.cc @@ -30,7 +30,6 @@ #ifdef ENVOY_ENABLE_QUIC #include "source/common/quic/client_connection_factory_impl.h" #include "source/common/quic/quic_transport_socket_factory.h" -#include "quiche/quic/core/crypto/quic_client_session_cache.h" #endif #include "source/extensions/transport_sockets/tls/context_config_impl.h" @@ -248,11 +247,7 @@ Network::ClientConnectionPtr HttpIntegrationTest::makeClientConnectionWithOption auto& quic_transport_socket_factory_ref = dynamic_cast(*quic_transport_socket_factory_); return Quic::createQuicNetworkConnection( - *quic_connection_persistent_info_, - std::make_shared( - std::make_unique( - quic_transport_socket_factory_ref.sslCtx()), - std::make_unique()), + *quic_connection_persistent_info_, quic_transport_socket_factory_ref.getCryptoConfig(), quic::QuicServerId( quic_transport_socket_factory_ref.clientContextConfig().serverNameIndication(), static_cast(port)), diff --git a/test/integration/http_integration.h b/test/integration/http_integration.h index b2fffc14d5ab..71d36ee6d973 100644 --- a/test/integration/http_integration.h +++ b/test/integration/http_integration.h @@ -5,7 +5,6 @@ #include #include "source/common/http/codec_client.h" -#include "source/common/http/http3/quic_client_connection_factory.h" #include "source/common/network/filter_impl.h" #include "test/common/http/http2/http2_frame.h" diff --git a/test/integration/quic_http_integration_test.cc b/test/integration/quic_http_integration_test.cc index 0ac27e43672c..d7fcc73b8ca7 100644 --- a/test/integration/quic_http_integration_test.cc +++ b/test/integration/quic_http_integration_test.cc @@ -187,7 +187,7 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, (host.empty() ? transport_socket_factory_->clientContextConfig().serverNameIndication() : host), static_cast(port), false}, - crypto_config_, &push_promise_index_, *dispatcher_, + transport_socket_factory_->getCryptoConfig(), &push_promise_index_, *dispatcher_, // Use smaller window than the default one to have test coverage of client codec buffer // exceeding high watermark. /*send_buffer_limit=*/2 * Http2::Utility::OptionsLimits::MIN_INITIAL_STREAM_WINDOW_SIZE, @@ -259,9 +259,6 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, // Latch quic_transport_socket_factory_ which is instantiated in initialize(). transport_socket_factory_ = static_cast(quic_transport_socket_factory_.get()); - crypto_config_ = std::make_shared( - std::make_unique(transport_socket_factory_->sslCtx()), - std::make_unique()); registerTestServerPorts({"http"}); ASSERT(&transport_socket_factory_->clientContextConfig()); @@ -355,7 +352,6 @@ class QuicHttpIntegrationTest : public HttpIntegrationTest, TestEnvoyQuicClientConnection* quic_connection_{nullptr}; std::list designated_connection_ids_; Quic::QuicClientTransportSocketFactory* transport_socket_factory_{nullptr}; - std::shared_ptr crypto_config_; bool validation_failure_on_path_response_{false}; }; diff --git a/test/integration/utility.cc b/test/integration/utility.cc index f3225458eea9..4836a62a2b44 100644 --- a/test/integration/utility.cc +++ b/test/integration/utility.cc @@ -25,7 +25,6 @@ #ifdef ENVOY_ENABLE_QUIC #include "source/common/quic/client_connection_factory_impl.h" #include "source/common/quic/quic_transport_socket_factory.h" -#include "quiche/quic/core/crypto/quic_client_session_cache.h" #endif #include "test/common/upstream/utility.h" @@ -227,10 +226,7 @@ IntegrationUtil::makeSingleRequest(const Network::Address::InstanceConstSharedPt local_address = std::make_shared("::1"); } Network::ClientConnectionPtr connection = Quic::createQuicNetworkConnection( - *persistent_info, - std::make_shared( - std::make_unique(quic_transport_socket_factory.sslCtx()), - std::make_unique()), + *persistent_info, quic_transport_socket_factory.getCryptoConfig(), quic::QuicServerId(quic_transport_socket_factory.clientContextConfig().serverNameIndication(), static_cast(addr->ip()->port())), *dispatcher, addr, local_address, quic_stat_names, {}, mock_stats_store); diff --git a/test/mocks/upstream/cluster_manager_factory.h b/test/mocks/upstream/cluster_manager_factory.h index b4328b31beb1..91f53900b44e 100644 --- a/test/mocks/upstream/cluster_manager_factory.h +++ b/test/mocks/upstream/cluster_manager_factory.h @@ -31,7 +31,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& source, ClusterConnectivityState& state)); + TimeSource& source, ClusterConnectivityState& state, + Http::PersistentQuicInfoPtr& quic_info)); MOCK_METHOD(Tcp::ConnectionPool::InstancePtr, allocateTcpConnPool, (Event::Dispatcher & dispatcher, HostConstSharedPtr host, ResourcePriority priority,