quic: make 0-RTT keys truly shared by extending persistent quic info …

…life time (envoyproxy#19862)

Commit Message: Move Quic::PersistentQuicInfoImpl instance into ClusterEntry to extends its life time.

Risk Level: low, quic connection pool only
Testing: existing tests pass
Fixes envoyproxy#19861
Part of envoyproxy#18715

Signed-off-by: Dan Zhang <[email protected]>

envoy/http/BUILD

@@ -203,3 +203,8 @@ envoy_cc_library(
         "//envoy/upstream:upstream_interface",
     ],
 )
+
+envoy_cc_library(
+    name = "persistent_quic_info_interface",
+    hdrs = ["persistent_quic_info.h"],
+)

envoy/http/persistent_quic_info.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <memory>
+
+namespace Envoy {
+namespace Http {
+
+// Store quic helpers which can be shared between connections and must live beyond the lifetime of
+// individual connections. When used in HTTP/3 upstream, it should be owned by cluster and shared
+// across its HTTP/3 connection pools. This an opaque placeholder is needed so that an
+// implementation can be passed around while the QUICHE members which are behind ENVOY_ENABLE_QUIC
+// preprocessor in the actual implementation can be hidden from the Envoy intefaces.
+struct PersistentQuicInfo {
+  virtual ~PersistentQuicInfo() = default;
+};
+
+using PersistentQuicInfoPtr = std::unique_ptr<PersistentQuicInfo>;
+
+} // namespace Http
+} // namespace Envoy

envoy/upstream/BUILD

@@ -26,6 +26,7 @@ envoy_cc_library(
         "//envoy/grpc:async_client_manager_interface",
         "//envoy/http:async_client_interface",
         "//envoy/http:conn_pool_interface",
+        "//envoy/http:persistent_quic_info_interface",
         "//envoy/local_info:local_info_interface",
         "//envoy/runtime:runtime_interface",
         "//envoy/secret:secret_manager_interface",

envoy/upstream/cluster_manager.h

@@ -17,6 +17,7 @@
 #include "envoy/config/subscription_factory.h"
 #include "envoy/grpc/async_client_manager.h"
 #include "envoy/http/conn_pool.h"
+#include "envoy/http/persistent_quic_info.h"
 #include "envoy/local_info/local_info.h"
 #include "envoy/runtime/runtime.h"
 #include "envoy/secret/secret_manager.h"
@@ -467,7 +468,8 @@ class ClusterManagerFactory {
                        alternate_protocol_options,
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
                    const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
-                   TimeSource& time_source, ClusterConnectivityState& state) PURE;
+                   TimeSource& time_source, ClusterConnectivityState& state,
+                   Http::PersistentQuicInfoPtr& quic_info) PURE;
 
   /**
    * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and

source/common/http/conn_pool_grid.cc

@@ -198,12 +198,13 @@ ConnectivityGrid::ConnectivityGrid(
     const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
     Upstream::ClusterConnectivityState& state, TimeSource& time_source,
     AlternateProtocolsCacheSharedPtr alternate_protocols, ConnectivityOptions connectivity_options,
-    Quic::QuicStatNames& quic_stat_names, Stats::Scope& scope)
+    Quic::QuicStatNames& quic_stat_names, Stats::Scope& scope, Http::PersistentQuicInfo& quic_info)
     : dispatcher_(dispatcher), random_generator_(random_generator), host_(host),
       priority_(priority), options_(options), transport_socket_options_(transport_socket_options),
       state_(state), next_attempt_duration_(std::chrono::milliseconds(kDefaultTimeoutMs)),
       time_source_(time_source), http3_status_tracker_(dispatcher_),
-      alternate_protocols_(alternate_protocols), quic_stat_names_(quic_stat_names), scope_(scope) {
+      alternate_protocols_(alternate_protocols), quic_stat_names_(quic_stat_names), scope_(scope),
+      quic_info_(quic_info) {
   // ProdClusterManagerFactory::allocateConnPool verifies the protocols are HTTP/1, HTTP/2 and
   // HTTP/3.
   AlternateProtocolsCache::Origin origin("https", host_->hostname(),
@@ -245,10 +246,10 @@ absl::optional<ConnectivityGrid::PoolIterator> ConnectivityGrid::createNextPool(
   // HTTP/3 is hard-coded as higher priority, H2 as secondary.
   ConnectionPool::InstancePtr pool;
   if (pools_.empty()) {
-    pool = Http3::allocateConnPool(dispatcher_, random_generator_, host_, priority_, options_,
-                                   transport_socket_options_, state_, quic_stat_names_,
-                                   *alternate_protocols_, scope_,
-                                   makeOptRefFromPtr<Http3::PoolConnectResultCallback>(this));
+    pool = Http3::allocateConnPool(
+        dispatcher_, random_generator_, host_, priority_, options_, transport_socket_options_,
+        state_, quic_stat_names_, *alternate_protocols_, scope_,
+        makeOptRefFromPtr<Http3::PoolConnectResultCallback>(this), quic_info_);
   } else {
     pool = std::make_unique<HttpConnPoolImplMixed>(dispatcher_, random_generator_, host_, priority_,
                                                    options_, transport_socket_options_, state_);

source/common/http/conn_pool_grid.h

@@ -139,7 +139,7 @@ class ConnectivityGrid : public ConnectionPool::Instance,
                    Upstream::ClusterConnectivityState& state, TimeSource& time_source,
                    AlternateProtocolsCacheSharedPtr alternate_protocols,
                    ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names,
-                   Stats::Scope& scope);
+                   Stats::Scope& scope, Http::PersistentQuicInfo& quic_info);
   ~ConnectivityGrid() override;
 
   // Event::DeferredDeletable
@@ -233,6 +233,7 @@ class ConnectivityGrid : public ConnectionPool::Instance,
 
   Quic::QuicStatNames& quic_stat_names_;
   Stats::Scope& scope_;
+  Http::PersistentQuicInfo& quic_info_;
 };
 
 } // namespace Http

source/common/http/http3/BUILD

@@ -13,20 +13,15 @@ envoy_cc_library(
     srcs = ["conn_pool.cc"],
     hdrs = ["conn_pool.h"],
     deps = [
-        ":quic_client_connection_factory_lib",
         "//envoy/event:dispatcher_interface",
+        "//envoy/http:persistent_quic_info_interface",
         "//envoy/upstream:upstream_interface",
         "//source/common/http:codec_client_lib",
         "//source/common/http:conn_pool_base_lib",
         "//source/common/quic:client_connection_factory_lib",
     ],
 )
 
-envoy_cc_library(
-    name = "quic_client_connection_factory_lib",
-    hdrs = ["quic_client_connection_factory.h"],
-)
-
 envoy_cc_library(
     name = "codec_stats_lib",
     hdrs = ["codec_stats.h"],

source/common/http/http3/conn_pool.cc

@@ -12,8 +12,6 @@
 #include "source/common/network/utility.h"
 #include "source/common/runtime/runtime_features.h"
 
-#include "quiche/quic/core/crypto/quic_client_session_cache.h"
-
 namespace Envoy {
 namespace Http {
 namespace Http3 {
@@ -76,11 +74,11 @@ Http3ConnPoolImpl::Http3ConnPoolImpl(
     const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
     Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state,
     CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector<Http::Protocol> protocol,
-    OptRef<PoolConnectResultCallback> connect_callback)
+    OptRef<PoolConnectResultCallback> connect_callback, Http::PersistentQuicInfo& quic_info)
     : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options,
                             random_generator, state, client_fn, codec_fn, protocol),
-      quic_info_(Quic::createPersistentQuicInfoForCluster(dispatcher, host->cluster())),
-      server_id_(getConfig(host->transportSocketFactory()).serverNameIndication(),
+      quic_info_(dynamic_cast<Quic::PersistentQuicInfoImpl&>(quic_info)),
+      server_id_(getConfig(host_->transportSocketFactory()).serverNameIndication(),
                  static_cast<uint16_t>(host_->address()->ip()->port()), false),
       connect_callback_(connect_callback) {}
 
@@ -93,31 +91,13 @@ void Http3ConnPoolImpl::onConnected(Envoy::ConnectionPool::ActiveClient&) {
 // Make sure all connections are torn down before quic_info_ is deleted.
 Http3ConnPoolImpl::~Http3ConnPoolImpl() { destructAllConnections(); }
 
-std::shared_ptr<quic::QuicCryptoClientConfig> Http3ConnPoolImpl::cryptoConfig() {
-  Envoy::Ssl::ClientContextSharedPtr context =
-      dynamic_cast<Quic::QuicClientTransportSocketFactory&>(host_->transportSocketFactory())
-          .sslCtx();
-  // If the secrets haven't been loaded, there is no crypto config.
-  if (context == nullptr) {
-    return nullptr;
-  }
-
-  // If the secret has been updated, update the proof source.
-  if (context.get() != client_context_.get()) {
-    client_context_ = context;
-    crypto_config_ = std::make_shared<quic::QuicCryptoClientConfig>(
-        std::make_unique<Quic::EnvoyQuicProofVerifier>(std::move(context)),
-        std::make_unique<quic::QuicClientSessionCache>());
-  }
-  // Return the latest client config.
-  return crypto_config_;
-}
-
 std::unique_ptr<Network::ClientConnection>
 Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names,
                                           OptRef<Http::AlternateProtocolsCache> rtt_cache,
                                           Stats::Scope& scope) {
-  std::shared_ptr<quic::QuicCryptoClientConfig> crypto_config = cryptoConfig();
+  std::shared_ptr<quic::QuicCryptoClientConfig> crypto_config =
+      dynamic_cast<Quic::QuicClientTransportSocketFactory&>(host_->transportSocketFactory())
+          .getCryptoConfig();
   if (crypto_config == nullptr) {
     return nullptr; // no secrets available yet.
   }
@@ -127,7 +107,7 @@ Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names,
     source_address = Network::Utility::getLocalAddress(host_address->ip()->version());
   }
 
-  return Quic::createQuicNetworkConnection(*quic_info_, std::move(crypto_config), server_id_,
+  return Quic::createQuicNetworkConnection(quic_info_, std::move(crypto_config), server_id_,
                                            dispatcher(), host()->address(), source_address,
                                            quic_stat_names, rtt_cache, scope);
 }
@@ -139,7 +119,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
                  const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
                  Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names,
                  OptRef<Http::AlternateProtocolsCache> rtt_cache, Stats::Scope& scope,
-                 OptRef<PoolConnectResultCallback> connect_callback) {
+                 OptRef<PoolConnectResultCallback> connect_callback,
+                 Http::PersistentQuicInfo& quic_info) {
   return std::make_unique<Http3ConnPoolImpl>(
       host, priority, dispatcher, options, transport_socket_options, random_generator, state,
       [&quic_stat_names, rtt_cache,
@@ -192,7 +173,7 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
                                                     pool->randomGenerator());
         return codec;
       },
-      std::vector<Protocol>{Protocol::Http3}, connect_callback);
+      std::vector<Protocol>{Protocol::Http3}, connect_callback, quic_info);
 }
 
 } // namespace Http3

source/common/http/http3/conn_pool.h

@@ -4,6 +4,7 @@
 #include <memory>
 
 #include "envoy/common/optref.h"
+#include "envoy/http/persistent_quic_info.h"
 #include "envoy/upstream/upstream.h"
 
 #include "source/common/http/codec_client.h"
@@ -126,7 +127,8 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
                     Random::RandomGenerator& random_generator,
                     Upstream::ClusterConnectivityState& state, CreateClientFn client_fn,
                     CreateCodecFn codec_fn, std::vector<Http::Protocol> protocol,
-                    OptRef<PoolConnectResultCallback> connect_callback);
+                    OptRef<PoolConnectResultCallback> connect_callback,
+                    Http::PersistentQuicInfo& quic_info);
 
   ~Http3ConnPoolImpl() override;
   ConnectionPool::Cancellable* newStream(Http::ResponseDecoder& response_decoder,
@@ -147,20 +149,11 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
 private:
   friend class Http3ConnPoolImplPeer;
 
-  // Returns the most recent crypto config from host_;
-  std::shared_ptr<quic::QuicCryptoClientConfig> cryptoConfig();
-
-  // Store quic helpers which can be shared between connections and must live
-  // beyond the lifetime of individual connections.
-  std::unique_ptr<Quic::PersistentQuicInfoImpl> quic_info_;
+  // Latches Quic helpers shared across the cluster
+  Quic::PersistentQuicInfoImpl& quic_info_;
   // server-id can change over the lifetime of Envoy but will be consistent for a
   // given connection pool.
   quic::QuicServerId server_id_;
-  // Latch the latest crypto config, to determine if it has updated since last
-  // checked.
-  Envoy::Ssl::ClientContextSharedPtr client_context_;
-  // If client_context_ changes, client config will be updated as well.
-  std::shared_ptr<quic::QuicCryptoClientConfig> crypto_config_;
   // If not nullopt, called when the handshake state changes.
   OptRef<PoolConnectResultCallback> connect_callback_;
 };
@@ -172,7 +165,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
                  const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
                  Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names,
                  OptRef<Http::AlternateProtocolsCache> rtt_cache, Stats::Scope& scope,
-                 OptRef<PoolConnectResultCallback> connect_callback);
+                 OptRef<PoolConnectResultCallback> connect_callback,
+                 Http::PersistentQuicInfo& quic_info);
 
 } // namespace Http3
 } // namespace Http

source/common/quic/BUILD

@@ -164,8 +164,8 @@ envoy_cc_library(
         ":envoy_quic_server_session_lib",
         ":envoy_quic_utils_lib",
         "//envoy/http:codec_interface",
+        "//envoy/http:persistent_quic_info_interface",
         "//envoy/registry",
-        "//source/common/http/http3:quic_client_connection_factory_lib",
         "//source/extensions/quic/crypto_stream:envoy_quic_crypto_client_stream_lib",
         "//source/extensions/transport_sockets/tls:ssl_socket_lib",
         "@com_github_google_quiche//:quic_core_http_spdy_session_lib",
@@ -405,13 +405,15 @@ envoy_cc_library(
     hdrs = ["quic_transport_socket_factory.h"],
     tags = ["nofips"],
     deps = [
+        ":envoy_quic_proof_verifier_lib",
         "//envoy/network:transport_socket_interface",
         "//envoy/server:transport_socket_config_interface",
         "//envoy/ssl:context_config_interface",
         "//source/common/common:assert_lib",
         "//source/common/network:transport_socket_options_lib",
         "//source/extensions/transport_sockets/tls:context_config_lib",
         "//source/extensions/transport_sockets/tls:ssl_socket_lib",
+        "@com_github_google_quiche//:quic_core_crypto_crypto_handshake_lib",
         "@envoy_api//envoy/extensions/transport_sockets/quic/v3:pkg_cc_proto",
     ],
 )

source/common/quic/client_connection_factory_impl.cc

@@ -1,9 +1,5 @@
 #include "source/common/quic/client_connection_factory_impl.h"
 
-#include "source/common/quic/quic_transport_socket_factory.h"
-
-#include "quiche/quic/core/crypto/quic_client_session_cache.h"
-
 namespace Envoy {
 namespace Quic {
 
@@ -46,21 +42,22 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
       quic::QuicUtils::CreateRandomConnectionId(), server_addr, info_impl->conn_helper_,
       info_impl->alarm_factory_, quic_versions, local_addr, dispatcher, nullptr);
 
+  // TODO (danzh) move this temporary config and initial RTT configuration to h3 pool.
+  quic::QuicConfig config = info_impl->quic_config_;
   // Update config with latest srtt, if available.
   if (rtt_cache.has_value()) {
     Http::AlternateProtocolsCache::Origin origin("https", server_id.host(), server_id.port());
     std::chrono::microseconds rtt = rtt_cache.value().get().getSrtt(origin);
     if (rtt.count() != 0) {
-      info_impl->quic_config_.SetInitialRoundTripTimeUsToSend(rtt.count());
+      config.SetInitialRoundTripTimeUsToSend(rtt.count());
     }
   }
 
   // QUICHE client session always use the 1st version to start handshake.
   return std::make_unique<EnvoyQuicClientSession>(
-      info_impl->quic_config_, quic_versions, std::move(connection), server_id,
-      std::move(crypto_config), &info_impl->push_promise_index_, dispatcher,
-      info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache,
-      scope);
+      config, quic_versions, std::move(connection), server_id, std::move(crypto_config),
+      &info_impl->push_promise_index_, dispatcher, info_impl->buffer_limit_,
+      info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, scope);
 }
 
 } // namespace Quic

source/common/quic/client_connection_factory_impl.h

@@ -1,10 +1,13 @@
 #pragma once
 
-#include "source/common/http/http3/quic_client_connection_factory.h"
+#include <memory>
+
+#include "envoy/http/persistent_quic_info.h"
+#include "envoy/upstream/upstream.h"
+
 #include "source/common/quic/envoy_quic_alarm_factory.h"
 #include "source/common/quic/envoy_quic_client_session.h"
 #include "source/common/quic/envoy_quic_connection_helper.h"
-#include "source/common/quic/envoy_quic_proof_verifier.h"
 #include "source/common/quic/envoy_quic_utils.h"
 #include "source/extensions/quic/crypto_stream/envoy_quic_crypto_client_stream.h"
 #include "source/extensions/transport_sockets/tls/ssl_socket.h"
@@ -16,6 +19,8 @@ namespace Envoy {
 namespace Quic {
 
 // Information which can be shared across connections, though not across threads.
+// TODO(danzh) considering exposing these QUICHE interfaces via base class virtual methods, so that
+// down casting can be avoided while passing around this object.
 struct PersistentQuicInfoImpl : public Http::PersistentQuicInfo {
   PersistentQuicInfoImpl(Event::Dispatcher& dispatcher, uint32_t buffer_limit);