quic: support half-way server's preferred address (envoyproxy#25356)

Signed-off-by: Dan Zhang <[email protected]>

Author: danzh2010

api/BUILD

@@ -268,6 +268,7 @@ proto_library(
         "//envoy/extensions/quic/connection_id_generator/v3:pkg",
         "//envoy/extensions/quic/crypto_stream/v3:pkg",
         "//envoy/extensions/quic/proof_source/v3:pkg",
+        "//envoy/extensions/quic/server_preferred_address/v3:pkg",
         "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg",
         "//envoy/extensions/regex_engines/v3:pkg",

api/envoy/config/listener/v3/quic_config.proto

@@ -9,6 +9,8 @@ import "envoy/config/core/v3/protocol.proto";
 import "google/protobuf/duration.proto";
 import "google/protobuf/wrappers.proto";
 
+import "xds/annotations/v3/status.proto";
+
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
 import "validate/validate.proto";
@@ -22,7 +24,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // [#protodoc-title: QUIC listener config]
 
 // Configuration specific to the UDP QUIC listener.
-// [#next-free-field: 9]
+// [#next-free-field: 10]
 message QuicProtocolOptions {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.api.v2.listener.QuicProtocolOptions";
@@ -68,4 +70,11 @@ message QuicProtocolOptions {
   // If not specified the :ref:`default one configured by <envoy_v3_api_msg_extensions.quic.connection_id_generator.v3.DeterministicConnectionIdGeneratorConfig>` will be used.
   // [#extension-category: envoy.quic.connection_id_generator]
   core.v3.TypedExtensionConfig connection_id_generator_config = 8;
+
+  // Configure the server's preferred address to advertise so that client can migrate to it. See :ref:`example <envoy_v3_api_msg_extensions.quic.server_preferred_address.v3.FixedServerPreferredAddressConfig>` which configures a pair of v4 and v6 preferred addresses.
+  // The current QUICHE implementation will advertise only one of the preferred IPv4 and IPv6 addresses based on the address family the client initially connects with, and only if the client is also QUICHE-based.
+  // If not specified, Envoy will not advertise any server's preferred address.
+  // [#extension-category: envoy.quic.server_preferred_address]
+  core.v3.TypedExtensionConfig server_preferred_address_config = 9
+      [(xds.annotations.v3.field_status).work_in_progress = true];
 }

api/envoy/extensions/quic/server_preferred_address/v3/BUILD

@@ -0,0 +1,12 @@
+# DO NOT EDIT. This file is generated by tools/proto_format/proto_sync.py.
+
+load("@envoy_api//bazel:api_build_system.bzl", "api_proto_package")
+
+licenses(["notice"])  # Apache 2
+
+api_proto_package(
+    deps = [
+        "@com_github_cncf_udpa//udpa/annotations:pkg",
+        "@com_github_cncf_udpa//xds/annotations/v3:pkg",
+    ],
+)

api/envoy/extensions/quic/server_preferred_address/v3/fixed_server_preferred_address_config.proto

@@ -0,0 +1,35 @@
+syntax = "proto3";
+
+package envoy.extensions.quic.server_preferred_address.v3;
+
+import "xds/annotations/v3/status.proto";
+
+import "udpa/annotations/status.proto";
+
+option java_package = "io.envoyproxy.envoy.extensions.quic.server_preferred_address.v3";
+option java_outer_classname = "FixedServerPreferredAddressConfigProto";
+option java_multiple_files = true;
+option go_package = "github.com/envoyproxy/go-control-plane/envoy/extensions/quic/server_preferred_address/v3;server_preferred_addressv3";
+option (udpa.annotations.file_status).package_version_status = ACTIVE;
+
+// [#protodoc-title: QUIC server preferred address config]
+// [#extension: envoy.quic.server_preferred_address.fixed]
+
+// Configuration for FixedServerPreferredAddressConfig.
+message FixedServerPreferredAddressConfig {
+  // [#comment:TODO(danzh2010): discuss with API shepherds before removing WiP status.]
+
+  option (xds.annotations.v3.message_status).work_in_progress = true;
+
+  oneof ipv4_type {
+    // String representation of IPv4 address, i.e. "127.0.0.2".
+    // If not specified, none will be configured.
+    string ipv4_address = 1;
+  }
+
+  oneof ipv6_type {
+    // String representation of IPv6 address, i.e. "::1".
+    // If not specified, none will be configured.
+    string ipv6_address = 2;
+  }
+}

api/versioning/BUILD

@@ -207,6 +207,7 @@ proto_library(
         "//envoy/extensions/quic/connection_id_generator/v3:pkg",
         "//envoy/extensions/quic/crypto_stream/v3:pkg",
         "//envoy/extensions/quic/proof_source/v3:pkg",
+        "//envoy/extensions/quic/server_preferred_address/v3:pkg",
         "//envoy/extensions/rate_limit_descriptors/expr/v3:pkg",
         "//envoy/extensions/rbac/matchers/upstream_ip_port/v3:pkg",
         "//envoy/extensions/regex_engines/v3:pkg",

docs/root/api-v3/config/quic/quic_extensions.rst

@@ -8,3 +8,4 @@ Quic extensions
   ../../extensions/quic/crypto_stream/v3/*
   ../../extensions/quic/proof_source/v3/*
   ../../extensions/quic/connection_id_generator/v3/*
+  ../../extensions/quic/server_preferred_address/v3/*

source/common/quic/BUILD

@@ -399,6 +399,7 @@ envoy_cc_library(
         ":envoy_quic_packet_writer_lib",
         ":envoy_quic_proof_source_factory_interface",
         ":envoy_quic_proof_source_lib",
+        ":envoy_quic_server_preferred_address_config_factory_interface",
         ":envoy_quic_utils_lib",
         "//envoy/network:listener_interface",
         "//source/common/network:listener_lib",
@@ -592,6 +593,17 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "envoy_quic_server_preferred_address_config_factory_interface",
+    hdrs = ["envoy_quic_server_preferred_address_config_factory.h"],
+    tags = ["nofips"],
+    deps = [
+        "//envoy/config:typed_config_interface",
+        "//envoy/network:address_interface",
+        "@com_github_google_quiche//:quic_platform_socket_address",
+    ],
+)
+
 envoy_cc_library(
     name = "quic_stats_gatherer",
     srcs = ["quic_stats_gatherer.cc"],

source/common/quic/active_quic_listener.cc

@@ -287,6 +287,20 @@ ActiveQuicListenerFactory::ActiveQuicListenerFactory(
       *Config::Utility::translateToFactoryConfig(cid_generator_config, validation_visitor,
                                                  cid_generator_config_factory));
 
+  if (config.has_server_preferred_address_config()) {
+    const envoy::config::core::v3::TypedExtensionConfig& server_preferred_address_config =
+        config.server_preferred_address_config();
+    auto& server_preferred_address_config_factory =
+        Config::Utility::getAndCheckFactory<EnvoyQuicServerPreferredAddressConfigFactory>(
+            server_preferred_address_config);
+    server_preferred_address_config_ =
+        server_preferred_address_config_factory.createServerPreferredAddressConfig(
+            *Config::Utility::translateToFactoryConfig(config.server_preferred_address_config(),
+                                                       validation_visitor,
+                                                       server_preferred_address_config_factory),
+            validation_visitor);
+  }
+
 #if defined(SO_ATTACH_REUSEPORT_CBPF) && defined(__linux__)
   if (!disable_kernel_bpf_packet_routing_for_test_) {
     if (concurrency_ > 1) {
@@ -312,6 +326,26 @@ Network::ConnectionHandler::ActiveUdpListenerPtr ActiveQuicListenerFactory::crea
     Network::SocketSharedPtr&& listen_socket_ptr, Event::Dispatcher& disptacher,
     Network::ListenerConfig& config) {
   ASSERT(crypto_server_stream_factory_.has_value());
+  if (server_preferred_address_config_ != nullptr) {
+    std::pair<quic::QuicSocketAddress, quic::QuicSocketAddress> addresses =
+        server_preferred_address_config_->getServerPreferredAddresses(
+            listen_socket_ptr->connectionInfoProvider().localAddress());
+    quic::QuicSocketAddress v4_address = addresses.first;
+    if (v4_address.IsInitialized()) {
+      ENVOY_BUG(v4_address.host().address_family() == quiche::IpAddressFamily::IP_V4,
+                absl::StrCat("Configured IPv4 server's preferred address isn't a v4 address:",
+                             v4_address.ToString()));
+      quic_config_.SetIPv4AlternateServerAddressToSend(v4_address);
+    }
+    quic::QuicSocketAddress v6_address = addresses.second;
+    if (v6_address.IsInitialized()) {
+      ENVOY_BUG(v6_address.host().address_family() == quiche::IpAddressFamily::IP_V6,
+                absl::StrCat("Configured IPv6 server's preferred address isn't a v6 address:",
+                             v4_address.ToString()));
+      quic_config_.SetIPv6AlternateServerAddressToSend(v6_address);
+    }
+  }
+
   return std::make_unique<ActiveQuicListener>(
       runtime, worker_index, concurrency_, disptacher, parent, std::move(listen_socket_ptr), config,
       quic_config_, kernel_worker_routing_, enabled_, quic_stat_names_,

source/common/quic/active_quic_listener.h

@@ -10,6 +10,7 @@
 #include "source/common/quic/envoy_quic_connection_id_generator_factory.h"
 #include "source/common/quic/envoy_quic_dispatcher.h"
 #include "source/common/quic/envoy_quic_proof_source_factory_interface.h"
+#include "source/common/quic/envoy_quic_server_preferred_address_config_factory.h"
 #include "source/common/runtime/runtime_protos.h"
 #include "source/server/active_udp_listener.h"
 
@@ -117,6 +118,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
   absl::optional<std::reference_wrapper<EnvoyQuicProofSourceFactoryInterface>>
       proof_source_factory_;
   EnvoyQuicConnectionIdGeneratorFactoryPtr quic_cid_generator_factory_;
+  EnvoyQuicServerPreferredAddressConfigPtr server_preferred_address_config_;
   quic::QuicConfig quic_config_;
   const uint32_t concurrency_;
   envoy::config::core::v3::RuntimeFeatureFlag enabled_;

source/common/quic/envoy_quic_client_connection.cc

@@ -127,6 +127,11 @@ void EnvoyQuicClientConnection::maybeMigratePort() {
     return;
   }
 
+  probeWithNewPort(peer_address(), quic::PathValidationReason::kPortMigration);
+}
+
+void EnvoyQuicClientConnection::probeWithNewPort(const quic::QuicSocketAddress& peer_address,
+                                                 quic::PathValidationReason reason) {
   const Network::Address::InstanceConstSharedPtr& current_local_address =
       connectionSocket()->connectionInfoProvider().localAddress();
   // Creates an IP address with unset port. The port will be set when the new socket is created.
@@ -148,13 +153,11 @@ void EnvoyQuicClientConnection::maybeMigratePort() {
       std::make_unique<Network::UdpDefaultWriter>(probing_socket->ioHandle()));
   quic::QuicSocketAddress self_address = envoyIpAddressToQuicSocketAddress(
       probing_socket->connectionInfoProvider().localAddress()->ip());
-  quic::QuicSocketAddress peer_address = envoyIpAddressToQuicSocketAddress(
-      probing_socket->connectionInfoProvider().remoteAddress()->ip());
 
   auto context = std::make_unique<EnvoyQuicPathValidationContext>(
       self_address, peer_address, std::move(writer), std::move(probing_socket));
   ValidatePath(std::move(context), std::make_unique<EnvoyPathValidationResultDelegate>(*this),
-               quic::PathValidationReason::kPortMigration);
+               reason);
 }
 
 void EnvoyQuicClientConnection::onPathValidationSuccess(
@@ -163,8 +166,16 @@ void EnvoyQuicClientConnection::onPathValidationSuccess(
       static_cast<EnvoyQuicClientConnection::EnvoyQuicPathValidationContext*>(context.get());
 
   auto probing_socket = envoy_context->releaseSocket();
-  if (MigratePath(envoy_context->self_address(), envoy_context->peer_address(),
-                  envoy_context->releaseWriter(), true)) {
+  if (envoy_context->peer_address() != peer_address()) {
+    OnServerPreferredAddressValidated(*envoy_context, true);
+    envoy_context->releaseWriter();
+  } else {
+    MigratePath(envoy_context->self_address(), envoy_context->peer_address(),
+                envoy_context->releaseWriter(), true);
+  }
+
+  if (self_address() == envoy_context->self_address() &&
+      peer_address() == envoy_context->peer_address()) {
     // probing_socket will be set as the new default socket. But old sockets are still able to
     // receive packets.
     setConnectionSocket(std::move(probing_socket));
@@ -231,7 +242,7 @@ void EnvoyQuicClientConnection::setNumPtosForPortMigration(uint32_t num_ptos_for
 }
 
 EnvoyQuicClientConnection::EnvoyQuicPathValidationContext::EnvoyQuicPathValidationContext(
-    quic::QuicSocketAddress& self_address, quic::QuicSocketAddress& peer_address,
+    const quic::QuicSocketAddress& self_address, const quic::QuicSocketAddress& peer_address,
     std::unique_ptr<EnvoyQuicPacketWriter> writer,
     std::unique_ptr<Network::ConnectionSocket> probing_socket)
     : QuicPathValidationContext(self_address, peer_address), writer_(std::move(writer)),
@@ -277,5 +288,11 @@ void EnvoyQuicClientConnection::OnCanWrite() {
   onWriteEventDone();
 }
 
+void EnvoyQuicClientConnection::probeAndMigrateToServerPreferredAddress(
+    const quic::QuicSocketAddress& server_preferred_address) {
+  probeWithNewPort(server_preferred_address,
+                   quic::PathValidationReason::kServerPreferredAddressMigration);
+}
+
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_client_connection.h

@@ -82,12 +82,17 @@ class EnvoyQuicClientConnection : public quic::QuicConnection,
 
   void setNumPtosForPortMigration(uint32_t num_ptos_for_path_degrading);
 
+  // Probes the given peer address. If the probing succeeds, start sending packets to this address
+  // instead.
+  void
+  probeAndMigrateToServerPreferredAddress(const quic::QuicSocketAddress& server_preferred_address);
+
 private:
   // Holds all components needed for a QUIC connection probing/migration.
   class EnvoyQuicPathValidationContext : public quic::QuicPathValidationContext {
   public:
-    EnvoyQuicPathValidationContext(quic::QuicSocketAddress& self_address,
-                                   quic::QuicSocketAddress& peer_address,
+    EnvoyQuicPathValidationContext(const quic::QuicSocketAddress& self_address,
+                                   const quic::QuicSocketAddress& peer_address,
                                    std::unique_ptr<EnvoyQuicPacketWriter> writer,
                                    std::unique_ptr<Network::ConnectionSocket> probing_socket);
 
@@ -131,6 +136,9 @@ class EnvoyQuicClientConnection : public quic::QuicConnection,
 
   void maybeMigratePort();
 
+  void probeWithNewPort(const quic::QuicSocketAddress& peer_address,
+                        quic::PathValidationReason reason);
+
   OptRef<PacketsToReadDelegate> delegate_;
   uint32_t packets_dropped_{0};
   Event::Dispatcher& dispatcher_;

source/common/quic/envoy_quic_client_session.cc

@@ -254,6 +254,11 @@ void EnvoyQuicClientSession::OnNewEncryptionKeyAvailable(
     raiseConnectionEvent(Network::ConnectionEvent::ConnectedZeroRtt);
   }
 }
+void EnvoyQuicClientSession::OnServerPreferredAddressAvailable(
+    const quic::QuicSocketAddress& server_preferred_address) {
+  static_cast<EnvoyQuicClientConnection*>(connection())
+      ->probeAndMigrateToServerPreferredAddress(server_preferred_address);
+}
 
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_client_session.h

@@ -82,6 +82,9 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
   // Notify any registered connection pool when new streams are available.
   void OnCanCreateNewOutgoingStream(bool) override;
 
+  void OnServerPreferredAddressAvailable(
+      const quic::QuicSocketAddress& server_preferred_address) override;
+
   using quic::QuicSpdyClientSession::PerformActionOnActiveStreams;
 
 protected:

source/common/quic/envoy_quic_dispatcher.cc

@@ -60,7 +60,9 @@ EnvoyQuicDispatcher::EnvoyQuicDispatcher(
       listener_stats_(listener_stats), per_worker_stats_(per_worker_stats), dispatcher_(dispatcher),
       listen_socket_(listen_socket), quic_stat_names_(quic_stat_names),
       crypto_server_stream_factory_(crypto_server_stream_factory),
-      quic_stats_(generateStats(listener_config.listenerScope())) {}
+      quic_stats_(generateStats(listener_config.listenerScope())),
+      connection_stats_({QUIC_CONNECTION_STATS(
+          POOL_COUNTER_PREFIX(listener_config.listenerScope(), "quic.connection"))}) {}
 
 void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_id,
                                              quic::QuicErrorCode error,
@@ -101,7 +103,8 @@ std::unique_ptr<quic::QuicSession> EnvoyQuicDispatcher::CreateQuicSession(
       quic_config, quic::ParsedQuicVersionVector{version}, std::move(quic_connection), this,
       session_helper(), crypto_config(), compressed_certs_cache(), dispatcher_,
       listener_config_->perConnectionBufferLimitBytes(), quic_stat_names_,
-      listener_config_->listenerScope(), crypto_server_stream_factory_, std::move(stream_info));
+      listener_config_->listenerScope(), crypto_server_stream_factory_, std::move(stream_info),
+      connection_stats_);
   if (filter_chain != nullptr) {
     // Setup filter chain before Initialize().
     const bool has_filter_initialized =

source/common/quic/envoy_quic_dispatcher.h

@@ -93,6 +93,7 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher {
   EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory_;
   FilterChainToConnectionMap connections_by_filter_chain_;
   QuicDispatcherStats quic_stats_;
+  QuicConnectionStats connection_stats_;
 };
 
 } // namespace Quic

source/common/quic/envoy_quic_server_preferred_address_config_factory.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "envoy/config/typed_config.h"
+#include "envoy/network/address.h"
+#include "envoy/protobuf/message_validator.h"
+
+#include "quiche/quic/platform/api/quic_socket_address.h"
+
+namespace Envoy {
+namespace Quic {
+
+// An interface to provide the server's preferred addresses at runtime.
+class EnvoyQuicServerPreferredAddressConfig {
+public:
+  virtual ~EnvoyQuicServerPreferredAddressConfig() = default;
+
+  /**
+   * Called during config loading.
+   * @param local_address the configured default listening address.
+   * Returns a pair of the server preferred addresses in form of {IPv4, IPv6} which will be used for
+   * the entire life time of the QUIC listener. An uninitialized address value means no preferred
+   * address for that address family.
+   */
+  virtual std::pair<quic::QuicSocketAddress, quic::QuicSocketAddress>
+  getServerPreferredAddresses(const Network::Address::InstanceConstSharedPtr& local_address) PURE;
+};
+
+using EnvoyQuicServerPreferredAddressConfigPtr =
+    std::unique_ptr<EnvoyQuicServerPreferredAddressConfig>;
+
+class EnvoyQuicServerPreferredAddressConfigFactory : public Config::TypedFactory {
+public:
+  std::string category() const override { return "envoy.quic.server_preferred_address"; }
+
+  /**
+   * Returns an EnvoyQuicServerPreferredAddressConfig object according to the given server preferred
+   * address config.
+   */
+  virtual EnvoyQuicServerPreferredAddressConfigPtr
+  createServerPreferredAddressConfig(const Protobuf::Message& config,
+                                     ProtobufMessage::ValidationVisitor& validation_visitor) PURE;
+};
+
+} // namespace Quic
+} // namespace Envoy