[+] Add support for multipath draft-11

demo/demo_client.c

@@ -40,7 +40,7 @@
 #define XQC_PACKET_TMP_BUF_LEN  1600
 #define MAX_BUF_SIZE            (100*1024*1024)
 #define XQC_INTEROP_TLS_GROUPS  "X25519:P-256:P-384:P-521"
-#define MAX_PATH_CNT            2
+#define MAX_PATH_CNT            16
 
 
 typedef enum xqc_demo_cli_alpn_type_s {
@@ -183,13 +183,17 @@ typedef struct xqc_demo_cli_quic_config_s {
     uint8_t mp_version;
 
     uint64_t init_max_path_id;
+    uint64_t extra_paths_num;
 
     /* support interop test */
     int is_interop_mode;
 
     uint8_t send_path_standby;
     xqc_msec_t path_status_timer_threshold;
 
+    xqc_msec_t path_cid_rotation;
+    xqc_msec_t path_cid_retirement;
+
     uint64_t least_available_cid_count;
 
     uint64_t idle_timeout;
@@ -419,6 +423,8 @@ typedef struct xqc_demo_cli_user_conn_s {
     struct event            *ev_delay_req;
     struct event            *ev_idle_restart;
     struct event            *ev_close_path;
+    struct event            *ev_cid_rotation;
+    struct event            *ev_path_cid_retirement;
     struct event            *ev_rebinding_p0;
     struct event            *ev_rebinding_p1;
 
@@ -430,6 +436,8 @@ typedef struct xqc_demo_cli_user_conn_s {
     xqc_msec_t              path_status_time;
     xqc_msec_t              path_status_timer_threshold;
 
+    int                     trigger_cid_rotation;
+    xqc_msec_t              cid_rotation_timer_threshold;
 
     xqc_msec_t              path_create_time;
     xqc_flag_t              remove_path_flag;
@@ -495,6 +503,11 @@ xqc_demo_cli_close_task(xqc_demo_cli_task_t *task)
         user_conn->ev_close_path = NULL;
     }
 
+    if (user_conn->ev_cid_rotation) {
+        event_del(user_conn->ev_cid_rotation);
+        user_conn->ev_cid_rotation = NULL;
+    }
+
     if (user_conn->ev_rebinding_p0) {
         event_del(user_conn->ev_rebinding_p0);
         user_conn->ev_rebinding_p0 =  NULL;
@@ -858,16 +871,48 @@ xqc_demo_cli_conn_create_path(const xqc_cid_t *cid, void *conn_user_data)
     xqc_demo_cli_user_conn_t *user_conn = conn_user_data;
     xqc_demo_cli_ctx_t *ctx = user_conn->ctx;
     uint64_t path_id;
-    int ret;
+    int ret, i;
     int backup = 0;
-    printf("ready to create path notify\n");
+    printf("ready to create path notify: trying to create %"PRIu64" paths, current path %i, ifcnt: %i, init_max_path_id: %"PRIu64"\n",
+           ctx->args->quic_cfg.extra_paths_num,
+           user_conn->total_path_cnt,
+           ctx->args->net_cfg.ifcnt,
+           ctx->args->quic_cfg.init_max_path_id);
 
-    if (user_conn->total_path_cnt < ctx->args->net_cfg.ifcnt
-        && user_conn->total_path_cnt < ctx->args->quic_cfg.init_max_path_id)
-    {
+    for (i = 0; i < ctx->args->quic_cfg.extra_paths_num; i++) {
+
+        if (user_conn->total_path_cnt < ctx->args->net_cfg.ifcnt
+            && user_conn->total_path_cnt <= ctx->args->quic_cfg.init_max_path_id)
+        {
 
-        if (user_conn->total_path_cnt == 1 && ctx->args->quic_cfg.mp_backup) {
-            backup = 1;
+            if (user_conn->total_path_cnt == 1 && ctx->args->quic_cfg.mp_backup) {
+                backup = 1;
+            }
+
+            ret = xqc_conn_create_path(ctx->engine, &(user_conn->cid), &path_id, backup);
+            if (ret < 0) {
+                printf("not support mp, xqc_conn_create_path err = %d path_id: %"PRIu64"\n", ret, path_id);
+                return;
+            }
+
+            printf("client created path: path_id %"PRIu64"\n", path_id);
+
+            ret = xqc_demo_cli_init_user_path(user_conn, user_conn->total_path_cnt, path_id);
+            if (ret < 0) {
+                printf("xqc_demo_cli_init_user_path fail: path_id %"PRIu64"\n", path_id);
+                xqc_conn_close_path(ctx->engine, &(user_conn->cid), path_id);
+                return;
+            }
+
+            user_conn->path_create_time = xqc_now();
+
+            if (user_conn->total_path_cnt == 2 && ctx->args->quic_cfg.mp_backup) {
+                if (backup == 1) {
+                    printf("Init No.%d path (id = %"PRIu64") to STANDBY state\n", 1, path_id);
+                }
+                printf("set No.%d path (id = %"PRIu64") to STANDBY state\n", 1, path_id);
+                xqc_conn_mark_path_standby(ctx->engine, &(user_conn->cid), path_id);
+            }
         }
 
         ret = xqc_conn_create_path(ctx->engine, &(user_conn->cid), &path_id, backup);
@@ -892,7 +937,6 @@ xqc_demo_cli_conn_create_path(const xqc_cid_t *cid, void *conn_user_data)
             printf("set No.%d path (id = %"PRIu64") to STANDBY state\n", 1, path_id);
             xqc_conn_mark_path_standby(ctx->engine, &(user_conn->cid), path_id);
         }
-
     }
 }
 
@@ -1467,6 +1511,11 @@ xqc_demo_cli_idle_callback(int fd, short what, void *arg)
                 user_conn->ev_close_path = NULL;
             }
 
+            if (user_conn->ev_cid_rotation) {
+                event_del(user_conn->ev_cid_rotation);
+                user_conn->ev_cid_rotation = NULL;
+            }
+
             if (user_conn->ev_rebinding_p0) {
                 event_del(user_conn->ev_rebinding_p0);
                 user_conn->ev_rebinding_p0 =  NULL;
@@ -1539,6 +1588,39 @@ xqc_demo_cli_close_path_timeout(int fd, short what, void *arg)
     }
 }
 
+static void
+xqc_demo_cli_trigger_path_cid_rotation(int fd, short what, void *arg)
+{
+    xqc_demo_cli_user_conn_t *user_conn = (xqc_demo_cli_user_conn_t *) arg;
+    printf("xqc_demo_cli_cid_rotation: active path: %d\n", user_conn->active_path_cnt);
+    if (user_conn->active_path_cnt > 0) {
+        uint64_t path_id = 1;
+        printf("trigger cid rotation on path: path_id %"PRIu64"\n", path_id);
+        int ret = xqc_conn_trigger_cid_rotation_on_path(user_conn->ctx->engine, &(user_conn->cid), path_id);
+        if (ret != XQC_OK) {
+            printf("trigger cid rotation on path: path_id %"PRIu64" failed\n", path_id);
+        } else {
+            user_conn->trigger_cid_rotation = 1;
+        }
+    }
+}
+
+static void
+xqc_demo_cli_trigger_path_cid_retirment(int fd, short what, void *arg)
+{
+    xqc_demo_cli_user_conn_t *user_conn = (xqc_demo_cli_user_conn_t *) arg;
+    uint64_t path_id = 1;
+
+    if (!user_conn->trigger_cid_rotation) {
+        printf("can't trigger cid retirement on path: path_id %"PRIu64" because the cid rotation failed\n", path_id);
+        return;
+    }
+
+    printf("trigger cid retirement on path: path_id %"PRIu64"\n", path_id);
+    xqc_conn_trigger_cid_retirement_on_path(user_conn->ctx->engine, &(user_conn->cid), path_id);
+}
+
+
 static void
 xqc_demo_cli_rebind_path0(int fd, short what, void *arg)
 {
@@ -1727,6 +1809,7 @@ xqc_demo_cli_init_args(xqc_demo_cli_client_args_t *args)
     args->quic_cfg.mp_version = XQC_MULTIPATH_10;
     args->quic_cfg.init_max_path_id = 2;
     args->quic_cfg.max_pkt_sz = 1200;
+    args->quic_cfg.extra_paths_num = 1;
 
     args->req_cfg.throttled_req = -1;
 
@@ -1745,7 +1828,7 @@ xqc_demo_cli_parse_server_addr(char *url, xqc_demo_cli_net_config_t *cfg)
     /* set hint for hostname resolve */
     struct addrinfo hints = {0};
     memset(&hints, 0, sizeof(struct addrinfo));
-    hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
+    hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6: set to AF_INET if only using IPv4 */
     hints.ai_socktype = SOCK_DGRAM; /* Datagram socket */
     hints.ai_flags = AI_PASSIVE;    /* For wildcard IP address */
     hints.ai_protocol = 0;          /* Any protocol */
@@ -1860,7 +1943,7 @@ void
 xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args)
 {
     int ch = 0;
-    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bZ:NQT:R:V:B:I:n:eEF:G:r:x:y:Y:f:")) != -1) {
+    while ((ch = getopt(argc, argv, "a:p:c:Ct:S:0m:A:D:l:L:k:K:U:u:dMoi:w:Ps:bz:Z:NQT:R:V:B:I:n:eEF:g:G:r:x:X:y:Y:f:")) != -1) {
         switch (ch) {
         /* server ip */
         case 'a':
@@ -2013,10 +2096,11 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
             printf("option adding interface: %s\n", optarg);
             if (args->net_cfg.ifcnt < MAX_PATH_CNT) {
                 strncpy(args->net_cfg.iflist[args->net_cfg.ifcnt++], optarg, strlen(optarg));
-            } else {
+            }
+            /*} else {
                 printf("too many interfaces (two at most)!\n");
                 exit(0);
-            }
+            }*/
             break;
 
         case 'w':
@@ -2075,6 +2159,22 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
             args->quic_cfg.path_status_timer_threshold = atoi(optarg) * 1000;
             break;
 
+        case 'z':
+            printf("option multipath trigger cid rotation: %s ms\n", optarg);
+            args->quic_cfg.path_cid_rotation = atoi(optarg);
+            break;
+
+        case 'g':
+            printf("option force a cid rotation after %s ms\n", optarg);
+            args->quic_cfg.path_cid_retirement = atoi(optarg);
+            break;
+
+        case 'X':
+            printf("option create path numbers: %s\n", optarg);
+            args->quic_cfg.extra_paths_num = atoi(optarg);
+            args->quic_cfg.init_max_path_id = args->quic_cfg.extra_paths_num;
+            break;
+
         case 'I':
             printf("option idle gap: %s\n", optarg);
             args->req_cfg.idle_gap = atoi(optarg);
@@ -2625,6 +2725,28 @@ xqc_demo_cli_start(xqc_demo_cli_user_conn_t *user_conn, xqc_demo_cli_client_args
         event_add(user_conn->ev_close_path, &tv);
     }
 
+    if (args->quic_cfg.path_cid_rotation) {
+        user_conn->ev_cid_rotation = event_new(user_conn->ctx->eb, -1, 0,
+                                               xqc_demo_cli_trigger_path_cid_rotation,
+                                               user_conn);
+        struct timeval tv = {
+                .tv_sec = args->quic_cfg.path_cid_rotation / 1000,
+                .tv_usec = (args->quic_cfg.path_cid_rotation % 1000) * 1000,
+        };
+        event_add(user_conn->ev_cid_rotation, &tv);
+    }
+
+    if (args->quic_cfg.path_cid_retirement) {
+        user_conn->ev_path_cid_retirement = event_new(user_conn->ctx->eb, -1, 0,
+                                                      xqc_demo_cli_trigger_path_cid_retirment,
+                                                      user_conn);
+        struct timeval tv = {
+                .tv_sec = args->quic_cfg.path_cid_retirement / 1000,
+                .tv_usec = (args->quic_cfg.path_cid_retirement % 1000) * 1000,
+        };
+        event_add(user_conn->ev_path_cid_retirement, &tv);
+    }
+
     if (args->net_cfg.rebind_p0) {
         user_conn->ev_rebinding_p0 = event_new(user_conn->ctx->eb, -1, 0, 
                                                xqc_demo_cli_rebind_path0, 

include/xquic/xqc_configure.h

@@ -9,3 +9,8 @@
 /* #undef XQC_ENABLE_MP_INTEROP */
 /* #undef XQC_NO_PID_PACKET_PROCESS */
 /* #undef XQC_PROTECT_POOL_MEM */
+/* #undef XQC_COMPAT_DUPLICATE */
+/* #undef XQC_ENABLE_FEC */
+/* #undef XQC_ENABLE_XOR */
+/* #undef XQC_ENABLE_RSC */
+/* #undef XQC_ENABLE_PKM */

include/xquic/xqc_errno.h

@@ -279,6 +279,12 @@ typedef enum {
     XQC_QPACK_ERR_MAX,
 } xqc_qpack_error_t;
 
+typedef enum {
+    XQC_PATH_NO_ERROR                   = 0x0,
+    XQC_PATH_APPLICATION_ABANDON        = 0x004150504142414E, /* Path abandon error code: APPLICATION_ABANDON */
+    XQC_PATH_RESOURCE_LIMIT_REACHED     = 0x0052534C494D4954, /* Path abandon error code: RESOURCE_LIMIT_REACHED */
+} xqc_multipath_error_t;
+
 #define QPACK_ERR_START 900
 static const int QPACK_ERR_CNT = XQC_QPACK_ERR_MAX - QPACK_ERR_START;
 

include/xquic/xquic.h

@@ -1246,7 +1246,8 @@ typedef struct xqc_linger_s {
 
 typedef enum {
     XQC_ERR_MULTIPATH_VERSION   = 0x00,
-    XQC_MULTIPATH_10            = 0x0a, 
+    XQC_MULTIPATH_10            = 0x0a,
+    XQC_MULTIPATH_11            = 0x0b,
 } xqc_multipath_version_t;
 
 typedef enum {
@@ -1456,7 +1457,7 @@ typedef enum {
 } xqc_0rtt_flag_t;
 
 
-#define XQC_MAX_PATHS_COUNT 8
+#define XQC_MAX_PATHS_COUNT 32
 #define XQC_CONN_INFO_LEN 400
 
 typedef struct xqc_path_metrics_s {
@@ -1504,6 +1505,7 @@ typedef struct xqc_conn_stats_s {
      * 0: 不支持MP
      * 1: 支持MP, 采用 Single PNS
      * 2: 支持MP, 采用 Multiple PNS
+     *
      */
     int                 enable_multipath;
 
@@ -2139,7 +2141,15 @@ XQC_EXPORT_PUBLIC_API
 xqc_int_t xqc_path_get_local_addr(xqc_connection_t *conn, uint64_t path_id,
     struct sockaddr *addr, socklen_t addr_cap, socklen_t *local_addr_len);
 
-
+
+/*
+ * These 2 APIs below is only used for IETF interop tests. Please don't use them for formal logic!
+ * */
+XQC_EXPORT_PUBLIC_API
+xqc_int_t xqc_conn_trigger_cid_rotation_on_path(xqc_engine_t *engine, const xqc_cid_t *scid, uint64_t path_id);
+
+XQC_EXPORT_PUBLIC_API
+xqc_int_t xqc_conn_trigger_cid_retirement_on_path(xqc_engine_t *engine, const xqc_cid_t *scid, uint64_t path_id);
 
 /**
  * @brief load balance cid encryption.

src/transport/xqc_conn.c

@@ -729,13 +729,17 @@ xqc_conn_create(xqc_engine_t *engine, xqc_cid_t *dcid, xqc_cid_t *scid,
     }
 
     if (xqc_conn_is_current_mp_version_supported(xc->conn_settings.multipath_version) != XQC_OK) {
-        xc->conn_settings.multipath_version = XQC_MULTIPATH_10;
+        xc->conn_settings.multipath_version = XQC_MULTIPATH_11;
     }
 
     if (xc->conn_settings.init_max_path_id == 0) {
         xc->conn_settings.init_max_path_id = XQC_DEFAULT_INIT_MAX_PATH_ID;
     }
 
+    if (xc->max_paths_count == 0) {
+        xc->max_paths_count = XQC_MAX_PATHS_COUNT;
+    }
+
     if (xc->conn_settings.probing_pkt_out_size == 0) {
         xc->conn_settings.probing_pkt_out_size = engine->default_conn_settings.probing_pkt_out_size;
     }
@@ -3230,7 +3234,7 @@ xqc_conn_info_print(xqc_connection_t *conn, xqc_conn_stats_t *conn_stats)
     init_cwnd = conn->conn_settings.cc_params.customize_on ? conn->conn_settings.cc_params.init_cwnd : 0;
 
     /* conn info */
-    ret = snprintf(buff, buff_size, "%s,%u,%u,%u,%u,%u,%u,"
+    ret = snprintf(buff, buff_size, "%s,%"PRIu64",%"PRIu64",%"PRIu64",%u,%u,%u,"
                    "%u,%u,%u,%u,%u,%u,%u,%"PRIu64",%"PRIu64",%"PRIu64",i%u,"
 #ifdef XQC_ENABLE_FEC
                    "%u,%u,%u,%u,%u,%u,%u,"
@@ -3790,10 +3794,10 @@ xqc_conn_update_flow_ctl_settings(xqc_connection_t *conn)
 }
 
 xqc_int_t
-xqc_conn_add_path_cid_sets(xqc_connection_t *conn, uint32_t start, uint32_t end)
+xqc_conn_add_path_cid_sets(xqc_connection_t *conn, uint64_t start, uint64_t end)
 {
     if (conn->enable_multipath) {
-        uint32_t path_id;
+        uint64_t path_id;
         xqc_int_t ret;
         /* add cid_set_inner for all paths */
         for (path_id = start; path_id <= end; path_id++) {
@@ -4623,7 +4627,7 @@ xqc_conn_get_available_path_id(xqc_connection_t *conn, uint64_t *path_id)
         /* principle: the next unused path ID has at least one unused DCID and one acked unused SCID */
         xqc_cid_set_inner_t *scid_inner_set = xqc_get_next_unused_path_cid_set(&conn->scid_set);
         if (scid_inner_set) {
-            xqc_cid_set_inner_t *dcid_inner_set = xqc_get_path_cid_set(&conn->scid_set, scid_inner_set->path_id);
+            xqc_cid_set_inner_t *dcid_inner_set = xqc_get_path_cid_set(&conn->dcid_set, scid_inner_set->path_id);
             if (dcid_inner_set) {
                 if (dcid_inner_set->unused_cnt > 0 && scid_inner_set->acked_unused > 0) {
                     if (path_id) {