REORG: quic: remove quic_cid_trees reference from proto_quic

Previous commit removed access/manipulation to QUIC CID global tree
outside of quic_cid module. This ensures that proper locking is always
performed.

This commit finalizes this cleanup by marking CID global tree as static
only to quic_cid source file. Initialization of this tree is removed
from proto_quic and now performed using dedicated initcalls
quic_alloc_global_cid_tree().

As a side change, complete CID global tree documentation, in particular
to explain CID global tree artificial splitting and ODCID handling.
Overall, the code is now clearer and safer.

include/haproxy/proto_quic.h

@@ -28,12 +28,6 @@
 extern struct protocol proto_quic4;
 extern struct protocol proto_quic6;
 
-struct quic_cid_tree {
-	struct eb_root root;
-	__decl_thread(HA_RWLOCK_T lock);
-};
-
 extern struct quic_dghdlr *quic_dghdlrs;
-extern struct quic_cid_tree *quic_cid_trees;
 
 #endif /* _HAPROXY_PROTO_QUIC_H  */

include/haproxy/quic_cid-t.h

@@ -3,6 +3,12 @@
 
 #include <import/ebtree-t.h>
 #include <haproxy/quic_tp-t.h>
+#include <haproxy/thread.h>
+
+struct quic_cid_tree {
+	struct eb_root root;
+	__decl_thread(HA_RWLOCK_T lock);
+};
 
 /* QUIC connection ID maximum length for version 1. */
 #define QUIC_CID_MAXLEN               20 /* bytes */

include/haproxy/quic_cid.h

@@ -11,9 +11,12 @@
 #include <haproxy/buf-t.h>
 #include <haproxy/chunk.h>
 #include <haproxy/quic_conn-t.h>
+#include <haproxy/quic_cid-t.h>
 #include <haproxy/quic_rx-t.h>
 #include <haproxy/proto_quic.h>
 
+extern struct quic_cid_tree *quic_cid_trees;
+
 struct quic_connection_id *new_quic_cid(struct eb_root *root,
                                         struct quic_conn *qc,
                                         const struct quic_cid *orig,

src/proto_quic.c

@@ -52,10 +52,6 @@
 /* per-thread quic datagram handlers */
 struct quic_dghdlr *quic_dghdlrs;
 
-/* global CID trees */
-#define QUIC_CID_TREES_CNT 256
-struct quic_cid_tree *quic_cid_trees;
-
 /* Size of the internal buffer of QUIC RX buffer at the fd level */
 #define QUIC_RX_BUFSZ  (1UL << 18)
 
@@ -704,17 +700,6 @@ static int quic_alloc_dghdlrs(void)
 		MT_LIST_INIT(&dghdlr->dgrams);
 	}
 
-	quic_cid_trees = calloc(QUIC_CID_TREES_CNT, sizeof(*quic_cid_trees));
-	if (!quic_cid_trees) {
-		ha_alert("Failed to allocate global quic CIDs trees.\n");
-		return 0;
-	}
-
-	for (i = 0; i < QUIC_CID_TREES_CNT; ++i) {
-		HA_RWLOCK_INIT(&quic_cid_trees[i].lock);
-		quic_cid_trees[i].root = EB_ROOT_UNIQUE;
-	}
-
 	return 1;
 }
 REGISTER_POST_CHECK(quic_alloc_dghdlrs);
@@ -729,8 +714,6 @@ static int quic_deallocate_dghdlrs(void)
 		free(quic_dghdlrs);
 	}
 
-	ha_free(&quic_cid_trees);
-
 	return 1;
 }
 REGISTER_POST_DEINIT(quic_deallocate_dghdlrs);

src/quic_cid.c

@@ -1,6 +1,10 @@
+#include <stdlib.h>
+
 #include <import/eb64tree.h>
 #include <import/ebmbtree.h>
 
+#include <haproxy/bug.h>
+#include <haproxy/errors.h>
 #include <haproxy/pool.h>
 #include <haproxy/quic_cid.h>
 #include <haproxy/quic_conn.h>
@@ -9,6 +13,30 @@
 #include <haproxy/trace.h>
 #include <haproxy/xxhash.h>
 
+/* *** QUIC CID handling general principles
+ *
+ * . CID global storage
+ * CIDs generated by haproxy and reuse by the peer as DCID are stored in a
+ * global tree. Tree access must only be done under lock protection.
+ *
+ * . CID global tree splitting
+ * To reduce thread contention, global CID tree is in reality splitted into 256
+ * distinct tree instances. Each CID is assigned to a single tree instance
+ * based on its content. Use quic_cid_tree_idx() to retrieve the expected tree
+ * location for a CID.
+ *
+ * . ODCID handling
+ * ODCID are never stored in global CID tree. This allows to reduce tree size
+ * as clients are expected to switch quickly to a new haproxy assigned CID.
+ * This new CID value is derived by haproxy from the ODCID plus a bunch of
+ * other parameters. If ODCID is reused by the client, first lookup in global
+ * CID tree won't be successful. In this case, derive operation is performed
+ * again before a new tree lookup.
+ */
+
+#define QUIC_CID_TREES_CNT 256
+struct quic_cid_tree *quic_cid_trees;
+
 /* Initialize the stateless reset token attached to <conn_id> connection ID.
  * Returns 1 if succeeded, 0 if not.
  */
@@ -353,3 +381,29 @@ int qc_build_new_connection_id_frm(struct quic_conn *qc,
 	TRACE_LEAVE(QUIC_EV_CONN_PRSHPKT, qc);
 	return ret;
 }
+
+static int quic_alloc_global_cid_tree(void)
+{
+	int i;
+
+	quic_cid_trees = calloc(QUIC_CID_TREES_CNT, sizeof(*quic_cid_trees));
+	if (!quic_cid_trees) {
+		ha_alert("Failed to allocate global quic CIDs trees.\n");
+		return 0;
+	}
+
+	for (i = 0; i < QUIC_CID_TREES_CNT; ++i) {
+		HA_RWLOCK_INIT(&quic_cid_trees[i].lock);
+		quic_cid_trees[i].root = EB_ROOT_UNIQUE;
+	}
+
+	return 1;
+}
+REGISTER_POST_CHECK(quic_alloc_global_cid_tree);
+
+static int quic_deallocate_global_cid_tree(void)
+{
+	ha_free(&quic_cid_trees);
+	return 1;
+}
+REGISTER_POST_DEINIT(quic_deallocate_global_cid_tree);