From 2bb6a55b69d5654f3dc81fe87e593d3167ee9dcd Mon Sep 17 00:00:00 2001 From: Nevin Date: Fri, 26 Jul 2024 02:48:09 +0800 Subject: [PATCH] kcp connection shares buffer while kcp2k is single-threaded and the peer's config never changes, there is no need to allocate 3 buffers for every peer. when there are many peers, it will result in high memory usage. --- kcp2k/kcp2k/highlevel/KcpPeer.cs | 28 ++++++++ kcp2k/kcp2k/highlevel/KcpServer.cs | 76 +++++++++++++++++++- kcp2k/kcp2k/highlevel/KcpServerConnection.cs | 23 ++++++ 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/kcp2k/kcp2k/highlevel/KcpPeer.cs b/kcp2k/kcp2k/highlevel/KcpPeer.cs index 8f0aac3..b27b0bf 100644 --- a/kcp2k/kcp2k/highlevel/KcpPeer.cs +++ b/kcp2k/kcp2k/highlevel/KcpPeer.cs @@ -172,6 +172,34 @@ protected KcpPeer(KcpConfig config, uint cookie) kcpSendBuffer = new byte[1 + reliableMax]; } + // SetupKcp creates and configures a new KCP instance. + // => useful to start from a fresh state every time the client connects + // => NoDelay, interval, wnd size are the most important configurations. + // let's force require the parameters so we don't forget it anywhere. + protected KcpPeer(KcpConfig config, uint cookie, byte[] rawSendBuffer, byte[] kcpMessageBuffer, byte[] kcpSendBuffer) + { + // initialize variable state in extra function so we can reuse it + // when reconnecting to reset state + Reset(config); + + // set the cookie after resetting state so it's not overwritten again. + // with log message for debugging in case of cookie issues. + this.cookie = cookie; + Log.Info($"[KCP] {GetType()}: created with cookie={cookie}"); + + // create mtu sized send buffer + this.rawSendBuffer = rawSendBuffer; + + // calculate max message sizes once + unreliableMax = UnreliableMaxMessageSize(config.Mtu); + reliableMax = ReliableMaxMessageSize(config.Mtu, config.ReceiveWindowSize); + + // create message buffers AFTER window size is set + // see comments on buffer definition for the "+1" part + this.kcpMessageBuffer = kcpMessageBuffer; + this.kcpSendBuffer = kcpSendBuffer; + } + // Reset all state once. // useful for KcpClient to reconned with a fresh kcp state. protected void Reset(KcpConfig config) diff --git a/kcp2k/kcp2k/highlevel/KcpServer.cs b/kcp2k/kcp2k/highlevel/KcpServer.cs index 5b8ce4d..bc8f7ba 100644 --- a/kcp2k/kcp2k/highlevel/KcpServer.cs +++ b/kcp2k/kcp2k/highlevel/KcpServer.cs @@ -10,6 +10,65 @@ namespace kcp2k { public class KcpServer { + // we need to subtract the channel and cookie bytes from every + // MaxMessageSize calculation. + // we also need to tell kcp to use MTU-1 to leave space for the byte. + public const int CHANNEL_HEADER_SIZE = 1; + public const int COOKIE_HEADER_SIZE = 4; + public const int METADATA_SIZE_RELIABLE = CHANNEL_HEADER_SIZE + COOKIE_HEADER_SIZE; + public const int METADATA_SIZE_UNRELIABLE = CHANNEL_HEADER_SIZE + COOKIE_HEADER_SIZE; + + // reliable channel (= kcp) MaxMessageSize so the outside knows largest + // allowed message to send. the calculation in Send() is not obvious at + // all, so let's provide the helper here. + // + // kcp does fragmentation, so max message is way larger than MTU. + // + // -> runtime MTU changes are disabled: mss is always MTU_DEF-OVERHEAD + // -> Send() checks if fragment count < rcv_wnd, so we use rcv_wnd - 1. + // NOTE that original kcp has a bug where WND_RCV default is used + // instead of configured rcv_wnd, limiting max message size to 144 KB + // https://github.com/skywind3000/kcp/pull/291 + // we fixed this in kcp2k. + // -> we add 1 byte KcpHeader enum to each message, so -1 + // + // IMPORTANT: max message is MTU * rcv_wnd, in other words it completely + // fills the receive window! due to head of line blocking, + // all other messages have to wait while a maxed size message + // is being delivered. + // => in other words, DO NOT use max size all the time like + // for batching. + // => sending UNRELIABLE max message size most of the time is + // best for performance (use that one for batching!) + static int ReliableMaxMessageSize_Unconstrained(int mtu, uint rcv_wnd) => + (mtu - Kcp.OVERHEAD - METADATA_SIZE_RELIABLE) * ((int)rcv_wnd - 1) - 1; + + // kcp encodes 'frg' as 1 byte. + // max message size can only ever allow up to 255 fragments. + // WND_RCV gives 127 fragments. + // WND_RCV * 2 gives 255 fragments. + // so we can limit max message size by limiting rcv_wnd parameter. + public static int ReliableMaxMessageSize(int mtu, uint rcv_wnd) => + ReliableMaxMessageSize_Unconstrained(mtu, Math.Min(rcv_wnd, Kcp.FRG_MAX)); + + // unreliable max message size is simply MTU - channel header - kcp header + public static int UnreliableMaxMessageSize(int mtu) => + mtu - METADATA_SIZE_UNRELIABLE - 1; + + // buffer to receive kcp's processed messages (avoids allocations). + // IMPORTANT: this is for KCP messages. so it needs to be of size: + // 1 byte header + MaxMessageSize content + readonly byte[] kcpMessageBuffer;// = new byte[1 + ReliableMaxMessageSize]; + + // send buffer for handing user messages to kcp for processing. + // (avoids allocations). + // IMPORTANT: needs to be of size: + // 1 byte header + MaxMessageSize content + readonly byte[] kcpSendBuffer;// = new byte[1 + ReliableMaxMessageSize]; + + // raw send buffer is exactly MTU. + readonly byte[] rawSendBuffer; + // callbacks // even for errors, to allow liraries to show popups etc. // instead of logging directly. @@ -63,6 +122,18 @@ public KcpServer(Action OnConnected, newClientEP = config.DualMode ? new IPEndPoint(IPAddress.IPv6Any, 0) : new IPEndPoint(IPAddress.Any, 0); + + // create mtu sized send buffer + rawSendBuffer = new byte[config.Mtu]; + + // calculate max message sizes once + // unreliableMax = UnreliableMaxMessageSize(config.Mtu); + var reliableMax = ReliableMaxMessageSize(config.Mtu, config.ReceiveWindowSize); + + // create message buffers AFTER window size is set + // see comments on buffer definition for the "+1" part + kcpMessageBuffer = new byte[1 + reliableMax]; + kcpSendBuffer = new byte[1 + reliableMax]; } public virtual bool IsActive() => socket != null; @@ -265,7 +336,10 @@ protected virtual KcpServerConnection CreateConnection(int connectionId) (data) => RawSend(connectionId, data), config, cookie, - newClientEP); + newClientEP, + rawSendBuffer, + kcpMessageBuffer, + kcpSendBuffer); return connection; diff --git a/kcp2k/kcp2k/highlevel/KcpServerConnection.cs b/kcp2k/kcp2k/highlevel/KcpServerConnection.cs index 922a83e..5c41a88 100644 --- a/kcp2k/kcp2k/highlevel/KcpServerConnection.cs +++ b/kcp2k/kcp2k/highlevel/KcpServerConnection.cs @@ -43,6 +43,29 @@ public KcpServerConnection( this.remoteEndPoint = remoteEndPoint; } + public KcpServerConnection( + Action OnConnected, + Action, KcpChannel> OnData, + Action OnDisconnected, + Action OnError, + Action> OnRawSend, + KcpConfig config, + uint cookie, + EndPoint remoteEndPoint, + byte[] rawSendBuffer, + byte[] kcpMessageBuffer, + byte[] kcpSendBuffer) + : base(config, cookie, rawSendBuffer, kcpMessageBuffer, kcpSendBuffer) + { + OnConnectedCallback = OnConnected; + OnDataCallback = OnData; + OnDisconnectedCallback = OnDisconnected; + OnErrorCallback = OnError; + RawSendCallback = OnRawSend; + + this.remoteEndPoint = remoteEndPoint; + } + // callbacks /////////////////////////////////////////////////////////// protected override void OnAuthenticated() {