diff --git a/neqo-bin/src/client/mod.rs b/neqo-bin/src/client/mod.rs
index 16137e70d3..2fa52fe1d6 100644
--- a/neqo-bin/src/client/mod.rs
+++ b/neqo-bin/src/client/mod.rs
@@ -382,6 +382,7 @@ struct Runner<'a, H: Handler> {
     handler: H,
     timeout: Option<Pin<Box<Sleep>>>,
     args: &'a Args,
+    recv_buf: Vec<u8>,
     send_buf: Vec<u8>,
 }
 
@@ -427,22 +428,17 @@ impl<'a, H: Handler> Runner<'a, H> {
     async fn process(&mut self) -> Result<(), io::Error> {
         let mut should_read = true;
         loop {
-            let output = crate::RECV_BUF.with_borrow_mut(
-                |recv_buf| -> Result<Output<&[u8]>, io::Error> {
-                    // TODO: Cleanup?
-                    let dgram = should_read
-                        .then(|| self.socket.recv(&self.local_addr, recv_buf))
-                        .transpose()?
-                        .flatten();
-                    should_read = dgram.is_some();
-
-                    Ok(self
-                        .client
-                        .process(dgram, Instant::now(), &mut self.send_buf))
-                },
-            )?;
-
-            match output {
+            // TODO: Cleanup?
+            let dgram = should_read
+                .then(|| self.socket.recv(&self.local_addr, &mut self.recv_buf))
+                .transpose()?
+                .flatten();
+            should_read = dgram.is_some();
+
+            match self
+                .client
+                .process(dgram, Instant::now(), &mut self.send_buf)
+            {
                 Output::Datagram(dgram) => {
                     self.socket.writable().await?;
                     self.socket.send(dgram)?;
@@ -580,6 +576,7 @@ pub async fn client(mut args: Args) -> Res<()> {
                     local_addr: real_local,
                     socket: &mut socket,
                     timeout: None,
+                    recv_buf: Vec::with_capacity(neqo_udp::RECV_BUF_SIZE),
                     send_buf: Vec::new(),
                 }
                 .run()
@@ -597,6 +594,7 @@ pub async fn client(mut args: Args) -> Res<()> {
                     local_addr: real_local,
                     socket: &mut socket,
                     timeout: None,
+                    recv_buf: Vec::with_capacity(neqo_udp::RECV_BUF_SIZE),
                     send_buf: Vec::new(),
                 }
                 .run()
diff --git a/neqo-bin/src/lib.rs b/neqo-bin/src/lib.rs
index a4701bb1a5..5051faae3d 100644
--- a/neqo-bin/src/lib.rs
+++ b/neqo-bin/src/lib.rs
@@ -8,7 +8,6 @@
 #![allow(clippy::missing_errors_doc)]
 
 use std::{
-    cell::RefCell,
     fmt::{self, Display},
     net::{SocketAddr, ToSocketAddrs},
     path::PathBuf,
@@ -25,10 +24,6 @@ pub mod client;
 pub mod server;
 pub mod udp;
 
-std::thread_local! {
-    static RECV_BUF: RefCell<Vec<u8>> = RefCell::new(vec![0; neqo_udp::RECV_BUF_SIZE]);
-}
-
 /// Firefox default value
 ///
 /// See `network.buffer.cache.size` pref <https://searchfox.org/mozilla-central/rev/f6e3b81aac49e602f06c204f9278da30993cdc8a/modules/libpref/init/all.js#3212>
diff --git a/neqo-bin/src/server/mod.rs b/neqo-bin/src/server/mod.rs
index 3998093c31..aa6a7701c4 100644
--- a/neqo-bin/src/server/mod.rs
+++ b/neqo-bin/src/server/mod.rs
@@ -210,6 +210,7 @@ pub struct ServerRunner {
     server: Box<dyn HttpServer>,
     timeout: Option<Pin<Box<Sleep>>>,
     sockets: Vec<(SocketAddr, crate::udp::Socket)>,
+    recv_buf: Vec<u8>,
     send_buf: Vec<u8>,
 }
 
@@ -225,6 +226,7 @@ impl ServerRunner {
             server,
             timeout: None,
             sockets,
+            recv_buf: Vec::with_capacity(neqo_udp::RECV_BUF_SIZE),
             send_buf: vec![],
         }
     }
@@ -232,26 +234,22 @@ impl ServerRunner {
     // TODO: Could as well call it UDP IO now, given that it does both in and output.
     async fn process(&mut self, mut inx: Option<usize>) -> Result<(), io::Error> {
         loop {
-            let output = crate::RECV_BUF.with_borrow_mut(
-                |recv_buf| -> Result<Output<&[u8]>, io::Error> {
-                    let mut dgram = if let Some(i) = inx {
-                        let (host, socket) = self.sockets.get_mut(i).unwrap();
-                        let dgram = socket.recv(host, recv_buf)?;
-                        if dgram.is_none() {
-                            // TODO: Better way to not try reading again?
-                            inx.take();
-                        }
-                        dgram
-                    } else {
-                        None
-                    };
-                    Ok(self
-                        .server
-                        .process(dgram.take(), (self.now)(), &mut self.send_buf))
-                },
-            )?;
-
-            match output {
+            let mut dgram = if let Some(i) = inx {
+                let (host, socket) = self.sockets.get_mut(i).unwrap();
+                let dgram = socket.recv(host, &mut self.recv_buf)?;
+                if dgram.is_none() {
+                    // TODO: Better way to not try reading again?
+                    inx.take();
+                }
+                dgram
+            } else {
+                None
+            };
+
+            match self
+                .server
+                .process(dgram.take(), (self.now)(), &mut self.send_buf)
+            {
                 Output::Datagram(dgram) => {
                     let socket = {
                         let addr = dgram.source();