diff --git a/quinn-proto/src/connection/mod.rs b/quinn-proto/src/connection/mod.rs
index 22baafa58..8b7ccea42 100644
--- a/quinn-proto/src/connection/mod.rs
+++ b/quinn-proto/src/connection/mod.rs
@@ -2301,7 +2301,10 @@ impl Connection {
             State::Established => {
                 match packet.header.space() {
                     SpaceId::Data => self.process_payload(now, remote, number.unwrap(), packet)?,
-                    _ => self.process_early_payload(now, packet)?,
+                    _ if packet.header.has_frames() => self.process_early_payload(now, packet)?,
+                    _ => {
+                        trace!("discarding unexpected pre-handshake packet");
+                    }
                 }
                 return Ok(());
             }
diff --git a/quinn-proto/src/packet.rs b/quinn-proto/src/packet.rs
index be132a74f..ee1babcc7 100644
--- a/quinn-proto/src/packet.rs
+++ b/quinn-proto/src/packet.rs
@@ -445,6 +445,18 @@ impl Header {
             VersionNegotiate { ref dst_cid, .. } => dst_cid,
         }
     }
+
+    /// Whether the payload of this packet contains QUIC frames
+    pub(crate) fn has_frames(&self) -> bool {
+        use Header::*;
+        match *self {
+            Initial(_) => true,
+            Long { .. } => true,
+            Retry { .. } => false,
+            Short { .. } => true,
+            VersionNegotiate { .. } => false,
+        }
+    }
 }
 
 pub(crate) struct PartialEncode {