diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index bf14524fe6d..cd32e72d762 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -689,15 +689,7 @@ func (app *App) Stop() error {
 			return
 		}
 
-		// First close h3server then close listeners unlike stdlib for several reasons:
-		// 1, udp has only a single socket, once closed, no more data can be read and
-		// written. In contrast, closing tcp listeners won't affect established connections.
-		// This have something to do with graceful shutdown when upstream implements it.
-		// 2, h3server will only close listeners it's registered (quic listeners). Closing
-		// listener first and these listeners maybe unregistered thus won't be closed. caddy
-		// distinguishes quic-listener and underlying datagram sockets.
-
-		if err := server.h3server.CloseGracefully(ctx); err != nil {
+		if err := server.h3server.Shutdown(ctx); err != nil {
 			app.logger.Error("HTTP/3 server shutdown",
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go
index f182974983b..96001c6f9c7 100644
--- a/modules/caddyhttp/server.go
+++ b/modules/caddyhttp/server.go
@@ -622,9 +622,6 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 				Tracer:   qlog.DefaultConnectionTracer,
 			},
 			IdleTimeout: time.Duration(s.IdleTimeout),
-			ConnContext: func(ctx context.Context, c quic.Connection) context.Context {
-				return context.WithValue(ctx, quicConnCtxKey, c)
-			},
 		}
 	}