diff --git a/modules/caddyhttp/app.go b/modules/caddyhttp/app.go
index 7dc2bee72fa..cd32e72d762 100644
--- a/modules/caddyhttp/app.go
+++ b/modules/caddyhttp/app.go
@@ -689,16 +689,7 @@ func (app *App) Stop() error {
 			return
 		}
 
-		// First close h3server then close listeners unlike stdlib for several reasons:
-		// 1, udp has only a single socket, once closed, no more data can be read and
-		// written. In contrast, closing tcp listeners won't affect established connections.
-		// This have something to do with graceful shutdown when upstream implements it.
-		// 2, h3server will only close listeners it's registered (quic listeners). Closing
-		// listener first and these listeners maybe unregistered thus won't be closed. caddy
-		// distinguishes quic-listener and underlying datagram sockets.
-
-		// TODO: CloseGracefully, once implemented upstream (see https://github.com/quic-go/quic-go/issues/2103)
-		if err := server.h3server.Close(); err != nil {
+		if err := server.h3server.Shutdown(ctx); err != nil {
 			app.logger.Error("HTTP/3 server shutdown",
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go
index 5aa7e0f6351..96001c6f9c7 100644
--- a/modules/caddyhttp/server.go
+++ b/modules/caddyhttp/server.go
@@ -614,22 +614,7 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 	// create HTTP/3 server if not done already
 	if s.h3server == nil {
 		s.h3server = &http3.Server{
-			// Currently when closing a http3.Server, only listeners are closed. But caddy reuses these listeners
-			// if possible, requests are still read and handled by the old handler. Close these connections manually.
-			// see issue: https://github.com/caddyserver/caddy/issues/6195
-			// Will interrupt ongoing requests.
-			// TODO: remove the handler wrap after http3.Server.CloseGracefully is implemented, see App.Stop
-			Handler: http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-				select {
-				case <-s.ctx.Done():
-					if quicConn, ok := request.Context().Value(quicConnCtxKey).(quic.Connection); ok {
-						//nolint:errcheck
-						quicConn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeRequestRejected), "")
-					}
-				default:
-					s.ServeHTTP(writer, request)
-				}
-			}),
+			Handler:        s,
 			TLSConfig:      tlsCfg,
 			MaxHeaderBytes: s.MaxHeaderBytes,
 			QUICConfig: &quic.Config{
@@ -637,9 +622,6 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 				Tracer:   qlog.DefaultConnectionTracer,
 			},
 			IdleTimeout: time.Duration(s.IdleTimeout),
-			ConnContext: func(ctx context.Context, c quic.Connection) context.Context {
-				return context.WithValue(ctx, quicConnCtxKey, c)
-			},
 		}
 	}
 
@@ -1099,10 +1081,6 @@ const (
 	// For referencing underlying net.Conn
 	ConnCtxKey caddy.CtxKey = "conn"
 
-	// For referencing underlying quic.Connection
-	// TODO: export if needed later
-	quicConnCtxKey caddy.CtxKey = "quic_conn"
-
 	// For tracking whether the client is a trusted proxy
 	TrustedProxyVarKey string = "trusted_proxy"