fix(overlay): detect document, respect overlay option (#1958)

Co-authored-by: neverland <[email protected]>

.changeset/itchy-lions-battle.md

@@ -0,0 +1,5 @@
+---
+'@rsbuild/core': patch
+---
+
+fix(overlay): detect document, respect overlay option

packages/core/src/client/hmr/index.ts

@@ -185,7 +185,9 @@ function onMessage(e: MessageEvent<string>) {
   const message = JSON.parse(e.data);
   switch (message.type) {
     case 'hash':
-      clearOverlay();
+      if (enableOverlay) {
+        clearOverlay();
+      }
       handleAvailableHash(message.data);
       break;
     case 'still-ok':

packages/core/src/client/hmr/overlay.ts

@@ -172,12 +172,25 @@ if (customElements && !customElements.get(overlayId)) {
   customElements.define(overlayId, ErrorOverlay);
 }
 
+const documentAvailable = typeof document !== 'undefined';
+
 export function createOverlay(err: string[]) {
+  if (!documentAvailable) {
+    console.info(
+      'Failed to display Rsbuild overlay since document is not available, considering turning off the `dev.client.overlay` option.',
+    );
+    return;
+  }
+
   clearOverlay();
   document.body.appendChild(new ErrorOverlay(err));
 }
 
 export function clearOverlay() {
+  if (!documentAvailable) {
+    return;
+  }
+
   // use NodeList's forEach api instead of dom.iterable
   // biome-ignore lint/complexity/noForEach: <explanation>
   document.querySelectorAll<ErrorOverlay>(overlayId).forEach((n) => n.close());