[GR-62543] Initialize the locking datastructures for the reference handler thread during early startup.

PullRequest: graal/20153

Author: christianhaeubl

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/graal/snippets/CEntryPointSnippets.java

@@ -79,6 +79,7 @@
 import com.oracle.svm.core.heap.PhysicalMemory;
 import com.oracle.svm.core.heap.ReferenceHandler;
 import com.oracle.svm.core.heap.ReferenceHandlerThread;
+import com.oracle.svm.core.heap.ReferenceInternals;
 import com.oracle.svm.core.heap.RestrictHeapAccess;
 import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
 import com.oracle.svm.core.imagelayer.ImageLayerSection;
@@ -434,6 +435,7 @@ private static int initializeIsolateInterruptibly1(CEntryPointCreateIsolateParam
          * result in deadlocks if ReferenceInternals.waitForReferenceProcessing() is called.
          */
         if (ReferenceHandler.useDedicatedThread()) {
+            ReferenceInternals.initializeLocking();
             ReferenceHandlerThread.start();
         }
 

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceHandlerThread.java

@@ -91,7 +91,7 @@ public void run() {
             if (t instanceof OutOfMemoryError && VMThreads.isTearingDown()) {
                 // Likely failed to allocate the InterruptedException, ignore either way.
             } else {
-                VMError.shouldNotReachHere("Reference processing and cleaners must handle all potential exceptions", t);
+                throw VMError.shouldNotReachHere("Reference processing and cleaners must handle all potential exceptions", t);
             }
         }
     }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java

@@ -35,6 +35,7 @@
 import com.oracle.svm.core.NeverInline;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.monitor.MonitorSupport;
 import com.oracle.svm.core.thread.VMOperation;
 import com.oracle.svm.core.util.BasedOnJDKClass;
 import com.oracle.svm.core.util.TimeUtils;
@@ -170,6 +171,14 @@ public static boolean hasQueue(Reference<?> instance) {
     private static final Object processPendingLock = new Object();
     private static boolean processPendingActive = false;
 
+    public static void initializeLocking() {
+        /*
+         * Initialize the locking datastructures as early as possible. This makes it less likely
+         * that the reference handler thread fails with an OutOfMemoryError because of locking.
+         */
+        MonitorSupport.singleton().ensureInitialized(processPendingLock);
+    }
+
     @NeverInline("Ensure that every exception can be caught, including implicit exceptions.")
     public static void waitForPendingReferences() throws InterruptedException {
         Heap.getHeap().waitForReferencePendingList();

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MonitorSupport.java

@@ -24,12 +24,13 @@
  */
 package com.oracle.svm.core.monitor;
 
-import jdk.graal.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.ImageSingletons;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.thread.ThreadStatus;
 
+import jdk.graal.compiler.api.replacements.Fold;
+
 /**
  * This interface provides functions related to monitor operations (the Java "synchronized" keyword
  * and related functions in the JDK) that are invoked by other parts of the VM.
@@ -112,4 +113,11 @@ public final void wait(Object obj, long timeoutMillis) throws InterruptedExcepti
      * state until the monitor has been successfully acquired.
      */
     public abstract int getParkedThreadStatus(Thread thread, boolean timed);
+
+    /**
+     * Ensures that all locking-related Java heap objects are allocated so that the given object can
+     * be used in {@code synchronized} statements and in calls to {@link Object#notify} and
+     * {@link Object#notifyAll} without triggering any locking-related Java heap allocations.
+     */
+    public abstract void ensureInitialized(Object obj);
 }

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/monitor/MultiThreadedMonitorSupport.java

@@ -202,6 +202,12 @@ public int getParkedThreadStatus(Thread thread, boolean timed) {
         return timed ? ThreadStatus.PARKED_TIMED : ThreadStatus.PARKED;
     }
 
+    @Override
+    public void ensureInitialized(Object obj) {
+        JavaMonitor monitor = getOrCreateMonitor(obj, MonitorInflationCause.VM_INTERNAL);
+        monitor.getOrCreateCondition(true);
+    }
+
     @SubstrateForeignCallTarget(stubCallingConvention = false)
     @Uninterruptible(reason = "Avoid stack overflow error before yellow zone has been activated", calleeMustBe = false)
     private static void slowPathMonitorEnter(Object obj) {