Populate first object table during planning.

Author: peter-hofer

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompactingOldGeneration.java

@@ -407,7 +407,7 @@ private void compact(Timers timers) {
                  * chunk during compaction. The remembered set bits are already set after planning.
                  */
                 if (!AlignedHeapChunk.isEmpty(chunk)) {
-                    RememberedSet.get().enableRememberedSetForChunk(chunk);
+                    RememberedSet.get().clearRememberedSet(chunk);
                 } // empty chunks will be freed or reset before reuse, no need to reinitialize here
 
                 chunk = HeapChunk.getNext(chunk);

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java

@@ -449,7 +449,7 @@ private Object copyAlignedObject(Object originalObj) {
             if (SerialGCOptions.useCompactingOldGen() && GCImpl.getGCImpl().isCompleteCollection()) {
                 /*
                  * In a compacting complete collection, the remembered set bit is set already during
-                 * marking and the first object table is built later during compaction.
+                 * marking and the first object table is built during planning.
                  */
             } else {
                 /*

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/compacting/PlanningVisitor.java

@@ -34,7 +34,9 @@
 import com.oracle.svm.core.genscavenge.HeapChunk;
 import com.oracle.svm.core.genscavenge.ObjectHeaderImpl;
 import com.oracle.svm.core.genscavenge.Space;
+import com.oracle.svm.core.genscavenge.remset.AlignedChunkRememberedSet;
 import com.oracle.svm.core.genscavenge.remset.BrickTable;
+import com.oracle.svm.core.genscavenge.remset.FirstObjectTable;
 import com.oracle.svm.core.hub.LayoutEncoding;
 
 import jdk.graal.compiler.word.Word;
@@ -56,6 +58,9 @@ public PlanningVisitor() {
     public void init(Space space) {
         allocChunk = space.getFirstAlignedHeapChunk();
         allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
+        if (!allocChunk.getShouldSweepInsteadOfCompact()) {
+            FirstObjectTable.initializeTable(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), AlignedChunkRememberedSet.getFirstObjectTableSize());
+        }
     }
 
     @Override
@@ -69,7 +74,7 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
         UnsignedWord brickIndex = Word.zero();
 
         /* Initialize the move info structure at the chunk's object start location. */
-        ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+        ObjectMoveInfo.setNewAddress(objSeq, objSeq);
         ObjectMoveInfo.setObjectSeqSize(objSeq, Word.zero());
         ObjectMoveInfo.setNextObjectSeqOffset(objSeq, Word.zero());
 
@@ -115,12 +120,43 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
                 }
 
                 objSeqSize = objSeqSize.add(objSize);
+                if (!sweeping) {
+                    if (allocPointer.add(objSeqSize).aboveThan(AlignedHeapChunk.getObjectsEnd(allocChunk))) {
+                        /* Out of space, move to the start of the next chunk. */
+                        allocChunk = HeapChunk.getNext(allocChunk);
+                        assert allocChunk.isNonNull();
+                        assert !allocChunk.getShouldSweepInsteadOfCompact();
+                        allocPointer = AlignedHeapChunk.getObjectsStart(allocChunk);
+
+                        /*
+                         * TODO: we should reset the FOT entries we already wrote in the last chunk
+                         * (but they should not be accessed, not even by heap verification)
+                         */
+
+                        /* Visit previous objects in sequence again to write new FOT entries. */
+                        FirstObjectTable.initializeTable(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), AlignedChunkRememberedSet.getFirstObjectTableSize());
+                        Pointer q = objSeq;
+                        while (q.notEqual(p)) {
+                            UnsignedWord offset = q.subtract(objSeq);
+                            UnsignedWord size = LayoutEncoding.getSizeFromObjectInlineInGC(q.toObject());
+                            FirstObjectTable.setTableForObject(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), offset, offset.add(size));
+                            q = q.add(size);
+                        }
+                    }
+
+                    Pointer allocEndOffset = allocPointer.add(objSeqSize).subtract(AlignedHeapChunk.getObjectsStart(allocChunk));
+                    FirstObjectTable.setTableForObject(AlignedChunkRememberedSet.getFirstObjectTableStart(allocChunk), allocEndOffset.subtract(objSize), allocEndOffset);
+                }
 
             } else { // not marked, i.e. not alive and start of a gap of yet unknown size
                 if (objSeqSize.notEqual(0)) { // end of an object sequence
-                    Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
-                    ObjectMoveInfo.setNewAddress(objSeq, newAddress);
                     ObjectMoveInfo.setObjectSeqSize(objSeq, objSeqSize);
+                    if (sweeping) {
+                        ObjectMoveInfo.setNewAddress(objSeq, objSeq);
+                    } else {
+                        ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+                        allocPointer = allocPointer.add(objSeqSize); // ensured enough memory above
+                    }
 
                     objSeqSize = Word.zero();
 
@@ -139,10 +175,15 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
 
         if (gapSize.notEqual(0)) { // truncate gap at chunk end
             chunk.setTopOffset(chunk.getTopOffset().subtract(gapSize));
+
         } else if (objSeqSize.notEqual(0)) {
-            Pointer newAddress = sweeping ? objSeq : allocate(objSeqSize);
-            ObjectMoveInfo.setNewAddress(objSeq, newAddress);
             ObjectMoveInfo.setObjectSeqSize(objSeq, objSeqSize);
+            if (sweeping) {
+                ObjectMoveInfo.setNewAddress(objSeq, objSeq);
+            } else {
+                ObjectMoveInfo.setNewAddress(objSeq, allocPointer);
+                allocPointer = allocPointer.add(objSeqSize); // ensured enough memory above
+            }
         }
 
         if (sweeping) {
@@ -168,18 +209,4 @@ public boolean visitChunk(AlignedHeapChunk.AlignedHeader chunk) {
 
         return true;
     }
-
-    private Pointer allocate(UnsignedWord size) {
-        Pointer p = allocPointer;
-        allocPointer = allocPointer.add(size);
-        if (allocPointer.aboveThan(AlignedHeapChunk.getObjectsEnd(allocChunk))) {
-            allocChunk = HeapChunk.getNext(allocChunk);
-            assert allocChunk.isNonNull();
-            assert !allocChunk.getShouldSweepInsteadOfCompact();
-
-            p = AlignedHeapChunk.getObjectsStart(allocChunk);
-            allocPointer = p.add(size);
-        }
-        return p;
-    }
 }

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java

@@ -53,7 +53,7 @@
 import jdk.graal.compiler.replacements.nodes.AssertionNode;
 import jdk.graal.compiler.word.Word;
 
-final class AlignedChunkRememberedSet {
+public final class AlignedChunkRememberedSet {
     private AlignedChunkRememberedSet() {
     }
 
@@ -63,7 +63,7 @@ public static int wordSize() {
     }
 
     @Fold
-    public static UnsignedWord getHeaderSize() {
+    static UnsignedWord getHeaderSize() {
         UnsignedWord headerSize = getFirstObjectTableLimitOffset();
         if (SerialGCOptions.useCompactingOldGen()) {
             // Compaction needs room for a ObjectMoveInfo structure before the first object.
@@ -74,7 +74,7 @@ public static UnsignedWord getHeaderSize() {
     }
 
     @Platforms(Platform.HOSTED_ONLY.class)
-    public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkPosition, List<ImageHeapObject> objects) {
+    static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkPosition, List<ImageHeapObject> objects) {
         // Completely clean the card table and the first object table.
         CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize());
         FirstObjectTable.initializeTable(getFirstObjectTableStart(chunk), getFirstObjectTableSize());
@@ -94,7 +94,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP
 
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) {
+    static void enableRememberedSetForObject(AlignedHeader chunk, Object obj, UnsignedWord objSize) {
         Pointer fotStart = getFirstObjectTableStart(chunk);
         Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
 
@@ -107,7 +107,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj,
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void enableRememberedSet(AlignedHeader chunk) {
+    static void enableRememberedSet(AlignedHeader chunk) {
         // Completely clean the card table and the first object table as further objects may be
         // added later on to this chunk.
         CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize());
@@ -124,7 +124,7 @@ public static void enableRememberedSet(AlignedHeader chunk) {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void clearRememberedSet(AlignedHeader chunk) {
+    static void clearRememberedSet(AlignedHeader chunk) {
         CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize());
     }
 
@@ -133,7 +133,7 @@ public static void clearRememberedSet(AlignedHeader chunk) {
      * the post-write barrier.
      */
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void dirtyCardForObject(Object object, boolean verifyOnly) {
+    static void dirtyCardForObject(Object object, boolean verifyOnly) {
         Pointer objectPointer = Word.objectToUntrackedPointer(object);
         AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer);
         Pointer cardTableStart = getCardTableStart(chunk);
@@ -146,7 +146,7 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static void walkDirtyObjects(AlignedHeader chunk, UninterruptibleObjectVisitor visitor, boolean clean) {
+    static void walkDirtyObjects(AlignedHeader chunk, UninterruptibleObjectVisitor visitor, boolean clean) {
         Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk);
         Pointer objectsLimit = HeapChunk.getTopPointer(chunk);
         UnsignedWord memorySize = objectsLimit.subtract(objectsStart);
@@ -212,7 +212,7 @@ private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end,
         }
     }
 
-    public static boolean verify(AlignedHeader chunk) {
+    static boolean verify(AlignedHeader chunk) {
         boolean success = true;
         success &= CardTable.verify(getCardTableStart(chunk), getCardTableEnd(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk));
         success &= FirstObjectTable.verify(getFirstObjectTableStart(chunk), AlignedHeapChunk.getObjectsStart(chunk), HeapChunk.getTopPointer(chunk));
@@ -242,7 +242,7 @@ static UnsignedWord getCardTableSize() {
     }
 
     @Fold
-    static UnsignedWord getFirstObjectTableSize() {
+    public static UnsignedWord getFirstObjectTableSize() {
         return getCardTableSize();
     }
 
@@ -294,7 +294,7 @@ private static Pointer getCardTableEnd(AlignedHeader chunk) {
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    private static Pointer getFirstObjectTableStart(AlignedHeader chunk) {
+    public static Pointer getFirstObjectTableStart(AlignedHeader chunk) {
         return getFirstObjectTableStart(HeapChunk.asPointer(chunk));
     }
 

substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java

@@ -24,7 +24,6 @@
  */
 package com.oracle.svm.core.genscavenge.remset;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.word.Pointer;
 import org.graalvm.word.UnsignedWord;
 
@@ -37,6 +36,8 @@
 import com.oracle.svm.core.log.Log;
 import com.oracle.svm.core.util.UnsignedUtils;
 
+import jdk.graal.compiler.word.Word;
+
 /**
  * A "first object table" to tell me the start of the first object that crosses onto a card
  * remembered set memory region.
@@ -109,7 +110,7 @@
  * <p>
  * Implementation note: Table entries are bytes but converted to and from ints with bounds checks.
  */
-final class FirstObjectTable {
+public final class FirstObjectTable {
     /**
      * The number of bytes of memory covered by an entry. Since the indexes into the CardTable are
      * used to index into the FirstObjectTable, these need to have the same value.
@@ -237,7 +238,7 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un
      */
     @AlwaysInline("GC performance")
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
+    static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) {
         Pointer result;
         Pointer firstObject = getFirstObject(tableStart, objectsStart, index);
         Pointer indexedMemoryStart = objectsStart.add(indexToMemoryOffset(index));
@@ -295,7 +296,7 @@ private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) {
         return indexOffset.subtract(entryOffset);
     }
 
-    public static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit) {
+    static boolean verify(Pointer tableStart, Pointer objectsStart, Pointer objectsLimit) {
         UnsignedWord indexLimit = getTableSizeForMemoryRange(objectsStart, objectsLimit);
         for (UnsignedWord index = Word.unsigned(0); index.belowThan(indexLimit); index = index.add(1)) {
             Pointer objStart = getFirstObject(tableStart, objectsStart, index);