Choose Non-moving Policy based on features

Cargo.toml

@@ -82,7 +82,7 @@ name = "main"
 harness = false
 
 [features]
-default = ["builtin_env_logger"]
+default = ["builtin_env_logger", "immortal_as_nonmoving"]
 
 # Built-in env_logger.  This feature is enabled by default.
 # The user can disable this default feature to remove `env_logger` from the dependencies.
@@ -224,14 +224,18 @@ malloc_jemalloc = ["dep:jemalloc-sys"]
 # is not compiled in default builds.
 malloc_native_mimalloc = []
 
-# If there are more groups, they should be inserted above this line
-# Group:end
-
 # Group:marksweepallocation
 # default is native allocator with lazy sweeping
 eager_sweeping = []
 # Use library malloc as the freelist allocator for mark sweep. This will makes mark sweep slower. As malloc may return addresses outside our
 # normal heap range, we will have to use chunk-based SFT table. Turning on this feature will use a different SFT map implementation on 64bits,
 # and will affect all the plans in the build. Please be aware of the consequence, and this is only meant to be experimental use.
 malloc_mark_sweep = []
+
+# Group:nonmovingspace
+immortal_as_nonmoving = []
+immix_as_nonmoving = []
+marksweep_as_nonmoving = []
+
+# If there are more groups, they should be inserted above this line
 # Group:end

src/plan/generational/immix/global.rs

@@ -87,9 +87,9 @@ impl<VM: VMBinding> Plan for GenImmix<VM> {
 
     fn last_collection_was_exhaustive(&self) -> bool {
         self.last_gc_was_full_heap.load(Ordering::Relaxed)
-            && ImmixSpace::<VM>::is_last_gc_exhaustive(
-                self.last_gc_was_defrag.load(Ordering::Relaxed),
-            )
+            && self
+                .immix_space
+                .is_last_gc_exhaustive(self.last_gc_was_defrag.load(Ordering::Relaxed))
     }
 
     fn collection_required(&self, space_full: bool, space: Option<SpaceStats<Self::VM>>) -> bool
@@ -131,7 +131,7 @@ impl<VM: VMBinding> Plan for GenImmix<VM> {
         if full_heap {
             self.immix_space.prepare(
                 full_heap,
-                crate::policy::immix::defrag::StatsForDefrag::new(self),
+                Some(crate::policy::immix::defrag::StatsForDefrag::new(self)),
             );
         }
     }
@@ -254,6 +254,7 @@ impl<VM: VMBinding> GenImmix<VM> {
                 // In GenImmix, young objects are not allocated in ImmixSpace directly.
                 #[cfg(feature = "vo_bit")]
                 mixed_age: false,
+                never_move_objects: cfg!(feature = "immix_non_moving"),
             },
         );
 

src/plan/global.rs

@@ -7,6 +7,7 @@ use crate::plan::tracing::ObjectQueue;
 use crate::plan::Mutator;
 use crate::policy::immortalspace::ImmortalSpace;
 use crate::policy::largeobjectspace::LargeObjectSpace;
+use crate::policy::immix::ImmixSpace;
 use crate::policy::space::{PlanCreateSpaceArgs, Space};
 #[cfg(feature = "vm_space")]
 use crate::policy::vmspace::VMSpace;
@@ -542,6 +543,15 @@ impl<VM: VMBinding> BasePlan<VM> {
     }
 }
 
+#[cfg(feature = "immortal_as_nonmoving")]
+pub type NonMovingSpace<VM> = crate::policy::immortalspace::ImmortalSpace<VM>;
+
+#[cfg(feature = "immix_as_nonmoving")]
+pub type NonMovingSpace<VM> = crate::policy::immix::ImmixSpace<VM>;
+
+#[cfg(feature = "marksweep_as_nonmoving")]
+pub type NonMovingSpace<VM> = crate::policy::marksweepspace::native_ms::MarkSweepSpace<VM>;
+
 /**
 CommonPlan is for representing state and features used by _many_ plans, but that are not fundamental to _all_ plans.  Examples include the Large Object Space and an Immortal space.  Features that are fundamental to _all_ plans must be included in BasePlan.
 */
@@ -553,7 +563,7 @@ pub struct CommonPlan<VM: VMBinding> {
     pub los: LargeObjectSpace<VM>,
     // TODO: We should use a marksweep space for nonmoving.
     #[space]
-    pub nonmoving: ImmortalSpace<VM>,
+    pub nonmoving: NonMovingSpace<VM>,
     #[parent]
     pub base: BasePlan<VM>,
 }
@@ -571,12 +581,7 @@ impl<VM: VMBinding> CommonPlan<VM> {
                 args.get_space_args("los", true, false, VMRequest::discontiguous()),
                 false,
             ),
-            nonmoving: ImmortalSpace::new(args.get_space_args(
-                "nonmoving",
-                true,
-                false,
-                VMRequest::discontiguous(),
-            )),
+            nonmoving: Self::new_nonmoving_space(&mut args),
             base: BasePlan::new(args),
         }
     }
@@ -591,14 +596,14 @@ impl<VM: VMBinding> CommonPlan<VM> {
     pub fn prepare(&mut self, tls: VMWorkerThread, full_heap: bool) {
         self.immortal.prepare();
         self.los.prepare(full_heap);
-        self.nonmoving.prepare();
+        self.prepare_nonmoving_space(full_heap);
         self.base.prepare(tls, full_heap)
     }
 
     pub fn release(&mut self, tls: VMWorkerThread, full_heap: bool) {
         self.immortal.release();
         self.los.release(full_heap);
-        self.nonmoving.release();
+        self.release_nonmoving_space(full_heap);
         self.base.release(tls, full_heap)
     }
 
@@ -610,9 +615,45 @@ impl<VM: VMBinding> CommonPlan<VM> {
         &self.los
     }
 
-    pub fn get_nonmoving(&self) -> &ImmortalSpace<VM> {
+    pub fn get_nonmoving(&self) -> &NonMovingSpace<VM> {
         &self.nonmoving
     }
+
+    fn new_nonmoving_space(args: &mut CreateSpecificPlanArgs<VM>) -> NonMovingSpace<VM> {
+        let space_args = args.get_space_args(
+            "nonmoving",
+            true,
+            false,
+            VMRequest::discontiguous(),
+        );
+        #[cfg(any(feature = "immortal_as_nonmoving", feature = "marksweep_as_nonmoving"))]
+        return NonMovingSpace::new(space_args);
+        #[cfg(feature = "immix_as_nonmoving")]
+        return NonMovingSpace::new(space_args, crate::policy::immix::ImmixSpaceArgs {
+            unlog_object_when_traced: false,
+            #[cfg(feature = "vo_bit")]
+            mixed_age: false,
+            never_move_objects: true,
+        });
+    }
+
+    fn prepare_nonmoving_space(&mut self, _full_heap: bool) {
+        #[cfg(feature = "immortal_as_nonmoving")]
+        self.nonmoving.prepare();
+        #[cfg(feature = "immix_as_nonmoving")]
+        self.nonmoving.prepare(_full_heap, None);
+        #[cfg(feature = "marksweep_as_nonmoving")]
+        self.nonmoving.prepare(_full_heap);
+    }
+
+    fn release_nonmoving_space(&mut self, _full_heap: bool) {
+        #[cfg(feature = "immortal_as_nonmoving")]
+        self.nonmoving.release();
+        #[cfg(feature = "immix_as_nonmoving")]
+        self.nonmoving.release(_full_heap);
+        #[cfg(feature = "marksweep_as_nonmoving")]
+        self.nonmoving.release();
+    }
 }
 
 use crate::policy::gc_work::TraceKind;

src/plan/immix/global.rs

@@ -38,7 +38,7 @@ pub struct Immix<VM: VMBinding> {
 
 /// The plan constraints for the immix plan.
 pub const IMMIX_CONSTRAINTS: PlanConstraints = PlanConstraints {
-    moves_objects: crate::policy::immix::DEFRAG,
+    moves_objects: true,
     // Max immix object size is half of a block.
     max_non_los_default_alloc_bytes: crate::policy::immix::MAX_IMMIX_OBJECT_SIZE,
     needs_prepare_mutator: false,
@@ -51,7 +51,8 @@ impl<VM: VMBinding> Plan for Immix<VM> {
     }
 
     fn last_collection_was_exhaustive(&self) -> bool {
-        ImmixSpace::<VM>::is_last_gc_exhaustive(self.last_gc_was_defrag.load(Ordering::Relaxed))
+        self.immix_space
+            .is_last_gc_exhaustive(self.last_gc_was_defrag.load(Ordering::Relaxed))
     }
 
     fn constraints(&self) -> &'static PlanConstraints {
@@ -86,7 +87,7 @@ impl<VM: VMBinding> Plan for Immix<VM> {
         self.common.prepare(tls, true);
         self.immix_space.prepare(
             true,
-            crate::policy::immix::defrag::StatsForDefrag::new(self),
+            Some(crate::policy::immix::defrag::StatsForDefrag::new(self)),
         );
     }
 
@@ -139,6 +140,7 @@ impl<VM: VMBinding> Immix<VM> {
                 unlog_object_when_traced: false,
                 #[cfg(feature = "vo_bit")]
                 mixed_age: false,
+                never_move_objects: cfg!(feature = "immix_non_moving"),
             },
         )
     }

src/plan/mutator_context.rs

@@ -5,7 +5,7 @@ use crate::plan::global::Plan;
 use crate::plan::AllocationSemantics;
 use crate::policy::space::Space;
 use crate::util::alloc::allocators::{AllocatorSelector, Allocators};
-use crate::util::alloc::Allocator;
+use crate::util::alloc::{Allocator, FreeListAllocator, ImmixAllocator};
 use crate::util::{Address, ObjectReference};
 use crate::util::{VMMutatorThread, VMWorkerThread};
 use crate::vm::VMBinding;
@@ -27,6 +27,20 @@ pub(crate) fn unreachable_prepare_func<VM: VMBinding>(
     unreachable!("`MutatorConfig::prepare_func` must not be called for the current plan.")
 }
 
+/// An mutator prepare implementation for plans that use [`crate::plan::global::CommonPlan`].
+pub(crate) fn common_prepare_func<VM: VMBinding>(mutator: &mut Mutator<VM>, _tls: VMWorkerThread) {
+    // Prepare the free list allocator used for non moving
+    #[cfg(feature = "marksweep_as_nonmoving")]
+    unsafe {
+        mutator
+            .allocators
+            .get_allocator_mut(mutator.config.allocator_mapping[AllocationSemantics::NonMoving])
+    }
+    .downcast_mut::<FreeListAllocator<VM>>()
+    .unwrap()
+    .prepare();
+}
+
 /// A place-holder implementation for `MutatorConfig::release_func` that should not be called.
 /// Currently only used by `NoGC`.
 pub(crate) fn unreachable_release_func<VM: VMBinding>(
@@ -36,6 +50,29 @@ pub(crate) fn unreachable_release_func<VM: VMBinding>(
     unreachable!("`MutatorConfig::release_func` must not be called for the current plan.")
 }
 
+/// An mutator release implementation for plans that use [`crate::plan::global::CommonPlan`].
+pub(crate) fn common_release_func<VM: VMBinding>(mutator: &mut Mutator<VM>, _tls: VMWorkerThread) {
+    // Release the free list allocator used for non moving
+    #[cfg(feature = "marksweep_as_nonmoving")]
+    unsafe {
+        mutator
+            .allocators
+            .get_allocator_mut(mutator.config.allocator_mapping[AllocationSemantics::NonMoving])
+    }
+    .downcast_mut::<FreeListAllocator<VM>>()
+    .unwrap()
+    .release();
+    #[cfg(feature = "immix_as_nonmoving")]
+    let immix_allocator = unsafe {
+        mutator
+            .allocators
+            .get_allocator_mut(mutator.config.allocator_mapping[AllocationSemantics::NonMoving])
+    }
+    .downcast_mut::<ImmixAllocator<VM>>()
+    .unwrap()
+    .reset();
+}
+
 /// A place-holder implementation for `MutatorConfig::release_func` that does nothing.
 pub(crate) fn no_op_release_func<VM: VMBinding>(_mutator: &mut Mutator<VM>, _tls: VMWorkerThread) {}
 
@@ -455,10 +492,21 @@ pub(crate) fn create_allocator_mapping(
         map[AllocationSemantics::Los] = AllocatorSelector::LargeObject(reserved.n_large_object);
         reserved.n_large_object += 1;
 
-        // TODO: This should be freelist allocator once we use marksweep for nonmoving space.
-        map[AllocationSemantics::NonMoving] =
-            AllocatorSelector::BumpPointer(reserved.n_bump_pointer);
-        reserved.n_bump_pointer += 1;
+        #[cfg(feature = "immix_as_nonmoving")]
+        {
+            map[AllocationSemantics::NonMoving] = AllocatorSelector::Immix(reserved.n_immix);
+            reserved.n_immix += 1;
+        }
+        #[cfg(feature = "immortal_as_nonmoving")]
+        {
+            map[AllocationSemantics::NonMoving] = AllocatorSelector::BumpPointer(reserved.n_bump_pointer);
+            reserved.n_bump_pointer += 1;
+        }
+        #[cfg(feature = "marksweep_as_nonmoving")]
+        {
+            map[AllocationSemantics::NonMoving] = AllocatorSelector::FreeList(reserved.n_free_list);
+            reserved.n_free_list += 1;
+        }
     }
 
     reserved.validate();
@@ -520,12 +568,30 @@ pub(crate) fn create_space_mapping<VM: VMBinding>(
             plan.common().get_los(),
         ));
         reserved.n_large_object += 1;
-        // TODO: This should be freelist allocator once we use marksweep for nonmoving space.
-        vec.push((
-            AllocatorSelector::BumpPointer(reserved.n_bump_pointer),
-            plan.common().get_nonmoving(),
-        ));
-        reserved.n_bump_pointer += 1;
+        #[cfg(feature = "immix_as_nonmoving")]
+        {
+            vec.push((
+                AllocatorSelector::Immix(reserved.n_immix),
+                plan.common().get_nonmoving(),
+            ));
+            reserved.n_immix += 1;
+        }
+        #[cfg(feature = "marksweep_as_nonmoving")]
+        {
+            vec.push((
+                AllocatorSelector::FreeList(reserved.n_free_list),
+                plan.common().get_nonmoving(),
+            ));
+            reserved.n_free_list += 1;
+        }
+        #[cfg(feature = "immortal_as_nonmoving")]
+        {
+            vec.push((
+                AllocatorSelector::BumpPointer(reserved.n_bump_pointer),
+                plan.common().get_nonmoving(),
+            ));
+            reserved.n_bump_pointer += 1;
+        }
     }
 
     reserved.validate();

src/plan/sticky/immix/global.rs

@@ -7,7 +7,6 @@ use crate::plan::PlanConstraints;
 use crate::policy::gc_work::TraceKind;
 use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN;
 use crate::policy::immix::ImmixSpace;
-use crate::policy::immix::PREFER_COPY_ON_NURSERY_GC;
 use crate::policy::immix::TRACE_KIND_FAST;
 use crate::policy::sft::SFT;
 use crate::policy::space::Space;
@@ -41,7 +40,7 @@ pub struct StickyImmix<VM: VMBinding> {
 
 /// The plan constraints for the sticky immix plan.
 pub const STICKY_IMMIX_CONSTRAINTS: PlanConstraints = PlanConstraints {
-    moves_objects: crate::policy::immix::DEFRAG || crate::policy::immix::PREFER_COPY_ON_NURSERY_GC,
+    moves_objects: true,
     needs_log_bit: true,
     barrier: crate::plan::BarrierSelector::ObjectBarrier,
     // We may trace duplicate edges in sticky immix (or any plan that uses object remembering barrier). See https://github.com/mmtk/mmtk-core/issues/743.
@@ -117,7 +116,7 @@ impl<VM: VMBinding> Plan for StickyImmix<VM> {
             // Prepare both large object space and immix space
             self.immix.immix_space.prepare(
                 false,
-                crate::policy::immix::defrag::StatsForDefrag::new(self),
+                Some(crate::policy::immix::defrag::StatsForDefrag::new(self)),
             );
             self.immix.common.los.prepare(false);
         } else {
@@ -164,7 +163,7 @@ impl<VM: VMBinding> Plan for StickyImmix<VM> {
 
     fn current_gc_may_move_object(&self) -> bool {
         if self.is_current_gc_nursery() {
-            PREFER_COPY_ON_NURSERY_GC
+            self.get_immix_space().prefer_copy_on_nursery_gc()
         } else {
             self.get_immix_space().in_defrag()
         }
@@ -263,7 +262,7 @@ impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> f
                     self.immix
                         .immix_space
                         .trace_object_without_moving(queue, object)
-                } else if crate::policy::immix::PREFER_COPY_ON_NURSERY_GC {
+                } else if self.immix.immix_space.prefer_copy_on_nursery_gc() {
                     let ret = self.immix.immix_space.trace_object_with_opportunistic_copy(
                         queue,
                         object,
@@ -330,6 +329,7 @@ impl<VM: VMBinding> StickyImmix<VM> {
                 // In StickyImmix, both young and old objects are allocated in the ImmixSpace.
                 #[cfg(feature = "vo_bit")]
                 mixed_age: true,
+                never_move_objects: cfg!(feature = "immix_non_moving"),
             },
         );
         Self {