[EXPERIMENT] Reduction of monomorphization load through type erasure

compiler/rustc_codegen_cranelift/src/abi/mod.rs

@@ -298,6 +298,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     func: &Operand<'tcx>,
     args: &[Operand<'tcx>],
     destination: Option<(Place<'tcx>, BasicBlock)>,
+    erased: bool,
 ) {
     let fn_ty = fx.monomorphize(func.ty(fx.mir, fx.tcx));
     let fn_sig =
@@ -306,7 +307,13 @@ pub(crate) fn codegen_terminator_call<'tcx>(
     let destination = destination.map(|(place, bb)| (codegen_place(fx, place), bb));
 
     // Handle special calls like instrinsics and empty drop glue.
-    let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
+    let instance = if erased {
+        if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
+            Some(ty::Instance::resolve_erased(fx.tcx, def_id, substs))
+        } else {
+            bug!()
+        }
+    } else if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
         let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
             .unwrap()
             .unwrap()

compiler/rustc_codegen_cranelift/src/base.rs

@@ -354,9 +354,17 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                 fn_span,
                 cleanup: _,
                 from_hir_call: _,
+                erased,
             } => {
                 fx.tcx.sess.time("codegen call", || {
-                    crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination)
+                    crate::abi::codegen_terminator_call(
+                        fx,
+                        *fn_span,
+                        func,
+                        args,
+                        *destination,
+                        *erased,
+                    )
                 });
             }
             TerminatorKind::InlineAsm {

compiler/rustc_codegen_ssa/src/mir/block.rs

@@ -498,6 +498,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         destination: &Option<(mir::Place<'tcx>, mir::BasicBlock)>,
         cleanup: Option<mir::BasicBlock>,
         fn_span: Span,
+        erased: bool,
     ) {
         let source_info = terminator.source_info;
         let span = source_info.span;
@@ -506,15 +507,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
         let callee = self.codegen_operand(&mut bx, func);
 
         let (instance, mut llfn) = match *callee.layout.ty.kind() {
-            ty::FnDef(def_id, substs) => (
-                Some(
+            ty::FnDef(def_id, substs) => {
+                let instance = if erased {
+                    ty::Instance::resolve_erased(bx.tcx(), def_id, substs)
+                } else {
                     ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs)
                         .unwrap()
                         .unwrap()
-                        .polymorphize(bx.tcx()),
-                ),
-                None,
-            ),
+                        .polymorphize(bx.tcx())
+                };
+                (Some(instance), None)
+            }
+            _ if erased => bug!("{} is not erasable", callee.layout.ty),
             ty::FnPtr(_) => (None, Some(callee.immediate())),
             _ => bug!("{} is not callable", callee.layout.ty),
         };
@@ -971,6 +975,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 cleanup,
                 from_hir_call: _,
                 fn_span,
+                erased,
             } => {
                 self.codegen_call_terminator(
                     helper,
@@ -981,6 +986,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     destination,
                     cleanup,
                     fn_span,
+                    erased,
                 );
             }
             mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => {

compiler/rustc_feature/src/builtin_attrs.rs

@@ -555,6 +555,7 @@ pub const BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
     // Internal attributes, Testing:
     // ==========================================================================
 
+    rustc_attr!(TEST, rustc_dyn, AssumedUsed, template!(Word)),
     rustc_attr!(TEST, rustc_outlives, Normal, template!(Word)),
     rustc_attr!(TEST, rustc_capture_analysis, Normal, template!(Word)),
     rustc_attr!(TEST, rustc_variance, Normal, template!(Word)),

compiler/rustc_middle/src/mir/mod.rs

@@ -185,6 +185,9 @@ pub struct Body<'tcx> {
 
     pub generator: Option<Box<GeneratorInfo<'tcx>>>,
 
+    /// Erased version for dyn_erased.
+    pub dyn_erased_body: Option<Box<Body<'tcx>>>,
+
     /// Declarations of locals.
     ///
     /// The first local is the return value pointer, followed by `arg_count`
@@ -273,6 +276,7 @@ impl<'tcx> Body<'tcx> {
                     generator_kind,
                 })
             }),
+            dyn_erased_body: None,
             local_decls,
             user_type_annotations,
             arg_count,
@@ -300,6 +304,7 @@ impl<'tcx> Body<'tcx> {
             basic_blocks,
             source_scopes: IndexVec::new(),
             generator: None,
+            dyn_erased_body: None,
             local_decls: IndexVec::new(),
             user_type_annotations: IndexVec::new(),
             arg_count: 0,

compiler/rustc_middle/src/mir/mono.rs

@@ -353,6 +353,7 @@ impl<'tcx> CodegenUnit<'tcx> {
                                 .map(|def_id| tcx.hir().local_def_id_to_hir_id(def_id)),
                             InstanceDef::VtableShim(..)
                             | InstanceDef::ReifyShim(..)
+                            | InstanceDef::ErasedShim(..)
                             | InstanceDef::Intrinsic(..)
                             | InstanceDef::FnPtrShim(..)
                             | InstanceDef::Virtual(..)

compiler/rustc_middle/src/mir/terminator.rs

@@ -190,6 +190,7 @@ pub enum TerminatorKind<'tcx> {
         /// This `Span` is the span of the function, without the dot and receiver
         /// (e.g. `foo(a, b)` in `x.foo(a, b)`
         fn_span: Span,
+        erased: bool,
     },
 
     /// Jump to the target if the condition has the expected value,
@@ -470,11 +471,11 @@ impl<'tcx> TerminatorKind<'tcx> {
             DropAndReplace { place, value, .. } => {
                 write!(fmt, "replace({:?} <- {:?})", place, value)
             }
-            Call { func, args, destination, .. } => {
+            Call { func, args, destination, erased, .. } => {
                 if let Some((destination, _)) = destination {
                     write!(fmt, "{:?} = ", destination)?;
                 }
-                write!(fmt, "{:?}(", func)?;
+                write!(fmt, "{:?}{}(", func, if *erased { "[erased]" } else { "" })?;
                 for (index, arg) in args.iter().enumerate() {
                     if index > 0 {
                         write!(fmt, ", ")?;

compiler/rustc_middle/src/mir/type_foldable.rs

@@ -41,7 +41,7 @@ impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> {
                 resume_arg: resume_arg.fold_with(folder),
                 drop,
             },
-            Call { func, args, destination, cleanup, from_hir_call, fn_span } => {
+            Call { func, args, destination, cleanup, from_hir_call, fn_span, erased } => {
                 let dest = destination.map(|(loc, dest)| (loc.fold_with(folder), dest));
 
                 Call {
@@ -51,6 +51,7 @@ impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> {
                     cleanup,
                     from_hir_call,
                     fn_span,
+                    erased,
                 }
             }
             Assert { cond, expected, msg, target, cleanup } => {

compiler/rustc_middle/src/mir/visit.rs

@@ -347,6 +347,7 @@ macro_rules! make_mir_visitor {
                         ty::InstanceDef::Intrinsic(_def_id) |
                         ty::InstanceDef::VtableShim(_def_id) |
                         ty::InstanceDef::ReifyShim(_def_id) |
+                        ty::InstanceDef::ErasedShim(_def_id) |
                         ty::InstanceDef::Virtual(_def_id, _) |
                         ty::InstanceDef::ClosureOnceShim { call_once: _def_id } |
                         ty::InstanceDef::DropGlue(_def_id, None) => {}
@@ -535,7 +536,8 @@ macro_rules! make_mir_visitor {
                         destination,
                         cleanup: _,
                         from_hir_call: _,
-                        fn_span: _
+                        fn_span: _,
+                        erased: _
                     } => {
                         self.visit_operand(func, location);
                         for arg in args {

compiler/rustc_middle/src/ty/fold.rs

@@ -86,6 +86,12 @@ pub trait TypeFoldable<'tcx>: fmt::Debug + Clone {
     fn references_error(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_ERROR)
     }
+    fn has_param_types(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_TY_PARAM)
+    }
+    fn has_param_consts(&self) -> bool {
+        self.has_type_flags(TypeFlags::HAS_CT_PARAM)
+    }
     fn has_param_types_or_consts(&self) -> bool {
         self.has_type_flags(TypeFlags::HAS_TY_PARAM | TypeFlags::HAS_CT_PARAM)
     }

compiler/rustc_middle/src/ty/instance.rs

@@ -60,6 +60,9 @@ pub enum InstanceDef<'tcx> {
     /// (the definition of the function itself).
     ReifyShim(DefId),
 
+    /// Compiler-generated untyped version of an item.
+    ErasedShim(DefId),
+
     /// `<fn() as FnTrait>::call_*` (generated `FnTrait` implementation for `fn()` pointers).
     ///
     /// `DefId` is `FnTrait::call_*`.
@@ -143,6 +146,7 @@ impl<'tcx> InstanceDef<'tcx> {
             InstanceDef::Item(def) => def.did,
             InstanceDef::VtableShim(def_id)
             | InstanceDef::ReifyShim(def_id)
+            | InstanceDef::ErasedShim(def_id)
             | InstanceDef::FnPtrShim(def_id, _)
             | InstanceDef::Virtual(def_id, _)
             | InstanceDef::Intrinsic(def_id)
@@ -158,6 +162,7 @@ impl<'tcx> InstanceDef<'tcx> {
             InstanceDef::Item(def) => def,
             InstanceDef::VtableShim(def_id)
             | InstanceDef::ReifyShim(def_id)
+            | InstanceDef::ErasedShim(def_id)
             | InstanceDef::FnPtrShim(def_id, _)
             | InstanceDef::Virtual(def_id, _)
             | InstanceDef::Intrinsic(def_id)
@@ -244,6 +249,7 @@ impl<'tcx> InstanceDef<'tcx> {
         match *self {
             InstanceDef::CloneShim(..)
             | InstanceDef::FnPtrShim(..)
+            | InstanceDef::ErasedShim(..)
             | InstanceDef::DropGlue(_, Some(_)) => false,
             InstanceDef::ClosureOnceShim { .. }
             | InstanceDef::DropGlue(..)
@@ -269,6 +275,7 @@ impl<'tcx> fmt::Display for Instance<'tcx> {
             InstanceDef::Item(_) => Ok(()),
             InstanceDef::VtableShim(_) => write!(f, " - shim(vtable)"),
             InstanceDef::ReifyShim(_) => write!(f, " - shim(reify)"),
+            InstanceDef::ErasedShim(_) => write!(f, " - shim(erased)"),
             InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"),
             InstanceDef::Virtual(_, num) => write!(f, " - virtual#{}", num),
             InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({})", ty),
@@ -436,6 +443,22 @@ impl<'tcx> Instance<'tcx> {
         Instance::resolve(tcx, ty::ParamEnv::reveal_all(), def_id, substs).unwrap().unwrap()
     }
 
+    pub fn resolve_erased(
+        tcx: TyCtxt<'tcx>,
+        def_id: DefId,
+        _substs: SubstsRef<'tcx>,
+    ) -> Instance<'tcx> {
+        let substs = InternalSubsts::for_item(tcx, def_id, |param, _| match param.kind {
+            ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
+            ty::GenericParamDefKind::Type { .. } => tcx.types.u8.into(),
+            ty::GenericParamDefKind::Const { .. } => {
+                panic!("Unsupported type-erased const params")
+            }
+        });
+        let def = ty::InstanceDef::ErasedShim(def_id);
+        ty::Instance { def, substs }
+    }
+
     pub fn fn_once_adapter_instance(
         tcx: TyCtxt<'tcx>,
         closure_did: DefId,

compiler/rustc_middle/src/ty/layout.rs

@@ -2455,18 +2455,26 @@ impl<'tcx> ty::Instance<'tcx> {
         // FIXME(davidtwco,eddyb): A `ParamEnv` should be passed through to this function.
         let ty = self.ty(tcx, ty::ParamEnv::reveal_all());
         match *ty.kind() {
-            ty::FnDef(..) => {
+            ty::FnDef(def_id, substs) => {
                 // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering
                 // parameters unused if they show up in the signature, but not in the `mir::Body`
                 // (i.e. due to being inside a projection that got normalized, see
                 // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping
                 // track of a polymorphization `ParamEnv` to allow normalizing later.
-                let mut sig = match *ty.kind() {
-                    ty::FnDef(def_id, substs) => tcx
-                        .normalize_erasing_regions(tcx.param_env(def_id), tcx.fn_sig(def_id))
-                        .subst(tcx, substs),
-                    _ => unreachable!(),
-                };
+                let mut sig = tcx.fn_sig(def_id);
+                if let ty::InstanceDef::ErasedShim(..) = self.def {
+                    let mir = tcx.optimized_mir(def_id);
+                    let inner_mir = &mir.dyn_erased_body.as_ref().unwrap();
+                    sig = sig.map_bound(|mut sig| {
+                        let mut inputs_and_output: Vec<_> =
+                            inner_mir.args_iter().map(|l| inner_mir.local_decls[l].ty).collect();
+                        inputs_and_output.push(inner_mir.return_ty());
+                        sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output[..]);
+                        sig
+                    });
+                }
+                let mut sig =
+                    tcx.normalize_erasing_regions(tcx.param_env(def_id), sig).subst(tcx, substs);
 
                 if let ty::InstanceDef::VtableShim(..) = self.def {
                     // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
@@ -2665,6 +2673,17 @@ where
             None
         };
 
+        let extra_args = if extra_args.is_empty() {
+            extra_args
+        } else if let ty::InstanceDef::ErasedShim(def_id) = instance.def {
+            // Remove extra arguments between Item and ErasedShim.
+            let raw_sig = cx.tcx().fn_sig(def_id);
+            let additions = sig.inputs().skip_binder().len() - raw_sig.inputs().skip_binder().len();
+            &extra_args[additions..]
+        } else {
+            extra_args
+        };
+
         let attrs = cx.tcx().codegen_fn_attrs(instance.def_id()).flags;
 
         call::FnAbi::new_internal(

compiler/rustc_middle/src/ty/mod.rs

@@ -1788,6 +1788,7 @@ impl<'tcx> TyCtxt<'tcx> {
             },
             ty::InstanceDef::VtableShim(..)
             | ty::InstanceDef::ReifyShim(..)
+            | ty::InstanceDef::ErasedShim(..)
             | ty::InstanceDef::Intrinsic(..)
             | ty::InstanceDef::FnPtrShim(..)
             | ty::InstanceDef::Virtual(..)

compiler/rustc_middle/src/ty/structural_impls.rs

@@ -626,6 +626,7 @@ impl<'a, 'tcx> Lift<'tcx> for ty::InstanceDef<'a> {
             ty::InstanceDef::Item(def_id) => Some(ty::InstanceDef::Item(def_id)),
             ty::InstanceDef::VtableShim(def_id) => Some(ty::InstanceDef::VtableShim(def_id)),
             ty::InstanceDef::ReifyShim(def_id) => Some(ty::InstanceDef::ReifyShim(def_id)),
+            ty::InstanceDef::ErasedShim(def_id) => Some(ty::InstanceDef::ErasedShim(def_id)),
             ty::InstanceDef::Intrinsic(def_id) => Some(ty::InstanceDef::Intrinsic(def_id)),
             ty::InstanceDef::FnPtrShim(def_id, ty) => {
                 Some(ty::InstanceDef::FnPtrShim(def_id, tcx.lift(ty)?))
@@ -814,6 +815,7 @@ impl<'tcx> TypeFoldable<'tcx> for ty::instance::Instance<'tcx> {
                 Item(def) => Item(def.fold_with(folder)),
                 VtableShim(did) => VtableShim(did.fold_with(folder)),
                 ReifyShim(did) => ReifyShim(did.fold_with(folder)),
+                ErasedShim(did) => ErasedShim(did.fold_with(folder)),
                 Intrinsic(did) => Intrinsic(did.fold_with(folder)),
                 FnPtrShim(did, ty) => FnPtrShim(did.fold_with(folder), ty.fold_with(folder)),
                 Virtual(did, i) => Virtual(did.fold_with(folder), i),
@@ -831,9 +833,11 @@ impl<'tcx> TypeFoldable<'tcx> for ty::instance::Instance<'tcx> {
         self.substs.visit_with(visitor)?;
         match self.def {
             Item(def) => def.visit_with(visitor),
-            VtableShim(did) | ReifyShim(did) | Intrinsic(did) | Virtual(did, _) => {
-                did.visit_with(visitor)
-            }
+            VtableShim(did)
+            | ReifyShim(did)
+            | ErasedShim(did)
+            | Intrinsic(did)
+            | Virtual(did, _) => did.visit_with(visitor),
             FnPtrShim(did, ty) | CloneShim(did, ty) => {
                 did.visit_with(visitor)?;
                 ty.visit_with(visitor)

compiler/rustc_mir/src/borrow_check/invalidation.rs

@@ -154,6 +154,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 cleanup: _,
                 from_hir_call: _,
                 fn_span: _,
+                erased: _,
             } => {
                 self.consume_operand(location, func);
                 for arg in args {

compiler/rustc_mir/src/borrow_check/mod.rs

@@ -698,6 +698,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
                 cleanup: _,
                 from_hir_call: _,
                 fn_span: _,
+                erased: _,
             } => {
                 self.consume_operand(loc, (func, span), flow_state);
                 for arg in args {

compiler/rustc_mir/src/dataflow/framework/direction.rs

@@ -467,7 +467,15 @@ impl Direction for Forward {
                 propagate(target, exit_state);
             }
 
-            Call { cleanup, destination, ref func, ref args, from_hir_call: _, fn_span: _ } => {
+            Call {
+                cleanup,
+                destination,
+                ref func,
+                ref args,
+                from_hir_call: _,
+                fn_span: _,
+                erased: _,
+            } => {
                 if let Some(unwind) = cleanup {
                     if dead_unwinds.map_or(true, |dead| !dead.contains(bb)) {
                         propagate(unwind, exit_state);

compiler/rustc_mir/src/dataflow/move_paths/builder.rs

@@ -405,6 +405,7 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 cleanup: _,
                 from_hir_call: _,
                 fn_span: _,
+                erased: _,
             } => {
                 self.gather_operand(func);
                 for arg in args {