Arbitrary self types v2: avoid dyn, generic ICE

Bug rust-lang#57276 (and several duplicates) is an ICE which occurs when a generic type
is used as a receiver which implements both Receiver and DispatchFromDyn.

This change proposes to detect this error condition and turn it into a regular
error rather than an ICE. Future changes could liberalise things here.
As this same code path currently produces an ICE, this seems to be strictly
better, and it seems defensible to inform the user that their
excessively generic type is not dyn-safe.

This is somewhat related to the stabilization of arbitrary self types in
PR rust-lang#135881, tracked in rust-lang#44874.

compiler/rustc_middle/src/traits/mod.rs

@@ -786,6 +786,9 @@ pub enum DynCompatibilityViolation {
 
     /// GAT
     GAT(Symbol, Span),
+
+    /// Type layout can't be determined
+    TooGeneric(Span),
 }
 
 impl DynCompatibilityViolation {
@@ -853,16 +856,18 @@ impl DynCompatibilityViolation {
             DynCompatibilityViolation::GAT(name, _) => {
                 format!("it contains the generic associated type `{name}`").into()
             }
+            DynCompatibilityViolation::TooGeneric(_span) => {
+                format!("it is too generic to determine type layout").into()
+            }
         }
     }
 
     pub fn solution(&self) -> DynCompatibilityViolationSolution {
         match self {
             DynCompatibilityViolation::SizedSelf(_)
             | DynCompatibilityViolation::SupertraitSelf(_)
-            | DynCompatibilityViolation::SupertraitNonLifetimeBinder(..) => {
-                DynCompatibilityViolationSolution::None
-            }
+            | DynCompatibilityViolation::SupertraitNonLifetimeBinder(..)
+            | DynCompatibilityViolation::TooGeneric(..) => DynCompatibilityViolationSolution::None,
             DynCompatibilityViolation::Method(
                 name,
                 MethodViolationCode::StaticMethod(Some((add_self_sugg, make_sized_sugg))),

compiler/rustc_trait_selection/src/traits/dyn_compatibility.rs

@@ -27,6 +27,7 @@ use super::elaborate;
 use crate::infer::TyCtxtInferExt;
 pub use crate::traits::DynCompatibilityViolation;
 use crate::traits::query::evaluate_obligation::InferCtxtExt;
+use crate::traits::ty::layout::LayoutError;
 use crate::traits::{MethodViolationCode, Obligation, ObligationCause, util};
 
 /// Returns the dyn-compatibility violations that affect HIR ty lowering.
@@ -112,7 +113,7 @@ fn dyn_compatibility_violations_for_trait(
     if violations.is_empty() {
         for item in tcx.associated_items(trait_def_id).in_definition_order() {
             if let ty::AssocKind::Fn = item.kind {
-                check_receiver_correct(tcx, trait_def_id, *item);
+                check_receiver_correct(tcx, trait_def_id, *item, &mut violations);
             }
         }
     }
@@ -501,9 +502,16 @@ fn virtual_call_violations_for_method<'tcx>(
 
 /// This code checks that `receiver_is_dispatchable` is correctly implemented.
 ///
+/// Any errors are appended to `violations`.
+///
 /// This check is outlined from the dyn-compatibility check to avoid cycles with
 /// layout computation, which relies on knowing whether methods are dyn-compatible.
-fn check_receiver_correct<'tcx>(tcx: TyCtxt<'tcx>, trait_def_id: DefId, method: ty::AssocItem) {
+fn check_receiver_correct<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    trait_def_id: DefId,
+    method: ty::AssocItem,
+    violations: &mut Vec<DynCompatibilityViolation>,
+) {
     if !is_vtable_safe_method(tcx, trait_def_id, method) {
         return;
     }
@@ -522,6 +530,9 @@ fn check_receiver_correct<'tcx>(tcx: TyCtxt<'tcx>, trait_def_id: DefId, method:
     let unit_receiver_ty = receiver_for_self_ty(tcx, receiver_ty, tcx.types.unit, method_def_id);
     match tcx.layout_of(typing_env.as_query_input(unit_receiver_ty)).map(|l| l.backend_repr) {
         Ok(BackendRepr::Scalar(..)) => (),
+        Err(LayoutError::TooGeneric(..)) => {
+            violations.push(DynCompatibilityViolation::TooGeneric(tcx.def_span(method_def_id)));
+        }
         abi => {
             tcx.dcx().span_delayed_bug(
                 tcx.def_span(method_def_id),
@@ -537,6 +548,9 @@ fn check_receiver_correct<'tcx>(tcx: TyCtxt<'tcx>, trait_def_id: DefId, method:
         receiver_for_self_ty(tcx, receiver_ty, trait_object_ty, method_def_id);
     match tcx.layout_of(typing_env.as_query_input(trait_object_receiver)).map(|l| l.backend_repr) {
         Ok(BackendRepr::ScalarPair(..)) => (),
+        Err(LayoutError::TooGeneric(..)) => {
+            violations.push(DynCompatibilityViolation::TooGeneric(tcx.def_span(method_def_id)));
+        }
         abi => {
             tcx.dcx().span_delayed_bug(
                 tcx.def_span(method_def_id),

tests/ui/self/dispatch-from-dyn-layout.rs

@@ -0,0 +1,11 @@
+#![feature(arbitrary_self_types, dispatch_from_dyn)]
+
+use std::ops::{Deref, DispatchFromDyn};
+
+trait Trait<T: Deref<Target=Self> + DispatchFromDyn<T>> {
+    fn foo(self: T) -> dyn Trait<T>;
+    //~^ ERROR: associated item referring to unboxed trait object for its own trait
+    //~| ERROR: the trait `Trait` is not dyn compatible
+}
+
+fn main() {}

tests/ui/self/dispatch-from-dyn-layout.stderr

@@ -0,0 +1,26 @@
+error: associated item referring to unboxed trait object for its own trait
+  --> $DIR/dispatch-from-dyn-layout.rs:6:24
+   |
+LL | trait Trait<T: Deref<Target=Self> + DispatchFromDyn<T>> {
+   |       ----- in this trait
+LL |     fn foo(self: T) -> dyn Trait<T>;
+   |                        ^^^^^^^^^^^^
+   |
+help: you might have meant to use `Self` to refer to the implementing type
+   |
+LL |     fn foo(self: T) -> Self;
+   |                        ~~~~
+
+error[E0038]: the trait `Trait` is not dyn compatible
+  --> $DIR/dispatch-from-dyn-layout.rs:6:24
+   |
+LL |     fn foo(self: T) -> dyn Trait<T>;
+   |                        ^^^^^^^^^^^^ `Trait` is not dyn compatible
+   |
+   = note: the trait is not dyn compatible because it is too generic to determine type layout
+   = note: for a trait to be dyn compatible it needs to allow building a vtable
+           for more information, visit <https://doc.rust-lang.org/reference/items/traits.html#object-safety>
+
+error: aborting due to 2 previous errors
+
+For more information about this error, try `rustc --explain E0038`.