Make it enable to safely inherit native types

Author: kngwyu

pyo3-derive-backend/src/pyclass.rs

@@ -16,6 +16,7 @@ pub struct PyClassArgs {
     pub name: Option<syn::Expr>,
     pub flags: Vec<syn::Expr>,
     pub base: syn::TypePath,
+    pub has_extends: bool,
     pub module: Option<syn::LitStr>,
 }
 
@@ -39,8 +40,9 @@ impl Default for PyClassArgs {
             module: None,
             // We need the 0 as value for the constant we're later building using quote for when there
             // are no other flags
-            flags: vec![parse_quote! {0}],
-            base: parse_quote! {pyo3::types::PyAny},
+            flags: vec![parse_quote! { 0 }],
+            base: parse_quote! { pyo3::types::PyAny },
+            has_extends: false,
         }
     }
 }
@@ -89,6 +91,7 @@ impl PyClassArgs {
                         path: exp.path.clone(),
                         qself: None,
                     };
+                    self.has_extends = true;
                 }
                 _ => {
                     return Err(syn::Error::new_spanned(
@@ -318,7 +321,7 @@ fn impl_class(
             }
         }
     }
-    // TODO: implement dict and weakref
+
     let weakref = if has_weakref {
         quote! { type WeakRef = pyo3::pyclass_slots::PyClassWeakRefSlot; }
     } else {
@@ -355,6 +358,11 @@ fn impl_class(
 
     let base = &attr.base;
     let flags = &attr.flags;
+    let extended = if attr.has_extends {
+        quote! { pyo3::type_flags::EXTENDED }
+    } else {
+        quote! { 0 }
+    };
 
     Ok(quote! {
         impl pyo3::type_object::PyTypeInfo for #cls {
@@ -365,7 +373,7 @@ fn impl_class(
             const NAME: &'static str = #cls_name;
             const MODULE: Option<&'static str> = #module;
             const DESCRIPTION: &'static str = #doc;
-            const FLAGS: usize = #(#flags)|*;
+            const FLAGS: usize = #(#flags)|* | #extended;
 
             #[inline]
             unsafe fn type_object() -> &'static mut pyo3::ffi::PyTypeObject {

src/conversion.rs

@@ -3,7 +3,7 @@
 //! Conversions between various states of rust and python types and their wrappers.
 use crate::err::{self, PyDowncastError, PyResult};
 use crate::object::PyObject;
-use crate::type_object::{PyConcreteObject, PyTypeInfo};
+use crate::type_object::{PyObjectLayout, PyTypeInfo};
 use crate::types::PyAny;
 use crate::types::PyTuple;
 use crate::{ffi, gil, Py, Python};

src/ffi/code.rs

@@ -31,13 +31,6 @@ pub struct PyCodeObject {
     pub co_extra: *mut c_void,
 }
 
-impl Default for PyCodeObject {
-    #[inline]
-    fn default() -> Self {
-        unsafe { ::std::mem::zeroed() }
-    }
-}
-
 /* Masks for co_flags */
 pub const CO_OPTIMIZED: c_int = 0x0001;
 pub const CO_NEWLOCALS: c_int = 0x0002;

src/ffi/complexobject.rs

@@ -27,19 +27,17 @@ extern "C" {
     pub fn PyComplex_ImagAsDouble(op: *mut PyObject) -> c_double;
 }
 
-#[cfg(not(Py_LIMITED_API))]
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct Py_complex {
     pub real: c_double,
     pub imag: c_double,
 }
 
-#[cfg(not(Py_LIMITED_API))]
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct PyComplexObject {
-    _ob_base: PyObject,
+    pub ob_base: PyObject,
     pub cval: Py_complex,
 }
 

src/ffi/datetime.rs

@@ -13,6 +13,22 @@ extern "C" {
     pub static mut PyDictValues_Type: PyTypeObject;
 }
 
+#[repr(C)]
+pub struct PyDictKeysObject {
+    _unused: [u8; 0],
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct PyDictObject {
+    pub ob_base: PyObject,
+    pub ma_used: Py_ssize_t,
+    #[cfg(Py_3_6)]
+    pub ma_version_tag: u64,
+    pub ma_keys: *mut PyDictKeysObject,
+    pub ma_values: *mut *mut PyObject,
+}
+
 #[inline]
 pub unsafe fn PyDict_Check(op: *mut PyObject) -> c_int {
     PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_DICT_SUBCLASS)

src/ffi/dictobject.rs

@@ -1,7 +1,6 @@
 use crate::ffi::object::*;
 use std::os::raw::{c_double, c_int};
 
-#[cfg(not(Py_LIMITED_API))]
 #[repr(C)]
 pub struct PyFloatObject {
     pub ob_base: PyObject,

src/ffi/floatobject.rs

@@ -5,13 +5,7 @@ use std::os::raw::c_int;
 #[repr(C)]
 #[derive(Copy, Clone)]
 pub struct PyListObject {
-    #[cfg(py_sys_config = "Py_TRACE_REFS")]
-    pub _ob_next: *mut PyObject,
-    #[cfg(py_sys_config = "Py_TRACE_REFS")]
-    pub _ob_prev: *mut PyObject,
-    pub ob_refcnt: Py_ssize_t,
-    pub ob_type: *mut PyTypeObject,
-    pub ob_size: Py_ssize_t,
+    pub ob_base: PyVarObject,
     pub ob_item: *mut *mut PyObject,
     pub allocated: Py_ssize_t,
 }

src/ffi/listobject.rs

@@ -6,8 +6,10 @@ use std::os::raw::{
 };
 
 /// This is an opaque type in the python c api
-#[repr(transparent)]
-pub struct PyLongObject(*mut c_void);
+#[repr(C)]
+pub struct PyLongObject {
+    _unused: [u8; 0],
+}
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {

src/ffi/longobject.rs

@@ -1,5 +1,5 @@
 use crate::ffi::object::*;
-use crate::ffi::pyport::Py_ssize_t;
+use crate::ffi::pyport::{Py_hash_t, Py_ssize_t};
 use std::os::raw::c_int;
 
 #[cfg_attr(windows, link(name = "pythonXY"))]
@@ -11,6 +11,28 @@ extern "C" {
     pub static mut PySetIter_Type: PyTypeObject;
 }
 
+pub const PySet_MINSIZE: usize = 8;
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct setentry {
+    pub key: *mut PyObject,
+    pub hash: Py_hash_t,
+}
+
+#[repr(C)]
+pub struct PySetObject {
+    pub ob_base: PyObject,
+    pub fill: Py_ssize_t,
+    pub used: Py_ssize_t,
+    pub mask: Py_ssize_t,
+    pub table: *mut setentry,
+    pub hash: Py_hash_t,
+    pub finger: Py_ssize_t,
+    pub smalltable: [setentry; PySet_MINSIZE],
+    pub weakreflist: *mut PyObject,
+}
+
 #[inline]
 #[cfg_attr(PyPy, link_name = "PyPyFrozenSet_CheckExact")]
 pub unsafe fn PyFrozenSet_CheckExact(ob: *mut PyObject) -> c_int {

src/ffi/setobject.rs

@@ -26,6 +26,14 @@ pub unsafe fn PySlice_Check(op: *mut PyObject) -> c_int {
     (Py_TYPE(op) == &mut PySlice_Type) as c_int
 }
 
+#[repr(C)]
+pub struct PySliceObject {
+    pub ob_base: PyObject,
+    pub start: *mut PyObject,
+    pub stop: *mut PyObject,
+    pub step: *mut PyObject,
+}
+
 #[cfg_attr(windows, link(name = "pythonXY"))]
 extern "C" {
     #[cfg_attr(PyPy, link_name = "PyPySlice_New")]

src/ffi/sliceobject.rs

@@ -3,7 +3,6 @@ use crate::ffi::pyport::Py_ssize_t;
 use std::os::raw::c_int;
 
 #[repr(C)]
-#[cfg(not(Py_LIMITED_API))]
 pub struct PyTupleObject {
     pub ob_base: PyVarObject,
     pub ob_item: [*mut PyObject; 1],

src/ffi/tupleobject.rs

@@ -4,7 +4,7 @@
 
 use crate::ffi;
 use crate::pyclass::{tp_free_fallback, PyClassAlloc};
-use crate::type_object::{PyConcreteObject, PyTypeInfo};
+use crate::type_object::{PyObjectLayout, PyTypeInfo};
 use crate::Python;
 use std::mem;
 use std::os::raw::c_void;
@@ -77,7 +77,7 @@ where
             ffi::PyObject_Init(obj, <Self as PyTypeInfo>::type_object());
             obj as _
         } else {
-            ffi::PyType_GenericAlloc(<Self as PyTypeInfo>::type_object(), 0) as _
+            crate::pyclass::default_alloc::<Self>() as _
         }
     }
 

src/freelist.rs

@@ -4,7 +4,7 @@ use crate::gil;
 use crate::object::PyObject;
 use crate::objectprotocol::ObjectProtocol;
 use crate::pyclass::{IntoInitializer, PyClass, PyClassShell};
-use crate::type_object::{PyConcreteObject, PyTypeInfo};
+use crate::type_object::{PyObjectLayout, PyTypeInfo};
 use crate::types::PyAny;
 use crate::{ffi, IntoPy};
 use crate::{AsPyPointer, FromPyObject, IntoPyPointer, Python, ToPyObject};
@@ -37,7 +37,9 @@ impl<T> Py<T> {
     /// Create new instance of T and move it under python management
     pub fn new(py: Python, value: impl IntoInitializer<T>) -> PyResult<Py<T>>
     where
-        T: PyClass + PyTypeInfo<ConcreteLayout = PyClassShell<T>>,
+        T: PyClass,
+        <T::BaseType as PyTypeInfo>::ConcreteLayout:
+            crate::type_object::PyObjectSizedLayout<T::BaseType>,
     {
         let initializer = value.into_initializer()?;
         let obj = unsafe { initializer.create_shell(py)? };

src/instance.rs

@@ -129,7 +129,7 @@ pub use crate::object::PyObject;
 pub use crate::objectprotocol::ObjectProtocol;
 pub use crate::pyclass::{PyClass, PyClassInitializer, PyClassShell};
 pub use crate::python::{prepare_freethreaded_python, Python};
-pub use crate::type_object::{type_flags, PyConcreteObject, PyTypeInfo};
+pub use crate::type_object::{type_flags, PyTypeInfo};
 
 // Re-exported for wrap_function
 #[doc(hidden)]

src/lib.rs

@@ -3,20 +3,33 @@ use crate::class::methods::{PyMethodDefType, PyMethodsProtocol};
 use crate::conversion::{AsPyPointer, FromPyPointer, ToPyObject};
 use crate::exceptions::RuntimeError;
 use crate::pyclass_slots::{PyClassDict, PyClassWeakRef};
-use crate::type_object::{type_flags, PyConcreteObject, PyTypeObject};
+use crate::type_object::{type_flags, PyObjectLayout, PyObjectSizedLayout, PyTypeObject};
 use crate::types::PyAny;
 use crate::{class, ffi, gil, PyErr, PyObject, PyResult, PyTypeInfo, Python};
 use std::ffi::CString;
 use std::mem::ManuallyDrop;
 use std::os::raw::c_void;
 use std::ptr::{self, NonNull};
 
+#[inline]
+pub(crate) unsafe fn default_alloc<T: PyTypeInfo>() -> *mut ffi::PyObject {
+    let tp_ptr = T::type_object();
+    if T::FLAGS & type_flags::EXTENDED != 0
+        && <T::BaseType as PyTypeInfo>::ConcreteLayout::IS_NATIVE_TYPE
+    {
+        let base_tp_ptr = <T::BaseType as PyTypeInfo>::type_object();
+        if let Some(base_new) = (*base_tp_ptr).tp_new {
+            return base_new(tp_ptr, ptr::null_mut(), ptr::null_mut());
+        }
+    }
+    let alloc = (*tp_ptr).tp_alloc.unwrap_or(ffi::PyType_GenericAlloc);
+    alloc(tp_ptr, 0)
+}
+
 /// A trait that enables custome alloc/dealloc implementations for pyclasses.
 pub trait PyClassAlloc: PyTypeInfo + Sized {
     unsafe fn alloc(_py: Python) -> *mut Self::ConcreteLayout {
-        let tp_ptr = Self::type_object();
-        let alloc = (*tp_ptr).tp_alloc.unwrap_or(ffi::PyType_GenericAlloc);
-        alloc(tp_ptr, 0) as _
+        default_alloc::<Self>() as _
     }
 
     unsafe fn dealloc(py: Python, self_: *mut Self::ConcreteLayout) {
@@ -49,7 +62,9 @@ pub unsafe fn tp_free_fallback(obj: *mut ffi::PyObject) {
     }
 }
 
-pub trait PyClass: PyTypeInfo + Sized + PyClassAlloc + PyMethodsProtocol {
+pub trait PyClass:
+    PyTypeInfo<ConcreteLayout = PyClassShell<Self>> + Sized + PyClassAlloc + PyMethodsProtocol
+{
     type Dict: PyClassDict;
     type WeakRef: PyClassWeakRef;
 }
@@ -88,7 +103,8 @@ pub struct PyClassShell<T: PyClass> {
 impl<T: PyClass> PyClassShell<T> {
     pub fn new_ref(py: Python, value: impl IntoInitializer<T>) -> PyResult<&Self>
     where
-        T: PyTypeInfo<ConcreteLayout = Self>,
+        <T::BaseType as PyTypeInfo>::ConcreteLayout:
+            crate::type_object::PyObjectSizedLayout<T::BaseType>,
     {
         unsafe {
             let initializer = value.into_initializer()?;
@@ -99,7 +115,8 @@ impl<T: PyClass> PyClassShell<T> {
 
     pub fn new_mut(py: Python, value: impl IntoInitializer<T>) -> PyResult<&mut Self>
     where
-        T: PyTypeInfo<ConcreteLayout = Self>,
+        <T::BaseType as PyTypeInfo>::ConcreteLayout:
+            crate::type_object::PyObjectSizedLayout<T::BaseType>,
     {
         unsafe {
             let initializer = value.into_initializer()?;
@@ -109,7 +126,11 @@ impl<T: PyClass> PyClassShell<T> {
     }
 
     #[doc(hidden)]
-    unsafe fn new(py: Python) -> PyResult<*mut Self> {
+    unsafe fn new(py: Python) -> PyResult<*mut Self>
+    where
+        <T::BaseType as PyTypeInfo>::ConcreteLayout:
+            crate::type_object::PyObjectSizedLayout<T::BaseType>,
+    {
         <T::BaseType as PyTypeObject>::init_type();
         T::init_type();
         let base = T::alloc(py);
@@ -123,8 +144,12 @@ impl<T: PyClass> PyClassShell<T> {
     }
 }
 
-impl<T: PyClass> PyConcreteObject<T> for PyClassShell<T> {
+impl<T: PyClass> PyObjectLayout<T> for PyClassShell<T> {
     const NEED_INIT: bool = std::mem::size_of::<T>() != 0;
+    const IS_NATIVE_TYPE: bool = false;
+    fn get_super(&mut self) -> Option<&mut <T::BaseType as PyTypeInfo>::ConcreteLayout> {
+        Some(&mut self.ob_base)
+    }
     unsafe fn internal_ref_cast(obj: &PyAny) -> &T {
         let shell = obj.as_ptr() as *const PyClassShell<T>;
         &(*shell).pyclass
@@ -142,11 +167,10 @@ impl<T: PyClass> PyConcreteObject<T> for PyClassShell<T> {
     unsafe fn py_init(&mut self, value: T) {
         self.pyclass = ManuallyDrop::new(value);
     }
-    fn get_super(&mut self) -> Option<&mut <T::BaseType as PyTypeInfo>::ConcreteLayout> {
-        Some(&mut self.ob_base)
-    }
 }
 
+impl<T: PyClass> PyObjectSizedLayout<T> for PyClassShell<T> {}
+
 impl<T: PyClass> AsPyPointer for PyClassShell<T> {
     fn as_ptr(&self) -> *mut ffi::PyObject {
         (self as *const _) as *mut _
@@ -208,8 +232,6 @@ where
 }
 
 /// An initializer for `PyClassShell<T>`.
-///
-/// **NOTE** If
 pub struct PyClassInitializer<T: PyTypeInfo> {
     init: Option<T>,
     super_init: Option<*mut PyClassInitializer<T::BaseType>>,
@@ -273,7 +295,9 @@ impl<T: PyTypeInfo> PyClassInitializer<T> {
 
     pub unsafe fn create_shell(self, py: Python) -> PyResult<*mut PyClassShell<T>>
     where
-        T: PyClass + PyTypeInfo<ConcreteLayout = PyClassShell<T>>,
+        T: PyClass,
+        <T::BaseType as PyTypeInfo>::ConcreteLayout:
+            crate::type_object::PyObjectSizedLayout<T::BaseType>,
     {
         let shell = PyClassShell::new(py)?;
         self.init_class(&mut *shell)?;