pyo3_path, part 2: add pyo3_path options and use them.

Author: birkenfeld

CHANGELOG.md

@@ -21,6 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add `Py::setattr` method. [#2009](https://github.com/PyO3/pyo3/pull/2009)
 - Add `PyCapsule`, exposing the [Capsule API](https://docs.python.org/3/c-api/capsule.html#capsules). [#1980](https://github.com/PyO3/pyo3/pull/1980)
+- All PyO3 proc-macros except the deprecated `#[pyproto]` now accept a supplemental attribute `#[pyo3(pyo3_path = "some::path")]` that specifies
+  where to find the `pyo3` crate, in case it has been renamed or is re-exported and not found at the crate root. [#2022](https://github.com/PyO3/pyo3/pull/2022)
 
 ### Changed
 

guide/src/faq.md

@@ -139,3 +139,23 @@ a: <builtins.Inner object at 0x0000020044FCC670>
 b: <builtins.Inner object at 0x0000020044FCC670>
 ```
 The downside to this approach is that any Rust code working on the `Outer` struct now has to acquire the GIL to do anything with its field.
+
+## I want to use the `pyo3` crate re-exported from from dependency but the proc-macros fail!
+
+All PyO3 proc-macros (`#[pyclass]`, `#[pyfunction]`, `#[derive(FromPyObject)]`
+and so on) expect the `pyo3` crate to be available under that name in your crate
+root, which is the normal situation when `pyo3` is a direct dependency of your
+crate.
+
+However, when the dependency is renamed, or your crate only indirectly depends
+on `pyo3`, you need to let the macro code know where to find the crate.  This is
+done with the `pyo3_path` attribute:
+
+```rust
+# use pyo3::prelude::*;
+# pub extern crate pyo3;
+# mod reexported { pub use ::pyo3; }
+#[pyclass]
+#[pyo3(pyo3_path = "reexported::pyo3")]
+struct MyClass;
+```

pyo3-macros-backend/src/attributes.rs

@@ -2,7 +2,7 @@ use syn::{
     parse::{Parse, ParseStream},
     punctuated::Punctuated,
     token::Comma,
-    Attribute, ExprPath, Ident, LitStr, Result, Token,
+    Attribute, ExprPath, Ident, LitStr, Path, Result, Token,
 };
 
 pub mod kw {
@@ -17,6 +17,7 @@ pub mod kw {
     syn::custom_keyword!(signature);
     syn::custom_keyword!(text_signature);
     syn::custom_keyword!(transparent);
+    syn::custom_keyword!(pyo3_path);
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -43,6 +44,19 @@ impl Parse for NameAttribute {
     }
 }
 
+/// For specifying the path to the pyo3 crate.
+#[derive(Clone, Debug, PartialEq)]
+pub struct PyO3PathAttribute(pub Path);
+
+impl Parse for PyO3PathAttribute {
+    fn parse(input: ParseStream) -> Result<Self> {
+        let _: kw::pyo3_path = input.parse()?;
+        let _: Token![=] = input.parse()?;
+        let string_literal: LitStr = input.parse()?;
+        string_literal.parse().map(PyO3PathAttribute)
+    }
+}
+
 #[derive(Clone, Debug, PartialEq)]
 pub struct TextSignatureAttribute {
     pub kw: kw::text_signature,

pyo3-macros-backend/src/from_pyobject.rs

@@ -1,4 +1,7 @@
-use crate::attributes::{self, get_pyo3_options, FromPyWithAttribute};
+use crate::{
+    attributes::{self, get_pyo3_options, FromPyWithAttribute, PyO3PathAttribute},
+    utils::get_pyo3_path,
+};
 use proc_macro2::TokenStream;
 use quote::quote;
 use syn::{
@@ -300,20 +303,25 @@ impl<'a> Container<'a> {
     }
 }
 
+#[derive(Default)]
 struct ContainerOptions {
     /// Treat the Container as a Wrapper, directly extract its fields from the input object.
     transparent: bool,
     /// Change the name of an enum variant in the generated error message.
     annotation: Option<syn::LitStr>,
+    /// Change the path for the pyo3 crate
+    pyo3_path: Option<PyO3PathAttribute>,
 }
 
 /// Attributes for deriving FromPyObject scoped on containers.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Debug)]
 enum ContainerPyO3Attribute {
     /// Treat the Container as a Wrapper, directly extract its fields from the input object.
     Transparent(attributes::kw::transparent),
     /// Change the name of an enum variant in the generated error message.
     ErrorAnnotation(LitStr),
+    /// Change the path for the pyo3 crate
+    PyO3Path(PyO3PathAttribute),
 }
 
 impl Parse for ContainerPyO3Attribute {
@@ -326,6 +334,8 @@ impl Parse for ContainerPyO3Attribute {
             let _: attributes::kw::annotation = input.parse()?;
             let _: Token![=] = input.parse()?;
             input.parse().map(ContainerPyO3Attribute::ErrorAnnotation)
+        } else if lookahead.peek(attributes::kw::pyo3_path) {
+            input.parse().map(ContainerPyO3Attribute::PyO3Path)
         } else {
             Err(lookahead.error())
         }
@@ -334,10 +344,8 @@ impl Parse for ContainerPyO3Attribute {
 
 impl ContainerOptions {
     fn from_attrs(attrs: &[Attribute]) -> Result<Self> {
-        let mut options = ContainerOptions {
-            transparent: false,
-            annotation: None,
-        };
+        let mut options = ContainerOptions::default();
+
         for attr in attrs {
             if let Some(pyo3_attrs) = get_pyo3_options(attr)? {
                 for pyo3_attr in pyo3_attrs {
@@ -356,6 +364,13 @@ impl ContainerOptions {
                             );
                             options.annotation = Some(lit_str);
                         }
+                        ContainerPyO3Attribute::PyO3Path(path) => {
+                            ensure_spanned!(
+                                options.pyo3_path.is_none(),
+                                path.0.span() => "`pyo3_path` may only be provided once"
+                            );
+                            options.pyo3_path = Some(path);
+                        }
                     }
                 }
             }
@@ -499,13 +514,18 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result<TokenStream> {
             .predicates
             .push(parse_quote!(#gen_ident: FromPyObject<#lt_param>))
     }
+    let options = ContainerOptions::from_attrs(&tokens.attrs)?;
+    let pyo3_path = get_pyo3_path(&options.pyo3_path);
     let derives = match &tokens.data {
         syn::Data::Enum(en) => {
+            if options.transparent || options.annotation.is_some() {
+                bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \
+                                                at top level for enums");
+            }
             let en = Enum::new(en, &tokens.ident)?;
             en.build()
         }
         syn::Data::Struct(st) => {
-            let options = ContainerOptions::from_attrs(&tokens.attrs)?;
             if let Some(lit_str) = &options.annotation {
                 bail_spanned!(lit_str.span() => "`annotation` is unsupported for structs");
             }
@@ -520,11 +540,15 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result<TokenStream> {
 
     let ident = &tokens.ident;
     Ok(quote!(
-        #[automatically_derived]
-        impl#trait_generics _pyo3::FromPyObject<#lt_param> for #ident#generics #where_clause {
-            fn extract(obj: &#lt_param _pyo3::PyAny) -> _pyo3::PyResult<Self>  {
-                #derives
+        const _: () = {
+            use #pyo3_path as _pyo3;
+
+            #[automatically_derived]
+            impl#trait_generics _pyo3::FromPyObject<#lt_param> for #ident#generics #where_clause {
+                fn extract(obj: &#lt_param _pyo3::PyAny) -> _pyo3::PyResult<Self>  {
+                    #derives
+                }
             }
-        }
+        };
     ))
 }

pyo3-macros-backend/src/method.rs

@@ -5,7 +5,7 @@ use crate::deprecations::Deprecation;
 use crate::params::{accept_args_kwargs, impl_arg_params};
 use crate::pyfunction::PyFunctionOptions;
 use crate::pyfunction::{PyFunctionArgPyO3Attributes, PyFunctionSignature};
-use crate::utils::{self, PythonDoc};
+use crate::utils::{self, get_pyo3_path, PythonDoc};
 use crate::{deprecations::Deprecations, pyfunction::Argument};
 use proc_macro2::{Span, TokenStream};
 use quote::ToTokens;
@@ -228,6 +228,7 @@ pub struct FnSpec<'a> {
     pub deprecations: Deprecations,
     pub convention: CallingConvention,
     pub text_signature: Option<TextSignatureAttribute>,
+    pub pyo3_path: syn::Path,
 }
 
 pub fn get_return_info(output: &syn::ReturnType) -> syn::Type {
@@ -254,12 +255,14 @@ pub fn parse_method_receiver(arg: &syn::FnArg) -> Result<SelfType> {
 impl<'a> FnSpec<'a> {
     /// Parser function signature and function attributes
     pub fn parse(
+        // Signature is mutable to remove the `Python` argument.
         sig: &'a mut syn::Signature,
         meth_attrs: &mut Vec<syn::Attribute>,
         options: PyFunctionOptions,
     ) -> Result<FnSpec<'a>> {
         let PyFunctionOptions {
             text_signature,
+            pyo3_path,
             name,
             mut deprecations,
             ..
@@ -278,6 +281,7 @@ impl<'a> FnSpec<'a> {
         let name = &sig.ident;
         let ty = get_return_info(&sig.output);
         let python_name = python_name.as_ref().unwrap_or(name).unraw();
+        let pyo3_path = get_pyo3_path(&pyo3_path);
 
         let doc = utils::get_doc(
             meth_attrs,
@@ -311,6 +315,7 @@ impl<'a> FnSpec<'a> {
             doc,
             deprecations,
             text_signature,
+            pyo3_path,
         })
     }
 
@@ -472,14 +477,16 @@ impl<'a> FnSpec<'a> {
         };
         let rust_call =
             quote! { _pyo3::callback::convert(#py, #rust_name(#self_arg #(#arg_names),*)) };
+        let pyo3_path = &self.pyo3_path;
         Ok(match self.convention {
             CallingConvention::Noargs => {
                 quote! {
                     unsafe extern "C" fn #ident (
-                        _slf: *mut _pyo3::ffi::PyObject,
-                        _args: *mut _pyo3::ffi::PyObject,
-                    ) -> *mut _pyo3::ffi::PyObject
+                        _slf: *mut #pyo3_path::ffi::PyObject,
+                        _args: *mut #pyo3_path::ffi::PyObject,
+                    ) -> *mut #pyo3_path::ffi::PyObject
                     {
+                        use #pyo3_path as _pyo3;
                         #deprecations
                         _pyo3::callback::handle_panic(|#py| {
                             #self_conversion
@@ -492,11 +499,12 @@ impl<'a> FnSpec<'a> {
                 let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, true)?;
                 quote! {
                     unsafe extern "C" fn #ident (
-                        _slf: *mut _pyo3::ffi::PyObject,
-                        _args: *const *mut _pyo3::ffi::PyObject,
-                        _nargs: _pyo3::ffi::Py_ssize_t,
-                        _kwnames: *mut _pyo3::ffi::PyObject) -> *mut _pyo3::ffi::PyObject
+                        _slf: *mut #pyo3_path::ffi::PyObject,
+                        _args: *const *mut #pyo3_path::ffi::PyObject,
+                        _nargs: #pyo3_path::ffi::Py_ssize_t,
+                        _kwnames: *mut #pyo3_path::ffi::PyObject) -> *mut #pyo3_path::ffi::PyObject
                     {
+                        use #pyo3_path as _pyo3;
                         #deprecations
                         _pyo3::callback::handle_panic(|#py| {
                             #self_conversion
@@ -519,10 +527,11 @@ impl<'a> FnSpec<'a> {
                 let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, false)?;
                 quote! {
                     unsafe extern "C" fn #ident (
-                        _slf: *mut _pyo3::ffi::PyObject,
-                        _args: *mut _pyo3::ffi::PyObject,
-                        _kwargs: *mut _pyo3::ffi::PyObject) -> *mut _pyo3::ffi::PyObject
+                        _slf: *mut #pyo3_path::ffi::PyObject,
+                        _args: *mut #pyo3_path::ffi::PyObject,
+                        _kwargs: *mut #pyo3_path::ffi::PyObject) -> *mut #pyo3_path::ffi::PyObject
                     {
+                        use #pyo3_path as _pyo3;
                         #deprecations
                         _pyo3::callback::handle_panic(|#py| {
                             #self_conversion
@@ -539,10 +548,11 @@ impl<'a> FnSpec<'a> {
                 let arg_convert_and_rust_call = impl_arg_params(self, cls, rust_call, &py, false)?;
                 quote! {
                     unsafe extern "C" fn #ident (
-                        subtype: *mut _pyo3::ffi::PyTypeObject,
-                        _args: *mut _pyo3::ffi::PyObject,
-                        _kwargs: *mut _pyo3::ffi::PyObject) -> *mut _pyo3::ffi::PyObject
+                        subtype: *mut #pyo3_path::ffi::PyTypeObject,
+                        _args: *mut #pyo3_path::ffi::PyObject,
+                        _kwargs: *mut #pyo3_path::ffi::PyObject) -> *mut #pyo3_path::ffi::PyObject
                     {
+                        use #pyo3_path as _pyo3;
                         #deprecations
                         use _pyo3::callback::IntoPyCallbackOutput;
                         _pyo3::callback::handle_panic(|#py| {

pyo3-macros-backend/src/module.rs

@@ -2,9 +2,12 @@
 //! Code generation for the function that initializes a python module and adds classes and function.
 
 use crate::{
-    attributes::{self, is_attribute_ident, take_attributes, take_pyo3_options, NameAttribute},
+    attributes::{
+        self, is_attribute_ident, take_attributes, take_pyo3_options, NameAttribute,
+        PyO3PathAttribute,
+    },
     pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
-    utils::PythonDoc,
+    utils::{get_pyo3_path, PythonDoc},
 };
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
@@ -16,17 +19,20 @@ use syn::{
     Ident, Path, Result,
 };
 
+#[derive(Default)]
 pub struct PyModuleOptions {
+    pyo3_path: Option<PyO3PathAttribute>,
     name: Option<syn::Ident>,
 }
 
 impl PyModuleOptions {
     pub fn from_attrs(attrs: &mut Vec<syn::Attribute>) -> Result<Self> {
-        let mut options: PyModuleOptions = PyModuleOptions { name: None };
+        let mut options: PyModuleOptions = Default::default();
 
         for option in take_pyo3_options(attrs)? {
             match option {
                 PyModulePyO3Option::Name(name) => options.set_name(name.0)?,
+                PyModulePyO3Option::PyO3Path(path) => options.set_pyo3_path(path)?,
             }
         }
 
@@ -42,20 +48,32 @@ impl PyModuleOptions {
         self.name = Some(name);
         Ok(())
     }
+
+    fn set_pyo3_path(&mut self, path: PyO3PathAttribute) -> Result<()> {
+        ensure_spanned!(
+            self.pyo3_path.is_none(),
+            path.0.span() => "`pyo3_path` may only be specified once"
+        );
+
+        self.pyo3_path = Some(path);
+        Ok(())
+    }
 }
 
 /// Generates the function that is called by the python interpreter to initialize the native
 /// module
 pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: PythonDoc) -> TokenStream {
     let name = options.name.unwrap_or_else(|| fnname.unraw());
+    let pyo3_path = get_pyo3_path(&options.pyo3_path);
     let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site());
 
     quote! {
         #[no_mangle]
         #[allow(non_snake_case)]
         /// This autogenerated function is called by the python interpreter when importing
         /// the module.
-        pub unsafe extern "C" fn #cb_name() -> *mut _pyo3::ffi::PyObject {
+        pub unsafe extern "C" fn #cb_name() -> *mut #pyo3_path::ffi::PyObject {
+            use #pyo3_path as _pyo3;
             use _pyo3::derive_utils::ModuleDef;
             static NAME: &str = concat!(stringify!(#name), "\0");
             static DOC: &str = #doc;
@@ -143,6 +161,7 @@ fn get_pyfn_attr(attrs: &mut Vec<syn::Attribute>) -> syn::Result<Option<PyFnArgs
 }
 
 enum PyModulePyO3Option {
+    PyO3Path(PyO3PathAttribute),
     Name(NameAttribute),
 }
 
@@ -151,6 +170,8 @@ impl Parse for PyModulePyO3Option {
         let lookahead = input.lookahead1();
         if lookahead.peek(attributes::kw::name) {
             input.parse().map(PyModulePyO3Option::Name)
+        } else if lookahead.peek(attributes::kw::pyo3_path) {
+            input.parse().map(PyModulePyO3Option::PyO3Path)
         } else {
             Err(lookahead.error())
         }