Merge pull request #2081 from davidhewitt/wrap-paths

macros: accept paths in wrap_x macros

Author: davidhewitt

CHANGELOG.md

@@ -12,10 +12,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Update MSRV to Rust 1.48. [#2004](https://github.com/PyO3/pyo3/pull/2004)
 - Update `indoc` optional dependency to 1.0. [#2004](https://github.com/PyO3/pyo3/pull/2004)
-- Update `paste` optional dependency to 1.0. [#2004](https://github.com/PyO3/pyo3/pull/2004)
 - Drop support for Python 3.6, remove `abi3-py36` feature. [#2006](https://github.com/PyO3/pyo3/pull/2006)
 - `pyo3-build-config` no longer enables the `resolve-config` feature by default. [#2008](https://github.com/PyO3/pyo3/pull/2008)
 - Update `inventory` optional dependency to 0.2. [#2019](https://github.com/PyO3/pyo3/pull/2019)
+- Drop `paste` dependency. [#2081](https://github.com/PyO3/pyo3/pull/2081)
 
 ### Added
 
@@ -24,6 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - All PyO3 proc-macros except the deprecated `#[pyproto]` now accept a supplemental attribute `#[pyo3(crate = "some::path")]` that specifies
   where to find the `pyo3` crate, in case it has been renamed or is re-exported and not found at the crate root. [#2022](https://github.com/PyO3/pyo3/pull/2022)
 - Expose `pyo3-build-config` APIs for cross-compiling and Python configuration discovery for use in other projects. [#1996](https://github.com/PyO3/pyo3/pull/1996)
+- Accept paths in `wrap_pyfunction` and `wrap_pymodule`. [#2081](https://github.com/PyO3/pyo3/pull/2081)
 
 ### Changed
 
@@ -55,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix memory leak in `PyErr::into_value`. [#2026](https://github.com/PyO3/pyo3/pull/2026)
 - Fix clippy warning `needless-option-as-deref` in code generated by `#[pyfunction]` and `#[pymethods]`. [#2040](https://github.com/PyO3/pyo3/pull/2040)
 - Fix undefined behavior in `PySlice::indices`. [#2061](https://github.com/PyO3/pyo3/pull/2061)
+- Use the Rust function path for `wrap_pymodule!` of a `#[pymodule]` with a `#[pyo3(name = "..")]` attribute, not the Python name. [#2081](https://github.com/PyO3/pyo3/pull/2081)
 
 ## [0.15.1] - 2021-11-19
 

Cargo.toml

@@ -22,7 +22,6 @@ parking_lot = "0.11.0"
 # support crates for macros feature
 pyo3-macros = { path = "pyo3-macros", version = "=0.15.1", optional = true }
 indoc = { version = "1.0.3", optional = true }
-paste = { version = "1.0.6", optional = true }
 unindent = { version = "0.1.4", optional = true }
 
 # support crate for multiple-pymethods feature
@@ -53,7 +52,7 @@ pyo3-build-config = { path = "pyo3-build-config", version = "0.15.1", features =
 default = ["macros"]
 
 # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
-macros = ["pyo3-macros", "indoc", "paste", "unindent"]
+macros = ["pyo3-macros", "indoc", "unindent"]
 
 # Enables multiple #[pymethods] per #[pyclass]
 multiple-pymethods = ["inventory", "pyo3-macros/multiple-pymethods"]

pyo3-macros-backend/src/lib.rs

@@ -22,11 +22,13 @@ mod pyfunction;
 mod pyimpl;
 mod pymethod;
 mod pyproto;
+mod wrap;
 
 pub use frompyobject::build_derive_from_pyobject;
-pub use module::{process_functions_in_module, py_init, PyModuleOptions};
+pub use module::{process_functions_in_module, pymodule_impl, PyModuleOptions};
 pub use pyclass::{build_py_class, build_py_enum, PyClassArgs};
 pub use pyfunction::{build_py_function, PyFunctionOptions};
 pub use pyimpl::{build_py_methods, PyClassMethodsType};
 pub use pyproto::build_py_proto;
 pub use utils::get_doc;
+pub use wrap::{wrap_pyfunction_impl, wrap_pymodule_impl, WrapPyFunctionArgs};

pyo3-macros-backend/src/module.rs

@@ -7,6 +7,7 @@ use crate::{
     },
     pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
     utils::{get_pyo3_crate, PythonDoc},
+    wrap::module_def_ident,
 };
 use proc_macro2::{Span, TokenStream};
 use quote::quote;
@@ -15,7 +16,7 @@ use syn::{
     parse::{Parse, ParseStream},
     spanned::Spanned,
     token::Comma,
-    Ident, Path, Result,
+    Ident, Path, Result, Visibility,
 };
 
 #[derive(Default)]
@@ -61,25 +62,32 @@ impl PyModuleOptions {
 
 /// Generates the function that is called by the python interpreter to initialize the native
 /// module
-pub fn py_init(fnname: &Ident, options: PyModuleOptions, doc: PythonDoc) -> TokenStream {
+pub fn pymodule_impl(
+    fnname: &Ident,
+    options: PyModuleOptions,
+    doc: PythonDoc,
+    visibility: &Visibility,
+) -> TokenStream {
     let name = options.name.unwrap_or_else(|| fnname.unraw());
     let krate = get_pyo3_crate(&options.krate);
     let cb_name = Ident::new(&format!("PyInit_{}", name), Span::call_site());
 
+    let module_def_name = module_def_ident(fnname);
+
     quote! {
         #[no_mangle]
         #[allow(non_snake_case)]
         /// This autogenerated function is called by the python interpreter when importing
         /// the module.
         pub unsafe extern "C" fn #cb_name() -> *mut #krate::ffi::PyObject {
-            use #krate as _pyo3;
-            use _pyo3::derive_utils::ModuleDef;
-            static NAME: &str = concat!(stringify!(#name), "\0");
-            static DOC: &str = #doc;
-            static MODULE_DEF: ModuleDef = unsafe { ModuleDef::new(NAME, DOC) };
-
-            _pyo3::callback::handle_panic(|_py| { MODULE_DEF.make_module(_py, #fnname) })
+            use #krate::{self as _pyo3, IntoPyPointer};
+            _pyo3::callback::handle_panic(|_py| ::std::result::Result::Ok(#module_def_name.make_module(_py)?.into_ptr()))
         }
+
+        #[doc(hidden)]
+        #visibility static #module_def_name: #krate::impl_::pymodule::ModuleDef = unsafe {
+            #krate::impl_::pymodule::ModuleDef::new(concat!(stringify!(#name), "\0"), #doc, #krate::impl_::pymodule::ModuleInitializer(#fnname))
+        };
     }
 }
 

pyo3-macros-backend/src/pyfunction.rs

@@ -9,6 +9,7 @@ use crate::{
     method::{self, CallingConvention, FnArg},
     pymethod::check_generic,
     utils::{self, ensure_not_async_fn, get_pyo3_crate},
+    wrap::function_wrapper_ident,
 };
 use proc_macro2::{Span, TokenStream};
 use quote::{format_ident, quote};
@@ -366,12 +367,6 @@ pub fn build_py_function(
     Ok(impl_wrap_pyfunction(ast, options)?.1)
 }
 
-/// Coordinates the naming of a the add-function-to-python-module function
-fn function_wrapper_ident(name: &Ident) -> Ident {
-    // Make sure this ident matches the one of wrap_pyfunction
-    format_ident!("__pyo3_get_function_{}", name)
-}
-
 /// Generates python wrapper over a function that allows adding it to a python module as a python
 /// function
 pub fn impl_wrap_pyfunction(

pyo3-macros-backend/src/wrap.rs

@@ -0,0 +1,67 @@
+use proc_macro2::TokenStream;
+use quote::{format_ident, quote};
+use syn::{parse::Parse, spanned::Spanned, Ident, Token};
+
+pub struct WrapPyFunctionArgs {
+    function: syn::Path,
+    comma_and_arg: Option<(Token![,], syn::Expr)>,
+}
+
+impl Parse for WrapPyFunctionArgs {
+    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
+        let function = input.parse()?;
+        let comma_and_arg = if !input.is_empty() {
+            Some((input.parse()?, input.parse()?))
+        } else {
+            None
+        };
+        Ok(Self {
+            function,
+            comma_and_arg,
+        })
+    }
+}
+
+pub fn wrap_pyfunction_impl(args: WrapPyFunctionArgs) -> syn::Result<TokenStream> {
+    let WrapPyFunctionArgs {
+        mut function,
+        comma_and_arg,
+    } = args;
+    let span = function.span();
+    let last_segment = function
+        .segments
+        .last_mut()
+        .ok_or_else(|| err_spanned!(span => "expected non-empty path"))?;
+
+    last_segment.ident = function_wrapper_ident(&last_segment.ident);
+
+    let output = if let Some((_, arg)) = comma_and_arg {
+        quote! { #function(#arg) }
+    } else {
+        quote! { &|arg| #function(arg) }
+    };
+    Ok(output)
+}
+
+pub fn wrap_pymodule_impl(mut module_path: syn::Path) -> syn::Result<TokenStream> {
+    let span = module_path.span();
+    let last_segment = module_path
+        .segments
+        .last_mut()
+        .ok_or_else(|| err_spanned!(span => "expected non-empty path"))?;
+
+    last_segment.ident = module_def_ident(&last_segment.ident);
+
+    Ok(quote! {
+
+        &|py| unsafe { #module_path.make_module(py).expect("failed to wrap pymodule") }
+    })
+}
+
+pub(crate) fn function_wrapper_ident(name: &Ident) -> Ident {
+    format_ident!("__pyo3_get_function_{}", name)
+}
+
+pub(crate) fn module_def_ident(name: &Ident) -> Ident {
+    format_ident!("__PYO3_PYMODULE_DEF_{}", name.to_string().to_uppercase())
+}

pyo3-macros/Cargo.toml

@@ -17,6 +17,7 @@ proc-macro = true
 multiple-pymethods = []
 
 [dependencies]
+proc-macro2 = { version = "1", default-features = false }
 quote = "1"
 syn = { version = "1", features = ["full", "extra-traits"] }
 pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.15.1" }

pyo3-macros/src/lib.rs

@@ -6,10 +6,12 @@
 extern crate proc_macro;
 
 use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
 use pyo3_macros_backend::{
     build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods,
-    build_py_proto, get_doc, process_functions_in_module, py_init, PyClassArgs, PyClassMethodsType,
-    PyFunctionOptions, PyModuleOptions,
+    build_py_proto, get_doc, process_functions_in_module, pymodule_impl, wrap_pyfunction_impl,
+    wrap_pymodule_impl, PyClassArgs, PyClassMethodsType, PyFunctionOptions, PyModuleOptions,
+    WrapPyFunctionArgs,
 };
 use quote::quote;
 use syn::{parse::Nothing, parse_macro_input};
@@ -46,7 +48,7 @@ pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream {
 
     let doc = get_doc(&ast.attrs, None);
 
-    let expanded = py_init(&ast.sig.ident, options, doc);
+    let expanded = pymodule_impl(&ast.sig.ident, options, doc, &ast.vis);
 
     quote!(
         #ast
@@ -68,7 +70,7 @@ pub fn pymodule(args: TokenStream, input: TokenStream) -> TokenStream {
 #[proc_macro_attribute]
 pub fn pyproto(_: TokenStream, input: TokenStream) -> TokenStream {
     let mut ast = parse_macro_input!(input as syn::ItemImpl);
-    let expanded = build_py_proto(&mut ast).unwrap_or_else(|e| e.to_compile_error());
+    let expanded = build_py_proto(&mut ast).unwrap_or_compile_error();
 
     quote!(
         #ast
@@ -180,7 +182,7 @@ pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream {
     let mut ast = parse_macro_input!(input as syn::ItemFn);
     let options = parse_macro_input!(attr as PyFunctionOptions);
 
-    let expanded = build_py_function(&mut ast, options).unwrap_or_else(|e| e.to_compile_error());
+    let expanded = build_py_function(&mut ast, options).unwrap_or_compile_error();
 
     quote!(
         #ast
@@ -192,21 +194,39 @@ pub fn pyfunction(attr: TokenStream, input: TokenStream) -> TokenStream {
 #[proc_macro_derive(FromPyObject, attributes(pyo3))]
 pub fn derive_from_py_object(item: TokenStream) -> TokenStream {
     let ast = parse_macro_input!(item as syn::DeriveInput);
-    let expanded = build_derive_from_pyobject(&ast).unwrap_or_else(|e| e.to_compile_error());
+    let expanded = build_derive_from_pyobject(&ast).unwrap_or_compile_error();
     quote!(
         #expanded
     )
     .into()
 }
 
+/// Wraps a Rust function annotated with [`#[pyfunction]`](macro@crate::pyfunction).
+///
+/// This can be used with `PyModule::add_function` to add free functions to a `PyModule` - see its
+/// documentation for more information.
+#[proc_macro]
+pub fn wrap_pyfunction(input: TokenStream) -> TokenStream {
+    let args = parse_macro_input!(input as WrapPyFunctionArgs);
+    wrap_pyfunction_impl(args).unwrap_or_compile_error().into()
+}
+
+/// Returns a function that takes a `Python` instance and returns a Python module.
+///
+/// Use this together with [`#[pymodule]`](macro@crate::pymodule) and `PyModule::add_wrapped`.
+#[proc_macro]
+pub fn wrap_pymodule(input: TokenStream) -> TokenStream {
+    let path = parse_macro_input!(input as syn::Path);
+    wrap_pymodule_impl(path).unwrap_or_compile_error().into()
+}
+
 fn pyclass_impl(
     attrs: TokenStream,
     mut ast: syn::ItemStruct,
     methods_type: PyClassMethodsType,
 ) -> TokenStream {
     let args = parse_macro_input!(attrs with PyClassArgs::parse_stuct_args);
-    let expanded =
-        build_py_class(&mut ast, &args, methods_type).unwrap_or_else(|e| e.to_compile_error());
+    let expanded = build_py_class(&mut ast, &args, methods_type).unwrap_or_compile_error();
 
     quote!(
         #ast
@@ -221,8 +241,7 @@ fn pyclass_enum_impl(
     methods_type: PyClassMethodsType,
 ) -> TokenStream {
     let args = parse_macro_input!(attrs with PyClassArgs::parse_enum_args);
-    let expanded =
-        build_py_enum(&mut ast, &args, methods_type).unwrap_or_else(|e| e.into_compile_error());
+    let expanded = build_py_enum(&mut ast, &args, methods_type).unwrap_or_compile_error();
 
     quote!(
         #ast
@@ -233,8 +252,7 @@ fn pyclass_enum_impl(
 
 fn pymethods_impl(input: TokenStream, methods_type: PyClassMethodsType) -> TokenStream {
     let mut ast = parse_macro_input!(input as syn::ItemImpl);
-    let expanded =
-        build_py_methods(&mut ast, methods_type).unwrap_or_else(|e| e.to_compile_error());
+    let expanded = build_py_methods(&mut ast, methods_type).unwrap_or_compile_error();
 
     quote!(
         #ast
@@ -250,3 +268,13 @@ fn methods_type() -> PyClassMethodsType {
         PyClassMethodsType::Specialization
     }
 }
+
+trait UnwrapOrCompileError {
+    fn unwrap_or_compile_error(self) -> TokenStream2;
+}
+
+impl UnwrapOrCompileError for syn::Result<TokenStream2> {
+    fn unwrap_or_compile_error(self) -> TokenStream2 {
+        self.unwrap_or_else(|e| e.into_compile_error())
+    }
+}

pytests/pyo3-pytests/src/datetime.rs

@@ -203,7 +203,7 @@ impl TzClass {
 }
 
 #[pymodule]
-fn datetime(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+pub fn datetime(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(make_date, m)?)?;
     m.add_function(wrap_pyfunction!(get_date_tuple, m)?)?;
     m.add_function(wrap_pyfunction!(date_from_timestamp, m)?)?;

pytests/pyo3-pytests/src/dict_iter.rs

@@ -3,7 +3,7 @@ use pyo3::prelude::*;
 use pyo3::types::PyDict;
 
 #[pymodule]
-fn dict_iter(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
+pub fn dict_iter(_py: Python<'_>, m: &PyModule) -> PyResult<()> {
     m.add_class::<DictSize>()?;
     Ok(())
 }

pytests/pyo3-pytests/src/lib.rs

@@ -12,31 +12,19 @@ pub mod path;
 pub mod pyclass_iter;
 pub mod subclassing;
 
-#[cfg(not(Py_LIMITED_API))]
-use buf_and_str::*;
-#[cfg(not(Py_LIMITED_API))]
-use datetime::*;
-use dict_iter::*;
-use misc::*;
-use objstore::*;
-use othermod::*;
-use path::*;
-use pyclass_iter::*;
-use subclassing::*;
-
 #[pymodule]
 fn pyo3_pytests(py: Python, m: &PyModule) -> PyResult<()> {
     #[cfg(not(Py_LIMITED_API))]
-    m.add_wrapped(wrap_pymodule!(buf_and_str))?;
+    m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?;
     #[cfg(not(Py_LIMITED_API))]
-    m.add_wrapped(wrap_pymodule!(datetime))?;
-    m.add_wrapped(wrap_pymodule!(dict_iter))?;
-    m.add_wrapped(wrap_pymodule!(misc))?;
-    m.add_wrapped(wrap_pymodule!(objstore))?;
-    m.add_wrapped(wrap_pymodule!(othermod))?;
-    m.add_wrapped(wrap_pymodule!(path))?;
-    m.add_wrapped(wrap_pymodule!(pyclass_iter))?;
-    m.add_wrapped(wrap_pymodule!(subclassing))?;
+    m.add_wrapped(wrap_pymodule!(datetime::datetime))?;
+    m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?;
+    m.add_wrapped(wrap_pymodule!(misc::misc))?;
+    m.add_wrapped(wrap_pymodule!(objstore::objstore))?;
+    m.add_wrapped(wrap_pymodule!(othermod::othermod))?;
+    m.add_wrapped(wrap_pymodule!(path::path))?;
+    m.add_wrapped(wrap_pymodule!(pyclass_iter::pyclass_iter))?;
+    m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?;
 
     // Inserting to sys.modules allows importing submodules nicely from Python
     // e.g. import pyo3_pytests.buf_and_str as bas