emit c-string literals on Rust 1.77 or later (#4269)

* emit c-string literals on Rust 1.77 or later

* only clone `PyO3CratePath` instead of the whole `Ctx`

Co-authored-by: mejrs <[email protected]>

---------

Co-authored-by: mejrs <[email protected]>
Co-authored-by: David Hewitt <[email protected]>

Author: 3 people

pyo3-build-config/src/lib.rs

@@ -141,6 +141,10 @@ pub fn print_feature_cfgs() {
         println!("cargo:rustc-cfg=invalid_from_utf8_lint");
     }
 
+    if rustc_minor_version >= 77 {
+        println!("cargo:rustc-cfg=c_str_lit");
+    }
+
     // Actually this is available on 1.78, but we should avoid
     // https://github.com/rust-lang/rust/issues/124651 just in case
     if rustc_minor_version >= 79 {
@@ -167,6 +171,7 @@ pub fn print_expected_cfgs() {
     println!("cargo:rustc-check-cfg=cfg(pyo3_disable_reference_pool)");
     println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)");
     println!("cargo:rustc-check-cfg=cfg(diagnostic_namespace)");
+    println!("cargo:rustc-check-cfg=cfg(c_str_lit)");
 
     // allow `Py_3_*` cfgs from the minimum supported version up to the
     // maximum minor version (+1 for development for the next)

pyo3-macros-backend/Cargo.toml

@@ -20,10 +20,13 @@ pyo3-build-config = { path = "../pyo3-build-config", version = "=0.22.0-dev", fe
 quote = { version = "1", default-features = false }
 
 [dependencies.syn]
-version = "2"
+version = "2.0.59" # for `LitCStr`
 default-features = false
 features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"]
 
+[build-dependencies]
+pyo3-build-config = { path = "../pyo3-build-config", version = "=0.22.0-dev" }
+
 [lints]
 workspace = true
 

pyo3-macros-backend/build.rs

@@ -0,0 +1,4 @@
+fn main() {
+    pyo3_build_config::print_expected_cfgs();
+    pyo3_build_config::print_feature_cfgs();
+}

pyo3-macros-backend/src/konst.rs

@@ -1,12 +1,12 @@
 use std::borrow::Cow;
+use std::ffi::CString;
 
-use crate::utils::Ctx;
+use crate::utils::{Ctx, LitCStr};
 use crate::{
     attributes::{self, get_pyo3_options, take_attributes, NameAttribute},
     deprecations::Deprecations,
 };
-use proc_macro2::{Ident, TokenStream};
-use quote::quote;
+use proc_macro2::{Ident, Span};
 use syn::{
     ext::IdentExt,
     parse::{Parse, ParseStream},
@@ -29,10 +29,9 @@ impl ConstSpec<'_> {
     }
 
     /// Null-terminated Python name
-    pub fn null_terminated_python_name(&self, ctx: &Ctx) -> TokenStream {
-        let Ctx { pyo3_path, .. } = ctx;
+    pub fn null_terminated_python_name(&self, ctx: &Ctx) -> LitCStr {
         let name = self.python_name().to_string();
-        quote!(#pyo3_path::ffi::c_str!(#name))
+        LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx)
     }
 }
 

pyo3-macros-backend/src/method.rs

@@ -1,12 +1,13 @@
 use std::borrow::Cow;
+use std::ffi::CString;
 use std::fmt::Display;
 
 use proc_macro2::{Span, TokenStream};
 use quote::{format_ident, quote, quote_spanned, ToTokens};
 use syn::{ext::IdentExt, spanned::Spanned, Ident, Result};
 
 use crate::deprecations::deprecate_trailing_option_default;
-use crate::utils::Ctx;
+use crate::utils::{Ctx, LitCStr};
 use crate::{
     attributes::{FromPyWithAttribute, TextSignatureAttribute, TextSignatureAttributeValue},
     deprecations::{Deprecation, Deprecations},
@@ -472,12 +473,10 @@ impl<'a> FnSpec<'a> {
         })
     }
 
-    pub fn null_terminated_python_name(&self, ctx: &Ctx) -> TokenStream {
-        let Ctx { pyo3_path, .. } = ctx;
-        let span = self.python_name.span();
-        let pyo3_path = pyo3_path.to_tokens_spanned(span);
+    pub fn null_terminated_python_name(&self, ctx: &Ctx) -> LitCStr {
         let name = self.python_name.to_string();
-        quote_spanned!(self.python_name.span() => #pyo3_path::ffi::c_str!(#name))
+        let name = CString::new(name).unwrap();
+        LitCStr::new(name, self.python_name.span(), ctx)
     }
 
     fn parse_fn_type(

pyo3-macros-backend/src/module.rs

@@ -7,10 +7,11 @@ use crate::{
     get_doc,
     pyclass::PyClassPyO3Option,
     pyfunction::{impl_wrap_pyfunction, PyFunctionOptions},
-    utils::Ctx,
+    utils::{Ctx, LitCStr},
 };
-use proc_macro2::TokenStream;
+use proc_macro2::{Span, TokenStream};
 use quote::quote;
+use std::ffi::CString;
 use syn::{
     ext::IdentExt,
     parse::{Parse, ParseStream},
@@ -403,10 +404,11 @@ fn module_initialization(name: &syn::Ident, ctx: &Ctx) -> TokenStream {
     let Ctx { pyo3_path, .. } = ctx;
     let pyinit_symbol = format!("PyInit_{}", name);
     let name = name.to_string();
+    let pyo3_name = LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx);
 
     quote! {
         #[doc(hidden)]
-        pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_path::ffi::c_str!(#name);
+        pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name;
 
         pub(super) struct MakeDef;
         #[doc(hidden)]

pyo3-macros-backend/src/pyclass.rs

@@ -22,7 +22,7 @@ use crate::pymethod::{
     SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, __RICHCMP__,
 };
 use crate::pyversions;
-use crate::utils::{self, apply_renaming_rule, PythonDoc};
+use crate::utils::{self, apply_renaming_rule, LitCStr, PythonDoc};
 use crate::utils::{is_abi3, Ctx};
 use crate::PyFunctionOptions;
 
@@ -2016,7 +2016,7 @@ impl<'a> PyClassImplsBuilder<'a> {
         let Ctx { pyo3_path, .. } = ctx;
         let cls = self.cls;
         let doc = self.doc.as_ref().map_or(
-            quote! {#pyo3_path::ffi::c_str!("")},
+            LitCStr::empty(ctx).to_token_stream(),
             PythonDoc::to_token_stream,
         );
         let is_basetype = self.attr.options.subclass.is_some();

pyo3-macros-backend/src/pymethod.rs

@@ -1,11 +1,12 @@
 use std::borrow::Cow;
+use std::ffi::CString;
 
 use crate::attributes::{NameAttribute, RenamingRule};
 use crate::deprecations::deprecate_trailing_option_default;
 use crate::method::{CallingConvention, ExtractErrorMode, PyArg};
 use crate::params::{check_arg_for_gil_refs, impl_regular_arg_param, Holders};
-use crate::utils::Ctx;
 use crate::utils::PythonDoc;
+use crate::utils::{Ctx, LitCStr};
 use crate::{
     method::{FnArg, FnSpec, FnType, SelfType},
     pyfunction::PyFunctionOptions,
@@ -870,8 +871,7 @@ pub enum PropertyType<'a> {
 }
 
 impl PropertyType<'_> {
-    fn null_terminated_python_name(&self, ctx: &Ctx) -> Result<TokenStream> {
-        let Ctx { pyo3_path, .. } = ctx;
+    fn null_terminated_python_name(&self, ctx: &Ctx) -> Result<LitCStr> {
         match self {
             PropertyType::Descriptor {
                 field,
@@ -892,7 +892,8 @@ impl PropertyType<'_> {
                         bail_spanned!(field.span() => "`get` and `set` with tuple struct fields require `name`");
                     }
                 };
-                Ok(quote_spanned!(field.span() => #pyo3_path::ffi::c_str!(#name)))
+                let name = CString::new(name).unwrap();
+                Ok(LitCStr::new(name, field.span(), ctx))
             }
             PropertyType::Function { spec, .. } => Ok(spec.null_terminated_python_name(ctx)),
         }

pyo3-macros-backend/src/utils.rs

@@ -1,6 +1,7 @@
 use crate::attributes::{CrateAttribute, RenamingRule};
 use proc_macro2::{Span, TokenStream};
 use quote::{quote, ToTokens};
+use std::ffi::CString;
 use syn::spanned::Spanned;
 use syn::{punctuated::Punctuated, Token};
 
@@ -67,14 +68,59 @@ pub fn option_type_argument(ty: &syn::Type) -> Option<&syn::Type> {
     None
 }
 
+// TODO: Replace usage of this by [`syn::LitCStr`] when on MSRV 1.77
+#[derive(Clone)]
+pub struct LitCStr {
+    lit: CString,
+    span: Span,
+    pyo3_path: PyO3CratePath,
+}
+
+impl LitCStr {
+    pub fn new(lit: CString, span: Span, ctx: &Ctx) -> Self {
+        Self {
+            lit,
+            span,
+            pyo3_path: ctx.pyo3_path.clone(),
+        }
+    }
+
+    pub fn empty(ctx: &Ctx) -> Self {
+        Self {
+            lit: CString::new("").unwrap(),
+            span: Span::call_site(),
+            pyo3_path: ctx.pyo3_path.clone(),
+        }
+    }
+}
+
+impl quote::ToTokens for LitCStr {
+    fn to_tokens(&self, tokens: &mut TokenStream) {
+        if cfg!(c_str_lit) {
+            syn::LitCStr::new(&self.lit, self.span).to_tokens(tokens);
+        } else {
+            let pyo3_path = &self.pyo3_path;
+            let lit = self.lit.to_str().unwrap();
+            tokens.extend(quote::quote_spanned!(self.span => #pyo3_path::ffi::c_str!(#lit)));
+        }
+    }
+}
+
 /// A syntax tree which evaluates to a nul-terminated docstring for Python.
 ///
 /// Typically the tokens will just be that string, but if the original docs included macro
 /// expressions then the tokens will be a concat!("...", "\n", "\0") expression of the strings and
-/// macro parts.
-/// contents such as parse the string contents.
+/// macro parts. contents such as parse the string contents.
+#[derive(Clone)]
+pub struct PythonDoc(PythonDocKind);
+
 #[derive(Clone)]
-pub struct PythonDoc(TokenStream);
+enum PythonDocKind {
+    LitCStr(LitCStr),
+    // There is currently no way to `concat!` c-string literals, we fallback to the `c_str!` macro in
+    // this case.
+    Tokens(TokenStream),
+}
 
 /// Collects all #[doc = "..."] attributes into a TokenStream evaluating to a null-terminated string.
 ///
@@ -125,7 +171,7 @@ pub fn get_doc(
         }
     }
 
-    let tokens = if !parts.is_empty() {
+    if !parts.is_empty() {
         // Doc contained macro pieces - return as `concat!` expression
         if !current_part.is_empty() {
             parts.push(current_part.to_token_stream());
@@ -140,17 +186,26 @@ pub fn get_doc(
             syn::token::Comma(Span::call_site()).to_tokens(tokens);
         });
 
-        tokens
+        PythonDoc(PythonDocKind::Tokens(
+            quote!(#pyo3_path::ffi::c_str!(#tokens)),
+        ))
     } else {
         // Just a string doc - return directly with nul terminator
-        current_part.to_token_stream()
-    };
-    PythonDoc(quote!(#pyo3_path::ffi::c_str!(#tokens)))
+        let docs = CString::new(current_part).unwrap();
+        PythonDoc(PythonDocKind::LitCStr(LitCStr::new(
+            docs,
+            Span::call_site(),
+            ctx,
+        )))
+    }
 }
 
 impl quote::ToTokens for PythonDoc {
     fn to_tokens(&self, tokens: &mut TokenStream) {
-        self.0.to_tokens(tokens)
+        match &self.0 {
+            PythonDocKind::LitCStr(lit) => lit.to_tokens(tokens),
+            PythonDocKind::Tokens(toks) => toks.to_tokens(tokens),
+        }
     }
 }
 
@@ -194,6 +249,7 @@ impl Ctx {
     }
 }
 
+#[derive(Clone)]
 pub enum PyO3CratePath {
     Given(syn::Path),
     Default,