Merge pull request #3342 from davidhewitt/3.12-ffi-check-fixes

fix ffi check failures for 3.12.0b4

Author: davidhewitt

.github/workflows/build.yml

@@ -25,7 +25,7 @@ on:
 
 jobs:
   build:
-    continue-on-error: ${{ contains(fromJSON('["3.7", "3.12-dev", "pypy3.7", "pypy3.10"]'), inputs.python-version) }}
+    continue-on-error: ${{ endsWith(inputs.python-version, '-dev') || contains(fromJSON('["3.7", "pypy3.7", "pypy3.10"]'), inputs.python-version) }}
     runs-on: ${{ inputs.os }}
     steps:
       - uses: actions/checkout@v3
@@ -138,7 +138,7 @@ jobs:
       - name: Run pyo3-ffi-check
         # pypy 3.7 and 3.8 are not PEP 3123 compliant so fail checks here, nor
         # is pypy 3.9 on windows
-        if: ${{ steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows')) }}
+        if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && inputs.python-version != 'pypy3.7' && inputs.python-version != 'pypy3.8' && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }}
         run: nox -s ffi-check
 
 

newsfragments/3342.changed.md

@@ -0,0 +1 @@
+Update `pyo3::ffi` struct definitions to be compatible with 3.12.0b4.

pyo3-ffi-check/macro/Cargo.toml

@@ -12,3 +12,4 @@ glob = "0.3"
 quote = "1"
 proc-macro2 = "1"
 scraper = "0.17"
+pyo3-build-config = { path = "../../pyo3-build-config" }

pyo3-ffi-check/macro/src/lib.rs

@@ -1,8 +1,14 @@
 use std::{env, fs, path::PathBuf};
 
 use proc_macro2::{Ident, Span, TokenStream, TokenTree};
+use pyo3_build_config::PythonVersion;
 use quote::quote;
 
+const PY_3_12: PythonVersion = PythonVersion {
+    major: 3,
+    minor: 12,
+};
+
 /// Macro which expands to multiple macro calls, one per pyo3-ffi struct.
 #[proc_macro]
 pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream {
@@ -130,16 +136,27 @@ pub fn for_all_fields(input: proc_macro::TokenStream) -> proc_macro::TokenStream
     let mut output = TokenStream::new();
 
     for el in html.select(&selector) {
-        let id = el
+        let field_name = el
             .value()
             .id()
             .unwrap()
             .strip_prefix("structfield.")
             .unwrap();
 
-        let field_ident = Ident::new(id, Span::call_site());
+        let field_ident = Ident::new(field_name, Span::call_site());
+
+        let bindgen_field_ident = if (pyo3_build_config::get().version >= PY_3_12)
+            && struct_name == "PyObject"
+            && field_name == "ob_refcnt"
+        {
+            // PyObject since 3.12 implements ob_refcnt as a union; bindgen creates
+            // an anonymous name for the field
+            Ident::new("__bindgen_anon_1", Span::call_site())
+        } else {
+            field_ident.clone()
+        };
 
-        output.extend(quote!(#macro_name!(#struct_name, #field_ident);));
+        output.extend(quote!(#macro_name!(#struct_name, #field_ident, #bindgen_field_ident);));
     }
 
     output.into()

pyo3-ffi-check/src/main.rs

@@ -47,9 +47,11 @@ fn main() {
     }
 
     macro_rules! check_field {
-        ($struct_name:ident, $field:ident) => {{
+        ($struct_name:ident, $field:ident, $bindgen_field:ident) => {{
+            #[allow(clippy::used_underscore_binding)]
             let pyo3_ffi_offset = memoffset::offset_of!(pyo3_ffi::$struct_name, $field);
-            let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $field);
+            #[allow(clippy::used_underscore_binding)]
+            let bindgen_offset = memoffset::offset_of!(bindings::$struct_name, $bindgen_field);
 
             if pyo3_ffi_offset != bindgen_offset {
                 failed = true;
@@ -79,7 +81,9 @@ fn main() {
     non_upper_case_globals,
     dead_code,
     improper_ctypes,
-    clippy::all
+    clippy::all,
+    // clippy fails with lots of errors if this is not set specifically
+    clippy::used_underscore_binding
 )]
 mod bindings {
     include!(concat!(env!("OUT_DIR"), "/bindings.rs"));

pyo3-ffi/src/cpython/code.rs

@@ -6,12 +6,30 @@ use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void};
 #[cfg(not(PyPy))]
 use std::ptr::addr_of_mut;
 
+#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))]
+opaque_struct!(_PyOpcache);
+
+pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 14;
+pub const _PY_MONITORING_EVENTS: usize = 16;
+
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _Py_Monitors {
+    pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS],
+}
+
 // skipped _Py_CODEUNIT
+
 // skipped _Py_OPCODE
 // skipped _Py_OPARG
 
-#[cfg(all(Py_3_8, not(PyPy), not(Py_3_11)))]
-opaque_struct!(_PyOpcache);
+// skipped _py_make_codeunit
+
+// skipped _py_set_opcode
+
+// skipped _Py_MAKE_CODEUNIT
+// skipped _Py_SET_OPCODE
 
 #[cfg(Py_3_12)]
 #[repr(C)]
@@ -23,6 +41,27 @@ pub struct _PyCoCached {
     pub _co_freevars: *mut PyObject,
 }
 
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCoLineInstrumentationData {
+    pub original_opcode: u8,
+    pub line_delta: i8,
+}
+
+#[cfg(Py_3_12)]
+#[repr(C)]
+#[derive(Copy, Clone)]
+pub struct _PyCoMonitoringData {
+    pub local_monitors: _Py_Monitors,
+    pub active_monitors: _Py_Monitors,
+    pub tools: *mut u8,
+    pub lines: *mut _PyCoLineInstrumentationData,
+    pub line_tools: *mut u8,
+    pub per_instruction_opcodes: *mut u8,
+    pub per_instruction_tools: *mut u8,
+}
+
 #[cfg(all(not(PyPy), not(Py_3_7)))]
 opaque_struct!(PyCodeObject);
 
@@ -97,8 +136,7 @@ pub struct PyCodeObject {
     pub co_flags: c_int,
     #[cfg(not(Py_3_12))]
     pub co_warmup: c_int,
-    #[cfg(Py_3_12)]
-    pub _co_linearray_entry_size: c_short,
+
     pub co_argcount: c_int,
     pub co_posonlyargcount: c_int,
     pub co_kwonlyargcount: c_int,
@@ -109,9 +147,12 @@ pub struct PyCodeObject {
     #[cfg(Py_3_12)]
     pub co_framesize: c_int,
     pub co_nlocals: c_int,
+    #[cfg(not(Py_3_12))]
     pub co_nplaincellvars: c_int,
     pub co_ncellvars: c_int,
     pub co_nfreevars: c_int,
+    #[cfg(Py_3_12)]
+    pub co_version: u32,
 
     pub co_localsplusnames: *mut PyObject,
     pub co_localspluskinds: *mut PyObject,
@@ -122,13 +163,15 @@ pub struct PyCodeObject {
     pub co_weakreflist: *mut PyObject,
     #[cfg(not(Py_3_12))]
     pub _co_code: *mut PyObject,
-    #[cfg(Py_3_12)]
-    pub _co_cached: *mut _PyCoCached,
     #[cfg(not(Py_3_12))]
     pub _co_linearray: *mut c_char,
-    pub _co_firsttraceable: c_int,
     #[cfg(Py_3_12)]
-    pub _co_linearray: *mut c_char,
+    pub _co_cached: *mut _PyCoCached,
+    #[cfg(Py_3_12)]
+    pub _co_instrumentation_version: u64,
+    #[cfg(Py_3_12)]
+    pub _co_monitoring: *mut _PyCoMonitoringData,
+    pub _co_firsttraceable: c_int,
     pub co_extra: *mut c_void,
     pub co_code_adaptive: [c_char; 1],
 }

pyo3-ffi/src/cpython/funcobject.rs

@@ -41,6 +41,8 @@ pub struct PyFunctionObject {
     pub func_weakreflist: *mut PyObject,
     pub func_module: *mut PyObject,
     pub func_annotations: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub func_typeparams: *mut PyObject,
     pub vectorcall: Option<crate::vectorcallfunc>,
     #[cfg(Py_3_11)]
     pub func_version: u32,

pyo3-ffi/src/cpython/genobject.rs

@@ -16,6 +16,7 @@ pub struct PyGenObject {
     pub gi_frame: *mut PyFrameObject,
     #[cfg(not(Py_3_10))]
     pub gi_running: c_int,
+    #[cfg(not(Py_3_12))]
     pub gi_code: *mut PyObject,
     pub gi_weakreflist: *mut PyObject,
     pub gi_name: *mut PyObject,

pyo3-ffi/src/cpython/object.rs

@@ -299,6 +299,8 @@ pub struct PyTypeObject {
 #[derive(Clone)]
 pub struct _specialization_cache {
     pub getitem: *mut PyObject,
+    #[cfg(Py_3_12)]
+    pub getitem_version: u32,
 }
 
 #[repr(C)]

pyo3-ffi/src/cpython/unicodeobject.rs

@@ -287,17 +287,19 @@ impl PyASCIIObject {
 
     /// Get the `kind` field of the [`PyASCIIObject`] state bitfield.
     ///
-    /// Returns one of: [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`],
-    /// [`PyUnicode_4BYTE_KIND`]
+    /// Returns one of:
+    #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")]
+    /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`].
     #[inline]
     pub unsafe fn kind(&self) -> c_uint {
         PyASCIIObjectState::from(self.state).kind()
     }
 
     /// Set the `kind` field of the [`PyASCIIObject`] state bitfield.
     ///
-    /// Calling this function with an argument that is not [`PyUnicode_WCHAR_KIND`], [`PyUnicode_1BYTE_KIND`],
-    /// [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid.
+    /// Calling this function with an argument that is not
+    #[cfg_attr(not(Py_3_12), doc = "[`PyUnicode_WCHAR_KIND`], ")]
+    /// [`PyUnicode_1BYTE_KIND`], [`PyUnicode_2BYTE_KIND`], or [`PyUnicode_4BYTE_KIND`] is invalid.
     #[inline]
     pub unsafe fn set_kind(&mut self, val: c_uint) {
         let mut state = PyASCIIObjectState::from(self.state);