diff --git a/.azure/build.yml b/.azure/build.yml index 1ebe2455..e3d8bf59 100644 --- a/.azure/build.yml +++ b/.azure/build.yml @@ -71,6 +71,10 @@ jobs: imageName: "ubuntu-latest" python.version: '3.12' jdk.version: '17' + linux-py3.13-jdk17: + imageName: "ubuntu-latest" + python.version: '3.13.0-rc.2' + jdk.version: '17' # Windows windows-py3.8-jdk8: imageName: "windows-2019" diff --git a/native/python/pyjp_class.cpp b/native/python/pyjp_class.cpp index fc633e11..4cdb8f1a 100644 --- a/native/python/pyjp_class.cpp +++ b/native/python/pyjp_class.cpp @@ -125,6 +125,13 @@ PyObject *PyJPClass_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) typenew->tp_new = PyJPException_Type->tp_new; } ((PyJPClass*) typenew)->m_Doc = nullptr; + +#if PY_VERSION_HEX >= 0x030d0000 + // This flag will try to place the dictionary are part of the object which + // adds an unknown number of bytes to the end of the object making it impossible + // to attach our needed data. If we kill the flag then we get usable behavior. + typenew->tp_flags &= ~Py_TPFLAGS_INLINE_VALUES; +#endif return (PyObject*) typenew; JP_PY_CATCH(nullptr); } diff --git a/native/python/pyjp_module.cpp b/native/python/pyjp_module.cpp index a2110efe..80d87f61 100644 --- a/native/python/pyjp_module.cpp +++ b/native/python/pyjp_module.cpp @@ -37,6 +37,7 @@ extern void PyJPNumber_initType(PyObject* module); extern void PyJPClassHints_initType(PyObject* module); extern void PyJPPackage_initType(PyObject* module); extern void PyJPChar_initType(PyObject* module); +extern void PyJPValue_initType(PyObject* module); static PyObject *PyJPModule_convertBuffer(JPPyBuffer& buffer, PyObject *dtype); @@ -739,6 +740,7 @@ PyMODINIT_FUNC PyInit__jpype() PyJPClassMagic = PyDict_New(); // Initialize each of the python extension types + PyJPValue_initType(module); PyJPClass_initType(module); PyJPObject_initType(module); diff --git a/native/python/pyjp_value.cpp b/native/python/pyjp_value.cpp index 1deecaa9..f3b167d6 100644 --- a/native/python/pyjp_value.cpp +++ b/native/python/pyjp_value.cpp @@ -16,12 +16,19 @@ #include "jpype.h" #include "pyjp.h" #include "jp_stringtype.h" +#include +#include #ifdef __cplusplus extern "C" { #endif +std::mutex mtx; + +// Create a dummy type which we will use only for allocation +PyTypeObject* PyJPAlloc_Type = nullptr; + /** * Allocate a new Python object with a slot for Java. * @@ -40,53 +47,37 @@ extern "C" PyObject* PyJPValue_alloc(PyTypeObject* type, Py_ssize_t nitems) { JP_PY_TRY("PyJPValue_alloc"); - // Modification from Python to add size elements - const size_t size = _PyObject_VAR_SIZE(type, nitems + 1) + sizeof (JPValue); - PyObject *obj = nullptr; - if (PyType_IS_GC(type)) - { - // Horrible kludge because python lacks an API for allocating a GC type with extra memory - // The private method _PyObject_GC_Alloc is no longer visible, so we are forced to allocate - // a different type with the extra memory and then hot swap the type to the real one. - PyTypeObject type2; - type2.tp_basicsize = size; - type2.tp_itemsize = 0; - type2.tp_name = nullptr; - type2.tp_flags = type->tp_flags; - type2.tp_traverse = type->tp_traverse; - - // Allocate the fake type - obj = PyObject_GC_New(PyObject, &type2); - - // Note the object will be inited twice which should not leak. (fingers crossed) + +#if PY_VERSION_HEX >= 0x030d0000 + // This flag will try to place the dictionary are part of the object which + // adds an unknown number of bytes to the end of the object making it impossible + // to attach our needed data. If we kill the flag then we get usable behavior. + if (PyType_HasFeature(type, Py_TPFLAGS_INLINE_VALUES)) { + PyErr_Format(PyExc_RuntimeError, "Unhandled object layout"); + return 0; } - else - { - obj = (PyObject*) PyObject_MALLOC(size); +#endif + + PyObject* obj = nullptr; + { + std::lock_guard lock(mtx); + // Mutate the allocator type + PyJPAlloc_Type->tp_flags = type->tp_flags; + PyJPAlloc_Type->tp_basicsize = type->tp_basicsize + sizeof (JPValue); + PyJPAlloc_Type->tp_itemsize = type->tp_itemsize; + + // Create a new allocation for the dummy type + obj = PyType_GenericAlloc(PyJPAlloc_Type, nitems); } - if (obj == nullptr) - return PyErr_NoMemory(); // GCOVR_EXCL_LINE - memset(obj, 0, size); + // Watch for memory errors + if (obj == nullptr) + return nullptr; - Py_ssize_t refcnt = ((PyObject*) type)->ob_refcnt; + // Polymorph the type to match our desired type obj->ob_type = type; + Py_INCREF(type); - if (type->tp_itemsize == 0) - PyObject_Init(obj, type); - else - PyObject_InitVar((PyVarObject *) obj, type, nitems); - - // This line is required to deal with Python bug (GH-11661) - // Some versions of Python fail to increment the reference counter of - // heap types properly. - if (refcnt == ((PyObject*) type)->ob_refcnt) - Py_INCREF(type); // GCOVR_EXCL_LINE - - if (PyType_IS_GC(type)) - { - PyObject_GC_Track(obj); - } JP_TRACE("alloc", type->tp_name, obj); return obj; JP_PY_CATCH(nullptr); @@ -107,17 +98,20 @@ Py_ssize_t PyJPValue_getJavaSlotOffset(PyObject* self) if (type == nullptr || type->tp_alloc != (allocfunc) PyJPValue_alloc || type->tp_finalize != (destructor) PyJPValue_finalize) + { return 0; - Py_ssize_t offset; - Py_ssize_t sz = 0; + } + Py_ssize_t offset = 0; + Py_ssize_t sz = 0; + #if PY_VERSION_HEX>=0x030c0000 // starting in 3.12 there is no longer ob_size in PyLong if (PyType_HasFeature(self->ob_type, Py_TPFLAGS_LONG_SUBCLASS)) sz = (((PyLongObject*)self)->long_value.lv_tag) >> 3; // Private NON_SIZE_BITS else #endif - if (type->tp_itemsize != 0) + if (type->tp_itemsize != 0) sz = Py_SIZE(self); // PyLong abuses ob_size with negative values prior to 3.12 if (sz < 0) @@ -221,7 +215,7 @@ PyObject* PyJPValue_str(PyObject* self) Py_INCREF(cache); return cache; } - auto jstr = (jstring) value->getValue().l; + auto jstr = (jstring) value->getValue().l; string str; str = frame.toStringUTF8(jstr); cache = JPPyString::fromStringUTF8(str).keep(); @@ -337,3 +331,38 @@ bool PyJPValue_isSetJavaSlot(PyObject* self) auto* slot = (JPValue*) (((char*) self) + offset); return slot->getClass() != nullptr; } + +/***************** Create a dummy type for use when allocating. ************************/ +static int PyJPAlloc_traverse(PyObject *self, visitproc visit, void *arg) +{ + return 0; +} + +static int PyJPAlloc_clear(PyObject *self) +{ + return 0; +} + + +static PyType_Slot allocSlots[] = { + { Py_tp_traverse, (void*) PyJPAlloc_traverse}, + { Py_tp_clear, (void*) PyJPAlloc_clear}, + {0, NULL} // Sentinel +}; + +static PyType_Spec allocSpec = { + "_jpype._JAlloc", + sizeof(PyObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + allocSlots +}; + +void PyJPValue_initType(PyObject* module) +{ + PyObject *bases = PyTuple_Pack(1, &PyBaseObject_Type); + PyJPAlloc_Type = (PyTypeObject*) PyType_FromSpecWithBases(&allocSpec, bases); + Py_DECREF(bases); + Py_INCREF(PyJPAlloc_Type); + JP_PY_CHECK(); +}