From 14e8ad8fed18470e8be98a2c928c5cb6ff9c3993 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:55:52 -0400 Subject: [PATCH 01/59] chore: fail fast and clearly when Py_LIMITED_API is defined Py_LIMITED_API is not supported yet; a build with it defined failed with pages of unrelated errors. Add a single #error in common.h, and reserve a bit for Py_LIMITED_API in the precompiled configuration check so later stable-ABI work can switch it on. See https://github.com/pybind/pybind11/discussions/6104 (B1). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/compiling.rst | 5 +++-- include/pybind11/detail/common.h | 5 +++++ include/pybind11/detail/internals.h | 14 ++++++++++---- tools/check_inl_headers.py | 1 + 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 5169f5feab..9e61949b02 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -436,8 +436,9 @@ Requirements and caveats: ``PYBIND11_SIMPLE_GIL_MANAGEMENT``, ``PYBIND11_DETAILED_ERROR_MESSAGES`` (defaults on in debug builds), ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, - ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, and - ``PYBIND11_TYPE_CREATION_VIA_SPEC``. A mismatch produces one + ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, + ``PYBIND11_TYPE_CREATION_VIA_SPEC``, and + ``Py_LIMITED_API``. A mismatch produces one readable undefined symbol at link time referencing ``pybind11_precompiled_config``. * Configuration macros that only change code inside the library (for example diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index e706708527..286e9d4905 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -14,6 +14,11 @@ # error "PYTHON < 3.9 IS UNSUPPORTED. pybind11 v3.0 was the last to support Python 3.8." #endif +#if defined(Py_LIMITED_API) +# error \ + "pybind11 does not support Py_LIMITED_API (the Python stable ABI) yet. Track progress at https://github.com/pybind/pybind11/discussions/6104" +#endif + // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // See also: https://github.com/python/cpython/blob/HEAD/Include/patchlevel.h /* -- start version constants -- */ diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 7e4b77ca84..5c3b7998d5 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -79,16 +79,21 @@ # else # define PYBIND11_PRECOMPILED_CFG_TD 0 # endif +# if defined(Py_LIMITED_API) +# define PYBIND11_PRECOMPILED_CFG_LA 1 +# else +# define PYBIND11_PRECOMPILED_CFG_LA 0 +# endif # if PYBIND11_TYPE_CREATION_VIA_SPEC # define PYBIND11_PRECOMPILED_CFG_TS 1 # else # define PYBIND11_PRECOMPILED_CFG_TS 0 # endif // PYBIND11_CONCAT does not macro-expand its arguments (## suppresses expansion). -# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, ts) \ - pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td##_ts##ts -# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td, ts) \ - PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, ts) +# define PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la, ts) \ + pybind11_precompiled_config_v##v##_gd##gd##_sg##sg##_de##de##_si##si##_td##td##_la##la##_ts##ts +# define PYBIND11_PRECOMPILED_CONFIG_NAME(v, gd, sg, de, si, td, la, ts) \ + PYBIND11_PRECOMPILED_CONFIG_NAME_(v, gd, sg, de, si, td, la, ts) # define PYBIND11_PRECOMPILED_CONFIG_CHECK \ PYBIND11_PRECOMPILED_CONFIG_NAME(PYBIND11_INTERNALS_VERSION, \ PYBIND11_PRECOMPILED_CFG_GD, \ @@ -96,6 +101,7 @@ PYBIND11_PRECOMPILED_CFG_DE, \ PYBIND11_PRECOMPILED_CFG_SI, \ PYBIND11_PRECOMPILED_CFG_TD, \ + PYBIND11_PRECOMPILED_CFG_LA, \ PYBIND11_PRECOMPILED_CFG_TS) # define PYBIND11_PRECOMPILED_CONFIG_GUARD \ ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK(); diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index aef0f0d3fe..7eef6d2b31 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -47,6 +47,7 @@ "PYBIND11_SIMPLE_GIL_MANAGEMENT", "PYBIND11_TYPE_CREATION_VIA_SPEC", "Py_GIL_DISABLED", + "Py_LIMITED_API", } # Only change code inside the library; documented in docs/compiling.rst. From 39c0c2ee8a2fa9c08d297ecb45ccb2b52765ee4d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:57:29 -0400 Subject: [PATCH 02/59] chore: add a stable-ABI tag to PYBIND11_INTERNALS_ID and PYBIND11_MODULE_LOCAL_ID Modules built with Py_LIMITED_API will use a different internals layout, so they must not share internals with regular modules in one process. The tag is empty today and becomes "_stable" under Py_LIMITED_API. It is not part of PYBIND11_PLATFORM_ABI_ID, so cpp_conduit keeps bridging both worlds. See https://github.com/pybind/pybind11/discussions/6104 (B2). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/internals.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 5c3b7998d5..85feb9faff 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -514,13 +514,26 @@ struct native_enum_record { static const char *attribute_name() { return "__pybind11_native_enum__"; } }; +// Modules built for the Python stable ABI (Py_LIMITED_API) use a different `internals` layout +// (opaque TSS keys, different type-object access), so they must not share internals with +// regular modules in the same process. The tag keeps the two universes apart. It is deliberately +// not part of PYBIND11_PLATFORM_ABI_ID: the C++ ABI is the same, so the cpp_conduit protocol +// still bridges the two. +#if defined(Py_LIMITED_API) +# define PYBIND11_INTERNALS_SABI_TAG "_stable" +#else +# define PYBIND11_INTERNALS_SABI_TAG "" +#endif + #define PYBIND11_INTERNALS_ID \ "__pybind11_internals_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE \ + PYBIND11_PLATFORM_ABI_ID PYBIND11_INTERNALS_SABI_TAG "__" #define PYBIND11_MODULE_LOCAL_ID \ "__pybind11_module_local_v" PYBIND11_TOSTRING(PYBIND11_INTERNALS_VERSION) \ - PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE PYBIND11_PLATFORM_ABI_ID "__" + PYBIND11_COMPILER_TYPE_LEADING_UNDERSCORE \ + PYBIND11_PLATFORM_ABI_ID PYBIND11_INTERNALS_SABI_TAG "__" /// We use this to figure out if there are or have been multiple subinterpreters active at any /// point. This must never go from true to false while any interpreter may be running in any From 27629eda86293a190e2fa15f139b6cd71f2dd7e1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 15:58:53 -0400 Subject: [PATCH 03/59] refactor: add detail::get_bases()/get_mro() and use them instead of tp_bases/tp_mro Direct reads of tp_bases/tp_mro are not possible under the Python stable ABI. Route all reads through two small helpers so the limited-API branch can be added in one place later. See https://github.com/pybind/pybind11/discussions/6104 (B3). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 4 ++-- include/pybind11/detail/type_caster_base-inl.h | 10 +++++++--- include/pybind11/pybind11-inl.h | 3 +-- include/pybind11/pytypes.h | 11 +++++++++++ 4 files changed, 21 insertions(+), 7 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 727ffaf867..6cc9839cab 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -329,7 +329,7 @@ PYBIND11_INLINE void traverse_offset_bases(void *valueptr, const detail::type_info *tinfo, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)) { - for (handle h : reinterpret_borrow(tinfo->type->tp_bases)) { + for (handle h : get_bases(tinfo->type)) { if (auto *parent_tinfo = get_type_info(reinterpret_cast(h.ptr()))) { for (auto &c : parent_tinfo->implicit_casts) { if (c.first == tinfo->cpptype) { @@ -672,7 +672,7 @@ PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { extern "C" PYBIND11_INLINE int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { // Look for a `get_buffer` implementation in this type's info or any bases (following MRO). type_info *tinfo = nullptr; - for (auto type : reinterpret_borrow(Py_TYPE(obj)->tp_mro)) { + for (auto type : get_mro(Py_TYPE(obj))) { tinfo = get_type_info((PyTypeObject *) type.ptr()); if (tinfo && tinfo->get_buffer) { break; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 872c97ad0c..addf38c4cd 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -87,7 +87,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void all_type_info_populate(PyTypeObject *t, std::vector &bases) { assert(bases.empty()); std::vector check; - for (handle parent : reinterpret_borrow(t->tp_bases)) { + for (handle parent : get_bases(t)) { check.push_back(reinterpret_cast(parent.ptr())); } auto const &type_dict = get_internals().registered_types_py; @@ -120,9 +120,13 @@ all_type_info_populate(PyTypeObject *t, std::vector &bases) { all_type_info_add_base_most_derived_first(bases, tinfo); } } - } else if (type->tp_bases) { + } else { // It's some python type, so keep follow its bases classes to look for one or more // registered types + tuple parents = get_bases(type); + if (parents.empty()) { + continue; + } if (i + 1 == check.size()) { // When we're at the end, we can pop off the current element to avoid growing // `check` when adding just one base (which is typical--i.e. when there is no @@ -130,7 +134,7 @@ all_type_info_populate(PyTypeObject *t, std::vector &bases) { check.pop_back(); i--; } - for (handle parent : reinterpret_borrow(type->tp_bases)) { + for (handle parent : parents) { check.push_back(reinterpret_cast(parent.ptr())); } } diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 0cfe9fc28c..0c32d7acc0 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -1337,8 +1337,7 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { } PYBIND11_INLINE void generic_type::mark_parents_nonsimple(PyTypeObject *value) { - auto t = reinterpret_borrow(value->tp_bases); - for (handle h : t) { + for (handle h : get_bases(value)) { auto *tinfo2 = get_type_info(reinterpret_cast(h.ptr())); if (tinfo2) { tinfo2->simple_type = false; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 88de78d024..021bcc95d5 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2465,6 +2465,17 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE // Meant to return a Python str, but this is not checked. +/// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). +inline tuple get_bases(PyTypeObject *type) { + if (type->tp_bases == nullptr) { + return tuple(); + } + return reinterpret_borrow(type->tp_bases); +} + +/// `type.__mro__` as a tuple. +inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } + inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { From a865adc96672b44430cfb8cfbbab46a70eb5a98d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:01:04 -0400 Subject: [PATCH 04/59] refactor: add detail::get_tp_name()/tp_name_equals() and use them instead of raw tp_name reads Direct reads of tp_name are not possible under the Python stable ABI. obj_class_name() now returns std::string. See https://github.com/pybind/pybind11/discussions/6104 (B4). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 8 +++----- include/pybind11/detail/class-inl.h | 6 +++--- .../pybind11/detail/function_record_pyobject.h | 8 +++----- include/pybind11/pytypes-inl.h | 15 ++------------- include/pybind11/pytypes.h | 15 ++++++++++++--- include/pybind11/stl.h | 3 +-- 6 files changed, 24 insertions(+), 31 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 891f1c9d18..71951be50f 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -475,10 +475,9 @@ class type_caster { private: // Test if an object is a NumPy boolean (without fetching the type). static bool is_numpy_bool(handle object) { - const char *type_name = Py_TYPE(object.ptr())->tp_name; // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support - return std::strcmp("numpy.bool", type_name) == 0 - || std::strcmp("numpy.bool_", type_name) == 0; + return tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool") + || tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool_"); } }; @@ -1170,8 +1169,7 @@ std::weak_ptr potentially_slicing_weak_ptr(handle obj) { if (caster.load(obj, /*convert=*/true)) { return caster.potentially_slicing_weak_ptr(); } - const char *obj_type_name = detail::obj_class_name(obj.ptr()); - throw type_error("\"" + std::string(obj_type_name) + throw type_error("\"" + detail::obj_class_name(obj.ptr()) + "\" object is not convertible to std::weak_ptr (with T = " + type_id() + ")"); } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 6cc9839cab..f00770efa0 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -21,13 +21,13 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_INLINE std::string get_fully_qualified_tp_name(PyTypeObject *type) { #if !defined(PYPY_VERSION) - return type->tp_name; + return get_tp_name(type); #else auto module_name = handle((PyObject *) type).attr("__module__").cast(); if (module_name == PYBIND11_BUILTINS_MODULE) - return type->tp_name; + return get_tp_name(type); else - return std::move(module_name) + "." + type->tp_name; + return std::move(module_name) + "." + get_tp_name(type); #endif } diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 2e66944b9c..1aa2902d19 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -105,12 +105,10 @@ inline PyTypeObject *get_function_record_PyTypeObject() { // This works across extension modules, and does not need the internals lock. // Note that tp_name is versioned. +// This works across extension modules. Note that tp_name is versioned. inline bool function_record_PyTypeObject_name_matches(PyTypeObject *obj_type) { - return std::strcmp(obj_type->tp_name, function_record_PyTypeObject_methods::tp_qualname_impl) - == 0 - || std::strcmp(obj_type->tp_name, - function_record_PyTypeObject_methods::tp_plainname_impl) - == 0; + return tp_name_equals(obj_type, function_record_PyTypeObject_methods::tp_qualname_impl) + || tp_name_equals(obj_type, function_record_PyTypeObject_methods::tp_plainname_impl); } inline bool is_function_record_PyObject(PyObject *obj) { diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index dce0fe5e6c..5e260ea909 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -28,13 +28,7 @@ PYBIND11_INLINE error_fetch_and_normalize::error_fetch_and_normalize(const char + " called while " "Python error indicator not set."); } - const char *exc_type_name_orig = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_orig == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the original active exception type."); - } - m_lazy_error_string = exc_type_name_orig; + m_lazy_error_string = detail::obj_class_name(m_type.ptr()); #if PY_VERSION_HEX >= 0x030C0000 // The presence of __notes__ is likely due to exception normalization // errors, although that is not necessarily true, therefore insert a @@ -56,12 +50,7 @@ PYBIND11_INLINE error_fetch_and_normalize::error_fetch_and_normalize(const char + " failed to normalize the " "active exception."); } - const char *exc_type_name_norm = detail::obj_class_name(m_type.ptr()); - if (exc_type_name_norm == nullptr) { - pybind11_fail("Internal error: " + std::string(called) - + " failed to obtain the name " - "of the normalized active exception type."); - } + std::string exc_type_name_norm = detail::obj_class_name(m_type.ptr()); if (exc_type_name_norm != m_lazy_error_string) { std::string msg = std::string(called) + ": MISMATCH of original and normalized " diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 021bcc95d5..d12a260791 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -500,12 +500,21 @@ T reinterpret_steal(handle h) { PYBIND11_NAMESPACE_BEGIN(detail) +/// The `tp_name` of a type: `module.Name` for static and pybind11 types, `Name` for Python +/// heap types. +inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } + +/// True if `get_tp_name(type) == name`, without allocating in the common case. +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + return std::strcmp(type->tp_name, name) == 0; +} + // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). -inline const char *obj_class_name(PyObject *obj) { +inline std::string obj_class_name(PyObject *obj) { if (PyType_Check(obj)) { - return reinterpret_cast(obj)->tp_name; + return get_tp_name(reinterpret_cast(obj)); } - return Py_TYPE(obj)->tp_name; + return get_tp_name(Py_TYPE(obj)); } std::string error_string(); diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 01be0b47c6..f573e566d3 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -79,9 +79,8 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, if (PyType_Check(obj)) { return false; } - const char *obj_tp_name = Py_TYPE(obj)->tp_name; for (const auto *tp_name : tp_names) { - if (std::strcmp(obj_tp_name, tp_name) == 0) { + if (tp_name_equals(Py_TYPE(obj), tp_name)) { return true; } } From d0e4536f9d7383d78a9dce11e0fdba0c823c5582 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:06:48 -0400 Subject: [PATCH 05/59] refactor: add detail::type_lookup() and use it instead of _PyType_Lookup _PyType_Lookup is a private CPython function and is not available under the stable ABI. Route the three users (metaclass getattro/setattro and the cpp_conduit instance-method check) through one helper. See https://github.com/pybind/pybind11/discussions/6104 (B7). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 19 +++++++++---------- include/pybind11/detail/cpp_conduit-inl.h | 4 ++-- include/pybind11/pytypes.h | 6 ++++++ 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index f00770efa0..02e3b5ad28 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -152,24 +152,24 @@ class pybind11_static_property(property): #endif // PYPY extern "C" PYBIND11_INLINE int pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { - // Use `_PyType_Lookup()` instead of `PyObject_GetAttr()` in order to get the raw + // Use `type_lookup()` instead of `PyObject_GetAttr()` in order to get the raw // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). - PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); + object descr = type_lookup((PyTypeObject *) obj, name); // The following assignment combinations are possible: // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` // 2. `Type.static_prop = other_static_prop` --> setattro: replace existing `static_prop` // 3. `Type.regular_attribute = value` --> setattro: regular attribute assignment auto *const static_prop = (PyObject *) get_internals().static_property_type; - const auto call_descr_set = (descr != nullptr) && (value != nullptr) - && (PyObject_IsInstance(descr, static_prop) != 0) + const auto call_descr_set = descr && (value != nullptr) + && (PyObject_IsInstance(descr.ptr(), static_prop) != 0) && (PyObject_IsInstance(value, static_prop) == 0); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. #if !defined(PYPY_VERSION) - return Py_TYPE(descr)->tp_descr_set(descr, obj, value); + return Py_TYPE(descr.ptr())->tp_descr_set(descr.ptr(), obj, value); #else - if (PyObject *result = PyObject_CallMethod(descr, "__set__", "OO", obj, value)) { + if (PyObject *result = PyObject_CallMethod(descr.ptr(), "__set__", "OO", obj, value)) { Py_DECREF(result); return 0; } else { @@ -183,10 +183,9 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { } extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { - PyObject *descr = _PyType_Lookup((PyTypeObject *) obj, name); - if (descr && PyInstanceMethod_Check(descr)) { - Py_INCREF(descr); - return descr; + object descr = type_lookup((PyTypeObject *) obj, name); + if (descr && PyInstanceMethod_Check(descr.ptr())) { + return descr.release().ptr(); } return PyType_Type.tp_getattro(obj, name); } diff --git a/include/pybind11/detail/cpp_conduit-inl.h b/include/pybind11/detail/cpp_conduit-inl.h index 6ec0fb4120..733b180cfa 100644 --- a/include/pybind11/detail/cpp_conduit-inl.h +++ b/include/pybind11/detail/cpp_conduit-inl.h @@ -31,8 +31,8 @@ PYBIND11_INLINE bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { } PYBIND11_INLINE bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { - PyObject *descr = _PyType_Lookup(type_obj, attr_name); - return ((descr != nullptr) && PyInstanceMethod_Check(descr)); + object descr = type_lookup(type_obj, attr_name); + return descr && PyInstanceMethod_Check(descr.ptr()); } PYBIND11_INLINE object try_get_cpp_conduit_method(PyObject *obj) { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index d12a260791..2dd157b28c 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2485,6 +2485,12 @@ inline tuple get_bases(PyTypeObject *type) { /// `type.__mro__` as a tuple. inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } +/// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. +/// Returns a null object if the attribute is not found; never raises. +inline object type_lookup(PyTypeObject *type, handle name) { + return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); +} + inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { From cff7bfa297a2c48d20a9dc4845f7abf94d116922 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:11:21 -0400 Subject: [PATCH 06/59] refactor: add PYBIND11_HAS_DIRECT_STRUCT_ACCESS for the struct-access fallbacks Gate the PyPy fallbacks that exist because cpyext does not expose CPython struct layouts (static-property __set__, bool conversion, tuple/list fast iterators) on one semantic macro. The stable ABI needs the same fallbacks. See https://github.com/pybind/pybind11/discussions/6104 (B9). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 4 ++-- include/pybind11/detail/class-inl.h | 2 +- include/pybind11/detail/common.h | 7 +++++++ include/pybind11/pytypes.h | 2 +- tools/check_inl_headers.py | 4 +++- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 71951be50f..27f2e7c51c 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -445,8 +445,8 @@ class type_caster { if (src.is_none()) { res = 0; // None is implicitly converted to False } -#if defined(PYPY_VERSION) - // On PyPy, check that "__bool__" attr exists +#if !defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + // Check that the "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 02e3b5ad28..871fd87197 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -166,7 +166,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { && (PyObject_IsInstance(value, static_prop) == 0); if (call_descr_set) { // Call `static_property.__set__()` instead of replacing the `static_property`. -#if !defined(PYPY_VERSION) +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) return Py_TYPE(descr.ptr())->tp_descr_set(descr.ptr(), obj, value); #else if (PyObject *result = PyObject_CallMethod(descr.ptr(), "__set__", "OO", obj, value)) { diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 286e9d4905..6254091a9f 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -265,6 +265,13 @@ # endif #endif +// Direct access to CPython object struct fields and type slots (tp_*, nb_*, ob_item, ...) is +// available. Not on PyPy (cpyext emulates only part of the layouts) and not under the stable +// ABI (the structs are opaque). Code that is gated off this falls back to the public API. +#if !defined(PYPY_VERSION) && !defined(Py_LIMITED_API) +# define PYBIND11_HAS_DIRECT_STRUCT_ACCESS +#endif + #include #include #include diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2dd157b28c..77766034f9 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -1169,7 +1169,7 @@ class dict_readonly { }; PYBIND11_NAMESPACE_END(iterator_policies) -#if !defined(PYPY_VERSION) +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) using tuple_iterator = generic_iterator; using list_iterator = generic_iterator; #else diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index 7eef6d2b31..850b78bbd9 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -38,10 +38,12 @@ "__clang__", } -# Encoded in PYBIND11_PRECOMPILED_CONFIG_CHECK. +# Encoded in PYBIND11_PRECOMPILED_CONFIG_CHECK (PYBIND11_HAS_DIRECT_STRUCT_ACCESS through the +# Py_LIMITED_API bit and the environment). GUARDED_MACROS = { "PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET", "PYBIND11_DETAILED_ERROR_MESSAGES", + "PYBIND11_HAS_DIRECT_STRUCT_ACCESS", "PYBIND11_HAS_SUBINTERPRETER_SUPPORT", "PYBIND11_INTERNALS_VERSION", "PYBIND11_SIMPLE_GIL_MANAGEMENT", From c6ad3dff2861b8aafed14041a2b2e48dcc0e5792 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:13:09 -0400 Subject: [PATCH 07/59] refactor: replace PyTuple/PyList/PyByteArray/PyCFunction macro accessors Use the function forms (available under the stable ABI) at the non-hot sites, and route the dispatcher and tuple/list caster sites through small detail:: wrappers that keep the macro forms when struct access is available. See https://github.com/pybind/pybind11/discussions/6104 (B10). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/cast.h | 4 ++-- include/pybind11/functional.h | 2 +- include/pybind11/pybind11-inl.h | 22 +++++++++++---------- include/pybind11/pybind11.h | 8 ++++---- include/pybind11/pytypes-inl.h | 2 +- include/pybind11/pytypes.h | 35 +++++++++++++++++++++++++++++---- include/pybind11/stl.h | 4 ++-- 7 files changed, 53 insertions(+), 24 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 27f2e7c51c..d9f4e4507a 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -847,7 +847,7 @@ class tuple_caster { tuple result(size); int counter = 0; for (auto &entry : entries) { - PyTuple_SET_ITEM(result.ptr(), counter++, entry.release().ptr()); + tuple_set_item(result.ptr(), counter++, entry.release().ptr()); } return result.release(); } @@ -1964,7 +1964,7 @@ typing::Tuple make_tuple(Args &&...args_) { tuple result(size); int counter = 0; for (auto &arg_value : args) { - PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr()); + detail::tuple_set_item(result.ptr(), counter++, arg_value.release().ptr()); } PYBIND11_WARNING_PUSH #ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index c06fabe0d6..cfa0a88a08 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -82,7 +82,7 @@ struct type_caster> { captured variables), in which case the roundtrip can be avoided. */ if (auto cfunc = func.cpp_function()) { - auto *cfunc_self = PyCFunction_GET_SELF(cfunc.ptr()); + auto *cfunc_self = PyCFunction_GetSelf(cfunc.ptr()); if (cfunc_self == nullptr) { PyErr_Clear(); } else { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 0c32d7acc0..e987ea1d89 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -482,7 +482,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, # else PyObject *co_varnames = PyObject_GetAttrString((PyObject *) f_code, "co_varnames"); # endif - PyObject *self_arg = PyTuple_GET_ITEM(co_varnames, 0); + PyObject *self_arg = PyTuple_GetItem(co_varnames, 0); Py_DECREF(co_varnames); PyObject *self_caller = dict_getitem(locals, self_arg); Py_DECREF(locals); @@ -611,9 +611,9 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto *self = PyCFunction_GET_SELF(rec->sibling.ptr()); + auto *self = PyCFunction_GetSelf(rec->sibling.ptr()); if (self == nullptr) { - pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GET_SELF"); + pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GetSelf"); } chain = detail::function_record_ptr_from_PyObject(self); if (chain && !chain->scope.is(rec->scope)) { @@ -674,8 +674,8 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec = reinterpret_cast( - PyCFunction_GET_SELF(m_ptr)); + auto *py_func_rec + = reinterpret_cast(PyCFunction_GetSelf(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { @@ -976,7 +976,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, // 2. Check kwargs and, failing that, defaults that may help complete the list small_vector used_kwargs( - kwnames_in ? static_cast(PyTuple_GET_SIZE(kwnames_in)) : 0, false); + kwnames_in ? static_cast(detail::tuple_size(kwnames_in)) : 0, false); size_t used_kwargs_count = 0; if (args_copied < num_args) { for (; args_copied < num_args; ++args_copied) { @@ -1052,7 +1052,8 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, // Cast values into handles before indexing into kwargs to ensure // well-defined evaluation order (MSVC C4866). handle arg_in_arr = args_in_arr[n_args_in + i], - kwname = PyTuple_GET_ITEM(kwnames_in, i); + kwname + = detail::tuple_get_item(kwnames_in, static_cast(i)); kwargs[kwname] = arg_in_arr; } } @@ -1206,19 +1207,20 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, msg += ""; } } - if (kwnames_in && PyTuple_GET_SIZE(kwnames_in) > 0) { + if (kwnames_in && detail::tuple_size(kwnames_in) > 0) { if (some_args) { msg += "; "; } msg += "kwargs: "; bool first = true; - for (size_t i = 0; i < static_cast(PyTuple_GET_SIZE(kwnames_in)); ++i) { + for (size_t i = 0; i < static_cast(detail::tuple_size(kwnames_in)); ++i) { if (first) { first = false; } else { msg += ", "; } - msg += reinterpret_borrow(PyTuple_GET_ITEM(kwnames_in, i)); + msg += reinterpret_borrow( + detail::tuple_get_item(kwnames_in, static_cast(i))); msg += '='; try { msg += pybind11::repr(args_in_arr[n_args_in + i]); diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4d574b9bd7..3a2bd9d7b2 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -562,15 +562,15 @@ class cpp_function : public function { * second lexicographic check. This is wildly expensive for huge argument lists, but those * are incredibly rare so we optimize for the vastly common case of just a couple of args. */ - auto n = PyTuple_GET_SIZE(haystack); + auto n = detail::tuple_size(haystack); auto s = reinterpret_steal(PyUnicode_InternFromString(needle)); for (ssize_t i = 0; i < n; ++i) { - if (PyTuple_GET_ITEM(haystack, i) == s.ptr()) { + if (detail::tuple_get_item(haystack, i) == s.ptr()) { return i; } } for (ssize_t i = 0; i < n; ++i) { - if (PyUnicode_Compare(PyTuple_GET_ITEM(haystack, i), s.ptr()) == 0) { + if (PyUnicode_Compare(detail::tuple_get_item(haystack, i), s.ptr()) == 0) { return i; } } @@ -1864,7 +1864,7 @@ class class_ : public detail::generic_type { return nullptr; } - handle func_self = PyCFunction_GET_SELF(h.ptr()); + handle func_self = PyCFunction_GetSelf(h.ptr()); if (!func_self) { throw error_already_set(); } diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index 5e260ea909..90d573e6ca 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -107,7 +107,7 @@ PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() } else { result += "\n__notes__ (len=" + std::to_string(len_notes) + "):"; for (ssize_t i = 0; i < len_notes; i++) { - PyObject *note = PyList_GET_ITEM(notes.ptr(), i); + PyObject *note = PyList_GetItem(notes.ptr(), i); auto note_bytes = reinterpret_steal( PyUnicode_AsEncodedString(note, "utf-8", "backslashreplace")); if (!note_bytes) { diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 77766034f9..ab7c36aa64 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -509,6 +509,33 @@ inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +// Unchecked tuple/list element access for hot paths. Function forms under the stable ABI. +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) +inline ssize_t tuple_size(PyObject *tup) { return PyTuple_GET_SIZE(tup); } +inline PyObject *tuple_get_item(PyObject *tup, ssize_t i) { return PyTuple_GET_ITEM(tup, i); } +/// Steals a reference to `item`; `tup` must be a fresh tuple. +inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { + PyTuple_SET_ITEM(tup, i, item); +} +inline ssize_t list_size(PyObject *lst) { return PyList_GET_SIZE(lst); } +inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GET_ITEM(lst, i); } +/// Steals a reference to `item`; `lst` must be a fresh list. +inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { + PyList_SET_ITEM(lst, i, item); +} +#else +inline ssize_t tuple_size(PyObject *tup) { return PyTuple_Size(tup); } +inline PyObject *tuple_get_item(PyObject *tup, ssize_t i) { return PyTuple_GetItem(tup, i); } +inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { + PyTuple_SetItem(tup, i, item); +} +inline ssize_t list_size(PyObject *lst) { return PyList_Size(lst); } +inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GetItem(lst, i); } +inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { + PyList_SetItem(lst, i, item); +} +#endif + // Equivalent to obj.__class__.__name__ (or obj.__name__ if obj is a class). inline std::string obj_class_name(PyObject *obj) { if (PyType_Check(obj)) { @@ -1649,8 +1676,8 @@ class bytearray : public object { size_t size() const { return static_cast(PyByteArray_Size(m_ptr)); } explicit operator std::string() const { - char *buffer = PyByteArray_AS_STRING(m_ptr); - ssize_t size = PyByteArray_GET_SIZE(m_ptr); + char *buffer = PyByteArray_AsString(m_ptr); + ssize_t size = PyByteArray_Size(m_ptr); return std::string(buffer, static_cast(size)); } }; @@ -1973,7 +2000,7 @@ class tuple : public object { return object::operator[](std::forward(o)); } detail::tuple_iterator begin() const { return {*this, 0}; } - detail::tuple_iterator end() const { return {*this, PyTuple_GET_SIZE(m_ptr)}; } + detail::tuple_iterator end() const { return {*this, detail::tuple_size(m_ptr)}; } }; // We need to put this into a separate function because the Intel compiler @@ -2063,7 +2090,7 @@ class list : public object { return object::operator[](std::forward(o)); } detail::list_iterator begin() const { return {*this, 0}; } - detail::list_iterator end() const { return {*this, PyList_GET_SIZE(m_ptr)}; } + detail::list_iterator end() const { return {*this, detail::list_size(m_ptr)}; } template void append(T &&val) /* py-non-const */ { if (PyList_Append(m_ptr, detail::object_or_cast(std::forward(val)).ptr()) != 0) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index f573e566d3..7a2ac29c7c 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -351,7 +351,7 @@ struct list_caster { if (!value_) { return handle(); } - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + list_set_item(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } @@ -461,7 +461,7 @@ struct array_caster { if (!value_) { return handle(); } - PyList_SET_ITEM(l.ptr(), index++, value_.release().ptr()); // steals a reference + list_set_item(l.ptr(), index++, value_.release().ptr()); // steals a reference } return l.release(); } From 7f7584e768bc231300659e3144aa700a53240407 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:19:00 -0400 Subject: [PATCH 08/59] refactor: forward to type/property slots through PyType_GetSlot() The metaclass and static-property implementations called the tp_* fields of PyType_Type and PyProperty_Type directly. Look the slots up once per module with PyType_GetSlot(), which works on static types since 3.10 and is part of the stable ABI; older versions and PyPy keep the direct reads. See https://github.com/pybind/pybind11/discussions/6104 (C2). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 41 ++++++++++++++++++++++++----- include/pybind11/detail/class.h | 8 ++++++ 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 871fd87197..503f8e83e8 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -48,16 +48,45 @@ type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, P } #endif + +// Slots of the static base types that the pybind11 metaclass and static property type forward +// to. PyType_GetSlot() works on static types since Python 3.10 and is part of the stable ABI; +// the function-local caches make each lookup happen once per module. Below 3.10 (and on PyPy, +// where PyType_GetSlot on static types is not reliable) the struct fields are read directly. +#if PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) +# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) \ + static const auto cached = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + return cached; +#else +# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) return type.field; +#endif + +PYBIND11_INLINE ternaryfunc type_type_call(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_call, tp_call, ternaryfunc)} PYBIND11_INLINE setattrofunc + type_type_setattro(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_setattro, tp_setattro, setattrofunc)} PYBIND11_INLINE getattrofunc + type_type_getattro(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_getattro, tp_getattro, getattrofunc)} PYBIND11_INLINE destructor + type_type_dealloc(){PYBIND11_BASE_TYPE_SLOT( + PyType_Type, Py_tp_dealloc, tp_dealloc, destructor)} PYBIND11_INLINE descrgetfunc + property_type_descr_get(){PYBIND11_BASE_TYPE_SLOT( + PyProperty_Type, Py_tp_descr_get, tp_descr_get, descrgetfunc)} PYBIND11_INLINE descrsetfunc + property_type_descr_set() { + PYBIND11_BASE_TYPE_SLOT(PyProperty_Type, Py_tp_descr_set, tp_descr_set, descrsetfunc) +} + +#undef PYBIND11_BASE_TYPE_SLOT + #if !defined(PYPY_VERSION) extern "C" PYBIND11_INLINE PyObject * pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls) { - return PyProperty_Type.tp_descr_get(self, cls, cls); + return property_type_descr_get()(self, cls, cls); } extern "C" PYBIND11_INLINE int pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { PyObject *cls = PyType_Check(obj) ? obj : (PyObject *) Py_TYPE(obj); - return PyProperty_Type.tp_descr_set(self, cls, value); + return property_type_descr_set()(self, cls, value); } # if PYBIND11_TYPE_CREATION_VIA_SPEC @@ -178,7 +207,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { #endif } else { // Replace existing attribute. - return PyType_Type.tp_setattro(obj, name, value); + return type_type_setattro()(obj, name, value); } } @@ -187,14 +216,14 @@ extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObj if (descr && PyInstanceMethod_Check(descr.ptr())) { return descr.release().ptr(); } - return PyType_Type.tp_getattro(obj, name); + return type_type_getattro()(obj, name); } extern "C" PYBIND11_INLINE PyObject * pybind11_meta_call(PyObject *type, PyObject *args, PyObject *kwargs) { // use the default metaclass call to create/initialize the object - PyObject *self = PyType_Type.tp_call(type, args, kwargs); + PyObject *self = type_type_call()(type, args, kwargs); if (self == nullptr) { return nullptr; } @@ -259,7 +288,7 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { } }); - PyType_Type.tp_dealloc(obj); + type_type_dealloc()(obj); } #if PYBIND11_TYPE_CREATION_VIA_SPEC diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 0492be064e..c0de2088b0 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -43,6 +43,14 @@ PyTypeObject * type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases); #endif +/// Slots of `type` and `property` that pybind11's own types forward to. +ternaryfunc type_type_call(); +setattrofunc type_type_setattro(); +getattrofunc type_type_getattro(); +destructor type_type_dealloc(); +descrgetfunc property_type_descr_get(); +descrsetfunc property_type_descr_set(); + #if !defined(PYPY_VERSION) /// `pybind11_static_property.__get__()`: Always pass the class instead of the instance. From b672a8f259a3886d447e0defcd4de5d676ab5cbf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 16:33:47 -0400 Subject: [PATCH 09/59] refactor: wrap instance tp_alloc/tp_free and the __dict__ slot pointer type_alloc()/type_free() use PyType_GetSlot() when direct slot access is unavailable; instance_dict_ptr() is the single place that reads the instance dictionary pointer. See https://github.com/pybind/pybind11/discussions/6104 (C7). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 48 +++++++++++++++++++---------- include/pybind11/detail/class.h | 10 ++++-- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 503f8e83e8..94ae3138a3 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -434,6 +434,36 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ return ret; } +PYBIND11_INLINE PyObject *type_alloc(PyTypeObject *type) { +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + return type->tp_alloc(type, 0); +#else + return reinterpret_cast(PyType_GetSlot(type, Py_tp_alloc))(type, 0); +#endif +} + +PYBIND11_INLINE void type_free(PyTypeObject *type, PyObject *self) { +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) + type->tp_free(self); +#else + reinterpret_cast(PyType_GetSlot(type, Py_tp_free))(self); +#endif +} + +PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { +#if defined(PYPY_VERSION_NUM) && (PYPY_VERSION_NUM & 0xFFFF0000) == 0x08000000 + // PyPy 8.0.0's _PyObject_GetDictPtr() can abort when called from tp_dealloc. + // The bug was fixed in 8.0.1, but all 8.0.x releases share an extension ABI, + // so an extension built against a fixed release can still run on 8.0.0. + // Keep this workaround for the entire 8.0.x series. + auto offset = Py_TYPE(self)->tp_dictoffset; + return offset > 0 ? reinterpret_cast(reinterpret_cast(self) + offset) + : nullptr; +#else + return _PyObject_GetDictPtr(self); +#endif +} + PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first @@ -443,7 +473,7 @@ PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { type->tp_basicsize = instance_size; } #endif - PyObject *self = type->tp_alloc(type, 0); + PyObject *self = type_alloc(type); auto *inst = reinterpret_cast(self); // Allocate the value/holder internals: inst->allocate_layout(); @@ -496,20 +526,6 @@ PYBIND11_INLINE void clear_patients(PyObject *self) { } } -PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { -#if defined(PYPY_VERSION_NUM) && (PYPY_VERSION_NUM & 0xFFFF0000) == 0x08000000 - // PyPy 8.0.0's _PyObject_GetDictPtr() can abort when called from tp_dealloc. - // The bug was fixed in 8.0.1, but all 8.0.x releases share an extension ABI, - // so an extension built against a fixed release can still run on 8.0.0. - // Keep this workaround for the entire 8.0.x series. - auto offset = Py_TYPE(self)->tp_dictoffset; - return offset > 0 ? reinterpret_cast(reinterpret_cast(self) + offset) - : nullptr; -#else - return _PyObject_GetDictPtr(self); -#endif -} - PYBIND11_INLINE void clear_instance(PyObject *self) { auto *instance = reinterpret_cast(self); @@ -572,7 +588,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { clear_instance(self); - type->tp_free(self); + type_free(type, self); // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index c0de2088b0..f49e3a6edd 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -118,6 +118,13 @@ void register_instance(instance *self, void *valptr, const type_info *tinfo); bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); +/// `type->tp_alloc(type, 0)` / `type->tp_free(self)`, also without direct slot access. +PyObject *type_alloc(PyTypeObject *type); +void type_free(PyTypeObject *type, PyObject *self); + +/// Pointer to the `__dict__` slot of an instance, or nullptr if its type has none. +PyObject **instance_dict_ptr(PyObject *self); + /// Instance creation function for all pybind11 types. It allocates the internal instance layout /// for holding C++ objects and holders. Allocation is done lazily (the first time the instance is /// cast to a reference or pointer), and initialization is done by an `__init__` function. @@ -136,9 +143,6 @@ void add_patient(PyObject *nurse, PyObject *patient); void clear_patients(PyObject *self); -/// Returns the address of the instance `__dict__` slot, or nullptr if there is none. -PyObject **instance_dict_ptr(PyObject *self); - /// Clears all internal data from the instance and removes it from registered instances in /// preparation for deallocation. void clear_instance(PyObject *self); From 651af1223b612c469ace132eb16fe8e3cc689ed2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:15:33 -0400 Subject: [PATCH 10/59] feat: support the Python stable ABI (Py_LIMITED_API >= 0x030C0000) Replace the Py_LIMITED_API #error with real gates and the missing pieces: - Require CPython 3.12+, reject PyPy/GraalPy and free-threaded builds, force PYBIND11_SIMPLE_GIL_MANAGEMENT and PYBIND11_TYPE_CREATION_VIA_SPEC, disable subinterpreter support; embed.h and chrono.h #error, and py::custom_type_setup is unavailable. - Heap-allocated Py_tss_t keys (the struct is opaque). - dynamic_attr types and pybind11_static_property append the __dict__ slot to the base layout (no Py_TPFLAGS_MANAGED_DICT); the offset is cached in type_info because _PyObject_GetDictPtr is unavailable. - A pybind11-provided instancemethod type (shared through the internals) replaces PyInstanceMethod_Type; bound methods go through types.MethodType. - Attribute-based fallbacks for tp_name, __bases__/__mro__, type lookup, frame inspection, docstrings on PyCFunctions, dict.setdefault, length hints, generators, complex numbers, and eval/exec (Py_CompileString). - The import-time version check accepts any interpreter from the Py_LIMITED_API version on. Modules built this way run unchanged on every CPython from the targeted version on. See docs/advanced/stable_abi.rst (next commits) for the feature matrix. See https://github.com/pybind/pybind11/discussions/6104 (D1). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/attr.h | 8 + include/pybind11/cast.h | 2 +- include/pybind11/chrono.h | 5 + include/pybind11/complex.h | 27 +- include/pybind11/detail/class-inl.h | 324 ++++++++++++++++-- include/pybind11/detail/class.h | 28 ++ include/pybind11/detail/common.h | 110 ++++-- include/pybind11/detail/init.h | 15 +- include/pybind11/detail/internals.h | 71 ++-- .../pybind11/detail/type_caster_base-inl.h | 15 +- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/embed.h | 3 + include/pybind11/eval.h | 51 ++- include/pybind11/gil_simple.h | 2 + include/pybind11/pybind11-inl.h | 52 ++- include/pybind11/pybind11.h | 12 +- include/pybind11/pytypes.h | 157 ++++++++- include/pybind11/stl.h | 9 +- tests/env.py | 7 + tests/pybind11_tests.cpp | 6 + tests/test_call_policies.cpp | 3 + tests/test_call_policies.py | 3 +- tests/test_kwargs_and_defaults.cpp | 4 +- tests/test_methods_and_attributes.py | 3 + 24 files changed, 803 insertions(+), 116 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index ebac419663..048d165987 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -97,6 +97,7 @@ struct metaclass { explicit metaclass(handle value) : value(value) {} }; +#if !defined(Py_LIMITED_API) /// Specifies a custom callback with signature `void (PyHeapTypeObject*)` that /// may be used to customize the Python type. /// @@ -106,6 +107,8 @@ struct metaclass { /// work with later versions of pybind11. You may wish to consult the /// implementation of `make_new_python_type` in `detail/classes.h` to understand /// the context in which the callback will be run. +/// +/// Not available under the stable ABI (`Py_LIMITED_API`): PyHeapTypeObject is opaque there. struct custom_type_setup { using callback = std::function; @@ -113,6 +116,7 @@ struct custom_type_setup { callback value; }; +#endif /// Annotation that marks a class as local to the module: struct module_local { @@ -327,7 +331,9 @@ struct type_record { handle metaclass; /// Custom type setup. +#if !defined(Py_LIMITED_API) custom_type_setup::callback custom_type_setup_callback; +#endif /// Multiple inheritance marker bool multiple_inheritance : 1; @@ -567,12 +573,14 @@ struct process_attribute : process_attribute_default static void init(const dynamic_attr &, type_record *r) { r->dynamic_attr = true; } }; +#if !defined(Py_LIMITED_API) template <> struct process_attribute { static void init(const custom_type_setup &value, type_record *r) { r->custom_type_setup_callback = value.value; } }; +#endif template <> struct process_attribute : process_attribute_default { diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index d9f4e4507a..cc7975e3b0 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -2418,7 +2418,7 @@ unpacking_collector collect_arguments(Args &&...args) { template template object object_api::operator()(Args &&...args) const { -#ifndef NDEBUG +#if !defined(NDEBUG) && !defined(Py_LIMITED_API) // PyGILState_Check: not in the stable ABI if (!PyGILState_Check()) { pybind11_fail("pybind11::object_api<>::operator() PyGILState_Check() failure."); } diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 84c9aef3b8..adf911421d 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -12,6 +12,11 @@ #include "pybind11.h" +#if defined(Py_LIMITED_API) +# error \ + "pybind11/chrono.h is not available under the stable ABI (Py_LIMITED_API): the datetime C API accesses struct fields." +#endif + #include #include #include diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 2d37a12277..5e2309bedb 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -57,10 +57,10 @@ class type_caster> { return false; } handle src_or_index = src; + object index; // Older PyPy versions need explicit __index__ conversion before PyComplex_AsCComplex(). // Preserve complex values and __complex__ precedence. #if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x08000000 - object index; if (!PyComplex_Check(src.ptr()) && !hasattr(type::handle_of(src), "__complex__") && PYBIND11_INDEX_CHECK(src.ptr())) { index = reinterpret_steal(PyNumber_Index(src.ptr())); @@ -73,12 +73,37 @@ class type_caster> { } } #endif +#if !defined(Py_LIMITED_API) Py_complex result = PyComplex_AsCComplex(src_or_index.ptr()); if (result.real == -1.0 && PyErr_Occurred()) { PyErr_Clear(); return false; } value = std::complex((T) result.real, (T) result.imag); +#else + // Py_complex is not part of the stable ABI. Before 3.13, PyComplex_RealAsDouble() does + // not call __complex__ itself. + if (!PyComplex_Check(src_or_index.ptr()) && hasattr(src_or_index, "__complex__")) { + index = reinterpret_steal( + PyObject_CallMethod(src_or_index.ptr(), "__complex__", nullptr)); + if (!index) { + PyErr_Clear(); + return false; + } + src_or_index = index; + } + double real = PyComplex_RealAsDouble(src_or_index.ptr()); + if (real == -1.0 && PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + double imag = PyComplex_ImagAsDouble(src_or_index.ptr()); + if (imag == -1.0 && PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + value = std::complex((T) real, (T) imag); +#endif return true; } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 94ae3138a3..21250d8966 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -16,6 +16,8 @@ #include "class.h" +#include + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -32,7 +34,7 @@ PYBIND11_INLINE std::string get_fully_qualified_tp_name(PyTypeObject *type) { } PYBIND11_INLINE PyTypeObject *type_incref(PyTypeObject *type) { - Py_INCREF(type); + Py_INCREF(reinterpret_cast(type)); return type; } @@ -48,7 +50,6 @@ type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, P } #endif - // Slots of the static base types that the pybind11 metaclass and static property type forward // to. PyType_GetSlot() works on static types since Python 3.10 and is part of the stable ABI; // the function-local caches make each lookup happen once per module. Below 3.10 (and on PyPy, @@ -91,9 +92,31 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { # if PYBIND11_TYPE_CREATION_VIA_SPEC +# if defined(Py_LIMITED_API) +// Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two +// functions find it through the type's `__dictoffset__`. +PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { + static const Py_ssize_t offset = handle(reinterpret_cast(Py_TYPE(self))) + .attr("__dictoffset__") + .cast(); + return reinterpret_cast(reinterpret_cast(self) + offset); +} +extern "C" PYBIND11_INLINE int +pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(*static_property_dict_ptr(self)); + Py_VISIT(Py_TYPE(self)); + return 0; +} +extern "C" PYBIND11_INLINE int pybind11_static_property_clear(PyObject *self) { + Py_CLEAR(*static_property_dict_ptr(self)); + return 0; +} +# endif + PYBIND11_INLINE PyTypeObject *make_static_property_type() { // Since Python-3.12 property-derived types are required to have dynamic attributes (to set // `__doc__`), hence the GC and dict slots. +# if !defined(Py_LIMITED_API) static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, {Py_tp_traverse, reinterpret_cast(pybind11_traverse)}, @@ -106,6 +129,26 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT, slots}; +# else + // Append the `__dict__` slot to the property layout. (PEP 697 relative offsets are not + // applied to `__dictoffset__` on 3.12, so the offset is absolute.) + const auto dictoffset = type_generic_getattr(&PyProperty_Type, "__basicsize__").cast(); + PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + PyType_Slot slots[] + = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, + {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, + {Py_tp_traverse, reinterpret_cast(pybind11_static_property_traverse)}, + {Py_tp_clear, reinterpret_cast(pybind11_static_property_clear)}, + {Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_static_property", + dictoffset + static_cast(sizeof(PyObject *)), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + slots}; +# endif return type_from_spec("make_static_property_type()", nullptr, &spec, @@ -213,7 +256,7 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { object descr = type_lookup((PyTypeObject *) obj, name); - if (descr && PyInstanceMethod_Check(descr.ptr())) { + if (descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr())) { return descr.release().ptr(); } return type_type_getattro()(obj, name); @@ -434,6 +477,161 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ return ret; } +#if defined(Py_LIMITED_API) + +// Stand-in for PyInstanceMethod_Type, which the stable ABI does not export. Like CPython's +// instancemethod: `__get__` binds through types.MethodType, calls and unknown attributes go to +// the wrapped function. The type is shared through the internals. +struct instancemethod_object { + PyObject_HEAD + PyObject *func; +}; + +PYBIND11_INLINE PyTypeObject *get_bound_method_type() { + static PyTypeObject *const type = [] { + auto types = reinterpret_steal(PyImport_ImportModule("types")); + if (!types) { + throw error_already_set(); + } + // Static builtin type: keep the reference forever. + return reinterpret_cast( + types.attr("MethodType").cast().release().ptr()); + }(); + return type; +} + +PYBIND11_INLINE bool is_bound_method(PyObject *obj) { + return Py_TYPE(obj) == get_bound_method_type(); +} + +PYBIND11_INLINE PyObject *bound_method_function(PyObject *obj) { + PyObject *func = PyObject_GetAttrString(obj, "__func__"); + if (func == nullptr) { + PyErr_Clear(); + return nullptr; + } + Py_DECREF(func); // the method object keeps the function alive + return func; +} + +extern "C" PYBIND11_INLINE PyObject * +instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { + if (obj == nullptr || obj == Py_None) { + Py_INCREF(self); + return self; + } + return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), + reinterpret_cast(self)->func, + obj, + nullptr); +} + +extern "C" PYBIND11_INLINE PyObject * +instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { + return PyObject_Call(reinterpret_cast(self)->func, args, kwargs); +} + +extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { + // Descriptors of the instancemethod type itself (`__func__`, `__class__`, ...) win; + // everything else (`__name__`, `__doc__`, `__module__`, ...) comes from the function. + object descr = type_lookup(Py_TYPE(self), name); + if (descr) { + auto get = reinterpret_cast( + PyType_GetSlot(Py_TYPE(descr.ptr()), Py_tp_descr_get)); + if (get != nullptr) { + return get(descr.ptr(), self, reinterpret_cast(Py_TYPE(self))); + } + } + return PyObject_GetAttr(reinterpret_cast(self)->func, name); +} + +extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { + PyObject *func = reinterpret_cast(self)->func; + auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); + if (!name) { + PyErr_Clear(); + return PyUnicode_FromFormat("", self); + } + return PyUnicode_FromFormat("", name.ptr(), self); +} + +extern "C" PYBIND11_INLINE int +instancemethod_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(reinterpret_cast(self)->func); + Py_VISIT(Py_TYPE(self)); + return 0; +} + +extern "C" PYBIND11_INLINE int instancemethod_clear(PyObject *self) { + Py_CLEAR(reinterpret_cast(self)->func); + return 0; +} + +extern "C" PYBIND11_INLINE void instancemethod_dealloc(PyObject *self) { + PyTypeObject *type = Py_TYPE(self); + PyObject_GC_UnTrack(self); + Py_CLEAR(reinterpret_cast(self)->func); + type_free(type, self); + Py_DECREF(reinterpret_cast(type)); +} + +PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { + return with_internals([](internals &internals) { + if (internals.instancemethod_type == nullptr) { + static PyMemberDef members[] = {{"__func__", + Py_T_OBJECT_EX, + offsetof(instancemethod_object, func), + Py_READONLY, + nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + static PyType_Slot slots[] + = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, + {Py_tp_call, reinterpret_cast(instancemethod_call)}, + {Py_tp_getattro, reinterpret_cast(instancemethod_getattro)}, + {Py_tp_repr, reinterpret_cast(instancemethod_repr)}, + {Py_tp_traverse, reinterpret_cast(instancemethod_traverse)}, + {Py_tp_clear, reinterpret_cast(instancemethod_clear)}, + {Py_tp_dealloc, reinterpret_cast(instancemethod_dealloc)}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + static PyType_Spec spec + = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", + static_cast(sizeof(instancemethod_object)), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, + slots}; + PyObject *type = PyType_FromSpec(&spec); + if (type == nullptr) { + pybind11_fail("get_instancemethod_type(): failure in PyType_FromSpec(): " + + error_string()); + } + internals.instancemethod_type = reinterpret_cast(type); + } + return internals.instancemethod_type; + }); +} + +PYBIND11_INLINE bool is_instancemethod(PyObject *obj) { + return Py_TYPE(obj) == get_instancemethod_type(); +} + +PYBIND11_INLINE PyObject *instancemethod_function(PyObject *obj) { + return reinterpret_cast(obj)->func; +} + +PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { + PyTypeObject *type = get_instancemethod_type(); + PyObject *self = type_alloc(type); + if (self == nullptr) { + return nullptr; + } + Py_INCREF(func); + reinterpret_cast(self)->func = func; + return self; +} + +#endif // Py_LIMITED_API + PYBIND11_INLINE PyObject *type_alloc(PyTypeObject *type) { #if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) return type->tp_alloc(type, 0); @@ -459,8 +657,19 @@ PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { auto offset = Py_TYPE(self)->tp_dictoffset; return offset > 0 ? reinterpret_cast(reinterpret_cast(self) + offset) : nullptr; -#else +#elif !defined(Py_LIMITED_API) return _PyObject_GetDictPtr(self); +#else + // Only `py::dynamic_attr()` types have a `__dict__` slot pybind11 manages (recorded in the + // type_info of the pybind11 type that introduced it); Python subclasses that add a managed + // dict clear it themselves before calling the base tp_dealloc. + for (const auto *tinfo : all_type_info(Py_TYPE(self))) { + if (tinfo->dictoffset > 0) { + return reinterpret_cast(reinterpret_cast(self) + + tinfo->dictoffset); + } + } + return nullptr; #endif } @@ -575,7 +784,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { PyObject_GC_UnTrack(self); } -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) // PyObject_ClearManagedDict() is available from Python 3.13+. It must be // called before tp_free() because on Python 3.14+ tp_free no longer // implicitly clears the managed dict, which would abandon the refcounts of @@ -592,7 +801,7 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { // This was not needed before Python 3.8 (Python issue 35810) // https://github.com/pybind/pybind11/issues/1946 - Py_DECREF(type); + Py_DECREF(reinterpret_cast(type)); } #if PYBIND11_TYPE_CREATION_VIA_SPEC @@ -668,14 +877,15 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { #endif // PYBIND11_TYPE_CREATION_VIA_SPEC extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit, void *arg) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) int ret = PyObject_VisitManagedDict(self, visit, arg); if (ret) { return ret; } #else - PyObject *&dict = *instance_dict_ptr(self); - Py_VISIT(dict); + if (PyObject **dict = instance_dict_ptr(self)) { + Py_VISIT(*dict); + } #endif // https://docs.python.org/3/c-api/typeobj.html#c.PyTypeObject.tp_traverse Py_VISIT(Py_TYPE(self)); @@ -683,11 +893,12 @@ extern "C" PYBIND11_INLINE int pybind11_traverse(PyObject *self, visitproc visit } extern "C" PYBIND11_INLINE int pybind11_clear(PyObject *self) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PY_VERSION_HEX >= 0x030D0000 && !defined(Py_LIMITED_API) PyObject_ClearManagedDict(self); #else - PyObject *&dict = *instance_dict_ptr(self); - Py_CLEAR(dict); + if (PyObject **dict = instance_dict_ptr(self)) { + Py_CLEAR(*dict); + } #endif return 0; } @@ -699,19 +910,21 @@ PYBIND11_INLINE PyGetSetDef *dynamic_attr_getset() { return getset; } +#if !defined(Py_LIMITED_API) PYBIND11_INLINE void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; -#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +# ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it -#else +# else type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; -#endif +# endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; type->tp_getset = dynamic_attr_getset(); } +#endif // !Py_LIMITED_API extern "C" PYBIND11_INLINE int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) { // Look for a `get_buffer` implementation in this type's info or any bases (following MRO). @@ -817,12 +1030,14 @@ extern "C" PYBIND11_INLINE void pybind11_releasebuffer(PyObject *, Py_buffer *vi delete (buffer_info *) view->internal; } +#if !defined(Py_LIMITED_API) PYBIND11_INLINE void enable_buffer_protocol(PyHeapTypeObject *heap_type) { heap_type->ht_type.tp_as_buffer = &heap_type->as_buffer; heap_type->as_buffer.bf_getbuffer = pybind11_getbuffer; heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer; } +#endif // !Py_LIMITED_API PYBIND11_INLINE type_names make_type_names(const type_record &rec, const object &module_) { type_names names; @@ -842,15 +1057,17 @@ PYBIND11_INLINE type_names make_type_names(const type_record &rec, const object return names; } +#if !defined(Py_LIMITED_API) // For a type nothing else can see yet: makes no Python calls that could run the GC, so it is // safe before PyType_Ready(). PYBIND11_INLINE void set_type_names(PyHeapTypeObject *heap_type, const type_names &names) { Py_XSETREF(heap_type->ht_name, names.name.inc_ref().ptr()); -#ifdef PYBIND11_BUILTIN_QUALNAME +# ifdef PYBIND11_BUILTIN_QUALNAME Py_XSETREF(heap_type->ht_qualname, names.qualname.inc_ref().ptr()); -#endif +# endif heap_type->ht_type.tp_name = names.full_name; } +#endif #if PYBIND11_TYPE_CREATION_VIA_SPEC // For a type that is already reachable (through its bases' __subclasses__()): the name objects @@ -860,20 +1077,32 @@ PYBIND11_INLINE void set_type_names(PyHeapTypeObject *heap_type, const type_name // type (CPython's own copy, the ht_name UTF-8 cache, or pybind11's interned full name), so the // pointer swap needs no synchronization. PYBIND11_INLINE void set_published_type_names(PyTypeObject *type, const type_names &names) { - auto *heap_type = reinterpret_cast(type); auto *type_obj = reinterpret_cast(type); const auto set_name = [type_obj](const char *attr, const object &value) { object descriptors = reinterpret_borrow(reinterpret_cast(&PyType_Type)) .attr("__dict__"); descriptors[attr].attr("__set__")(handle(type_obj), value); }; - if (!names.name.equal(handle(heap_type->ht_name))) { +# if !defined(Py_LIMITED_API) + auto *heap_type = reinterpret_cast(type); + object current_name = reinterpret_borrow(heap_type->ht_name); + object current_qualname = reinterpret_borrow(heap_type->ht_qualname); +# else + object current_name = type_generic_getattr(type, "__name__"); + object current_qualname = type_generic_getattr(type, "__qualname__"); +# endif + if (!names.name.equal(current_name)) { set_name("__name__", names.name); } - if (!names.qualname.equal(handle(heap_type->ht_qualname))) { + if (!names.qualname.equal(current_qualname)) { set_name("__qualname__", names.qualname); } +# if !defined(Py_LIMITED_API) type->tp_name = names.full_name; +# else + // tp_name is CPython's copy of the spec name, which already is the fully qualified name. + (void) names.full_name; +# endif } #endif @@ -887,16 +1116,26 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { // PyType_FromMetaclass() cannot run a pre-PyType_Ready callback, rejects metaclasses with a // custom tp_new, and refuses a metaclass that is not at least as derived as the metaclass of // each base. The legacy path accepts all three. +# if !defined(Py_LIMITED_API) bool base_metaclass_mismatch = false; for (handle b : rec.bases) { base_metaclass_mismatch |= PyType_IsSubtype(metaclass, Py_TYPE(b.ptr())) == 0; } if (rec.custom_type_setup_callback || base_metaclass_mismatch - || metaclass->tp_new != PyType_Type.tp_new + || PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new) || (metaclass != internals.default_metaclass && PyType_IsSubtype(metaclass, internals.default_metaclass) == 0)) { return make_new_python_type_legacy(rec); } +# else + // Without the legacy path, a less derived py::metaclass() (e.g. `type`) is resolved to the + // most derived one by PyType_FromMetaclass(), i.e. the class gets pybind11_type. + if (PyType_GetSlot(metaclass, Py_tp_new) != PyType_GetSlot(&PyType_Type, Py_tp_new)) { + pybind11_fail(std::string(rec.name) + + ": py::metaclass() must not define __new__ under the stable ABI " + "(Py_LIMITED_API)"); + } +# endif object module_ = get_module_name_if_available(rec.scope); // PyType_FromMetaclass() copies the name, splits it at the last dot, and warns if there is @@ -922,8 +1161,34 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { if (!rec.is_final) { flags |= Py_TPFLAGS_BASETYPE; } + int basicsize = 0; // inherit the instance layout from the base +# if defined(Py_LIMITED_API) + // No Py_TPFLAGS_MANAGED_DICT in the stable ABI: the `__dict__` slot is appended to the base + // layout unless a base has one already. (PEP 697 relative offsets are not applied to + // `__dictoffset__` on 3.12, so the offset is absolute.) + PyMemberDef dict_member[] = {{"__dictoffset__", Py_T_PYSSIZET, 0, Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; +# endif if (rec.dynamic_attr) { - flags |= Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_MANAGED_DICT; + flags |= Py_TPFLAGS_HAVE_GC; +# if !defined(Py_LIMITED_API) + flags |= Py_TPFLAGS_MANAGED_DICT; +# else + bool base_has_dict = false; + int base_basicsize = 0; + for (handle b : bases) { + auto *b_type = reinterpret_cast(b.ptr()); + base_has_dict + |= type_generic_getattr(b_type, "__dictoffset__").cast() != 0; + base_basicsize = std::max(base_basicsize, + type_generic_getattr(b_type, "__basicsize__").cast()); + } + if (!base_has_dict) { + dict_member[0].offset = base_basicsize; + basicsize = base_basicsize + static_cast(sizeof(PyObject *)); + slots.push_back({Py_tp_members, reinterpret_cast(dict_member)}); + } +# endif slots.push_back({Py_tp_traverse, reinterpret_cast(pybind11_traverse)}); slots.push_back({Py_tp_clear, reinterpret_cast(pybind11_clear)}); slots.push_back({Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}); @@ -934,8 +1199,7 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { } slots.push_back({0, nullptr}); - // basicsize 0: inherit the instance layout from the base - PyType_Spec spec = {spec_name.c_str(), 0, 0, flags, slots.data()}; + PyType_Spec spec = {spec_name.c_str(), basicsize, 0, flags, slots.data()}; auto *type = type_from_spec(rec.name, metaclass, &spec, base.ptr()); // CPython keeps its own copy of the spec name in _ht_tpname and frees that, not tp_name. set_published_type_names(type, names); @@ -945,7 +1209,7 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { if (rec.scope) { setattr(rec.scope, rec.name, reinterpret_cast(type)); } else { - Py_INCREF(type); // Keep it alive forever (reference leak) + Py_INCREF(reinterpret_cast(type)); // Keep it alive forever (reference leak) } if (module_) { // Needed by pydoc @@ -963,6 +1227,7 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { #endif // PYBIND11_TYPE_CREATION_VIA_SPEC +#if !defined(Py_LIMITED_API) PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { object module_ = get_module_name_if_available(rec.scope); auto names = make_type_names(rec, module_); @@ -971,11 +1236,11 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { if (rec.doc && options::show_user_defined_docstrings()) { /* Allocate memory for docstring (Python will free this later on) */ size_t size = std::strlen(rec.doc) + 1; -#if PY_VERSION_HEX >= 0x030D0000 +# if PY_VERSION_HEX >= 0x030D0000 tp_doc = static_cast(PyMem_MALLOC(size)); -#else +# else tp_doc = (char *) PyObject_MALLOC(size); -#endif +# endif std::memcpy((void *) tp_doc, rec.doc, size); } @@ -1042,7 +1307,7 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { if (rec.scope) { setattr(rec.scope, rec.name, reinterpret_cast(type)); } else { - Py_INCREF(type); // Keep it alive forever (reference leak) + Py_INCREF(reinterpret_cast(type)); // Keep it alive forever (reference leak) } if (module_) { // Needed by pydoc @@ -1053,6 +1318,7 @@ PYBIND11_INLINE PyObject *make_new_python_type_legacy(const type_record &rec) { return reinterpret_cast(type); } +#endif // !Py_LIMITED_API PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index f49e3a6edd..434f449063 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -53,6 +53,12 @@ descrsetfunc property_type_descr_set(); #if !defined(PYPY_VERSION) +# if defined(Py_LIMITED_API) +PyObject **static_property_dict_ptr(PyObject *self); +extern "C" int pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg); +extern "C" int pybind11_static_property_clear(PyObject *self); +# endif + /// `pybind11_static_property.__get__()`: Always pass the class instead of the instance. extern "C" PyObject *pybind11_static_get(PyObject *self, PyObject * /*ob*/, PyObject *cls); @@ -122,6 +128,20 @@ bool deregister_instance(instance *self, void *valptr, const type_info *tinfo); PyObject *type_alloc(PyTypeObject *type); void type_free(PyTypeObject *type, PyObject *self); +#if defined(Py_LIMITED_API) +/// Stand-ins for the instancemethod and method C APIs (see detail/class-inl.h). The +/// `is_*`/`*_function` functions are also declared in pytypes.h. +PyTypeObject *get_bound_method_type(); +PyTypeObject *get_instancemethod_type(); +extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); +extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); +extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); +extern "C" PyObject *instancemethod_repr(PyObject *self); +extern "C" int instancemethod_traverse(PyObject *self, visitproc visit, void *arg); +extern "C" int instancemethod_clear(PyObject *self); +extern "C" void instancemethod_dealloc(PyObject *self); +#endif + /// Pointer to the `__dict__` slot of an instance, or nullptr if its type has none. PyObject **instance_dict_ptr(PyObject *self); @@ -167,8 +187,10 @@ extern "C" int pybind11_clear(PyObject *self); /// The `__dict__` descriptor for types with dynamic attributes. PyGetSetDef *dynamic_attr_getset(); +#if !defined(Py_LIMITED_API) /// Give instances of this type a `__dict__` and opt into garbage collection. void enable_dynamic_attributes(PyHeapTypeObject *heap_type); +#endif /// buffer_protocol: Fill in the view as specified by flags. extern "C" int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags); @@ -176,8 +198,10 @@ extern "C" int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags); /// buffer_protocol: Release the resources of the buffer. extern "C" void pybind11_releasebuffer(PyObject *, Py_buffer *view); +#if !defined(Py_LIMITED_API) /// Give this type a buffer interface. void enable_buffer_protocol(PyHeapTypeObject *heap_type); +#endif struct type_names { object name; @@ -185,7 +209,9 @@ struct type_names { const char *full_name = nullptr; // interned, lives forever }; type_names make_type_names(const type_record &rec, const object &module_); +#if !defined(Py_LIMITED_API) void set_type_names(PyHeapTypeObject *heap_type, const type_names &names); +#endif #if PYBIND11_TYPE_CREATION_VIA_SPEC void set_published_type_names(PyTypeObject *type, const type_names &names); #endif @@ -194,10 +220,12 @@ void set_published_type_names(PyTypeObject *type, const type_names &names); Return value: New reference. */ PyObject *make_new_python_type(const type_record &rec); +#if !defined(Py_LIMITED_API) /// The hand-filled PyHeapTypeObject path: the only one before CPython 3.12 or with /// PYBIND11_TYPE_CREATION_VIA_SPEC=0. Otherwise still used for `py::custom_type_setup` and for /// metaclasses that PyType_FromMetaclass() rejects. PyObject *make_new_python_type_legacy(const type_record &rec); +#endif PYBIND11_WARNING_POP diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 6254091a9f..8c25d086cd 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -14,9 +14,37 @@ # error "PYTHON < 3.9 IS UNSUPPORTED. pybind11 v3.0 was the last to support Python 3.8." #endif +// Python stable ABI (PEP 384) support. Requires CPython 3.12+ (PyType_FromMetaclass) and a +// matching Py_LIMITED_API value. See docs/advanced/stable_abi.rst for the feature matrix. #if defined(Py_LIMITED_API) -# error \ - "pybind11 does not support Py_LIMITED_API (the Python stable ABI) yet. Track progress at https://github.com/pybind/pybind11/discussions/6104" +# if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) +# error "Py_LIMITED_API is only supported with CPython." +# endif +# if PY_VERSION_HEX < 0x030C0000 || Py_LIMITED_API + 0 < 0x030C0000 +# error \ + "pybind11 requires Py_LIMITED_API >= 0x030C0000 and CPython >= 3.12 for the stable ABI." +# endif +# if defined(Py_GIL_DISABLED) +# error "The free-threaded build has no stable ABI; do not define Py_LIMITED_API." +# endif +# if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) +# define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals +# endif +# if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) +# define PYBIND11_TYPE_CREATION_VIA_SPEC \ + 1 // the only type-creation path without struct access +# elif !PYBIND11_TYPE_CREATION_VIA_SPEC +# error \ + "The legacy type-creation path (PYBIND11_TYPE_CREATION_VIA_SPEC=0) needs struct access." +# endif +# if !defined(PYBIND11_HAS_SUBINTERPRETER_SUPPORT) +# define PYBIND11_HAS_SUBINTERPRETER_SUPPORT 0 // reads thread/interpreter state fields +# elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT +# error "Subinterpreter support is not available under Py_LIMITED_API." +# endif +# if !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) +# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET // no Py_TPFLAGS_MANAGED_DICT +# endif #endif // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html @@ -260,7 +288,7 @@ # if PY_VERSION_HEX < 0x030C0000 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) # error "PYBIND11_TYPE_CREATION_VIA_SPEC requires CPython 3.12 or newer." # endif -# if defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) +# if defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) && !defined(Py_LIMITED_API) # error "PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET needs the legacy type creation path" # endif #endif @@ -273,6 +301,7 @@ #endif #include +#include #include #include #include @@ -310,9 +339,10 @@ # define PYBIND11_HAS_SPAN 1 #endif -// See description of PR #4246: +// See description of PR #4246. PyGILState_Check() is not part of the stable ABI. #if !defined(PYBIND11_NO_ASSERT_GIL_HELD_INCREF_DECREF) && !defined(NDEBUG) \ - && !defined(PYPY_VERSION) && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) + && !defined(PYPY_VERSION) && !defined(Py_LIMITED_API) \ + && !defined(PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF) # define PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF #endif @@ -374,9 +404,15 @@ // behavior. /// Compatibility macros for Python 2 / Python 3 versions TODO: remove -#define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) -#define PYBIND11_INSTANCE_METHOD_CHECK PyInstanceMethod_Check -#define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyInstanceMethod_GET_FUNCTION +#if !defined(Py_LIMITED_API) +# define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) PyInstanceMethod_New(ptr) +# define PYBIND11_INSTANCE_METHOD_CHECK PyInstanceMethod_Check +# define PYBIND11_INSTANCE_METHOD_GET_FUNCTION PyInstanceMethod_GET_FUNCTION +#else // PyInstanceMethod_Type is not part of the stable ABI; see detail/class-inl.h +# define PYBIND11_INSTANCE_METHOD_NEW(ptr, class_) ::pybind11::detail::instancemethod_new(ptr) +# define PYBIND11_INSTANCE_METHOD_CHECK ::pybind11::detail::is_instancemethod +# define PYBIND11_INSTANCE_METHOD_GET_FUNCTION ::pybind11::detail::instancemethod_function +#endif #define PYBIND11_BYTES_CHECK PyBytes_Check #define PYBIND11_BYTES_FROM_STRING PyBytes_FromString #define PYBIND11_BYTES_FROM_STRING_AND_SIZE PyBytes_FromStringAndSize @@ -433,22 +469,42 @@ } while (0) #endif -#define PYBIND11_CHECK_PYTHON_VERSION \ - { \ - const char *compiled_ver \ - = PYBIND11_TOSTRING(PY_MAJOR_VERSION) "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ - const char *runtime_ver = Py_GetVersion(); \ - size_t len = std::strlen(compiled_ver); \ - if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ - || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ - PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for Python %s, " \ - "but the interpreter version is incompatible: %s.", \ - compiled_ver, \ - runtime_ver); \ - return nullptr; \ - } \ - } +#if !defined(Py_LIMITED_API) +# define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *compiled_ver \ + = PYBIND11_TOSTRING(PY_MAJOR_VERSION) "." PYBIND11_TOSTRING(PY_MINOR_VERSION); \ + const char *runtime_ver = Py_GetVersion(); \ + size_t len = std::strlen(compiled_ver); \ + if (std::strncmp(runtime_ver, compiled_ver, len) != 0 \ + || (runtime_ver[len] >= '0' && runtime_ver[len] <= '9')) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for Python %s, " \ + "but the interpreter version is incompatible: %s.", \ + compiled_ver, \ + runtime_ver); \ + return nullptr; \ + } \ + } +#else +// A stable-ABI module runs on every CPython from the Py_LIMITED_API version on (abi3). +# define PYBIND11_CHECK_PYTHON_VERSION \ + { \ + const char *runtime_ver = Py_GetVersion(); \ + long major = std::strtol(runtime_ver, nullptr, 10); \ + const char *dot = std::strchr(runtime_ver, '.'); \ + long minor = dot ? std::strtol(dot + 1, nullptr, 10) : 0; \ + if (major != ((Py_LIMITED_API) >> 24) || minor < (((Py_LIMITED_API) >> 16) & 0xFF)) { \ + PyErr_Format(PyExc_ImportError, \ + "Python version mismatch: module was compiled for the Python %d.%d " \ + "stable ABI, but the interpreter version is incompatible: %s.", \ + (int) ((Py_LIMITED_API) >> 24), \ + (int) (((Py_LIMITED_API) >> 16) & 0xFF), \ + runtime_ver); \ + return nullptr; \ + } \ + } +#endif #define PYBIND11_CATCH_INIT_EXCEPTIONS \ catch (pybind11::error_already_set & e) { \ @@ -1458,8 +1514,10 @@ inline void silence_unused_warnings(Args &&...) {} # define PYBIND11_DETAILED_ERROR_MESSAGES #endif -// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. -#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) +// CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. (The +// stable ABI does not have it either; see the Py_LIMITED_API block at the top of this file.) +#if (PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION)) \ + && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index cce3eb5607..af23890d57 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) { + if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -399,8 +399,10 @@ struct factory { [func] #endif (value_and_holder &v_h, Args... args) { - construct( - v_h, func(std::forward(args)...), Py_TYPE(v_h.inst) != v_h.type->type); + construct(v_h, + func(std::forward(args)...), + Py_TYPE(reinterpret_cast(v_h.inst)) + != v_h.type->type); }, is_new_style_constructor(), extra...); @@ -447,7 +449,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(v_h.inst) == v_h.type->type) { + if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -535,8 +537,9 @@ struct pickle_factory { [func] #endif (value_and_holder &v_h, ArgState state) { - setstate( - v_h, func(std::forward(state)), Py_TYPE(v_h.inst) != v_h.type->type); + setstate(v_h, + func(std::forward(state)), + Py_TYPE(reinterpret_cast(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 85feb9faff..4fb8b10d61 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -121,29 +121,41 @@ using ExceptionTranslator = void (*)(std::exception_ptr); // The old Python Thread Local Storage (TLS) API is deprecated in Python 3.7 in favor of the new // Thread Specific Storage (TSS) API. -// Avoid unnecessary allocation of `Py_tss_t`, since we cannot use -// `Py_LIMITED_API` anyway. -#define PYBIND11_TLS_KEY_REF Py_tss_t & -#if defined(__clang__) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("clang diagnostic push") /**/ \ - _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var = Py_tss_NEEDS_INIT; \ - _Pragma("clang diagnostic pop") -#elif defined(__GNUC__) && !defined(__INTEL_COMPILER) -# define PYBIND11_TLS_KEY_INIT(var) \ - _Pragma("GCC diagnostic push") /**/ \ - _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ - Py_tss_t var = Py_tss_NEEDS_INIT; \ - _Pragma("GCC diagnostic pop") +// `Py_tss_t` is opaque under `Py_LIMITED_API`, so the key is heap-allocated there; otherwise it +// is embedded to avoid the allocation. +#if defined(Py_LIMITED_API) +using tss_key_ptr = Py_tss_t *; // lets PYBIND11_TLS_KEY_INIT(mutable key_) parse +# define PYBIND11_TLS_KEY_REF Py_tss_t * +# define PYBIND11_TLS_KEY_INIT(var) tss_key_ptr var = nullptr; +# define PYBIND11_TLS_KEY_CREATE(var) \ + (((var) = PyThread_tss_alloc()) != nullptr && PyThread_tss_create(var) == 0) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(key) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set((key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set((key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_free(key) #else -# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +# define PYBIND11_TLS_KEY_REF Py_tss_t & +# if defined(__clang__) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("clang diagnostic push") /**/ \ + _Pragma("clang diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("clang diagnostic pop") +# elif defined(__GNUC__) && !defined(__INTEL_COMPILER) +# define PYBIND11_TLS_KEY_INIT(var) \ + _Pragma("GCC diagnostic push") /**/ \ + _Pragma("GCC diagnostic ignored \"-Wmissing-field-initializers\"") /**/ \ + Py_tss_t var = Py_tss_NEEDS_INIT; \ + _Pragma("GCC diagnostic pop") +# else +# define PYBIND11_TLS_KEY_INIT(var) Py_tss_t var = Py_tss_NEEDS_INIT; +# endif +# define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) +# define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) +# define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) +# define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) +# define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) #endif -#define PYBIND11_TLS_KEY_CREATE(var) (PyThread_tss_create(&(var)) == 0) -#define PYBIND11_TLS_GET_VALUE(key) PyThread_tss_get(&(key)) -#define PYBIND11_TLS_REPLACE_VALUE(key, value) PyThread_tss_set(&(key), (value)) -#define PYBIND11_TLS_DELETE_VALUE(key) PyThread_tss_set(&(key), nullptr) -#define PYBIND11_TLS_FREE(key) PyThread_tss_delete(&(key)) /// A smart-pointer-like wrapper around a thread-specific value. get/set of the pointer applies to /// the current thread only. @@ -217,7 +229,11 @@ void translate_exception(std::exception_ptr p); inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); -#elif PY_VERSION_HEX < 0x030D0000 +#elif defined(Py_LIMITED_API) && Py_LIMITED_API < 0x030D0000 + // The stable ABI has no "current thread state or null" function before 3.13. All callers + // left under the stable ABI (the interpreter state dict lookup) hold the GIL. + return PyThreadState_Get(); +#elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) return _PyThreadState_UncheckedGet(); #else return PyThreadState_GetUnchecked(); @@ -390,6 +406,11 @@ struct internals { PyTypeObject *static_property_type = nullptr; PyTypeObject *default_metaclass = nullptr; PyObject *instance_base = nullptr; +#if defined(Py_LIMITED_API) + // Replacement for PyInstanceMethod_Type, which is not part of the stable ABI. Shared so + // that every module recognizes the methods of every other module's classes. + PyTypeObject *instancemethod_type = nullptr; +#endif // Unused if PYBIND11_SIMPLE_GIL_MANAGEMENT is defined: thread_specific_storage tstate; #if PYBIND11_INTERNALS_VERSION <= 11 @@ -485,6 +506,12 @@ struct type_info { std::forward_list alias_chain; #endif +#if defined(Py_LIMITED_API) + /* Offset of the `__dict__` slot of instances for `py::dynamic_attr()` types (0 if none); + * `_PyObject_GetDictPtr` is not available under the stable ABI. */ + Py_ssize_t dictoffset = 0; +#endif + /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index addf38c4cd..a593b5b2e7 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -288,7 +288,8 @@ find_registered_python_instance(void *src, const detail::type_info *tinfo) { return with_instance_map(src, [&](instance_map &instances) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { - for (auto *instance_type : detail::all_type_info(Py_TYPE(it_i->second))) { + for (auto *instance_type : + detail::all_type_info(Py_TYPE(reinterpret_cast(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { auto *wrapper = reinterpret_cast(it_i->second); if (try_incref(wrapper)) { @@ -305,7 +306,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { // Optimize common case: - if (!find_type || Py_TYPE(this) == find_type->type) { + if (!find_type || Py_TYPE(reinterpret_cast(this)) == find_type->type) { return value_and_holder(this, find_type, 0, 0); } @@ -320,10 +321,10 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" - + get_fully_qualified_tp_name(find_type->type) - + "' is not a pybind11 base of the given `" - + get_fully_qualified_tp_name(Py_TYPE(this)) + "' instance"); + pybind11_fail( + "pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(reinterpret_cast(this))) + "' instance"); #else pybind11_fail( "pybind11::detail::instance::get_value_and_holder: " @@ -333,7 +334,7 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void instance::allocate_layout() { - const auto &tinfo = all_type_info(Py_TYPE(this)); + const auto &tinfo = all_type_info(Py_TYPE(reinterpret_cast(this))); const size_t n_types = tinfo.size(); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 0eec34eb17..114d2fc52b 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -142,7 +142,7 @@ struct values_and_holders { public: explicit values_and_holders(instance *inst) - : inst{inst}, tinfo(all_type_info(Py_TYPE(inst))) {} + : inst{inst}, tinfo(all_type_info(Py_TYPE(reinterpret_cast(inst)))) {} explicit values_and_holders(PyObject *obj) : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index c333892ff5..9a8f4a7916 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -15,6 +15,9 @@ #if defined(PYPY_VERSION) # error Embedding the interpreter is not supported with PyPy #endif +#if defined(Py_LIMITED_API) +# error "pybind11/embed.h is not available under the stable ABI (Py_LIMITED_API)." +#endif #define PYBIND11_EMBEDDED_MODULE_IMPL(name) \ extern "C" PyObject *pybind11_init_impl_##name(); \ diff --git a/include/pybind11/eval.h b/include/pybind11/eval.h index ee24fd3584..ff0796e880 100644 --- a/include/pybind11/eval.h +++ b/include/pybind11/eval.h @@ -19,11 +19,12 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) inline void ensure_builtins_in_globals(object &global) { -#if defined(PYPY_VERSION) +#if defined(PYPY_VERSION) || defined(Py_LIMITED_API) // Running exec and eval adds `builtins` module under `__builtins__` key to // globals if not yet present. Python 3.8 made PyRun_String behave // similarly. Let's also do that for older versions, for consistency. This - // was missing from PyPy3.8 7.3.7. + // was missing from PyPy3.8 7.3.7, and PyEval_EvalCode (the stable-ABI path + // below) does not do it either. if (!global.contains("__builtins__")) global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE); #else @@ -31,6 +32,23 @@ inline void ensure_builtins_in_globals(object &global) { #endif } +#if defined(Py_LIMITED_API) +// PyRun_String() is not part of the stable ABI: compile and evaluate in two steps. +inline PyObject *run_string( + const char *source, const char *filename, int start, PyObject *global, PyObject *local) { + auto code = reinterpret_steal(Py_CompileString(source, filename, start)); + if (!code) { + return nullptr; + } + return PyEval_EvalCode(code.ptr(), global, local); +} +#else +inline PyObject *run_string( + const char *source, const char * /*filename*/, int start, PyObject *global, PyObject *local) { + return PyRun_String(source, start, global, local); +} +#endif + PYBIND11_NAMESPACE_END(detail) enum eval_mode { @@ -70,7 +88,8 @@ object eval(const str &expr, object global = globals(), object local = object()) pybind11_fail("invalid evaluation mode"); } - PyObject *result = PyRun_String(buffer.c_str(), start, global.ptr(), local.ptr()); + PyObject *result + = detail::run_string(buffer.c_str(), "", start, global.ptr(), local.ptr()); if (!result) { throw error_already_set(); } @@ -130,14 +149,31 @@ object eval_file(str fname, object global = globals(), object local = object()) pybind11_fail("invalid evaluation mode"); } - int closeFile = 1; std::string fname_str = (std::string) fname; +# if defined(Py_LIMITED_API) + // PyRun_FileEx() and Py_fopen() are not part of the stable ABI: read the file from Python. + object source; + try { + object io_open = module_::import("io").attr("open"); + object file = io_open(fname, "rb"); + source = file.attr("read")(); + file.attr("close")(); + } catch (error_already_set &) { + pybind11_fail("File \"" + fname_str + "\" could not be opened!"); + } + if (!global.contains("__file__")) { + global["__file__"] = std::move(fname); + } + PyObject *result = detail::run_string( + PyBytes_AsString(source.ptr()), fname_str.c_str(), start, global.ptr(), local.ptr()); +# else + int closeFile = 1; FILE *f = -# if PY_VERSION_HEX >= 0x030E0000 +# if PY_VERSION_HEX >= 0x030E0000 Py_fopen(fname.ptr(), "r"); -# else +# else _Py_fopen_obj(fname.ptr(), "r"); -# endif +# endif if (!f) { PyErr_Clear(); pybind11_fail("File \"" + fname_str + "\" could not be opened!"); @@ -149,6 +185,7 @@ object eval_file(str fname, object global = globals(), object local = object()) PyObject *result = PyRun_FileEx(f, fname_str.c_str(), start, global.ptr(), local.ptr(), closeFile); +# endif if (!result) { throw error_already_set(); diff --git a/include/pybind11/gil_simple.h b/include/pybind11/gil_simple.h index 9ff428ad84..61a0fe4269 100644 --- a/include/pybind11/gil_simple.h +++ b/include/pybind11/gil_simple.h @@ -26,7 +26,9 @@ class gil_scoped_release_simple { public: // PRECONDITION: The GIL must be held when this constructor is called. gil_scoped_release_simple() { +#if !defined(Py_LIMITED_API) // PyGILState_Check is not part of the stable ABI assert(PyGILState_Check()); +#endif state = PyEval_SaveThread(); } gil_scoped_release_simple(const gil_scoped_release_simple &) = delete; diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index e987ea1d89..98b8ade2bb 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -464,7 +464,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, /* Don't call dispatch code if invoked from overridden function. Unfortunately this doesn't work on PyPy and GraalPy. */ -#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) +#if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) && !defined(Py_LIMITED_API) PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); if (frame != nullptr) { PyCodeObject *f_code = PyFrame_GetCode(frame); @@ -497,6 +497,31 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, Py_DECREF(frame); } +#elif defined(Py_LIMITED_API) + // Same check through attribute access: the frame and code object layouts are not part of + // the stable ABI. + PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get()); + if (frame != nullptr) { + auto frame_obj = reinterpret_steal(reinterpret_cast(frame)); + auto f_code + = reinterpret_steal(reinterpret_cast(PyFrame_GetCode(frame))); + if (std::string(str(f_code.attr("co_name"))) == name + && f_code.attr("co_argcount").cast() > 0) { + object self_arg = f_code.attr("co_varnames")[int_(0)]; + auto locals + = reinterpret_steal(PyObject_GetAttrString(frame_obj.ptr(), "f_locals")); + auto self_caller + = locals + ? reinterpret_steal(PyObject_GetItem(locals.ptr(), self_arg.ptr())) + : object(); + if (!self_caller) { + PyErr_Clear(); + } else if (self_caller.is(self)) { + return function(); + } + } + } + #else /* PyPy currently doesn't provide a detailed cpyext emulation of frame objects, so we have to emulate this using Python. This @@ -735,13 +760,26 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u } } - auto *func = reinterpret_cast(m_ptr); // Install docstring if it's non-empty (when at least one option is enabled) auto *doc = signatures.empty() ? nullptr : PYBIND11_COMPAT_STRDUP(signatures.c_str()); +#if !defined(Py_LIMITED_API) + auto *func = reinterpret_cast(m_ptr); std::free(const_cast(PYBIND11_PYCFUNCTION_GET_DOC(func))); PYBIND11_PYCFUNCTION_SET_DOC(func, doc); +#else + // PyCFunctionObject is opaque, but the function reads its `__doc__` from the PyMethodDef, + // which pybind11 owns: the one record in the chain that allocated it has `def` set. + for (auto *it = chain_start; it != nullptr; it = it->next) { + if (it->def) { + std::free(const_cast(it->def->ml_doc)); + it->def->ml_doc = doc; + break; + } + } +#endif if (rec->is_method) { + PyObject *func = m_ptr; m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); if (!m_ptr) { pybind11_fail( @@ -1293,6 +1331,9 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { tinfo->simple_ancestors = true; tinfo->module_local = rec.module_local; tinfo->holder_enum_v = rec.holder_enum_v; +#if defined(Py_LIMITED_API) + tinfo->dictoffset = handle(m_ptr).attr("__dictoffset__").cast(); +#endif with_internals([&](internals &internals) { auto tindex = std::type_index(*rec.type); @@ -1384,9 +1425,10 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic, [](handle arg) -> std::string { std::string docstring; dict entries = arg.attr("__entries"); - if ((reinterpret_cast(arg.ptr()))->tp_doc) { - docstring - += std::string(reinterpret_cast(arg.ptr())->tp_doc); + // tp_doc: the class docstring; `__doc__` itself is this property. + if (const auto *tp_doc = static_cast(PyType_GetSlot( + reinterpret_cast(arg.ptr()), Py_tp_doc))) { + docstring += tp_doc; docstring += "\n\n"; } docstring += "Members:"; diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 3a2bd9d7b2..464fb768d7 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -825,10 +825,10 @@ class generic_type : public object { void install_buffer_funcs(buffer_info *(*get_buffer)(PyObject *, void *), void *get_buffer_data) { - auto *type = reinterpret_cast(m_ptr); - auto *tinfo = detail::get_type_info(&type->ht_type); + auto *type = reinterpret_cast(m_ptr); + auto *tinfo = detail::get_type_info(type); - if (!type->ht_type.tp_as_buffer) { + if (PyType_GetSlot(type, Py_bf_getbuffer) == nullptr) { pybind11_fail("To be able to register buffer protocol support for the type '" + get_fully_qualified_tp_name(tinfo->type) + "' the associated class<>(..) invocation must " @@ -2067,8 +2067,10 @@ class enum_ : public class_ { def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { - detail::initimpl::setstate( - v_h, static_cast(arg), Py_TYPE(v_h.inst) != v_h.type->type); + detail::initimpl::setstate(v_h, + static_cast(arg), + Py_TYPE(reinterpret_cast(v_h.inst)) + != v_h.type->type); }, detail::is_new_style_constructor(), pybind11::name("__setstate__"), diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index ab7c36aa64..8a440170e6 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -500,6 +500,7 @@ T reinterpret_steal(handle h) { PYBIND11_NAMESPACE_BEGIN(detail) +#if !defined(Py_LIMITED_API) /// The `tp_name` of a type: `module.Name` for static and pybind11 types, `Name` for Python /// heap types. inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } @@ -508,6 +509,52 @@ inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +#else +bool type_is_managed_by_our_internals(PyTypeObject *type_obj); // detail/cpp_conduit.h + +inline std::string utf8_or_empty(const object &o) { + Py_ssize_t size = 0; + const char *data + = o && PyUnicode_Check(o.ptr()) ? PyUnicode_AsUTF8AndSize(o.ptr(), &size) : nullptr; + if (data == nullptr) { + PyErr_Clear(); + return std::string(); + } + return std::string(data, static_cast(size)); +} + +/// Reconstructs `tp_name`, which is not readable under the stable ABI: `__module__.__name__` +/// for static types outside builtins and for pybind11 types, the bare `__name__` otherwise +/// (builtins and Python heap types). +inline std::string get_tp_name(PyTypeObject *type) { + std::string name = utf8_or_empty(reinterpret_steal(PyType_GetName(type))); + if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE) && !type_is_managed_by_our_internals(type)) { + return name; + } + std::string module_ = utf8_or_empty( + reinterpret_steal(PyObject_GetAttrString((PyObject *) type, "__module__"))); + if (module_.empty() || module_ == PYBIND11_BUILTINS_MODULE) { + return name; + } + return module_ + '.' + name; +} + +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + // Cheap pre-check on the unqualified name before reconstructing the full one. + const char *dot = std::strrchr(name, '.'); + auto type_name = reinterpret_steal(PyType_GetName(type)); + const char *type_name_utf8 + = type_name ? PyUnicode_AsUTF8AndSize(type_name.ptr(), nullptr) : nullptr; + if (type_name_utf8 == nullptr) { + PyErr_Clear(); + return false; + } + if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) != 0) { + return false; + } + return get_tp_name(type) == name; +} +#endif // Unchecked tuple/list element access for hot paths. Function forms under the stable ABI. #if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) @@ -773,13 +820,32 @@ inline ssize_t hash(handle obj) { /// @} python_builtins PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_LIMITED_API) +// Replacements for the instancemethod and method APIs, which are not part of the stable ABI. +// Defined in detail/class-inl.h (the instancemethod type is created with the other internals +// types). +bool is_instancemethod(PyObject *obj); +PyObject *instancemethod_function(PyObject *obj); // borrowed reference +PyObject *instancemethod_new(PyObject *func); // new reference +bool is_bound_method(PyObject *obj); +PyObject *bound_method_function(PyObject *obj); // borrowed reference +#endif + inline handle get_function(handle value) { if (value) { +#if !defined(Py_LIMITED_API) if (PyInstanceMethod_Check(value.ptr())) { value = PyInstanceMethod_GET_FUNCTION(value.ptr()); } else if (PyMethod_Check(value.ptr())) { value = PyMethod_GET_FUNCTION(value.ptr()); } +#else + if (is_instancemethod(value.ptr())) { + value = instancemethod_function(value.ptr()); + } else if (is_bound_method(value.ptr())) { + value = bound_method_function(value.ptr()); + } +#endif } return value; } @@ -830,13 +896,20 @@ inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { #endif } +// Returns a borrowed reference (the dict holds the value). inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *defaultobj) { PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); } +#if !defined(Py_LIMITED_API) PyObject *rv = PyDict_SetDefault(v, kv, defaultobj); +#else + // dict.setdefault() is one C-level call, so it is just as atomic as PyDict_SetDefault. + PyObject *rv = PyObject_CallMethod(v, "setdefault", "OO", kv, defaultobj); + Py_XDECREF(rv); +#endif Py_DECREF(kv); if (rv == nullptr) { throw error_already_set(); @@ -1123,6 +1196,7 @@ struct arrow_proxy { T *operator->() const { return &value; } }; +#if defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) /// Lightweight iterator policy using just a simple pointer: see ``PySequence_Fast_ITEMS`` class sequence_fast_readonly { protected: @@ -1145,6 +1219,7 @@ class sequence_fast_readonly { private: PyObject **ptr; }; +#endif /// Full read and write access using the sequence protocol: see ``detail::sequence_accessor`` class sequence_slow_readwrite { @@ -1229,7 +1304,28 @@ inline bool PyUnicode_Check_Permissive(PyObject *o) { # define PYBIND11_STR_CHECK_FUN PyUnicode_Check #endif +#if !defined(Py_LIMITED_API) inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == &PyStaticMethod_Type; } +inline PyObject *PyStaticMethod_New(PyObject *callable) { return ::PyStaticMethod_New(callable); } +#else +// PyStaticMethod_Type is not exported by the stable ABI; fetch it from builtins once. +inline PyTypeObject *get_staticmethod_type() { + static PyTypeObject *const type = [] { + PyObject *builtins = PyEval_GetBuiltins(); + PyObject *t = builtins ? PyDict_GetItemString(builtins, "staticmethod") : nullptr; + if (t == nullptr) { + pybind11_fail("pybind11::detail::get_staticmethod_type(): lookup failed"); + } + return reinterpret_cast(t); // immortal builtin + }(); + return type; +} +inline bool PyStaticMethod_Check(PyObject *o) { return Py_TYPE(o) == get_staticmethod_type(); } +inline PyObject *PyStaticMethod_New(PyObject *callable) { + return PyObject_CallFunctionObjArgs( + reinterpret_cast(get_staticmethod_type()), callable, nullptr); +} +#endif class kwargs_proxy : public handle { public: @@ -2184,7 +2280,10 @@ class function : public object { class staticmethod : public object { public: - PYBIND11_OBJECT_CVT(staticmethod, object, detail::PyStaticMethod_Check, PyStaticMethod_New) + PYBIND11_OBJECT_CVT(staticmethod, + object, + detail::PyStaticMethod_Check, + detail::PyStaticMethod_New) }; class buffer : public object { @@ -2340,7 +2439,20 @@ inline size_t len(handle h) { /// Get the length hint of a Python object. /// Returns 0 when this cannot be determined. inline size_t len_hint(handle h) { +#if !defined(Py_LIMITED_API) ssize_t result = PyObject_LengthHint(h.ptr(), 0); +#else + // PyObject_LengthHint is not part of the stable ABI: try len(), then __length_hint__(). + ssize_t result = PyObject_Length(h.ptr()); + if (result < 0) { + PyErr_Clear(); + result = -1; + if (auto hint = reinterpret_steal( + PyObject_CallMethod(h.ptr(), "__length_hint__", nullptr))) { + result = PyLong_AsSsize_t(hint.ptr()); + } + } +#endif if (result < 0) { // Sometimes a length can't be determined at all (eg generators) // In which case simply return 0 @@ -2501,6 +2613,7 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE // Meant to return a Python str, but this is not checked. +#if !defined(Py_LIMITED_API) /// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). inline tuple get_bases(PyTypeObject *type) { if (type->tp_bases == nullptr) { @@ -2517,6 +2630,48 @@ inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type inline object type_lookup(PyTypeObject *type, handle name) { return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); } +#else +// Generic attribute lookup on a type object: resolves the `type.__xxx__` descriptors without +// going through the metaclass's tp_getattro (pybind11's own metaclass calls back into here). +inline object type_generic_getattr(PyTypeObject *type, const char *name) { + auto name_obj = reinterpret_steal(PyUnicode_InternFromString(name)); + auto value = name_obj ? reinterpret_steal( + PyObject_GenericGetAttr((PyObject *) type, name_obj.ptr())) + : object(); + if (!value) { + PyErr_Clear(); + } + return value; +} + +inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { + object value = type_generic_getattr(type, name); + if (!value || !PyTuple_Check(value.ptr())) { + return tuple(); + } + return reinterpret_steal(value.release()); +} + +inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } +inline tuple get_mro(PyTypeObject *type) { return get_type_tuple_attr(type, "__mro__"); } + +// No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). +inline object type_lookup(PyTypeObject *type, handle name) { + for (handle base : get_mro(type)) { + object dict + = type_generic_getattr(reinterpret_cast(base.ptr()), "__dict__"); + if (!dict) { + continue; + } + auto value = reinterpret_steal(PyObject_GetItem(dict.ptr(), name.ptr())); + if (value) { + return value; + } + PyErr_Clear(); + } + return object(); +} +#endif inline object get_module_name_if_available(handle scope) { if (scope) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index 7a2ac29c7c..d1a5eb652d 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -93,9 +93,14 @@ inline bool object_is_convertible_to_std_vector(const handle &src) { return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); } // Allow generators, set/frozenset and several common iterable types. - return (PyGen_Check(src.ptr()) != 0) || (PyAnySet_Check(src.ptr()) != 0) +#if !defined(Py_LIMITED_API) + if (PyGen_Check(src.ptr()) != 0) { + return true; + } +#endif + return (PyAnySet_Check(src.ptr()) != 0) || object_is_instance_with_one_of_tp_names( - src.ptr(), {"dict_keys", "dict_values", "dict_items", "map", "zip"}); + src.ptr(), {"generator", "dict_keys", "dict_values", "dict_items", "map", "zip"}); } inline bool object_is_convertible_to_std_set(const handle &src, bool convert) { diff --git a/tests/env.py b/tests/env.py index d205d7ef0e..bbebf53935 100644 --- a/tests/env.py +++ b/tests/env.py @@ -32,6 +32,13 @@ def _is_musl() -> bool: # Compile-time config (what the binary was built for) PY_GIL_DISABLED = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) +try: + import pybind11_tests + + # Built with Py_LIMITED_API (stable ABI); some features and messages differ. + LIMITED_API = bool(pybind11_tests.LIMITED_API) +except ImportError: # the extra_* test suites do not build pybind11_tests + LIMITED_API = False # Runtime state (what's actually happening now) sys_is_gil_enabled = getattr(sys, "_is_gil_enabled", lambda: True) diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 3f342a0e9c..03873d7698 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,12 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + m.attr("LIMITED_API") = +#if defined(Py_LIMITED_API) + true; +#else + false; +#endif // Free threaded Python uses UINT32_MAX for immortal objects. m.attr("PYBIND11_SIMPLE_GIL_MANAGEMENT") = #if defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) diff --git a/tests/test_call_policies.cpp b/tests/test_call_policies.cpp index bf498be846..b5134f9855 100644 --- a/tests/test_call_policies.cpp +++ b/tests/test_call_policies.cpp @@ -254,6 +254,8 @@ TEST_SUBMODULE(call_policies, m) { #if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) // `py::call_guard()` should work in PyPy/GraalPy as well, // but it's unclear how to test it without `PyGILState_GetThisThreadState`. + // (Not under the stable ABI either: get_thread_state_unchecked() then requires the GIL.) +# if !defined(Py_LIMITED_API) auto report_gil_status = []() { auto is_gil_held = false; if (auto *tstate = py::detail::get_thread_state_unchecked()) { @@ -277,6 +279,7 @@ TEST_SUBMODULE(call_policies, m) { return report_gil_status(); }, py::call_guard()); +# endif #endif // test_keep_alive_failed_overload diff --git a/tests/test_call_policies.py b/tests/test_call_policies.py index 935160866b..04d29f5c4c 100644 --- a/tests/test_call_policies.py +++ b/tests/test_call_policies.py @@ -290,7 +290,8 @@ def test_call_guard_rejected_cast(with_hooks): @pytest.mark.skipif( - "env.PYPY or env.GRAALPY", reason="GIL state check requires CPython" + not hasattr(m, "call_guard_cast_without_gil"), + reason="GIL state check requires CPython without Py_LIMITED_API", ) def test_call_guard_cast_without_gil(): state = m.CallGuardState() diff --git a/tests/test_kwargs_and_defaults.cpp b/tests/test_kwargs_and_defaults.cpp index 831947f160..7f672bf592 100644 --- a/tests/test_kwargs_and_defaults.cpp +++ b/tests/test_kwargs_and_defaults.cpp @@ -175,7 +175,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { py::tuple t(a.size()); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i] = REFCNT(PyTuple_GetItem(a.ptr(), static_cast(i))); } return t; }); @@ -185,7 +185,7 @@ TEST_SUBMODULE(kwargs_and_defaults, m) { t[0] = o.ref_count(); for (size_t i = 0; i < a.size(); i++) { // Use raw Python API here to avoid an extra, intermediate incref on the tuple item: - t[i + 1] = REFCNT(PyTuple_GET_ITEM(a.ptr(), static_cast(i))); + t[i + 1] = REFCNT(PyTuple_GetItem(a.ptr(), static_cast(i))); } return t; }); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 99bbc8c2cd..a671e55bd5 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -221,6 +221,9 @@ def check_self(self): instance.static_cls = check_self +@pytest.mark.skipif( + "env.LIMITED_API", reason="PyType_FromMetaclass picks the most derived metaclass" +) def test_metaclass_override(): """Overriding pybind11's default metaclass changes the behavior of `static_property`""" From 4bddb32fc1ef15ec4a885b5628ef9b517b17a698 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:19:22 -0400 Subject: [PATCH 11/59] feat(cmake): add STABLE_ABI to pybind11_add_module() pybind11_add_module(... STABLE_ABI) builds the module against the Python stable ABI through python_add_library(USE_SABI) (CMake 3.26+, FindPython mode) and names it *.abi3.so / *.pyd. PYBIND11_STABLE_ABI makes it the default for a build tree and NO_STABLE_ABI opts a target out; PYBIND11_STABLE_ABI_VERSION (default 3.12) selects the targeted version. SHARED helper libraries get the same Py_LIMITED_API definition. The classic (FindPythonLibs) mode reports a clear error. pybind11_precompile(STABLE_ABI) compiles the precompiled library against the limited API; one build tree cannot mix stable-ABI and regular precompiled modules. The test suite gains PYBIND11_TEST_STABLE_ABI, which builds every test module as abi3 and skips the features that the stable ABI cannot offer. See https://github.com/pybind/pybind11/discussions/6104 (D2). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/compiling.rst | 14 ++++ tests/CMakeLists.txt | 27 ++++++- tests/test_cmake_build/CMakeLists.txt | 10 ++- tools/pybind11Common.cmake | 25 +++++- tools/pybind11NewTools.cmake | 107 ++++++++++++++++++++++++-- tools/pybind11Tools.cmake | 12 ++- 6 files changed, 176 insertions(+), 19 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 9e61949b02..9a8ffd7d73 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -349,6 +349,7 @@ function with the following signature: pybind11_add_module( [MODULE | SHARED] [EXCLUDE_FROM_ALL] [NO_EXTRAS] [THIN_LTO] [OPT_SIZE] [PRECOMPILE | NO_PRECOMPILE] + [STABLE_ABI | NO_STABLE_ABI] source1 [source2 ...]) This function behaves very much like CMake's builtin ``add_library`` (in fact, @@ -405,6 +406,15 @@ optimizations remain disabled. .. _ThinLTO: http://clang.llvm.org/docs/ThinLTO.html +``STABLE_ABI`` builds the module against the Python stable ABI +(``Py_LIMITED_API``), so that one ``.abi3`` module runs on every CPython from +the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` +to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make +it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` +to opt a target out. This needs CMake 3.26+, the FindPython mode +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available +for free-threaded builds, PyPy, or GraalPy. + .. _precompile-mode: Pre-compiling part of pybind11 @@ -469,6 +479,10 @@ Requirements and caveats: * The library is static and per-build-tree; it is never installed or shared between projects. Each extension module links its own copy, which keeps pybind11's per-module state the same as in header-only mode. +* With ``STABLE_ABI`` the library is compiled against the limited API too. + One build tree cannot mix stable-ABI and regular precompiled modules; the + first ``PRECOMPILE`` target decides, and a later mismatch is a configure + error. * Not available with ``PYBIND11_NOPYTHON`` (the library needs Python headers). diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4cbc364e79..4dd970771c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -72,12 +72,17 @@ option(DOWNLOAD_EIGEN "Download EIGEN" OFF) option(PYBIND11_CUDA_TESTS "Enable building CUDA tests" OFF) option(PYBIND11_TEST_SMART_HOLDER "Change the default to smart holder" OFF) option(PYBIND11_TEST_PRECOMPILE "Build the test modules against the precompiled library" OFF) +option(PYBIND11_TEST_STABLE_ABI "Build the test modules for the stable ABI (Py_LIMITED_API)" OFF) if(PYBIND11_TEST_PRECOMPILE) # Picked up by every pybind11_add_module() call below; any definition missing # from the precompiled library shows up as a link error across the test modules. set(PYBIND11_PRECOMPILE ON) endif() +if(PYBIND11_TEST_STABLE_ABI) + # Picked up by every pybind11_add_module() call below. + set(PYBIND11_STABLE_ABI ON) +endif() # These modules redefine PYBIND11_INTERNALS_VERSION inside their own TU, so they # cannot share the precompiled library. set(PYBIND11_NO_PRECOMPILE_TARGETS exo_planet_pybind11 cross_module_gil_utils) @@ -220,6 +225,19 @@ if(PYBIND11_TEST_FILTER) pybind11_filter_tests(PYBIND11_TEST_FILES ${PYBIND11_TEST_FILTER}) endif() +# Features that are not available under the stable ABI (see docs/advanced/stable_abi.rst). +if(PYBIND11_TEST_STABLE_ABI) + pybind11_filter_tests( + PYBIND11_TEST_FILES + test_chrono.cpp + test_custom_type_setup.cpp + test_multiple_interpreters.py + test_class_release_gil_before_calling_cpp_dtor.cpp # PyGILState_Check + MESSAGE + "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" + ) +endif() + # Skip tests for CUDA check: # /pybind11/tests/test_constants_and_functions.cpp(125): # error: incompatible exception specifications @@ -604,7 +622,7 @@ add_custom_target( WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" USES_TERMINAL) -if(NOT PYBIND11_CUDA_TESTS) +if(NOT PYBIND11_CUDA_TESTS AND NOT PYBIND11_TEST_STABLE_ABI) set(PYBIND11_MULTIPLE_INTERPRETERS_TEST_MODULES mod_per_interpreter_gil mod_shared_interpreter_gil mod_per_interpreter_gil_with_singleton) @@ -699,7 +717,12 @@ if(NOT PYBIND11_CUDA_TESTS) add_subdirectory(pure_cpp) # Test C++ code that depends on Python, such as embedding the interpreter. Provides the `cpptest` target. - add_subdirectory(test_with_catch) + if(PYBIND11_TEST_STABLE_ABI) + message(STATUS "Skipping embed tests under the stable ABI") + add_custom_target(cpptest) # Dummy target: embed.h is not available under Py_LIMITED_API. + else() + add_subdirectory(test_with_catch) + endif() # Test CMake build using functions and targets from subdirectory or installed location add_subdirectory(test_cmake_build) diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 33a78a6b1d..1d0e4af566 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -23,7 +23,7 @@ function(pybind11_add_build_test name) # Forward option-driven defines so the nested projects can verify they propagate # through the pybind11_headers usage requirements. foreach(opt IN ITEMS PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION - PYBIND11_SIMPLE_GIL_MANAGEMENT) + PYBIND11_SIMPLE_GIL_MANAGEMENT PYBIND11_STABLE_ABI) if(${opt}) list(APPEND build_options "-D${opt}=ON") endif() @@ -72,8 +72,9 @@ pybind11_add_build_test(subdirectory_target) pybind11_add_build_test(subdirectory_precompile) if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy") - message(STATUS "Skipping embed test on PyPy or GraalPy") + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy" + OR PYBIND11_STABLE_ABI) + message(STATUS "Skipping embed test on PyPy, GraalPy, or under the stable ABI") else() pybind11_add_build_test(subdirectory_embed) endif() @@ -91,7 +92,8 @@ if(PYBIND11_INSTALL) if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy")) + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "graalpy" + OR PYBIND11_STABLE_ABI)) pybind11_add_build_test(installed_embed INSTALL) endif() endif() diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 0073d3634d..8d204afd39 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -488,9 +488,19 @@ endfunction() # built from the consumer's project with the consumer's flags; each extension # module links its own copy, preserving pybind11's per-module state. Modules # using it must compile with PYBIND11_PRECOMPILED, which the PUBLIC compile -# definition below provides automatically. +# definition below provides automatically. With STABLE_ABI the library is +# compiled against the limited API (Py_LIMITED_API); one build tree cannot mix +# stable-ABI and regular precompiled modules. function(pybind11_precompile) + cmake_parse_arguments(ARG "STABLE_ABI" "" "" ${ARGN}) if(TARGET pybind11_precompiled) + get_target_property(_existing_sabi pybind11_precompiled PYBIND11_STABLE_ABI) + if(NOT "${_existing_sabi}" STREQUAL "${ARG_STABLE_ABI}") + message( + FATAL_ERROR + "pybind11::precompiled was created for STABLE_ABI=${_existing_sabi}; one build tree " + "cannot mix stable-ABI and regular precompiled modules.") + endif() return() endif() @@ -512,6 +522,11 @@ function(pybind11_precompile) add_library(pybind11_precompiled STATIC EXCLUDE_FROM_ALL ${_pybind11_precompile_sources}) add_library(pybind11::precompiled ALIAS pybind11_precompiled) target_compile_definitions(pybind11_precompiled PUBLIC PYBIND11_PRECOMPILED) + set_target_properties(pybind11_precompiled PROPERTIES PYBIND11_STABLE_ABI "${ARG_STABLE_ABI}") + if(ARG_STABLE_ABI) + _pybind11_stable_abi_hex(_sabi_hex) + target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + endif() # pybind11::module (not just pybind11::pybind11): the library must compile with the # interpreter's ABI macros (e.g. Py_GIL_DISABLED, which FindPython attaches to # Python::Module); on free-threaded Windows they select the correct autolink library. @@ -535,9 +550,13 @@ endfunction() # Link pybind11::precompiled when the PRECOMPILE keyword or the global # PYBIND11_PRECOMPILE variable requests it, unless NO_PRECOMPILE opts out. -function(_pybind11_maybe_precompile target_name precompile no_precompile) +function(_pybind11_maybe_precompile target_name precompile no_precompile stable_abi) if((precompile OR PYBIND11_PRECOMPILE) AND NOT no_precompile) - pybind11_precompile() + if(stable_abi) + pybind11_precompile(STABLE_ABI) + else() + pybind11_precompile() + endif() target_link_libraries(${target_name} PRIVATE pybind11::precompiled) endif() endfunction() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 175268d113..3735ff2cbb 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -35,11 +35,15 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(_pybind11_interp_component Interpreter) endif() - # Development.Module support (required for manylinux) started in 3.18 + # Development.Module support (required for manylinux) started in 3.18; + # Development.SABIModule (stable ABI, see the STABLE_ABI option) in 3.26. if(CMAKE_VERSION VERSION_LESS 3.18) set(_pybind11_dev_component Development) - else() + elseif(CMAKE_VERSION VERSION_LESS 3.26) set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed) + else() + set(_pybind11_dev_component Development.Module OPTIONAL_COMPONENTS Development.Embed + Development.SABIModule) endif() # Callers need to be able to access Python_EXECUTABLE @@ -252,12 +256,64 @@ else() PROPERTY INTERFACE_LINK_LIBRARIES pybind11::python_link_helper) endif() +# The Py_LIMITED_API value for STABLE_ABI modules (CPython 3.12 is the minimum pybind11 supports). +set(PYBIND11_STABLE_ABI_VERSION + "3.12" + CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") + +# PYBIND11_STABLE_ABI_VERSION as the Py_LIMITED_API hex value. +function(_pybind11_stable_abi_hex out_var) + string(REPLACE "." ";" _parts "${PYBIND11_STABLE_ABI_VERSION}") + list(GET _parts 0 _major) + list(GET _parts 1 _minor) + math(EXPR _hex "(${_major} << 24) | (${_minor} << 16)" OUTPUT_FORMAT HEXADECIMAL) + set(${out_var} + "${_hex}" + PARENT_SCOPE) +endfunction() + +# Fail with an explanation if a STABLE_ABI module cannot be built in this configuration. +function(_pybind11_check_stable_abi target_name lib_type) + if(CMAKE_VERSION VERSION_LESS 3.26) + message(FATAL_ERROR "${target_name}: STABLE_ABI requires CMake 3.26 or newer (USE_SABI).") + endif() + if(lib_type STREQUAL "STATIC") + message(FATAL_ERROR "${target_name}: STABLE_ABI is not supported for STATIC libraries.") + endif() + if(PYBIND11_STABLE_ABI_VERSION VERSION_LESS 3.12) + message(FATAL_ERROR "${target_name}: PYBIND11_STABLE_ABI_VERSION must be 3.12 or newer, " + "got ${PYBIND11_STABLE_ABI_VERSION}.") + endif() + if(DEFINED ${_Python}_INTERPRETER_ID AND NOT "${${_Python}_INTERPRETER_ID}" STREQUAL "Python") + message(FATAL_ERROR "${target_name}: STABLE_ABI requires CPython, found " + "${${_Python}_INTERPRETER_ID}.") + endif() + if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS PYBIND11_STABLE_ABI_VERSION) + message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " + ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") + endif() + if("${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" OR "${PYTHON_MODULE_EXTENSION}" MATCHES + "^\\.cpython-[0-9]+t") + message(FATAL_ERROR "${target_name}: the free-threaded build has no stable ABI.") + endif() + if(NOT TARGET ${_Python}::SABIModule) + message( + FATAL_ERROR + "${target_name}: STABLE_ABI needs the Development.SABIModule component " + "of FindPython. If your project calls find_package(Python) itself, add " + "Development.SABIModule to its components.") + endif() +endfunction() + # WITHOUT_SOABI and WITH_SOABI will disable the custom extension handling used by pybind11. # WITH_SOABI is passed on to python_add_library. function(pybind11_add_module target_name) cmake_parse_arguments( - PARSE_ARGV 1 ARG - "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI;PRECOMPILE;NO_PRECOMPILE" "" + PARSE_ARGV + 1 + ARG + "STATIC;SHARED;MODULE;THIN_LTO;OPT_SIZE;NO_EXTRAS;WITHOUT_SOABI;PRECOMPILE;NO_PRECOMPILE;STABLE_ABI;NO_STABLE_ABI" + "" "") if(ARG_STATIC) @@ -268,10 +324,22 @@ function(pybind11_add_module target_name) set(lib_type MODULE) endif() + # STABLE_ABI keyword, or the PYBIND11_STABLE_ABI variable as the default; NO_STABLE_ABI opts out. + set(stable_abi OFF) + set(use_sabi "") + if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) + _pybind11_check_stable_abi(${target_name} ${lib_type}) + set(stable_abi ON) + if(lib_type STREQUAL "MODULE") + # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). + set(use_sabi USE_SABI ${PYBIND11_STABLE_ABI_VERSION}) + endif() + endif() + if("${_Python}" STREQUAL "Python") - python_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") - python3_add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + python3_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) else() message(FATAL_ERROR "Cannot detect FindPython version: ${_Python}") endif() @@ -280,17 +348,27 @@ function(pybind11_add_module target_name) if(lib_type STREQUAL "MODULE") target_link_libraries(${target_name} PRIVATE pybind11::module) + elseif(stable_abi) + # A SHARED helper library that stable-ABI modules link: same ABI, no embedding. + _pybind11_stable_abi_hex(_sabi_hex) + target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_sabi_hex}") + target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) else() target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() - _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}") + _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}" + "${stable_abi}") _pybind11_default_hidden_visibility(${target_name}) # If we don't pass a WITH_SOABI or WITHOUT_SOABI, use our own default handling of extensions if(NOT ARG_WITHOUT_SOABI AND NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) - pybind11_extension(${target_name}) + if(stable_abi) + pybind11_extension_stable_abi(${target_name}) + else() + pybind11_extension(${target_name}) + endif() endif() if(ARG_NO_EXTRAS) @@ -331,3 +409,16 @@ function(pybind11_extension name) DEBUG_POSTFIX "${PYTHON_MODULE_DEBUG_POSTFIX}" SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() + +# Stable-ABI modules carry the version-independent "abi3" tag (none on Windows). +function(pybind11_extension_stable_abi name) + if(CMAKE_SYSTEM_NAME STREQUAL "Windows") + set(_ext ".pyd") + else() + set(_ext "${CMAKE_SHARED_MODULE_SUFFIX}") + endif() + if(DEFINED ${_Python}_SOSABI AND NOT "${${_Python}_SOSABI}" STREQUAL "") + set(_ext ".${${_Python}_SOSABI}${_ext}") + endif() + set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${_ext}") +endfunction() diff --git a/tools/pybind11Tools.cmake b/tools/pybind11Tools.cmake index 9a7e5c6217..01286c77dc 100644 --- a/tools/pybind11Tools.cmake +++ b/tools/pybind11Tools.cmake @@ -140,9 +140,17 @@ endfunction() # function(pybind11_add_module target_name) set(options - "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE;PRECOMPILE;NO_PRECOMPILE") + "MODULE;SHARED;EXCLUDE_FROM_ALL;NO_EXTRAS;SYSTEM;THIN_LTO;OPT_SIZE;PRECOMPILE;NO_PRECOMPILE;STABLE_ABI;NO_STABLE_ABI" + ) cmake_parse_arguments(ARG "${options}" "" "" ${ARGN}) + if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) + message( + FATAL_ERROR + "${target_name}: STABLE_ABI requires the FindPython mode (PYBIND11_FINDPYTHON=ON) " + "and CMake 3.26 or newer.") + endif() + if(ARG_MODULE AND ARG_SHARED) message(FATAL_ERROR "Can't be both MODULE and SHARED") elseif(ARG_SHARED) @@ -161,7 +169,7 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::module) - _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}") + _pybind11_maybe_precompile(${target_name} "${ARG_PRECOMPILE}" "${ARG_NO_PRECOMPILE}" OFF) if(ARG_SYSTEM) message( From 3160e95f115a1dada500601d408ab4f9bb21e094 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:19:22 -0400 Subject: [PATCH 12/59] feat(setup_helpers): Pybind11Extension(py_limited_api=True) py_limited_api=True (or a version such as "3.13") defines Py_LIMITED_API for the targeted version in addition to setuptools' abi3 file naming, so one wheel serves every CPython from that version on. See https://github.com/pybind/pybind11/discussions/6104 (D3). Assisted-by: ClaudeCode:claude-fable-5-1 --- pybind11/setup_helpers.py | 41 +++++++++++ tests/extra_setuptools/test_setuphelper.py | 80 ++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 248997a237..3053014382 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -41,6 +41,7 @@ import contextlib import os import platform +import re import shlex import shutil import sys @@ -86,6 +87,30 @@ # directory into your path if it sits beside your setup.py. +def _limited_api_hex(value: Any) -> str: + """ + The Py_LIMITED_API value for ``py_limited_api``: True selects 3.12 (the + minimum pybind11 supports), a string such as "3.13" or "cp313" selects + that version, and an int or hex string is passed through. + """ + if value is True: + return "0x030C0000" + if isinstance(value, int): + return f"0x{value:08X}" + text = str(value) + if text.lower().startswith("0x"): + return text + match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) + if not match: + msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" + raise ValueError(msg) + major, minor = (int(part) for part in match.groups()) + if (major, minor) < (3, 12): + msg = f"pybind11 supports the stable ABI from Python 3.12 on, got {value!r}" + raise ValueError(msg) + return f"0x{major:02X}{minor:02X}0000" + + class Pybind11Extension(_Extension): """ Build a C++11+ Extension module with pybind11. This automatically adds the @@ -108,6 +133,12 @@ class Pybind11Extension(_Extension): If you want to add pybind11 headers manually, for example for an exact git checkout, then set ``include_pybind11=False``. + + ``py_limited_api=True`` builds against the Python stable ABI (3.12, or + pass a version string such as ``"3.13"``): ``Py_LIMITED_API`` is defined + and setuptools names the module ``*.abi3.so``, so one wheel serves every + CPython from that version on. See the pybind11 documentation for the + features that are not available under the stable ABI. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -128,6 +159,16 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: include_pybind11 = kwargs.pop("include_pybind11", True) + py_limited_api = kwargs.get("py_limited_api", False) + if py_limited_api: + # setuptools only uses the flag for the .abi3 file name; the + # define selects the limited API in the headers. + kwargs["py_limited_api"] = True + macros = list(kwargs.get("define_macros", []) or []) + if not any(name == "Py_LIMITED_API" for name, _ in macros): + macros.append(("Py_LIMITED_API", _limited_api_hex(py_limited_api))) + kwargs["define_macros"] = macros + super().__init__(*args, **kwargs) # Flags are added in place, so do not modify the caller's lists diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 10a150db65..e0a9c5f50d 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -3,6 +3,7 @@ import os import subprocess import sys +import sysconfig from textwrap import dedent import pytest @@ -110,6 +111,85 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): ) +@pytest.mark.skipif( + sys.implementation.name != "cpython" + or sys.version_info < (3, 12) + or sysconfig.get_config_var("Py_GIL_DISABLED"), + reason="stable ABI needs CPython 3.12+ with the GIL", +) +def test_stable_abi_setup_py(monkeypatch, tmpdir): + monkeypatch.chdir(tmpdir) + monkeypatch.syspath_prepend(MAIN_DIR) + + (tmpdir / "setup.py").write_text( + dedent( + f"""\ + import sys + sys.path.append({MAIN_DIR!r}) + + from setuptools import setup + from pybind11.setup_helpers import build_ext, Pybind11Extension + + ext_modules = [ + Pybind11Extension( + "stable_abi_setup", + ["main.cpp"], + cxx_std=17, + py_limited_api=True, + ), + ] + + setup( + name="stable_abi_setup_package", + cmdclass={{"build_ext": build_ext}}, + ext_modules=ext_modules, + ) + """ + ), + encoding="ascii", + ) + + (tmpdir / "main.cpp").write_text( + dedent( + """\ + #include + + #if !defined(Py_LIMITED_API) + # error "Py_LIMITED_API should be defined" + #endif + + struct Point { int x; }; + + PYBIND11_MODULE(stable_abi_setup, m, pybind11::mod_gil_used()) { + pybind11::class_(m, "Point") + .def(pybind11::init()) + .def_readwrite("x", &Point::x); + } + """ + ), + encoding="ascii", + ) + + subprocess.check_call([sys.executable, "setup.py", "build_ext", "--inplace"]) + + built = [f for f in tmpdir.listdir() if f.basename.startswith("stable_abi_setup")] + assert len(built) == 1 + assert ".abi3" in built[0].basename or built[0].ext == ".pyd" + + (tmpdir / "test.py").write_text( + dedent( + """\ + import stable_abi_setup + assert stable_abi_setup.Point(4).x == 4 + """ + ), + encoding="ascii", + ) + subprocess.check_call( + [sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr + ) + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) From 457d05058f38c9ca29bf28bd44db1b2c949139e0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:20:36 -0400 Subject: [PATCH 13/59] ci: build and test the suite for the stable ABI Build every test module as abi3 on Linux, macOS, and Windows (one job also with the precompiled library), and run the modules built against the 3.12 headers unchanged on 3.13 and 3.14. See https://github.com/pybind/pybind11/discussions/6104 (D4). Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 93 ++++++++++++++++++++++++++++++++++++++++ tests/conftest.py | 10 +++-- tests/pybind11_tests.cpp | 3 ++ 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa74569a78..8c0634e9fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,6 +135,99 @@ jobs: python-version: ${{ matrix.python-version }} cmake-args: ${{ matrix.cmake-args }} + # Stable ABI (Py_LIMITED_API): every test module is built as *.abi3. + stable-abi: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + include: + - runs-on: ubuntu-latest + python-version: '3.12' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + # precompiled: the -inl.h files must not read any non-limited struct field + - runs-on: ubuntu-latest + python-version: '3.14' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON + - runs-on: macos-latest + python-version: '3.13' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DCMAKE_CXX_STANDARD=17 + - runs-on: windows-2022 + python-version: '3.12' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + + name: 🧱 + uses: ./.github/workflows/reusable-standard.yml + with: + runs-on: ${{ matrix.runs-on }} + python-version: ${{ matrix.python-version }} + cmake-args: ${{ matrix.cmake-args }} + + # The point of the stable ABI: build the suite once against the 3.12 headers + # and run the same .abi3 modules on newer interpreters. + stable-abi-forward: + if: github.event.pull_request.draft == false + name: 🧱 abi3 on newer Pythons + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=tests/test_chrono.py + --ignore=tests/test_custom_type_setup.py + --ignore=tests/test_multiple_interpreters.py + --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.12 (build) + id: py312 + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Setup Python 3.13 and 3.14 (run) + uses: actions/setup-python@v7 + with: + python-version: | + 3.13 + 3.14 + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py312.outputs.python-path }}" python3.13 python3.14; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.12, stable ABI) + run: > + cmake -S. -Bbuild -Werror=dev -GNinja + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYTHON_EXECUTABLE=${{ steps.py312.outputs.python-path }} + + - name: Build + run: cmake --build build + + - name: Python tests (3.12) + run: cmake --build build --target pytest + + - name: Python tests (3.13, same binaries) + working-directory: build/tests + run: python3.13 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + + - name: Python tests (3.14, same binaries) + working-directory: build/tests + run: python3.14 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/tests/conftest.py b/tests/conftest.py index 9d9815b88c..66403c1577 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -294,14 +294,16 @@ def pytest_report_header(): @pytest.fixture def backport_typehints() -> Callable[[SanitizedString], SanitizedString]: d = {} - if sys.version_info < (3, 13): + # The spellings are chosen at compile time (a stable-ABI module may run on a newer Python). + compiled = tuple(pybind11_tests.COMPILED_PYTHON_VERSION) + if compiled < (3, 13): d["typing_extensions.TypeIs"] = "typing.TypeIs" d["typing_extensions.CapsuleType"] = "types.CapsuleType" - if sys.version_info < (3, 12): + if compiled < (3, 12): d["typing_extensions.Buffer"] = "collections.abc.Buffer" - if sys.version_info < (3, 11): + if compiled < (3, 11): d["typing_extensions.Never"] = "typing.Never" - if sys.version_info < (3, 10): + if compiled < (3, 10): d["typing_extensions.TypeGuard"] = "typing.TypeGuard" def backport(sanitized_string: SanitizedString) -> SanitizedString: diff --git a/tests/pybind11_tests.cpp b/tests/pybind11_tests.cpp index 03873d7698..aaaacae459 100644 --- a/tests/pybind11_tests.cpp +++ b/tests/pybind11_tests.cpp @@ -89,6 +89,9 @@ PYBIND11_MODULE(pybind11_tests, m, py::mod_gil_not_used()) { #endif m.attr("cpp_std") = cpp_std(); m.attr("PYBIND11_INTERNALS_ID") = PYBIND11_INTERNALS_ID; + // The Python version the headers were compiled against; differs from the runtime version for + // stable-ABI modules. + m.attr("COMPILED_PYTHON_VERSION") = py::make_tuple(PY_MAJOR_VERSION, PY_MINOR_VERSION); m.attr("LIMITED_API") = #if defined(Py_LIMITED_API) true; From 28b56e53ac3d98816fca5e1e7b5e240d9d050cca Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 17:20:36 -0400 Subject: [PATCH 14/59] docs: describe stable ABI (abi3) support How to enable it with CMake, setuptools, or by hand; the feature matrix; ABI isolation between stable-ABI and regular modules; and the implementation differences that affect performance. See https://github.com/pybind/pybind11/discussions/6104 (D5). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 120 +++++++++++++++++++++++++++++++++++ docs/compiling.rst | 3 +- docs/index.rst | 1 + 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 docs/advanced/stable_abi.rst diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst new file mode 100644 index 0000000000..1bbfad2a50 --- /dev/null +++ b/docs/advanced/stable_abi.rst @@ -0,0 +1,120 @@ +Python stable ABI (abi3) +######################## + +CPython's `stable ABI `_ lets +one extension module run on every CPython release from the version it was +built for. pybind11 supports it from CPython 3.12 on: define +``Py_LIMITED_API`` to ``0x030C0000`` (or a newer version) and ship one +``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python +version. + +Enabling it +=========== + +With CMake, add ``STABLE_ABI`` to ``pybind11_add_module`` (CMake 3.26+ and +the FindPython mode are required), or set ``PYBIND11_STABLE_ABI`` to make it +the default for a build tree: + +.. code-block:: cmake + + pybind11_add_module(example STABLE_ABI example.cpp) + +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``) selects the targeted +version. ``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled +library is then compiled against the limited API too, and one build tree +cannot mix stable-ABI and regular precompiled modules. + +With setuptools, pass ``py_limited_api=True`` (or a version such as +``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and +``Py_LIMITED_API`` is defined for you. + +With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every +translation unit, link no version-specific Python library (``python3.lib`` on +Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). + +pybind11 rejects the combination with PyPy, GraalPy, and free-threaded +CPython at compile time: they have no stable ABI. + +What changes under the stable ABI +================================= + +The C++ API of pybind11 is the same; the differences are in what is +available and in how a few things are implemented. + +.. list-table:: + :header-rows: 1 + :widths: 30 20 50 + + * - Feature + - Stable ABI + - Notes + * - Classes, functions, casters, STL, numpy, Eigen + - supported + - + * - ``py::dynamic_attr()`` + - supported + - The ``__dict__`` slot is appended to the instance (no + ``Py_TPFLAGS_MANAGED_DICT``). + * - Buffer protocol + - supported + - Through the ``Py_bf_*`` type slots. + * - ``py::gil_scoped_acquire`` / ``release`` + - supported + - ``PYBIND11_SIMPLE_GIL_MANAGEMENT`` is forced; ``disarm()`` and + thread-state disassociation are not available. + * - ``py::metaclass(handle)`` + - restricted + - The metaclass must not define ``__new__``, and a metaclass less + derived than ``pybind11_type`` resolves to ``pybind11_type``. + * - ``py::custom_type_setup`` + - not available + - Its callback receives a ``PyHeapTypeObject *``, which is opaque. + * - ``pybind11/embed.h`` + - not available + - Embedding uses non-limited initialization functions. + * - ``pybind11/subinterpreter.h`` + - not available + - Reads thread- and interpreter-state fields. + * - ``pybind11/chrono.h`` + - not available + - The ``datetime`` C API accesses struct fields. + * - GIL-held assertions (``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``) + - not available + - ``PyGILState_Check()`` is not part of the stable ABI. + +ABI isolation +============= + +A stable-ABI module and a regular module can be loaded in one process, but +they do not share pybind11's internal state: the internals ID carries a +``_stable`` tag, so each kind of module registers its own types and +instances. Types bound in one kind of module are opaque to the other, except +through the ``pybind11_conduit_v1`` protocol +(``include/pybind11/conduit/README.txt``), which bridges the two: the C++ ABI +is unchanged, so ``PYBIND11_PLATFORM_ABI_ID`` is the same. + +Implementation notes and performance +==================================== + +Without access to CPython's struct layouts, pybind11 uses public functions +where it used to read fields directly. The costs are small and are confined +to specific operations: + +* Type objects are created with ``PyType_FromMetaclass()`` and + ``PyType_Spec`` (the ``PYBIND11_TYPE_CREATION_VIA_SPEC`` path, also + available as an opt-in for regular builds on CPython 3.12+). +* Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with + ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances + is found through the type's cached dictionary offset. +* Attribute access on bound *classes* (not instances) walks ``__mro__`` and + the class dictionaries instead of using ``_PyType_Lookup()``. +* Methods are bound through a pybind11-provided ``instancemethod`` type and + ``types.MethodType`` instead of ``PyInstanceMethod_Type`` / + ``PyMethod_New()``. +* Tuple and list element access uses the checked function forms; type names + in error messages are reconstructed from ``__module__`` and ``__name__``. +* ``py::eval``/``py::exec`` compile with ``Py_CompileString()`` and run with + ``PyEval_EvalCode()``; ``py::eval_file`` reads the file through Python. + +Modules built for the stable ABI check the interpreter version at import: +any CPython from the targeted version on is accepted. diff --git a/docs/compiling.rst b/docs/compiling.rst index 9a8ffd7d73..6895c518cf 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -413,7 +413,8 @@ to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode (``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available -for free-threaded builds, PyPy, or GraalPy. +for free-threaded builds, PyPy, or GraalPy. See :doc:`advanced/stable_abi` +for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/docs/index.rst b/docs/index.rst index 77b097c574..a25684f54f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -35,6 +35,7 @@ advanced/cast/index advanced/pycpp/index advanced/embedding + advanced/stable_abi advanced/misc advanced/deprecated From f7caed315cd55ebe90ccbd335178038b3808fe47 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 18:22:27 -0400 Subject: [PATCH 15/59] fix(cmake): export SABIModule target and Python_SOSABI to callers When the consumer sets Python_ARTIFACTS_INTERACTIVE before add_subdirectory(pybind11), FindPython runs in pybind11's directory scope. Python::SABIModule was not promoted to global and Python_SOSABI was not cached, so STABLE_ABI modules either failed to configure or lost the .abi3 suffix. Add a test_cmake_build case that covers this. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_cmake_build/CMakeLists.txt | 5 +++ .../subdirectory_stable_abi/CMakeLists.txt | 37 +++++++++++++++++++ tests/test_cmake_build/test.py | 5 +++ tools/pybind11NewTools.cmake | 8 ++++ 4 files changed, 55 insertions(+) create mode 100644 tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index 1d0e4af566..c145122f2e 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -78,6 +78,11 @@ if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" else() pybind11_add_build_test(subdirectory_embed) endif() +if(PYBIND11_STABLE_ABI + AND CMAKE_VERSION VERSION_GREATER_EQUAL 3.26 + AND NOT CMAKE_GENERATOR_PLATFORM MATCHES "Win32") + pybind11_add_build_test(subdirectory_stable_abi) +endif() if(PYBIND11_INSTALL) add_custom_target( diff --git a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt new file mode 100644 index 0000000000..d835c7c2ae --- /dev/null +++ b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt @@ -0,0 +1,37 @@ +cmake_minimum_required(VERSION 3.26...4.2) + +project(test_subdirectory_stable_abi CXX) + +# Allow PYTHON_EXECUTABLE if in FINDPYTHON mode and building pybind11's tests +# (makes transition easier while we support both modes). +if(DEFINED PYTHON_EXECUTABLE AND NOT DEFINED Python_EXECUTABLE) + set(Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +endif() + +# Keep FindPython in pybind11's directory scope; the results must still reach us. +set(Python_ARTIFACTS_INTERACTIVE TRUE) +set(PYBIND11_FINDPYTHON NEW) + +add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) +pybind11_add_module(test_subdirectory_stable_abi STABLE_ABI ../main.cpp) +set_target_properties(test_subdirectory_stable_abi PROPERTIES OUTPUT_NAME test_cmake_build) + +if(DEFINED Python_EXECUTABLE) + set(_Python_EXECUTABLE "${Python_EXECUTABLE}") +elseif(DEFINED PYTHON_EXECUTABLE) + set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}") +else() + message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)") +endif() + +add_custom_target( + check_subdirectory_stable_abi + ${CMAKE_COMMAND} + -E + env + PYTHONPATH=$ + PYBIND11_EXPECT_ABI3=$ + ${_Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/../test.py + ${PROJECT_NAME} + DEPENDS test_subdirectory_stable_abi) diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index bb4c20e0cb..627c8530ea 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import sys import test_cmake_build @@ -7,4 +8,8 @@ assert isinstance(__file__, str) # Test this is properly set assert test_cmake_build.add(1, 2) == 3 + +expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") +if expect_abi3: + assert ".abi3." in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 3735ff2cbb..4f490572b3 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -73,6 +73,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) if(TARGET Python::Module) set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE) endif() + if(TARGET Python::SABIModule) + set_property(TARGET Python::SABIModule PROPERTY IMPORTED_GLOBAL TRUE) + endif() endif() # Explicitly export version for callers (including our own functions) @@ -89,6 +92,11 @@ if(NOT Python_FOUND AND NOT Python3_FOUND) set(Python_VERSION_PATCH "${Python_VERSION_PATCH}" CACHE INTERNAL "") + if(DEFINED Python_SOSABI) + set(Python_SOSABI + "${Python_SOSABI}" + CACHE INTERNAL "") + endif() endif() endif() From e6a7932c19c4dad5f1608055dd84a00e392c345f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:34:24 -0400 Subject: [PATCH 16/59] feat: support the free-threaded stable ABI (abi3t, CPython 3.15+) Accept Py_LIMITED_API >= 0x030F0000 together with Py_GIL_DISABLED (PEP 803). PyObject and PyModuleDef are then incomplete types, so pybind11: - keeps `instance` and its other heap-type data as PEP 697 type data and reaches it through get_instance()/instance_object() instead of casts; - exports modules through the PEP 793 PyModExport_ hook; the PYBIND11_MODULE macro is unchanged, create_extension_module() is gone; - locks the internals with critical sections on a private object (PyMutex is not in the stable ABI) and keeps one weak reference per registered instance in place of PyUnstable_TryIncRef(); - tags the internals "_stable_ft", since abi3t modules also load on GIL-enabled interpreters. numpy.h is not available under abi3t: it mirrors NumPy's object layouts. CMake raises the stable ABI version to 3.15 on free-threaded Python and Pybind11Extension(py_limited_api=True) does the same. The test suite is built as .abi3t modules in CI for 3.15t and also run on GIL-enabled 3.15. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 73 +++++++++++++ docs/advanced/stable_abi.rst | 49 +++++++-- docs/compiling.rst | 5 +- include/pybind11/cast.h | 2 +- include/pybind11/detail/class-inl.h | 79 ++++++++++---- include/pybind11/detail/class.h | 4 +- include/pybind11/detail/common.h | 102 +++++++++++++----- .../detail/function_record_pyobject.h | 20 +++- include/pybind11/detail/init.h | 13 ++- include/pybind11/detail/internals-inl.h | 2 +- include/pybind11/detail/internals.h | 44 ++++++-- .../pybind11/detail/type_caster_base-inl.h | 35 ++++-- include/pybind11/detail/type_caster_base.h | 19 ++-- include/pybind11/detail/typeid.h | 8 ++ include/pybind11/detail/value_and_holder.h | 2 +- include/pybind11/numpy.h | 4 + include/pybind11/pybind11-inl.h | 18 ++-- include/pybind11/pybind11.h | 41 ++++++- include/pybind11/pytypes.h | 9 +- .../pybind11/trampoline_self_life_support.h | 6 +- pybind11/setup_helpers.py | 20 ++-- tests/CMakeLists.txt | 13 +++ tests/cross_module_gil_utils.cpp | 52 ++++++--- ...s_module_interleaved_error_already_set.cpp | 38 +++++-- tests/exo_planet_c_api.cpp | 14 +++ tests/extra_setuptools/test_setuphelper.py | 4 +- tests/test_cmake_build/test.py | 2 +- tests/test_modules.cpp | 5 + tests/test_stl_binders.cpp | 8 +- tests/test_stl_binders.py | 9 +- tools/check_inl_headers.py | 1 + tools/pybind11NewTools.cmake | 44 ++++++-- 32 files changed, 589 insertions(+), 156 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8c0634e9fd..687b2c25a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,13 @@ jobs: - runs-on: windows-2022 python-version: '3.12' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + # free-threaded stable ABI (abi3t, PEP 803): PyObject is opaque + - runs-on: ubuntu-latest + python-version: '3.15t' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON + - runs-on: macos-latest + python-version: '3.15t' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON name: 🧱 uses: ./.github/workflows/reusable-standard.yml @@ -228,6 +235,72 @@ jobs: working-directory: build/tests run: python3.14 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + # abi3t modules also load on GIL-enabled interpreters: build the suite with + # 3.15t and run the same .abi3t modules on 3.15. + stable-abi-forward-ft: + if: github.event.pull_request.draft == false + name: 🧱 abi3t on GIL-enabled Python + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=tests/test_chrono.py + --ignore=tests/test_custom_type_setup.py + --ignore=tests/test_multiple_interpreters.py + --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=tests/test_numpy_array.py + --ignore=tests/test_numpy_dtypes.py + --ignore=tests/test_numpy_scalars.py + --ignore=tests/test_numpy_vectorize.py + --ignore=tests/test_eigen_matrix.py + --ignore=tests/test_eigen_tensor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.15t (build) + id: py315t + uses: actions/setup-python@v7 + with: + python-version: '3.15t' + allow-prereleases: true + + - name: Setup Python 3.15 (run) + uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py315t.outputs.python-path }}" python3.15; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.15t, stable ABI) + run: > + cmake -S. -Bbuild -Werror=dev -GNinja + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYTHON_EXECUTABLE=${{ steps.py315t.outputs.python-path }} + + - name: Build + run: cmake --build build + + - name: Python tests (3.15t) + run: cmake --build build --target pytest + + - name: Python tests (3.15, same binaries) + working-directory: build/tests + run: python3.15 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 1bbfad2a50..85543baf9a 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -1,5 +1,5 @@ -Python stable ABI (abi3) -######################## +Python stable ABI (abi3, abi3t) +############################### CPython's `stable ABI `_ lets one extension module run on every CPython release from the version it was @@ -8,6 +8,12 @@ built for. pybind11 supports it from CPython 3.12 on: define ``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python version. +Free-threaded CPython has its own stable ABI from 3.15 on, ``abi3t`` +(:pep:`803`). pybind11 supports it too: build against the free-threaded +headers with ``Py_LIMITED_API`` set to ``0x030F0000`` or newer, and ship one +``*.abi3t.so`` that every free-threaded *and* GIL-enabled CPython from 3.15 +on can load. See `abi3t`_ below for what differs. + Enabling it =========== @@ -19,21 +25,23 @@ the default for a build tree: pybind11_add_module(example STABLE_ABI example.cpp) -``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``) selects the targeted -version. ``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` on +free-threaded Python) selects the targeted version. ``PRECOMPILE`` can be +combined with ``STABLE_ABI``; the precompiled library is then compiled against the limited API too, and one build tree cannot mix stable-ABI and regular precompiled modules. With setuptools, pass ``py_limited_api=True`` (or a version such as -``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and -``Py_LIMITED_API`` is defined for you. +``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` +(``*.abi3t.so`` on free-threaded Python) and ``Py_LIMITED_API`` is defined +for you. With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every translation unit, link no version-specific Python library (``python3.lib`` on Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). pybind11 rejects the combination with PyPy, GraalPy, and free-threaded -CPython at compile time: they have no stable ABI. +CPython before 3.15 at compile time: they have no stable ABI. What changes under the stable ABI ================================= @@ -48,9 +56,12 @@ available and in how a few things are implemented. * - Feature - Stable ABI - Notes - * - Classes, functions, casters, STL, numpy, Eigen + * - Classes, functions, casters, STL - supported - + * - numpy, Eigen + - supported (abi3 only) + - ``numpy.h`` mirrors NumPy's object layouts, which abi3t hides. * - ``py::dynamic_attr()`` - supported - The ``__dict__`` slot is appended to the instance (no @@ -82,12 +93,32 @@ available and in how a few things are implemented. - not available - ``PyGILState_Check()`` is not part of the stable ABI. +abi3t +===== + +Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and +``PyMutex`` and ``PyUnstable_TryIncRef()`` are not available. pybind11 then: + +* stores its per-instance data as :pep:`697` type data + (``PyObject_GetTypeData()``) instead of embedding ``PyObject_HEAD``; +* exports the module through the :pep:`793` ``PyModExport_`` hook + instead of ``PyInit_``, so ``PYBIND11_MODULE`` is unchanged but + ``py::module_::create_extension_module()`` is not available; +* locks its internals with critical sections on a private object, and keeps + one weak reference per registered instance so that the instance registry + can hand out strong references safely (``PyWeakref_GetRef()``). + +``pybind11/numpy.h`` (and therefore the Eigen headers) is not available. The +internals tag is ``_stable_ft``: an abi3t module and an abi3 module loaded +into the same GIL-enabled interpreter do not share internals (see below). + ABI isolation ============= A stable-ABI module and a regular module can be loaded in one process, but they do not share pybind11's internal state: the internals ID carries a -``_stable`` tag, so each kind of module registers its own types and +``_stable`` (abi3) or ``_stable_ft`` (abi3t) tag, so each kind of module +registers its own types and instances. Types bound in one kind of module are opaque to the other, except through the ``pybind11_conduit_v1`` protocol (``include/pybind11/conduit/README.txt``), which bridges the two: the C++ ABI diff --git a/docs/compiling.rst b/docs/compiling.rst index 6895c518cf..4f2143172c 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -412,8 +412,9 @@ the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode -(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers; it is not available -for free-threaded builds, PyPy, or GraalPy. See :doc:`advanced/stable_abi` +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers (3.15+ for +free-threaded builds, which produce ``.abi3t`` modules); it is not available +for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index cc7975e3b0..605b83c4b4 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -395,7 +395,7 @@ class type_caster : public type_caster { const auto &bases = all_type_info(reinterpret_cast(type::handle_of(h).ptr())); if (bases.size() == 1) { // Only allowing loading from a single-value type - value = values_and_holders(reinterpret_cast(h.ptr())).begin()->value_ptr(); + value = values_and_holders(get_instance(h.ptr())).begin()->value_ptr(); return true; } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 21250d8966..b24d12f2ec 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -416,7 +416,7 @@ PYBIND11_INLINE void traverse_offset_bases(void *valueptr, } } -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) PYBIND11_INLINE void enable_try_inc_ref(PyObject *obj) { # if PY_VERSION_HEX >= 0x030E00A4 PyUnstable_EnableTryIncRef(obj); @@ -441,8 +441,15 @@ PYBIND11_INLINE void enable_try_inc_ref(PyObject *obj) { #endif PYBIND11_INLINE bool register_instance_impl(void *ptr, instance *self) { assert(ptr); -#ifdef Py_GIL_DISABLED - enable_try_inc_ref(reinterpret_cast(self)); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + if (self->registry_weakref == nullptr) { + self->registry_weakref = PyWeakref_NewRef(instance_object(self), nullptr); + if (self->registry_weakref == nullptr) { + throw error_already_set(); + } + } +#elif defined(Py_GIL_DISABLED) + enable_try_inc_ref(instance_object(self)); #endif with_instance_map(ptr, [&](instance_map &instances) { instances.emplace(ptr, self); }); return true; // unused, but gives the same signature as the deregister func @@ -483,10 +490,20 @@ PYBIND11_INLINE bool deregister_instance(instance *self, void *valptr, const typ // instancemethod: `__get__` binds through types.MethodType, calls and unknown attributes go to // the wrapped function. The type is shared through the internals. struct instancemethod_object { +# if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +# endif PyObject *func; }; +PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { +# if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(self, Py_TYPE(self))); +# else + return reinterpret_cast(self); +# endif +} + PYBIND11_INLINE PyTypeObject *get_bound_method_type() { static PyTypeObject *const type = [] { auto types = reinterpret_steal(PyImport_ImportModule("types")); @@ -521,14 +538,14 @@ instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { return self; } return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), - reinterpret_cast(self)->func, + instancemethod_data(self)->func, obj, nullptr); } extern "C" PYBIND11_INLINE PyObject * instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { - return PyObject_Call(reinterpret_cast(self)->func, args, kwargs); + return PyObject_Call(instancemethod_data(self)->func, args, kwargs); } extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { @@ -542,11 +559,11 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyO return get(descr.ptr(), self, reinterpret_cast(Py_TYPE(self))); } } - return PyObject_GetAttr(reinterpret_cast(self)->func, name); + return PyObject_GetAttr(instancemethod_data(self)->func, name); } extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { - PyObject *func = reinterpret_cast(self)->func; + PyObject *func = instancemethod_data(self)->func; auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); if (!name) { PyErr_Clear(); @@ -557,20 +574,20 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { extern "C" PYBIND11_INLINE int instancemethod_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(reinterpret_cast(self)->func); + Py_VISIT(instancemethod_data(self)->func); Py_VISIT(Py_TYPE(self)); return 0; } extern "C" PYBIND11_INLINE int instancemethod_clear(PyObject *self) { - Py_CLEAR(reinterpret_cast(self)->func); + Py_CLEAR(instancemethod_data(self)->func); return 0; } extern "C" PYBIND11_INLINE void instancemethod_dealloc(PyObject *self) { PyTypeObject *type = Py_TYPE(self); PyObject_GC_UnTrack(self); - Py_CLEAR(reinterpret_cast(self)->func); + Py_CLEAR(instancemethod_data(self)->func); type_free(type, self); Py_DECREF(reinterpret_cast(type)); } @@ -581,7 +598,7 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { static PyMemberDef members[] = {{"__func__", Py_T_OBJECT_EX, offsetof(instancemethod_object, func), - Py_READONLY, + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, nullptr}, {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] @@ -596,7 +613,7 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", - static_cast(sizeof(instancemethod_object)), + PYBIND11_TYPE_DATA_SIZE(instancemethod_object), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, slots}; @@ -616,7 +633,7 @@ PYBIND11_INLINE bool is_instancemethod(PyObject *obj) { } PYBIND11_INLINE PyObject *instancemethod_function(PyObject *obj) { - return reinterpret_cast(obj)->func; + return instancemethod_data(obj)->func; } PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { @@ -626,7 +643,7 @@ PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { return nullptr; } Py_INCREF(func); - reinterpret_cast(self)->func = func; + instancemethod_data(self)->func = func; return self; } @@ -673,6 +690,18 @@ PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { #endif } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +PYBIND11_INLINE Py_ssize_t instance_data_offset() { + // The same for every pybind11 type: the type data of pybind11_object, whose base is object. + static const Py_ssize_t offset = [] { + handle base = get_internals().instance_base; + auto basicsize = base.attr("__basicsize__").cast(); + return basicsize - PyType_GetTypeDataSize(reinterpret_cast(base.ptr())); + }(); + return offset; +} +#endif + PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { #if defined(PYPY_VERSION) // PyPy gets tp_basicsize wrong (issue 2482) under multiple inheritance when the first @@ -683,7 +712,7 @@ PYBIND11_INLINE PyObject *make_new_instance(PyTypeObject *type) { } #endif PyObject *self = type_alloc(type); - auto *inst = reinterpret_cast(self); + auto *inst = get_instance(self); // Allocate the value/holder internals: inst->allocate_layout(); @@ -703,7 +732,7 @@ extern "C" PYBIND11_INLINE int pybind11_object_init(PyObject *self, PyObject *, } PYBIND11_INLINE void add_patient(PyObject *nurse, PyObject *patient) { - auto *instance = reinterpret_cast(nurse); + auto *instance = get_instance(nurse); instance->has_patients = true; Py_INCREF(patient); @@ -711,7 +740,7 @@ PYBIND11_INLINE void add_patient(PyObject *nurse, PyObject *patient) { } PYBIND11_INLINE void clear_patients(PyObject *self) { - auto *instance = reinterpret_cast(self); + auto *instance = get_instance(self); std::vector patients; with_internals([&](internals &internals) { @@ -736,7 +765,7 @@ PYBIND11_INLINE void clear_patients(PyObject *self) { } PYBIND11_INLINE void clear_instance(PyObject *self) { - auto *instance = reinterpret_cast(self); + auto *instance = get_instance(self); // Deallocate any values/holders, if present: for (auto &v_h : values_and_holders(instance)) { @@ -760,6 +789,9 @@ PYBIND11_INLINE void clear_instance(PyObject *self) { // Deallocate the value/holder layout internals: instance->deallocate_layout(); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + Py_CLEAR(instance->registry_weakref); +#endif if (instance->weakrefs) { PyObject_ClearWeakRefs(self); } @@ -808,9 +840,12 @@ extern "C" PYBIND11_INLINE void pybind11_object_dealloc(PyObject *self) { PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { /* Support weak references (needed for the keep_alive feature) */ - static PyMemberDef members[] = { - {"__weaklistoffset__", Py_T_PYSSIZET, offsetof(instance, weakrefs), Py_READONLY, nullptr}, - {nullptr, 0, 0, 0, nullptr}}; + static PyMemberDef members[] = {{"__weaklistoffset__", + Py_T_PYSSIZET, + offsetof(instance, weakrefs), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, + {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] = {{Py_tp_new, reinterpret_cast(pybind11_object_new)}, {Py_tp_init, reinterpret_cast(pybind11_object_init)}, @@ -818,7 +853,7 @@ PYBIND11_INLINE PyObject *make_object_base_type(PyTypeObject *metaclass) { {Py_tp_members, reinterpret_cast(members)}, {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_object", - static_cast(sizeof(instance)), + PYBIND11_TYPE_DATA_SIZE(instance), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots}; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 434f449063..28fae30189 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -113,7 +113,7 @@ void traverse_offset_bases(void *valueptr, instance *self, bool (*f)(void * /*parentptr*/, instance * /*self*/)); -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) void enable_try_inc_ref(PyObject *obj); #endif @@ -133,6 +133,8 @@ void type_free(PyTypeObject *type, PyObject *self); /// `is_*`/`*_function` functions are also declared in pytypes.h. PyTypeObject *get_bound_method_type(); PyTypeObject *get_instancemethod_type(); +struct instancemethod_object; +instancemethod_object *instancemethod_data(PyObject *self); extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 8c25d086cd..f4c98558dc 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -25,7 +25,11 @@ "pybind11 requires Py_LIMITED_API >= 0x030C0000 and CPython >= 3.12 for the stable ABI." # endif # if defined(Py_GIL_DISABLED) -# error "The free-threaded build has no stable ABI; do not define Py_LIMITED_API." +# if Py_LIMITED_API + 0 < 0x030F0000 || !defined(_Py_OPAQUE_PYOBJECT) +# error "The free-threaded stable ABI (abi3t) needs Py_LIMITED_API >= 0x030F0000." +# endif +// PEP 803: PyObject and PyModuleDef are incomplete types; see docs/advanced/stable_abi.rst. +# define PYBIND11_OPAQUE_PYOBJECT # endif # if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals @@ -47,6 +51,16 @@ # endif #endif +// Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative +// PyType_Spec::basicsize and member offsets relative to that data. +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_TYPE_DATA_SIZE(T) (-static_cast(sizeof(T))) +# define PYBIND11_MEMBER_OFFSET_FLAGS Py_RELATIVE_OFFSET +#else +# define PYBIND11_TYPE_DATA_SIZE(T) (static_cast(sizeof(T))) +# define PYBIND11_MEMBER_OFFSET_FLAGS 0 +#endif + // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html // See also: https://github.com/python/cpython/blob/HEAD/Include/patchlevel.h /* -- start version constants -- */ @@ -555,29 +569,49 @@ Note that this is run once for each (sub-)interpreter the module is imported int possibly concurrently. The PyModuleDef is allowed to be static, but the PyObject* resulting from PyModuleDef_Init should be treated like any other PyObject (so not shared across interpreters). */ -#define PYBIND11_MODULE_PYINIT(name, ...) \ - static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_PRECOMPILED_CONFIG_GUARD \ - try { \ - static ::pybind11::detail::slots_array mod_def_slots \ - = ::pybind11::detail::init_slots( \ - &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ - static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ - /* m_name */ PYBIND11_TOSTRING(name), \ - /* m_doc */ nullptr, \ - /* m_size */ 0, \ - /* m_methods */ nullptr, \ - /* m_slots */ mod_def_slots.data(), \ - /* m_traverse */ nullptr, \ - /* m_clear */ nullptr, \ - /* m_free */ nullptr}; \ - return PyModuleDef_Init(&def); \ - } \ - PYBIND11_CATCH_INIT_EXCEPTIONS \ - return nullptr; \ - } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type, so the module is described by a static +// PySlot array instead. +# define PYBIND11_MODULE_PYINIT(name, ...) \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ + extern "C" PYBIND11_MAYBE_UNUSED PYBIND11_EXPORT PySlot *PyModExport_##name(void); \ + extern "C" PYBIND11_EXPORT PySlot *PyModExport_##name(void) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ + try { \ + static ::pybind11::detail::export_slots_array mod_slots \ + = ::pybind11::detail::init_export_slots( \ + PYBIND11_TOSTRING(name), &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ + return mod_slots.data(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ + } +#else +# define PYBIND11_MODULE_PYINIT(name, ...) \ + static int PYBIND11_CONCAT(pybind11_exec_, name)(PyObject *); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ + try { \ + static ::pybind11::detail::slots_array mod_def_slots \ + = ::pybind11::detail::init_slots( \ + &PYBIND11_CONCAT(pybind11_exec_, name), ##__VA_ARGS__); \ + static PyModuleDef def{/* m_base */ PyModuleDef_HEAD_INIT, \ + /* m_name */ PYBIND11_TOSTRING(name), \ + /* m_doc */ nullptr, \ + /* m_size */ 0, \ + /* m_methods */ nullptr, \ + /* m_slots */ mod_def_slots.data(), \ + /* m_traverse */ nullptr, \ + /* m_clear */ nullptr, \ + /* m_free */ nullptr}; \ + return PyModuleDef_Init(&def); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ + } +#endif #define PYBIND11_MODULE_EXEC(name, variable) \ static void PYBIND11_CONCAT(pybind11_init_, name)(::pybind11::module_ &); \ @@ -736,7 +770,9 @@ struct nonsimple_values_and_holders { /// The 'instance' type which needs to be standard layout (need to be able to use 'offsetof') struct instance { +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +#endif /// Storage for pointers and holder; see simple_layout, below, for a description union { void *simple_value_holder[1 + instance_simple_holder_in_ptrs()]; @@ -786,6 +822,10 @@ struct instance { /// `old_style_placement_new` in `docs/upgrade.rst` and referenced from /// `docs/advanced/classes.rst`. bool old_style_init_active : 1; +#if defined(PYBIND11_OPAQUE_PYOBJECT) + /// Weak reference used by the instance registry: the limited API has no PyUnstable_TryIncRef. + PyObject *registry_weakref; +#endif /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) @@ -808,6 +848,20 @@ struct instance { static_assert(std::is_standard_layout::value, "Internal error: `pybind11::detail::instance` is not standard layout!"); +#if defined(PYBIND11_OPAQUE_PYOBJECT) +/// Offset of the `instance` data (PEP 697 type data) from the start of the Python object. +Py_ssize_t instance_data_offset(); +inline instance *get_instance(PyObject *obj) { + return reinterpret_cast(reinterpret_cast(obj) + instance_data_offset()); +} +inline PyObject *instance_object(instance *inst) { + return reinterpret_cast(reinterpret_cast(inst) - instance_data_offset()); +} +#else +inline instance *get_instance(PyObject *obj) { return reinterpret_cast(obj); } +inline PyObject *instance_object(instance *inst) { return reinterpret_cast(inst); } +#endif + // Some older compilers (e.g. gcc 9.4.0) require // static_assert(always_false::value, "..."); // instead of diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 1aa2902d19..3f229dca9f 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -20,10 +20,20 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) struct function_record_PyObject { +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PyObject_HEAD +#endif function_record *cpp_func_rec; }; +inline function_record_PyObject *function_record_data(PyObject *obj) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); +#else + return reinterpret_cast(obj); +#endif +} + PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) PyObject *tp_new_impl(PyTypeObject *type, PyObject *args, PyObject *kwds); @@ -85,7 +95,7 @@ static PyType_Slot function_record_PyType_Slots[] = { static PyType_Spec function_record_PyType_Spec = {function_record_PyTypeObject_methods::tp_qualname_impl, - sizeof(function_record_PyObject), + PYBIND11_TYPE_DATA_SIZE(function_record_PyObject), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HEAPTYPE, function_record_PyType_Slots}; @@ -128,7 +138,7 @@ inline bool is_function_record_PyObject(PyObject *obj) { inline function_record *function_record_ptr_from_PyObject(PyObject *obj) { if (is_function_record_PyObject(obj)) { - return (reinterpret_cast(obj))->cpp_func_rec; + return function_record_data(obj)->cpp_func_rec; } return nullptr; } @@ -145,12 +155,12 @@ inline function_record *function_record_ptr_from_dispatcher_self(PyObject *self) } inline object function_record_PyObject_New() { - auto *py_func_rec = PyObject_New(function_record_PyObject, get_function_record_PyTypeObject()); + PyObject *py_func_rec = PyObject_New(PyObject, get_function_record_PyTypeObject()); if (py_func_rec == nullptr) { throw error_already_set(); } - py_func_rec->cpp_func_rec = nullptr; // For clarity/purity. Redundant in practice. - return reinterpret_steal(reinterpret_cast(py_func_rec)); + function_record_data(py_func_rec)->cpp_func_rec = nullptr; // For clarity. Redundant. + return reinterpret_steal(py_func_rec); } PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index af23890d57..9c7c6ee258 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { + if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -401,8 +401,7 @@ struct factory { (value_and_holder &v_h, Args... args) { construct(v_h, func(std::forward(args)...), - Py_TYPE(reinterpret_cast(v_h.inst)) - != v_h.type->type); + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); @@ -449,7 +448,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(reinterpret_cast(v_h.inst)) == v_h.type->type) { + if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -482,9 +481,9 @@ void setstate(value_and_holder &v_h, std::pair &&result, bool need_alias) return; } // Our tests never run into an unset dict, but being careful here for now (see #5658) - auto dict = getattr(reinterpret_cast(v_h.inst), "__dict__", none()); + auto dict = getattr(instance_object(v_h.inst), "__dict__", none()); if (dict.is_none()) { - setattr(reinterpret_cast(v_h.inst), "__dict__", d); + setattr(instance_object(v_h.inst), "__dict__", d); } else { // Keep the original object dict and just update it if (PyDict_Update(dict.ptr(), d.ptr()) < 0) { @@ -539,7 +538,7 @@ struct pickle_factory { (value_and_holder &v_h, ArgState state) { setstate(v_h, func(std::forward(state)), - Py_TYPE(reinterpret_cast(v_h.inst)) != v_h.type->type); + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals-inl.h b/include/pybind11/detail/internals-inl.h index 586f78ad50..ec8daf054c 100644 --- a/include/pybind11/detail/internals-inl.h +++ b/include/pybind11/detail/internals-inl.h @@ -251,7 +251,7 @@ PYBIND11_INLINE size_t num_registered_instances() { size_t count = 0; for (size_t i = 0; i <= internals.instance_shards_mask; ++i) { auto &shard = internals.instance_shards[i]; - std::unique_lock lock(shard.mutex); + pycritical_section lock(shard.mutex); count += shard.registered_instances.size(); } return count; diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 4fb8b10d61..b4403cf7ba 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -229,10 +229,9 @@ void translate_exception(std::exception_ptr p); inline PyThreadState *get_thread_state_unchecked() { #if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) return PyThreadState_GET(); -#elif defined(Py_LIMITED_API) && Py_LIMITED_API < 0x030D0000 - // The stable ABI has no "current thread state or null" function before 3.13. All callers - // left under the stable ABI (the interpreter state dict lookup) hold the GIL. - return PyThreadState_Get(); +#elif defined(Py_LIMITED_API) + // PyThreadState_GetUnchecked() is not part of the stable ABI. + return PyGILState_GetThisThreadState(); #elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) return _PyThreadState_UncheckedGet(); #else @@ -308,7 +307,34 @@ struct override_hash { using instance_map = std::unordered_multimap; -#ifdef Py_GIL_DISABLED +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PyMutex is not part of the stable ABI, but critical sections are (3.15+). The lock is the +// per-object mutex of a plain `object()`, which is kept for the lifetime of the process. +class pymutex { + friend class pycritical_section; + PyObject *lock_object; + +public: + pymutex() + : lock_object(PyObject_CallNoArgs(reinterpret_cast(&PyBaseObject_Type))) { + if (lock_object == nullptr) { + throw error_already_set(); + } + } +}; + +class pycritical_section { + PyCriticalSection cs{}; + +public: + explicit pycritical_section(pymutex &m) { PyCriticalSection_Begin(&cs, m.lock_object); } + ~pycritical_section() { PyCriticalSection_End(&cs); } + pycritical_section(const pycritical_section &) = delete; + pycritical_section &operator=(const pycritical_section &) = delete; + pycritical_section(pycritical_section &&) = delete; + pycritical_section &operator=(pycritical_section &&) = delete; +}; +#elif defined(Py_GIL_DISABLED) // Wrapper around PyMutex to provide BasicLockable semantics class pymutex { friend class pycritical_section; @@ -350,7 +376,9 @@ class pycritical_section { pycritical_section(pycritical_section &&) = delete; pycritical_section &operator=(pycritical_section &&) = delete; }; +#endif +#ifdef Py_GIL_DISABLED // Instance map shards are used to reduce mutex contention in free-threaded Python. struct instance_map_shard { instance_map registered_instances; @@ -546,7 +574,9 @@ struct native_enum_record { // regular modules in the same process. The tag keeps the two universes apart. It is deliberately // not part of PYBIND11_PLATFORM_ABI_ID: the C++ ABI is the same, so the cpp_conduit protocol // still bridges the two. -#if defined(Py_LIMITED_API) +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_INTERNALS_SABI_TAG "_stable_ft" // abi3t modules also load on GIL builds +#elif defined(Py_LIMITED_API) # define PYBIND11_INTERNALS_SABI_TAG "_stable" #else # define PYBIND11_INTERNALS_SABI_TAG "" @@ -949,7 +979,7 @@ inline auto with_instance_map(const void *ptr, const F &cb) auto idx = static_cast(hash & internals.instance_shards_mask); auto &shard = internals.instance_shards[idx]; - std::unique_lock lock(shard.mutex); + pycritical_section lock(shard.mutex); return cb(shard.registered_instances); #else (void) ptr; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index a593b5b2e7..84c7e27026 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -242,7 +242,12 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_type_handle(const std::type_in PYBIND11_INLINE bool try_incref(PyObject *obj) { // Tries to increment the reference count of an object if it's not zero. -#if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // Not used: the registry goes through PyWeakref_GetRef() instead. + assert(Py_REFCNT(obj) > 0); + Py_INCREF(obj); + return true; +#elif defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 return PyUnstable_TryIncRef(obj); #elif defined(Py_GIL_DISABLED) // See @@ -289,12 +294,20 @@ find_registered_python_instance(void *src, const detail::type_info *tinfo) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { for (auto *instance_type : - detail::all_type_info(Py_TYPE(reinterpret_cast(it_i->second)))) { + detail::all_type_info(Py_TYPE(instance_object(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { - auto *wrapper = reinterpret_cast(it_i->second); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + PyObject *wrapper = nullptr; + if (it_i->second->registry_weakref != nullptr + && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { + return handle(wrapper); + } +#else + auto *wrapper = instance_object(it_i->second); if (try_incref(wrapper)) { return handle(wrapper); } +#endif } } } @@ -306,7 +319,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder instance::get_value_and_holder(const type_info *find_type /*= nullptr default in common.h*/, bool throw_if_missing /*= true in common.h*/) { // Optimize common case: - if (!find_type || Py_TYPE(reinterpret_cast(this)) == find_type->type) { + if (!find_type || Py_TYPE(instance_object(this)) == find_type->type) { return value_and_holder(this, find_type, 0, 0); } @@ -321,10 +334,10 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } #if defined(PYBIND11_DETAILED_ERROR_MESSAGES) - pybind11_fail( - "pybind11::detail::instance::get_value_and_holder: `" - + get_fully_qualified_tp_name(find_type->type) + "' is not a pybind11 base of the given `" - + get_fully_qualified_tp_name(Py_TYPE(reinterpret_cast(this))) + "' instance"); + pybind11_fail("pybind11::detail::instance::get_value_and_holder: `" + + get_fully_qualified_tp_name(find_type->type) + + "' is not a pybind11 base of the given `" + + get_fully_qualified_tp_name(Py_TYPE(instance_object(this))) + "' instance"); #else pybind11_fail( "pybind11::detail::instance::get_value_and_holder: " @@ -334,7 +347,7 @@ instance::get_value_and_holder(const type_info *find_type /*= nullptr default in } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void instance::allocate_layout() { - const auto &tinfo = all_type_info(Py_TYPE(reinterpret_cast(this))); + const auto &tinfo = all_type_info(Py_TYPE(instance_object(this))); const size_t n_types = tinfo.size(); @@ -405,7 +418,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return handle(reinterpret_cast(it->second)); + return handle(instance_object(it->second)); } } } @@ -486,7 +499,7 @@ type_caster_generic::cast(const cast_sources &srcs, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *wrapper = reinterpret_cast(inst.ptr()); + auto *wrapper = get_instance(inst.ptr()); wrapper->owned = false; void *&valueptr = values_and_holders(wrapper).begin()->value_ptr(); diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 114d2fc52b..5799b4f95c 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -142,12 +142,12 @@ struct values_and_holders { public: explicit values_and_holders(instance *inst) - : inst{inst}, tinfo(all_type_info(Py_TYPE(reinterpret_cast(inst)))) {} + : inst{inst}, tinfo(all_type_info(Py_TYPE(instance_object(inst)))) {} explicit values_and_holders(PyObject *obj) : inst{nullptr}, tinfo(all_type_info(Py_TYPE(obj))) { if (!tinfo.empty()) { - inst = reinterpret_cast(obj); + inst = get_instance(obj); } } @@ -433,7 +433,7 @@ handle smart_holder_from_unique_ptr(std::unique_ptr &&src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); valueptr = src_raw_void_ptr; @@ -501,7 +501,7 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr &src, } auto inst = reinterpret_steal(make_new_instance(tinfo->type)); - auto *inst_raw_ptr = reinterpret_cast(inst.ptr()); + auto *inst_raw_ptr = get_instance(inst.ptr()); inst_raw_ptr->owned = true; void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr(); valueptr = src_raw_void_ptr; @@ -542,7 +542,7 @@ struct shared_ptr_parent_life_support { struct shared_ptr_trampoline_self_life_support { PyObject *self; explicit shared_ptr_trampoline_self_life_support(instance *inst) - : self{reinterpret_cast(inst)} { + : self{instance_object(inst)} { gil_scoped_acquire gil; Py_INCREF(self); } @@ -581,7 +581,7 @@ struct load_helper : value_and_holder_helper { void maybe_set_python_instance_is_alias(handle src) { if (was_populated) { - python_instance_is_alias = reinterpret_cast(src.ptr())->is_alias; + python_instance_is_alias = get_instance(src.ptr())->is_alias; } } @@ -845,7 +845,7 @@ class type_caster_generic { // Case 1: If src is an exact type match for the target type then we can reinterpret_cast // the instance's value pointer to the target type: if (srctype == typeinfo->type) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); + this_.load_value(get_instance(src.ptr())->get_value_and_holder()); return true; } // Case 2: We have a derived class @@ -860,7 +860,7 @@ class type_caster_generic { // is extremely common, we handle it specially to avoid the loop iterator and type // pointer lookup overhead) if (bases.size() == 1 && (no_cpp_mi || bases.front()->type == typeinfo->type)) { - this_.load_value(reinterpret_cast(src.ptr())->get_value_and_holder()); + this_.load_value(get_instance(src.ptr())->get_value_and_holder()); return true; } // Case 2b: the python type inherits from multiple C++ bases. Check the bases to see @@ -870,8 +870,7 @@ class type_caster_generic { for (auto *base : bases) { if (no_cpp_mi ? PyType_IsSubtype(base->type, typeinfo->type) : base->type == typeinfo->type) { - this_.load_value( - reinterpret_cast(src.ptr())->get_value_and_holder(base)); + this_.load_value(get_instance(src.ptr())->get_value_and_holder(base)); return true; } } diff --git a/include/pybind11/detail/typeid.h b/include/pybind11/detail/typeid.h index c8160ad135..bcfe32305d 100644 --- a/include/pybind11/detail/typeid.h +++ b/include/pybind11/detail/typeid.h @@ -54,6 +54,14 @@ std::string type_id() { return detail::clean_type_id(typeid(T).name()); } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// typeid() needs a complete type; PyObject is opaque under the abi3t stable ABI. +template <> +inline std::string type_id() { + return "PyObject"; +} +#endif + PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) #ifndef PYBIND11_PRECOMPILED diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index b24551e678..98ab73c378 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -84,7 +84,7 @@ struct value_and_holder { // for `tp_traverse` and `tp_clear` implementations. // WARNING: The caller is responsible for ensuring that the `reinterpret_cast` is valid. inline bool is_holder_constructed(PyObject *obj) { - auto *const instance = reinterpret_cast(obj); + auto *const instance = get_instance(obj); return instance->get_value_and_holder().holder_constructed(); } diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index fdaaecc6b5..12b0aa943c 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,6 +10,10 @@ #pragma once #include "pybind11.h" + +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# error "numpy.h mirrors NumPy object layouts; it is not available with the abi3t stable ABI." +#endif #include "detail/common.h" #include "complex.h" #include "gil_safe_call_once.h" diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 98b8ade2bb..5a5b92f080 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -199,7 +199,7 @@ PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) PYBIND11_INLINE void tp_dealloc_impl(PyObject *self) { // Save type before PyObject_Free invalidates self. auto *type = Py_TYPE(self); - auto *py_func_rec = reinterpret_cast(self); + auto *py_func_rec = function_record_data(self); cpp_function::destruct(py_func_rec->cpp_func_rec); py_func_rec->cpp_func_rec = nullptr; // PyObject_New increments the heap type refcount and allocates via @@ -665,8 +665,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u rec->def->ml_flags = METH_FASTCALL | METH_KEYWORDS; object py_func_rec = detail::function_record_PyObject_New(); - (reinterpret_cast(py_func_rec.ptr()))->cpp_func_rec - = unique_rec.release(); + detail::function_record_data(py_func_rec.ptr())->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); object scope_module = detail::get_scope_module(rec->scope); @@ -699,8 +698,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec - = reinterpret_cast(PyCFunction_GetSelf(m_ptr)); + auto *py_func_rec = detail::function_record_data(PyCFunction_GetSelf(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { @@ -862,7 +860,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, auto *const tinfo = get_type_info(reinterpret_cast(overloads->scope.ptr())); - auto *const pi = reinterpret_cast(parent.ptr()); + auto *const pi = get_instance(parent.ptr()); self_value_and_holder = pi->get_value_and_holder(tinfo, true); // If this value is already registered it must mean __init__ is invoked multiple times; @@ -1293,7 +1291,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self, return nullptr; } if (overloads->is_constructor && !self_value_and_holder.holder_constructed()) { - auto *pi = reinterpret_cast(parent.ptr()); + auto *pi = get_instance(parent.ptr()); self_value_and_holder.type->init_instance(pi, nullptr); } return result.ptr(); @@ -1606,6 +1604,7 @@ module_::add_object(const char *name, handle obj, bool overwrite) { PyModule_AddObject(ptr(), name, obj.inc_ref().ptr() /* steals a reference */); } +#if !defined(PYBIND11_OPAQUE_PYOBJECT) PYBIND11_INLINE module_ module_::create_extension_module(const char *name, const char *doc, PyModuleDef *def, @@ -1628,14 +1627,15 @@ PYBIND11_INLINE module_ module_::create_extension_module(const char *name, pybind11_fail("Internal error in module_::create_extension_module()"); } if (gil_not_used.flag()) { -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif +# endif } // TODO: Should be reinterpret_steal for Python 3, but Python also steals it again when // returned from PyInit_... // For Python 2, reinterpret_borrow was correct. return reinterpret_borrow(m); } +#endif PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 464fb768d7..0248631ac9 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -720,6 +720,41 @@ inline slots_array init_slots(int (*exec_fn)(PyObject *), Options &&...options) return mod_def_slots; } +#if defined(PYBIND11_OPAQUE_PYOBJECT) +/// PEP 793 slot array; large enough for every slot below plus the zero sentinel. +using export_slots_array = std::array; + +template +inline export_slots_array +init_export_slots(const char *name, int (*exec_fn)(PyObject *), Options &&...options) noexcept { + export_slots_array slots{}; + size_t next_slot = 0; + auto data_slot = [&](int id, const void *value, uint16_t flags) { + slots[next_slot].sl_id = static_cast(id); + slots[next_slot].sl_flags = flags; + slots[next_slot].sl_ptr = const_cast(value); + ++next_slot; + }; + auto func_slot = [&](int id, _Py_funcptr_t fn) { + slots[next_slot].sl_id = static_cast(id); + slots[next_slot].sl_func = fn; + ++next_slot; + }; + PyABIInfo_VAR(abi_info); + data_slot(Py_mod_name, name, PySlot_INTPTR); + data_slot(Py_mod_abi, &abi_info, PySlot_STATIC); + func_slot(Py_mod_create, reinterpret_cast<_Py_funcptr_t>(&cached_create_module)); + if (exec_fn != nullptr) { + func_slot(Py_mod_exec, reinterpret_cast<_Py_funcptr_t>(exec_fn)); + } + data_slot(Py_mod_multiple_interpreters, multi_interp_slot(options...), PySlot_INTPTR); + if (gil_not_used_option(options...)) { + data_slot(Py_mod_gil, Py_MOD_GIL_NOT_USED, PySlot_INTPTR); + } + return slots; +} +#endif + PYBIND11_NAMESPACE_END(detail) /// Wrapper for Python extension modules @@ -727,11 +762,13 @@ class module_ : public object { public: PYBIND11_OBJECT_DEFAULT(module_, object, PyModule_Check) +#if !defined(PYBIND11_OPAQUE_PYOBJECT) /// Create a new top-level Python module with the given name and docstring PYBIND11_DEPRECATED("Use PYBIND11_MODULE or module_::create_extension_module instead") explicit module_(const char *name, const char *doc = nullptr) { *this = create_extension_module(name, doc, new PyModuleDef()); } +#endif /** \rst Create Python binding for a new function within the module scope. ``Func`` @@ -782,6 +819,7 @@ class module_ : public object { // DEPRECATED (since PR #5688): Use PyModuleDef directly instead. using module_def = PyModuleDef; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) /** \rst Create a new top-level module that can be used as the main module of a C extension. @@ -791,6 +829,7 @@ class module_ : public object { const char *doc, PyModuleDef *def, mod_gil_not_used gil_not_used = mod_gil_used()); +#endif }; PYBIND11_NAMESPACE_BEGIN(detail) @@ -2069,7 +2108,7 @@ class enum_ : public class_ { [](detail::value_and_holder &v_h, Scalar arg) { detail::initimpl::setstate(v_h, static_cast(arg), - Py_TYPE(reinterpret_cast(v_h.inst)) + Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); }, detail::is_new_style_constructor(), diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 8a440170e6..2945d2f037 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -73,7 +73,14 @@ using tuple_accessor = accessor; /// Tag and check to identify a class which implements the Python object API class pyobject_tag {}; template -using is_pyobject = std::is_base_of>; +struct is_pyobject : std::is_base_of> {}; +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PyObject is an incomplete type under the abi3t stable ABI; type traits cannot inspect it. +template <> +struct is_pyobject : std::false_type {}; +template <> +struct is_pyobject : std::false_type {}; +#endif /** \rst A mixin class which adds common functions to `handle`, `object` and various accessors. diff --git a/include/pybind11/trampoline_self_life_support.h b/include/pybind11/trampoline_self_life_support.h index a3c14bbd86..281421440c 100644 --- a/include/pybind11/trampoline_self_life_support.h +++ b/include/pybind11/trampoline_self_life_support.h @@ -25,12 +25,12 @@ struct trampoline_self_life_support { trampoline_self_life_support() = default; void activate_life_support(const detail::value_and_holder &v_h_) { - Py_INCREF((PyObject *) v_h_.inst); + Py_INCREF(instance_object(v_h_.inst)); v_h = v_h_; } void deactivate_life_support() { - Py_DECREF((PyObject *) v_h.inst); + Py_DECREF(instance_object(v_h.inst)); v_h = detail::value_and_holder(); } @@ -42,7 +42,7 @@ struct trampoline_self_life_support { v_h.value_ptr() = nullptr; v_h.holder().release_disowned(); detail::deregister_instance(v_h.inst, value_void_ptr, v_h.type); - Py_DECREF((PyObject *) v_h.inst); // Must be after deregister. + Py_DECREF(instance_object(v_h.inst)); // Must be after deregister. PyGILState_Release(threadstate); } } diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 3053014382..77e02926d7 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -89,12 +89,15 @@ def _limited_api_hex(value: Any) -> str: """ - The Py_LIMITED_API value for ``py_limited_api``: True selects 3.12 (the - minimum pybind11 supports), a string such as "3.13" or "cp313" selects - that version, and an int or hex string is passed through. + The Py_LIMITED_API value for ``py_limited_api``: True selects the minimum + pybind11 supports (3.12, or 3.15 on free-threaded Python, where the stable + ABI is abi3t), a string such as "3.13" or "cp313" selects that version, and + an int or hex string is passed through. """ + free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + minimum = (3, 15) if free_threaded else (3, 12) if value is True: - return "0x030C0000" + return f"0x{minimum[0]:02X}{minimum[1]:02X}0000" if isinstance(value, int): return f"0x{value:08X}" text = str(value) @@ -105,8 +108,13 @@ def _limited_api_hex(value: Any) -> str: msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" raise ValueError(msg) major, minor = (int(part) for part in match.groups()) - if (major, minor) < (3, 12): - msg = f"pybind11 supports the stable ABI from Python 3.12 on, got {value!r}" + if (major, minor) < minimum: + what = ( + "the free-threaded stable ABI (abi3t)" + if free_threaded + else "the stable ABI" + ) + msg = f"pybind11 supports {what} from Python {minimum[0]}.{minimum[1]} on, got {value!r}" raise ValueError(msg) return f"0x{major:02X}{minor:02X}0000" diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4dd970771c..71c805444c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -236,6 +236,19 @@ if(PYBIND11_TEST_STABLE_ABI) MESSAGE "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" ) + # numpy.h mirrors NumPy object layouts, which the abi3t stable ABI hides (PEP 803). + if("${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + pybind11_filter_tests( + PYBIND11_TEST_FILES + test_numpy_array.cpp + test_numpy_dtypes.cpp + test_numpy_scalars.cpp + test_numpy_vectorize.cpp + test_eigen_matrix.cpp + test_eigen_tensor.cpp + MESSAGE + "Skipping numpy and eigen tests under the free-threaded stable ABI (abi3t)") + endif() endif() # Skip tests for CUDA check: diff --git a/tests/cross_module_gil_utils.cpp b/tests/cross_module_gil_utils.cpp index 39112668a4..0d0f877eb3 100644 --- a/tests/cross_module_gil_utils.cpp +++ b/tests/cross_module_gil_utils.cpp @@ -77,35 +77,59 @@ void gil_acquire_nested() { constexpr char kModuleName[] = "cross_module_gil_utils"; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; +#endif } // namespace #define ADD_FUNCTION(Name, ...) \ PyModule_AddObject(m, Name, PyLong_FromVoidPtr(reinterpret_cast(&__VA_ARGS__))); +static int cross_module_gil_utils_exec(PyObject *m) { + static_assert(sizeof(&gil_acquire) == sizeof(void *), + "Function pointer must have the same size as void*"); + ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) + ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) + ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", + gil_acquire_nested) + ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", + gil_acquire_inner) + ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", + gil_acquire_nested) + return 0; +} + +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" PYBIND11_EXPORT PySlot *PyModExport_cross_module_gil_utils() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] + = {PySlot_PTR(Py_mod_name, kModuleName), + {Py_mod_exec, 0, {0}, {reinterpret_cast(&cross_module_gil_utils_exec)}}, + PySlot_PTR(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_gil_utils() { PyObject *m = PyModule_Create(&moduledef); if (m != nullptr) { - static_assert(sizeof(&gil_acquire) == sizeof(void *), - "Function pointer must have the same size as void*"); -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif - ADD_FUNCTION("gil_acquire_funcaddr", gil_acquire) - ADD_FUNCTION("gil_multi_acquire_release_funcaddr", gil_multi_acquire_release) - ADD_FUNCTION("gil_acquire_inner_custom_funcaddr", - gil_acquire_inner) - ADD_FUNCTION("gil_acquire_nested_custom_funcaddr", - gil_acquire_nested) - ADD_FUNCTION("gil_acquire_inner_pybind11_funcaddr", - gil_acquire_inner) - ADD_FUNCTION("gil_acquire_nested_pybind11_funcaddr", - gil_acquire_nested) +# endif + if (cross_module_gil_utils_exec(m) != 0) { + Py_DECREF(m); + return nullptr; + } } return m; } +#endif diff --git a/tests/cross_module_interleaved_error_already_set.cpp b/tests/cross_module_interleaved_error_already_set.cpp index 65a4463b8c..c549df7529 100644 --- a/tests/cross_module_interleaved_error_already_set.cpp +++ b/tests/cross_module_interleaved_error_already_set.cpp @@ -32,23 +32,45 @@ void interleaved_error_already_set() { constexpr char kModuleName[] = "cross_module_interleaved_error_already_set"; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, kModuleName, nullptr, 0, nullptr, nullptr, nullptr, nullptr, nullptr}; +#endif + +int module_exec(PyObject *m) { + static_assert(sizeof(&interleaved_error_already_set) == sizeof(void *), + "Function pointer must have the same size as void *"); + return PyModule_AddObject( + m, + "funcaddr", + PyLong_FromVoidPtr(reinterpret_cast(&interleaved_error_already_set))); +} } // namespace +#if defined(PYBIND11_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" PYBIND11_EXPORT PySlot *PyModExport_cross_module_interleaved_error_already_set() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] = {PySlot_PTR(Py_mod_name, kModuleName), + {Py_mod_exec, 0, {0}, {reinterpret_cast(&module_exec)}}, + PySlot_PTR(Py_mod_gil, Py_MOD_GIL_NOT_USED), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" PYBIND11_EXPORT PyObject *PyInit_cross_module_interleaved_error_already_set() { PyObject *m = PyModule_Create(&moduledef); if (m != nullptr) { - static_assert(sizeof(&interleaved_error_already_set) == sizeof(void *), - "Function pointer must have the same size as void *"); -#ifdef Py_GIL_DISABLED +# ifdef Py_GIL_DISABLED PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); -#endif - PyModule_AddObject( - m, - "funcaddr", - PyLong_FromVoidPtr(reinterpret_cast(&interleaved_error_already_set))); +# endif + if (module_exec(m) != 0) { + Py_DECREF(m); + return nullptr; + } } return m; } +#endif diff --git a/tests/exo_planet_c_api.cpp b/tests/exo_planet_c_api.cpp index 1510107175..75ed042982 100644 --- a/tests/exo_planet_c_api.cpp +++ b/tests/exo_planet_c_api.cpp @@ -47,6 +47,7 @@ PyMethodDef ThisMethodDef[] = {{"GetLuggage", wrapGetLuggage, METH_O, nullptr}, {"GetPoints", wrapGetPoints, METH_O, nullptr}, {nullptr, nullptr, 0, nullptr}}; +#if !defined(_Py_OPAQUE_PYOBJECT) struct PyModuleDef ThisModuleDef = { PyModuleDef_HEAD_INIT, // m_base "exo_planet_c_api", // m_name @@ -58,6 +59,7 @@ struct PyModuleDef ThisModuleDef = { nullptr, // m_clear nullptr // m_free }; +#endif } // namespace @@ -67,6 +69,17 @@ struct PyModuleDef ThisModuleDef = { # define EXO_PLANET_C_API_EXPORT __attribute__((visibility("default"))) #endif +#if defined(_Py_OPAQUE_PYOBJECT) +// PEP 793 export hook: PyModuleDef is an incomplete type under the abi3t stable ABI. +extern "C" EXO_PLANET_C_API_EXPORT PySlot *PyModExport_exo_planet_c_api() { + PyABIInfo_VAR(abi_info); + static PySlot slots[] = {PySlot_PTR(Py_mod_name, "exo_planet_c_api"), + PySlot_PTR_STATIC(Py_mod_methods, ThisMethodDef), + PySlot_PTR_STATIC(Py_mod_abi, &abi_info), + {0, 0, {0}, {nullptr}}}; + return slots; +} +#else extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { PyObject *m = PyModule_Create(&ThisModuleDef); if (m == nullptr) { @@ -74,3 +87,4 @@ extern "C" EXO_PLANET_C_API_EXPORT PyObject *PyInit_exo_planet_c_api() { } return m; } +#endif diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index e0a9c5f50d..9e1e86c6f0 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -114,8 +114,8 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): @pytest.mark.skipif( sys.implementation.name != "cpython" or sys.version_info < (3, 12) - or sysconfig.get_config_var("Py_GIL_DISABLED"), - reason="stable ABI needs CPython 3.12+ with the GIL", + or (sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 15)), + reason="stable ABI needs CPython 3.12+, or 3.15+ when free-threaded (abi3t)", ) def test_stable_abi_setup_py(monkeypatch, tmpdir): monkeypatch.chdir(tmpdir) diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 627c8530ea..9e4af55920 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -11,5 +11,5 @@ expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") if expect_abi3: - assert ".abi3." in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" + assert ".abi3" in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") diff --git a/tests/test_modules.cpp b/tests/test_modules.cpp index 842a3bc4b8..a70c45f086 100644 --- a/tests/test_modules.cpp +++ b/tests/test_modules.cpp @@ -77,8 +77,13 @@ TEST_SUBMODULE(modules, m) { class Dupe3 {}; class DupeException {}; +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // PyModuleDef is an incomplete type under the abi3t stable ABI. + auto dm = py::reinterpret_steal(PyModule_New("dummy")); +#else // Go ahead and leak, until we have a non-leaking py::module_ constructor auto dm = py::module_::create_extension_module("dummy", nullptr, new PyModuleDef); +#endif auto failures = py::list(); py::class_(dm, "Dupe1"); diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index f399ec0e43..bad5fe86de 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -7,8 +7,10 @@ BSD-style license that can be found in the LICENSE file. */ -#include #include +#if !defined(PYBIND11_OPAQUE_PYOBJECT) +# include +#endif #include "pybind11_tests.h" @@ -238,10 +240,12 @@ TEST_SUBMODULE(stl_binders, m) { double y; bool z; }; +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // needs numpy.h's generic format_descriptor m.def("create_undeclstruct", [m]() mutable { py::bind_vector>( m, "VectorUndeclStruct", py::buffer_protocol()); }); +#endif // Bind recursive container types py::bind_vector(m, "RecursiveVector"); @@ -253,6 +257,7 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector(m, "UserVectorLike"); py::bind_map(m, "UserMapLike"); +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // The rest depends on numpy: try { py::module_::import("numpy"); @@ -272,4 +277,5 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector>(m, "VectorStruct", py::buffer_protocol()); m.def("get_vectorstruct", [] { return std::vector{{false, 5, 3.0, true}, {true, 30, -1e4, false}}; }); +#endif } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index edc6d28924..523a5b991e 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -118,13 +118,16 @@ def test_vector_buffer(): v = m.VectorUChar(mv[::2]) assert v[1] == 3 - with pytest.raises(RuntimeError) as excinfo: - m.create_undeclstruct() # Undeclared struct contents, no buffer interface - assert "NumPy type info missing for " in str(excinfo.value) + if hasattr(m, "create_undeclstruct"): # not built with the abi3t stable ABI + with pytest.raises(RuntimeError) as excinfo: + m.create_undeclstruct() # Undeclared struct contents, no buffer interface + assert "NumPy type info missing for " in str(excinfo.value) def test_vector_buffer_numpy(): np = pytest.importorskip("numpy") + if not hasattr(m, "VectorStruct"): + pytest.skip("numpy.h is not available with the abi3t stable ABI") a = np.array([1, 2, 3, 4], dtype=np.int32) with pytest.raises(TypeError): m.VectorInt(a) diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index 850b78bbd9..2f22de63b7 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -46,6 +46,7 @@ "PYBIND11_HAS_DIRECT_STRUCT_ACCESS", "PYBIND11_HAS_SUBINTERPRETER_SUPPORT", "PYBIND11_INTERNALS_VERSION", + "PYBIND11_OPAQUE_PYOBJECT", # implied by Py_LIMITED_API with Py_GIL_DISABLED "PYBIND11_SIMPLE_GIL_MANAGEMENT", "PYBIND11_TYPE_CREATION_VIA_SPEC", "Py_GIL_DISABLED", diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 4f490572b3..5618776876 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -269,9 +269,37 @@ set(PYBIND11_STABLE_ABI_VERSION "3.12" CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") -# PYBIND11_STABLE_ABI_VERSION as the Py_LIMITED_API hex value. +# True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). +function(_pybind11_python_is_free_threaded out_var) + if(${_Python}_FREE_THREADED + OR "${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + set(${out_var} + ON + PARENT_SCOPE) + else() + set(${out_var} + OFF + PARENT_SCOPE) + endif() +endfunction() + +# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 on free-threaded Python (abi3t starts there). +function(_pybind11_stable_abi_version out_var) + set(_version "${PYBIND11_STABLE_ABI_VERSION}") + _pybind11_python_is_free_threaded(_free_threaded) + if(_free_threaded AND _version VERSION_LESS 3.15) + set(_version "3.15") + endif() + set(${out_var} + "${_version}" + PARENT_SCOPE) +endfunction() + +# The effective stable ABI version as the Py_LIMITED_API hex value. function(_pybind11_stable_abi_hex out_var) - string(REPLACE "." ";" _parts "${PYBIND11_STABLE_ABI_VERSION}") + _pybind11_stable_abi_version(_version) + string(REPLACE "." ";" _parts "${_version}") list(GET _parts 0 _major) list(GET _parts 1 _minor) math(EXPR _hex "(${_major} << 24) | (${_minor} << 16)" OUTPUT_FORMAT HEXADECIMAL) @@ -300,9 +328,12 @@ function(_pybind11_check_stable_abi target_name lib_type) message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") endif() - if("${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" OR "${PYTHON_MODULE_EXTENSION}" MATCHES - "^\\.cpython-[0-9]+t") - message(FATAL_ERROR "${target_name}: the free-threaded build has no stable ABI.") + _pybind11_python_is_free_threaded(_free_threaded) + if(_free_threaded + AND DEFINED ${_Python}_VERSION + AND ${_Python}_VERSION VERSION_LESS 3.15) + message(FATAL_ERROR "${target_name}: the free-threaded stable ABI (abi3t) needs Python " + ">= 3.15, found ${${_Python}_VERSION}.") endif() if(NOT TARGET ${_Python}::SABIModule) message( @@ -340,7 +371,8 @@ function(pybind11_add_module target_name) set(stable_abi ON) if(lib_type STREQUAL "MODULE") # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). - set(use_sabi USE_SABI ${PYBIND11_STABLE_ABI_VERSION}) + _pybind11_stable_abi_version(_sabi_version) + set(use_sabi USE_SABI ${_sabi_version}) endif() endif() From 74086767a6f280e713f56e14313c971a92cf68a3 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:43:27 -0400 Subject: [PATCH 17/59] feat(numpy): support numpy.h under the abi3t stable ABI Under abi3t the size of PyObject_HEAD is not known at compile time, so the proxy structs drop it and the accessors go through the field getters NumPy 2.5 added for the same purpose (_PyArray_GET_ITEM_DATA and _PyDataType_GET_ITEM_DATA); void scalars are read through the buffer protocol. Older NumPy versions are rejected at run time. The numpy and Eigen tests run again under abi3t. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 6 --- docs/advanced/stable_abi.rst | 11 +++-- include/pybind11/numpy.h | 80 +++++++++++++++++++++++++++++++----- tests/CMakeLists.txt | 13 ------ tests/test_stl_binders.cpp | 8 +--- tests/test_stl_binders.py | 9 ++-- 6 files changed, 78 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 687b2c25a7..6ce0d8c843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -248,12 +248,6 @@ jobs: --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py - --ignore=tests/test_numpy_array.py - --ignore=tests/test_numpy_dtypes.py - --ignore=tests/test_numpy_scalars.py - --ignore=tests/test_numpy_vectorize.py - --ignore=tests/test_eigen_matrix.py - --ignore=tests/test_eigen_tensor.py steps: - uses: actions/checkout@v7 diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 85543baf9a..d17f05c8c6 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -56,12 +56,10 @@ available and in how a few things are implemented. * - Feature - Stable ABI - Notes - * - Classes, functions, casters, STL + * - Classes, functions, casters, STL, numpy, Eigen - supported - - - * - numpy, Eigen - - supported (abi3 only) - - ``numpy.h`` mirrors NumPy's object layouts, which abi3t hides. + - Under abi3t, ``numpy.h`` needs NumPy 2.5+ at run time (it reads + array fields through NumPy's abi3t accessors). * - ``py::dynamic_attr()`` - supported - The ``__dict__`` slot is appended to the instance (no @@ -108,7 +106,8 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and one weak reference per registered instance so that the instance registry can hand out strong references safely (``PyWeakref_GetRef()``). -``pybind11/numpy.h`` (and therefore the Eigen headers) is not available. The +``pybind11/numpy.h`` reads NumPy's object fields through the accessors NumPy +2.5 added for abi3t, so older NumPy versions are rejected at run time. The internals tag is ``_stable_ft``: an abi3t module and an abi3 module loaded into the same GIL-enabled interpreter do not share internals (see below). diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 12b0aa943c..34374c1f5b 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -10,10 +10,6 @@ #pragma once #include "pybind11.h" - -#if defined(PYBIND11_OPAQUE_PYOBJECT) -# error "numpy.h mirrors NumPy object layouts; it is not available with the abi3t stable ABI." -#endif #include "detail/common.h" #include "complex.h" #include "gil_safe_call_once.h" @@ -75,20 +71,28 @@ struct handle_type_name { template struct npy_format_descriptor; -// NumPy 2.5.2+ aligns the first field after PyObject_HEAD to 8 bytes on Python 3.15+. +// The proxies mirror NumPy's object layouts. Under the abi3t stable ABI PyObject is opaque and +// its size differs between free-threaded and GIL-enabled interpreters; NumPy >= 2.5 then +// defines the same structs without the header and returns pointers to them from its C API +// (`_PyArray_GET_ITEM_DATA`, `_PyDataType_GET_ITEM_DATA`), which the accessors below use. // NumPy 2.5.2, the first release supporting Python 3.15, aligns the first // field after PyObject_HEAD to 8 bytes on Python 3.15+. // This assumes a supported NumPy/Python pairing: older NumPy builds on // 32-bit free-threaded Python 3.15 have an incompatible layout. -#if PY_VERSION_HEX >= 0x030F0000 +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_NUMPY_PROXY_HEAD +# define PYBIND11_NPY_FIRST_FIELD +#elif PY_VERSION_HEX >= 0x030F0000 +# define PYBIND11_NUMPY_PROXY_HEAD PyObject_HEAD # define PYBIND11_NPY_FIRST_FIELD alignas(8) #else +# define PYBIND11_NUMPY_PROXY_HEAD PyObject_HEAD # define PYBIND11_NPY_FIRST_FIELD #endif /* NumPy 1 proxy (always includes legacy fields) */ struct PyArrayDescr1_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PYBIND11_NPY_FIRST_FIELD PyObject *typeobj; char kind; char type; @@ -103,7 +107,7 @@ struct PyArrayDescr1_Proxy { }; struct PyArrayDescr_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PYBIND11_NPY_FIRST_FIELD PyObject *typeobj; char kind; char type; @@ -115,7 +119,7 @@ struct PyArrayDescr_Proxy { /* NumPy 2 proxy, including legacy fields */ struct PyArrayDescr2_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PYBIND11_NPY_FIRST_FIELD PyObject *typeobj; char kind; char type; @@ -135,7 +139,7 @@ struct PyArrayDescr2_Proxy { }; struct PyArray_Proxy { - PyObject_HEAD + PYBIND11_NUMPY_PROXY_HEAD PYBIND11_NPY_FIRST_FIELD char *data; int nd; ssize_t *dimensions; @@ -147,12 +151,14 @@ struct PyArray_Proxy { #undef PYBIND11_NPY_FIRST_FIELD +#if !defined(PYBIND11_OPAQUE_PYOBJECT) struct PyVoidScalarObject_Proxy { PyObject_VAR_HEAD char *obval; PyArrayDescr_Proxy *descr; int flags; PyObject *base; }; +#endif struct numpy_type_info { PyObject *dtype_ptr; @@ -334,6 +340,10 @@ struct npy_api { PyObject *(*PyArray_Squeeze_)(PyObject *); // Unused. Not removed because that affects ABI of the class. int (*PyArray_SetBaseObject_)(PyObject *, PyObject *); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + PyArray_Proxy *(*_PyArray_GET_ITEM_DATA_)(const PyObject *); + PyArrayDescr_Proxy *(*_PyDataType_GET_ITEM_DATA_)(const PyObject *); +#endif PyObject *(*PyArray_Resize_)(PyObject *, PyArray_Dims *, int, int); PyObject *(*PyArray_Newshape_)(PyObject *, PyArray_Dims *, int); PyObject *(*PyArray_View_)(PyObject *, PyObject *, PyObject *); @@ -361,7 +371,10 @@ struct npy_api { API_PyArray_View = 137, API_PyArray_DescrConverter = 174, API_PyArray_EquivTypes = 182, - API_PyArray_SetBaseObject = 282 + API_PyArray_SetBaseObject = 282, + // NumPy >= 2.5: field access for abi3t, where PyObject_HEAD has no fixed size. + API__PyArray_GET_ITEM_DATA = 369, + API__PyDataType_GET_ITEM_DATA = 372 }; static npy_api lookup() { @@ -399,6 +412,14 @@ struct npy_api { DECL_NPY_API(PyArray_DescrConverter); DECL_NPY_API(PyArray_EquivTypes); DECL_NPY_API(PyArray_SetBaseObject); +#if defined(PYBIND11_OPAQUE_PYOBJECT) + if (api.PyArray_RUNTIME_VERSION_ < 0x16) { + pybind11_fail( + "pybind11 numpy support with the abi3t stable ABI requires numpy >= 2.5"); + } + DECL_NPY_API(_PyArray_GET_ITEM_DATA); + DECL_NPY_API(_PyDataType_GET_ITEM_DATA); +#endif #undef DECL_NPY_API return api; @@ -550,6 +571,31 @@ static constexpr int normalized_dtype_num[npy_api::NPY_VOID_ + 1] = { npy_api::NPY_VOID_, }; +#if defined(PYBIND11_OPAQUE_PYOBJECT) +inline PyArray_Proxy *array_proxy(void *ptr) { + return npy_api::get()._PyArray_GET_ITEM_DATA_(static_cast(ptr)); +} + +inline const PyArray_Proxy *array_proxy(const void *ptr) { + return npy_api::get()._PyArray_GET_ITEM_DATA_(static_cast(ptr)); +} + +inline PyArrayDescr_Proxy *array_descriptor_proxy(PyObject *ptr) { + return npy_api::get()._PyDataType_GET_ITEM_DATA_(ptr); +} + +inline const PyArrayDescr_Proxy *array_descriptor_proxy(const PyObject *ptr) { + return npy_api::get()._PyDataType_GET_ITEM_DATA_(ptr); +} + +inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { + return reinterpret_cast(array_descriptor_proxy(ptr)); +} + +inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { + return reinterpret_cast(array_descriptor_proxy(ptr)); +} +#else inline PyArray_Proxy *array_proxy(void *ptr) { return reinterpret_cast(ptr); } inline const PyArray_Proxy *array_proxy(const void *ptr) { @@ -571,6 +617,7 @@ inline const PyArrayDescr1_Proxy *array_descriptor1_proxy(const PyObject *ptr) { inline const PyArrayDescr2_Proxy *array_descriptor2_proxy(const PyObject *ptr) { return reinterpret_cast(ptr); } +#endif inline bool check_flags(const void *ptr, int flag) { return (flag == (array_proxy(ptr)->flags & flag)); @@ -1810,7 +1857,18 @@ struct npy_format_descriptor { } if (auto descr = reinterpret_steal(api.PyArray_DescrFromScalar_(obj))) { if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // The void scalar is opaque; its buffer is the scalar's own storage. + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0) { + PyErr_Clear(); + return false; + } + value = view.buf; + PyBuffer_Release(&view); +#else value = ((PyVoidScalarObject_Proxy *) obj)->obval; +#endif return true; } } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 71c805444c..4dd970771c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -236,19 +236,6 @@ if(PYBIND11_TEST_STABLE_ABI) MESSAGE "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" ) - # numpy.h mirrors NumPy object layouts, which the abi3t stable ABI hides (PEP 803). - if("${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") - pybind11_filter_tests( - PYBIND11_TEST_FILES - test_numpy_array.cpp - test_numpy_dtypes.cpp - test_numpy_scalars.cpp - test_numpy_vectorize.cpp - test_eigen_matrix.cpp - test_eigen_tensor.cpp - MESSAGE - "Skipping numpy and eigen tests under the free-threaded stable ABI (abi3t)") - endif() endif() # Skip tests for CUDA check: diff --git a/tests/test_stl_binders.cpp b/tests/test_stl_binders.cpp index bad5fe86de..f399ec0e43 100644 --- a/tests/test_stl_binders.cpp +++ b/tests/test_stl_binders.cpp @@ -7,10 +7,8 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include -#if !defined(PYBIND11_OPAQUE_PYOBJECT) -# include -#endif #include "pybind11_tests.h" @@ -240,12 +238,10 @@ TEST_SUBMODULE(stl_binders, m) { double y; bool z; }; -#if !defined(PYBIND11_OPAQUE_PYOBJECT) // needs numpy.h's generic format_descriptor m.def("create_undeclstruct", [m]() mutable { py::bind_vector>( m, "VectorUndeclStruct", py::buffer_protocol()); }); -#endif // Bind recursive container types py::bind_vector(m, "RecursiveVector"); @@ -257,7 +253,6 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector(m, "UserVectorLike"); py::bind_map(m, "UserMapLike"); -#if !defined(PYBIND11_OPAQUE_PYOBJECT) // The rest depends on numpy: try { py::module_::import("numpy"); @@ -277,5 +272,4 @@ TEST_SUBMODULE(stl_binders, m) { py::bind_vector>(m, "VectorStruct", py::buffer_protocol()); m.def("get_vectorstruct", [] { return std::vector{{false, 5, 3.0, true}, {true, 30, -1e4, false}}; }); -#endif } diff --git a/tests/test_stl_binders.py b/tests/test_stl_binders.py index 523a5b991e..edc6d28924 100644 --- a/tests/test_stl_binders.py +++ b/tests/test_stl_binders.py @@ -118,16 +118,13 @@ def test_vector_buffer(): v = m.VectorUChar(mv[::2]) assert v[1] == 3 - if hasattr(m, "create_undeclstruct"): # not built with the abi3t stable ABI - with pytest.raises(RuntimeError) as excinfo: - m.create_undeclstruct() # Undeclared struct contents, no buffer interface - assert "NumPy type info missing for " in str(excinfo.value) + with pytest.raises(RuntimeError) as excinfo: + m.create_undeclstruct() # Undeclared struct contents, no buffer interface + assert "NumPy type info missing for " in str(excinfo.value) def test_vector_buffer_numpy(): np = pytest.importorskip("numpy") - if not hasattr(m, "VectorStruct"): - pytest.skip("numpy.h is not available with the abi3t stable ABI") a = np.array([1, 2, 3, 4], dtype=np.int32) with pytest.raises(TypeError): m.VectorInt(a) From 090bb27cff474a7502a5dde0243e82fed9254685 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 21:46:42 -0400 Subject: [PATCH 18/59] feat(chrono): support chrono.h under the stable ABI The datetime C API is not part of the limited API. Under Py_LIMITED_API, read the fields as attributes and build objects by calling the datetime types; regular builds keep the C macros. The chrono test runs under the stable ABI again. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 2 - docs/advanced/stable_abi.rst | 5 +- include/pybind11/chrono.h | 221 +++++++++++++++++++++++++---------- tests/CMakeLists.txt | 4 +- 4 files changed, 161 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ce0d8c843..22803214a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,7 +179,6 @@ jobs: timeout-minutes: 60 env: PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_chrono.py --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py @@ -244,7 +243,6 @@ jobs: timeout-minutes: 60 env: PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_chrono.py --ignore=tests/test_custom_type_setup.py --ignore=tests/test_multiple_interpreters.py --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index d17f05c8c6..0eb32c65eb 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -85,8 +85,9 @@ available and in how a few things are implemented. - not available - Reads thread- and interpreter-state fields. * - ``pybind11/chrono.h`` - - not available - - The ``datetime`` C API accesses struct fields. + - supported + - The ``datetime`` C API is not in the stable ABI; fields are read as + attributes and objects are built by calling the ``datetime`` types. * - GIL-held assertions (``PYBIND11_ASSERT_GIL_HELD_INCREF_DECREF``) - not available - ``PyGILState_Check()`` is not part of the stable ABI. diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index adf911421d..7abd03636b 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -12,19 +12,126 @@ #include "pybind11.h" -#if defined(Py_LIMITED_API) -# error \ - "pybind11/chrono.h is not available under the stable ABI (Py_LIMITED_API): the datetime C API accesses struct fields." -#endif - #include #include #include -#include +#if !defined(Py_LIMITED_API) +# include +#endif PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) +#if defined(Py_LIMITED_API) +// The datetime C API is not part of the stable ABI: use the Python-level types and attributes. +struct datetime_types { + PyTypeObject *datetime, *date, *time, *timedelta; +}; + +inline const datetime_types &get_datetime_types() { + // The type objects are kept for the lifetime of the process. + static const datetime_types types = [] { + module_ m = module_::import("datetime"); + auto get = [&](const char *name) { + return reinterpret_cast(m.attr(name).cast().release().ptr()); + }; + return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; + }(); + return types; +} + +inline void datetime_import() { (void) get_datetime_types(); } +inline bool is_timedelta(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().timedelta) != 0; +} +inline bool is_datetime(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().datetime) != 0; +} +inline bool is_date(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().date) != 0; +} +inline bool is_time(handle h) { + return PyObject_TypeCheck(h.ptr(), get_datetime_types().time) != 0; +} +inline int datetime_field(handle h, const char *name) { return h.attr(name).cast(); } +# define PYBIND11_DATETIME_FIELD(fn, attr) \ + inline int fn(handle h) { return datetime_field(h, #attr); } +# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) + +inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + return PyObject_CallFunction(reinterpret_cast(get_datetime_types().timedelta), + "iii", + days, + seconds, + microseconds); +} +inline PyObject *make_datetime( + int year, int month, int day, int hour, int minute, int second, int microsecond, int fold) { + auto cls + = reinterpret_borrow(reinterpret_cast(get_datetime_types().datetime)); + return cls(year, month, day, hour, minute, second, microsecond, arg("fold") = fold) + .release() + .ptr(); +} +inline PyObject *datetime_from_timestamp(std::time_t tt) { + auto cls + = reinterpret_borrow(reinterpret_cast(get_datetime_types().datetime)); + return cls.attr("fromtimestamp")(tt).release().ptr(); +} +#else +inline void datetime_import() { + if (!PyDateTimeAPI) { + PyDateTime_IMPORT; + } +} +inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } +inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } +inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } +inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } +# define PYBIND11_DATETIME_FIELD(fn, attr) +# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) \ + inline int fn(handle h) { return c_macro(h.ptr()); } +inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + return PyDelta_FromDSU(days, seconds, microseconds); +} +inline PyObject *make_datetime( + int year, int month, int day, int hour, int minute, int second, int microsecond, int fold) { + return PyDateTime_FromDateAndTimeAndFold( + year, month, day, hour, minute, second, microsecond, fold); +} +inline PyObject *datetime_from_timestamp(std::time_t tt) { + auto args = make_tuple(tt); + return PyDateTime_FromTimestamp(args.ptr()); +} +#endif + +// One accessor per field: attribute lookup under the stable ABI, the C macro otherwise. +#define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ + PYBIND11_DATETIME_FIELD(fn, attr) \ + PYBIND11_DATETIME_FIELD_C(fn, c_macro) +PYBIND11_DATETIME_ACCESSOR(timedelta_days, days, PyDateTime_DELTA_GET_DAYS) +PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, seconds, PyDateTime_DELTA_GET_SECONDS) +PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, microseconds, PyDateTime_DELTA_GET_MICROSECONDS) +PYBIND11_DATETIME_ACCESSOR(datetime_year, year, PyDateTime_GET_YEAR) +PYBIND11_DATETIME_ACCESSOR(datetime_month, month, PyDateTime_GET_MONTH) +PYBIND11_DATETIME_ACCESSOR(datetime_day, day, PyDateTime_GET_DAY) +PYBIND11_DATETIME_ACCESSOR(datetime_hour, hour, PyDateTime_DATE_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(datetime_minute, minute, PyDateTime_DATE_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(datetime_second, second, PyDateTime_DATE_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, microsecond, PyDateTime_DATE_GET_MICROSECOND) +PYBIND11_DATETIME_ACCESSOR(datetime_fold, fold, PyDateTime_DATE_GET_FOLD) +PYBIND11_DATETIME_ACCESSOR(date_year, year, PyDateTime_GET_YEAR) +PYBIND11_DATETIME_ACCESSOR(date_month, month, PyDateTime_GET_MONTH) +PYBIND11_DATETIME_ACCESSOR(date_day, day, PyDateTime_GET_DAY) +PYBIND11_DATETIME_ACCESSOR(time_hour, hour, PyDateTime_TIME_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(time_minute, minute, PyDateTime_TIME_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(time_second, second, PyDateTime_TIME_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(time_microsecond, microsecond, PyDateTime_TIME_GET_MICROSECOND) +PYBIND11_DATETIME_ACCESSOR(time_fold, fold, PyDateTime_TIME_GET_FOLD) +#undef PYBIND11_DATETIME_ACCESSOR +#undef PYBIND11_DATETIME_FIELD +#undef PYBIND11_DATETIME_FIELD_C + template class duration_caster { public: @@ -37,20 +144,16 @@ class duration_caster { bool load(handle src, bool) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); if (!src) { return false; } // If invoked with datetime.delta object - if (PyDelta_Check(src.ptr())) { + if (is_timedelta(src)) { value = type(duration_cast>( - days(PyDateTime_DELTA_GET_DAYS(src.ptr())) - + seconds(PyDateTime_DELTA_GET_SECONDS(src.ptr())) - + microseconds(PyDateTime_DELTA_GET_MICROSECONDS(src.ptr())))); + days(timedelta_days(src)) + seconds(timedelta_seconds(src)) + + microseconds(timedelta_microseconds(src)))); return true; } // If invoked with a float we assume it is seconds and convert @@ -84,10 +187,7 @@ class duration_caster { // Works out if it is a duration or time_point and get the duration auto d = get_duration(src); - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); // Declare these special duration types so the conversions happen with the correct // primitive types (int) @@ -99,7 +199,7 @@ class duration_caster { auto subd = d - dd; auto ss = duration_cast(subd); auto us = duration_cast(subd - ss); - return PyDelta_FromDSU(dd.count(), ss.count(), us.count()); + return make_timedelta(dd.count(), ss.count(), us.count()); } PYBIND11_TYPE_CASTER(type, const_name("datetime.timedelta")); @@ -113,10 +213,7 @@ class type_caster> bool load(handle src, bool) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); if (!src) { return false; @@ -126,35 +223,35 @@ class type_caster> microseconds msecs; int fold = 0; - if (PyDateTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_DATE_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_DATE_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_DATE_GET_HOUR(src.ptr()); - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + if (is_datetime(src)) { + cal.tm_sec = datetime_second(src); + cal.tm_min = datetime_minute(src); + cal.tm_hour = datetime_hour(src); + cal.tm_mday = datetime_day(src); + cal.tm_mon = datetime_month(src) - 1; + cal.tm_year = datetime_year(src) - 1900; cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_DATE_GET_MICROSECOND(src.ptr())); - fold = PyDateTime_DATE_GET_FOLD(src.ptr()); - } else if (PyDate_Check(src.ptr())) { + msecs = microseconds(datetime_microsecond(src)); + fold = datetime_fold(src); + } else if (is_date(src)) { cal.tm_sec = 0; cal.tm_min = 0; cal.tm_hour = 0; - cal.tm_mday = PyDateTime_GET_DAY(src.ptr()); - cal.tm_mon = PyDateTime_GET_MONTH(src.ptr()) - 1; - cal.tm_year = PyDateTime_GET_YEAR(src.ptr()) - 1900; + cal.tm_mday = date_day(src); + cal.tm_mon = date_month(src) - 1; + cal.tm_year = date_year(src) - 1900; cal.tm_isdst = -1; msecs = microseconds(0); - } else if (PyTime_Check(src.ptr())) { - cal.tm_sec = PyDateTime_TIME_GET_SECOND(src.ptr()); - cal.tm_min = PyDateTime_TIME_GET_MINUTE(src.ptr()); - cal.tm_hour = PyDateTime_TIME_GET_HOUR(src.ptr()); + } else if (is_time(src)) { + cal.tm_sec = time_second(src); + cal.tm_min = time_minute(src); + cal.tm_hour = time_hour(src); cal.tm_mday = 1; // This date (day, month, year) = (1, 0, 70) cal.tm_mon = 0; // represents 1-Jan-1970, which is the first cal.tm_year = 70; // earliest available date for Python's datetime cal.tm_isdst = -1; - msecs = microseconds(PyDateTime_TIME_GET_MICROSECOND(src.ptr())); - fold = PyDateTime_TIME_GET_FOLD(src.ptr()); + msecs = microseconds(time_microsecond(src)); + fold = time_fold(src); } else { return false; } @@ -176,14 +273,14 @@ class type_caster> // Use Python's local-time disambiguation, preserving fold while ignoring // tzinfo and datetime subclass overrides. Keep microseconds separate to // avoid losing precision through timestamp()'s floating-point result. - auto dt = reinterpret_steal(PyDateTime_FromDateAndTimeAndFold(cal.tm_year + 1900, - cal.tm_mon + 1, - cal.tm_mday, - cal.tm_hour, - cal.tm_min, - cal.tm_sec, - 0, - fold)); + auto dt = reinterpret_steal(make_datetime(cal.tm_year + 1900, + cal.tm_mon + 1, + cal.tm_mday, + cal.tm_hour, + cal.tm_min, + cal.tm_sec, + 0, + fold)); if (!dt) { throw error_already_set(); } @@ -197,10 +294,7 @@ class type_caster> handle /* parent */) { using namespace std::chrono; - // Lazy initialise the PyDateTime import - if (!PyDateTimeAPI) { - PyDateTime_IMPORT; - } + datetime_import(); // Get out microseconds, and make sure they are positive, to avoid bug in eastern // hemisphere time zones (cfr. https://github.com/pybind/pybind11/issues/2417) @@ -217,19 +311,18 @@ class type_caster> = system_clock::to_time_t(time_point_cast(src - us)); // Python determines the local calendar fields and fold from whole seconds. - auto args = make_tuple(tt); - auto dt = reinterpret_steal(PyDateTime_FromTimestamp(args.ptr())); + auto dt = reinterpret_steal(datetime_from_timestamp(tt)); if (!dt) { throw error_already_set(); } - return PyDateTime_FromDateAndTimeAndFold(PyDateTime_GET_YEAR(dt.ptr()), - PyDateTime_GET_MONTH(dt.ptr()), - PyDateTime_GET_DAY(dt.ptr()), - PyDateTime_DATE_GET_HOUR(dt.ptr()), - PyDateTime_DATE_GET_MINUTE(dt.ptr()), - PyDateTime_DATE_GET_SECOND(dt.ptr()), - us.count(), - PyDateTime_DATE_GET_FOLD(dt.ptr())); + return make_datetime(datetime_year(dt), + datetime_month(dt), + datetime_day(dt), + datetime_hour(dt), + datetime_minute(dt), + datetime_second(dt), + us.count(), + datetime_fold(dt)); } PYBIND11_TYPE_CASTER(type, const_name("datetime.datetime")); }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4dd970771c..de0cfb4a74 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -229,13 +229,11 @@ endif() if(PYBIND11_TEST_STABLE_ABI) pybind11_filter_tests( PYBIND11_TEST_FILES - test_chrono.cpp test_custom_type_setup.cpp test_multiple_interpreters.py test_class_release_gil_before_calling_cpp_dtor.cpp # PyGILState_Check MESSAGE - "Skipping chrono, custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI" - ) + "Skipping custom_type_setup, subinterpreter and PyGILState_Check tests under the stable ABI") endif() # Skip tests for CUDA check: From 731a66fb53a120a353078d34d2229a4d6ff1bbeb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:09:25 -0400 Subject: [PATCH 19/59] perf(stable-abi): use the default type getattro under the limited API Attribute reads on bound classes (and Python subclasses of them) went through pybind11_meta_getattro, which walks the MRO with attribute calls and raised KeyError for every miss: ~20x slower than `type`. pybind11's own instancemethod type already returns itself from `__get__` when accessed on a class, so the metaclass does not need to intercept reads. Drop the slot under Py_LIMITED_API and speed up the remaining type_lookup() users (setattro, conduit) with interned names and PyMapping_GetOptionalItem. Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 5 ++-- include/pybind11/detail/class-inl.h | 15 ++++++----- include/pybind11/pytypes.h | 37 +++++++++++++++++++++++----- tests/test_methods_and_attributes.py | 17 +++++++++++++ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 0eb32c65eb..c794219053 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -137,8 +137,9 @@ to specific operations: * Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances is found through the type's cached dictionary offset. -* Attribute access on bound *classes* (not instances) walks ``__mro__`` and - the class dictionaries instead of using ``_PyType_Lookup()``. +* Attribute *assignment* on bound classes (``Type.static_prop = value``) walks + ``__mro__`` and the class dictionaries instead of using ``_PyType_Lookup()``; + attribute reads use the metaclass default and are not affected. * Methods are bound through a pybind11-provided ``instancemethod`` type and ``types.MethodType`` instead of ``PyInstanceMethod_Type`` / ``PyMethod_New()``. diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index b24d12f2ec..28d596c1ab 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -337,12 +337,15 @@ extern "C" PYBIND11_INLINE void pybind11_meta_dealloc(PyObject *obj) { #if PYBIND11_TYPE_CREATION_VIA_SPEC PYBIND11_INLINE PyTypeObject *make_default_metaclass() { - static PyType_Slot slots[] - = {{Py_tp_call, reinterpret_cast(pybind11_meta_call)}, - {Py_tp_setattro, reinterpret_cast(pybind11_meta_setattro)}, - {Py_tp_getattro, reinterpret_cast(pybind11_meta_getattro)}, - {Py_tp_dealloc, reinterpret_cast(pybind11_meta_dealloc)}, - {0, nullptr}}; + static PyType_Slot slots[] = { + {Py_tp_call, reinterpret_cast(pybind11_meta_call)}, + {Py_tp_setattro, reinterpret_cast(pybind11_meta_setattro)}, +# if !defined(Py_LIMITED_API) + // Not needed with pybind11's own instancemethod: its `__get__` returns itself on a class. + {Py_tp_getattro, reinterpret_cast(pybind11_meta_getattro)}, +# endif + {Py_tp_dealloc, reinterpret_cast(pybind11_meta_dealloc)}, + {0, nullptr}}; static PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_type", 0, // inherit from type 0, diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 2945d2f037..5fdc7627a4 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2660,20 +2660,45 @@ inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { } inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } -inline tuple get_mro(PyTypeObject *type) { return get_type_tuple_attr(type, "__mro__"); } + +inline PyObject *interned_name(const char *name) { + PyObject *obj = PyUnicode_InternFromString(name); + if (obj == nullptr) { + throw error_already_set(); + } + return obj; // kept alive on purpose +} + +inline tuple get_mro(PyTypeObject *type) { + static PyObject *name = interned_name("__mro__"); + auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name)); + if (!value || !PyTuple_Check(value.ptr())) { + PyErr_Clear(); + return tuple(); + } + return reinterpret_steal(value.release()); +} // No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). inline object type_lookup(PyTypeObject *type, handle name) { + static PyObject *dict_name = interned_name("__dict__"); for (handle base : get_mro(type)) { - object dict - = type_generic_getattr(reinterpret_cast(base.ptr()), "__dict__"); + auto dict = reinterpret_steal(PyObject_GenericGetAttr(base.ptr(), dict_name)); if (!dict) { + PyErr_Clear(); continue; } - auto value = reinterpret_steal(PyObject_GetItem(dict.ptr(), name.ptr())); - if (value) { - return value; +# if Py_LIMITED_API >= 0x030D0000 + PyObject *value = nullptr; + if (PyMapping_GetOptionalItem(dict.ptr(), name.ptr(), &value) == 1) { + return reinterpret_steal(value); + } +# else + PyObject *value = PyObject_GetItem(dict.ptr(), name.ptr()); + if (value != nullptr) { + return reinterpret_steal(value); } +# endif PyErr_Clear(); } return object(); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index a671e55bd5..8ad08f919a 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -90,6 +90,23 @@ def test_methods_and_attributes(): assert cstats.move_assignments == 0 +def test_unbound_method_access(): + """`Class.method` returns the raw instancemethod, also on Python subclasses""" + + class Sub(m.ExampleMandA): + pass + + unbound = m.ExampleMandA.internal1 + assert Sub.internal1 is unbound + assert m.ExampleMandA.__dict__["internal1"] is unbound + assert unbound(m.ExampleMandA(7)) == 7 + assert Sub.__name__ == "Sub" + + # Aliasing through the class keeps the instancemethod + Sub.alias = m.ExampleMandA.internal1 + assert Sub(9).alias() == 9 + + def test_copy_method(): """Issue #443: calling copied methods fails in Python 3""" From 6001633cdf33f347e4f466133ca035f91b4af5b2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:14:03 -0400 Subject: [PATCH 20/59] perf(stable-abi): bind methods with PyObject_Vectorcall Avoids the varargs collection in PyObject_CallFunctionObjArgs on every `instance.method` access under the limited API (36 -> 32 ns here; the regular build's PyMethod_New path is 22 ns). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 28d596c1ab..9b81f98f5e 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -540,10 +540,9 @@ instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject * /*type*/) { Py_INCREF(self); return self; } - return PyObject_CallFunctionObjArgs(reinterpret_cast(get_bound_method_type()), - instancemethod_data(self)->func, - obj, - nullptr); + PyObject *args[] = {instancemethod_data(self)->func, obj}; + return PyObject_Vectorcall( + reinterpret_cast(get_bound_method_type()), args, 2, nullptr); } extern "C" PYBIND11_INLINE PyObject * From 65fa5bffb3d48bf3fdb3c0f657a3c18539f61e2c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:32:36 -0400 Subject: [PATCH 21/59] refactor(stable-abi): share type attribute, slot and spec helpers Collapse the ad hoc stable-ABI code paths into shared helpers: - type_dictoffset/type_basicsize with cached interned names replace seven __dictoffset__/__basicsize__ reads; attr.h uses the cached type_info::dictoffset - type_from_spec replaces five "PyType_FromMetaclass or fail" copies - type_data replaces the two PyObject_GetTypeData wrappers - value_and_holder::type_is_exact replaces five alias checks - tp_name_is_one_of fetches the type name once per candidate list - PYBIND11_BASE_TYPE_SLOT defines whole functions (readable layout) - chrono.h: one accessor macro per branch, interned field names - getattr(obj, name, handle()) replaces hand-rolled lookups Cheaper paths: type_lookup returns a borrowed handle with _PyType_Lookup, the bool caster reads nb_bool via PyType_GetSlot under the stable ABI, and PYBIND11_CHECK_PYTHON_VERSION compares Py_Version instead of parsing Py_GetVersion(). Remove dead code: unused list_get_item, the unreachable try_incref stub under PYBIND11_OPAQUE_PYOBJECT, the inert force-define of PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET, a redundant PyGen_Check branch and a dead preprocessor condition. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- include/pybind11/attr-inl.h | 4 +- include/pybind11/cast.h | 12 ++- include/pybind11/chrono.h | 37 ++++---- include/pybind11/detail/class-inl.h | 87 +++++++---------- include/pybind11/detail/class.h | 6 ++ include/pybind11/detail/common.h | 43 +++++---- include/pybind11/detail/cpp_conduit-inl.h | 4 +- .../detail/function_record_pyobject.h | 6 +- include/pybind11/detail/init.h | 12 +-- include/pybind11/detail/internals.h | 2 +- .../pybind11/detail/type_caster_base-inl.h | 23 ++--- include/pybind11/detail/type_caster_base.h | 6 ++ include/pybind11/detail/value_and_holder.h | 4 + include/pybind11/pybind11-inl.h | 5 +- include/pybind11/pybind11.h | 6 +- include/pybind11/pytypes-inl.h | 15 +-- include/pybind11/pytypes.h | 93 ++++++++++++------- include/pybind11/stl.h | 12 +-- tools/pybind11Common.cmake | 6 +- 19 files changed, 194 insertions(+), 189 deletions(-) diff --git a/include/pybind11/attr-inl.h b/include/pybind11/attr-inl.h index b3716bc3cb..1782e48e89 100644 --- a/include/pybind11/attr-inl.h +++ b/include/pybind11/attr-inl.h @@ -49,7 +49,9 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void type_record::add_base(const std::typ bases.append(reinterpret_cast(base_info->type)); -#ifdef PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET +#if defined(Py_LIMITED_API) + dynamic_attr |= base_info->dictoffset != 0; +#elif defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) dynamic_attr |= base_info->type->tp_dictoffset != 0; #else dynamic_attr |= (PyType_GetFlags(base_info->type) & Py_TPFLAGS_MANAGED_DICT) != 0; diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 605b83c4b4..9ecaaa5123 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -445,11 +445,18 @@ class type_caster { if (src.is_none()) { res = 0; // None is implicitly converted to False } -#if !defined(PYBIND11_HAS_DIRECT_STRUCT_ACCESS) +#if defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) // Check that the "__bool__" attr exists else if (hasattr(src, PYBIND11_BOOL_ATTR)) { res = PyObject_IsTrue(src.ptr()); } +#elif defined(Py_LIMITED_API) + // Same as below through the stable ABI (PyType_GetSlot works on static types since + // 3.10). + else if (auto *nb_bool + = reinterpret_cast(PyType_GetSlot(Py_TYPE(src.ptr()), Py_nb_bool))) { + res = nb_bool(src.ptr()); + } #else // Alternate approach for CPython: this does the same as the above, but optimized // using the CPython API so as to avoid an unneeded attribute lookup. @@ -476,8 +483,7 @@ class type_caster { // Test if an object is a NumPy boolean (without fetching the type). static bool is_numpy_bool(handle object) { // Name changed to `numpy.bool` in NumPy 2, `numpy.bool_` is needed for 1.x support - return tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool") - || tp_name_equals(Py_TYPE(object.ptr()), "numpy.bool_"); + return tp_name_is_one_of(Py_TYPE(object.ptr()), {"numpy.bool", "numpy.bool_"}); } }; diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 7abd03636b..ec62ac9eb5 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -33,7 +33,7 @@ inline const datetime_types &get_datetime_types() { static const datetime_types types = [] { module_ m = module_::import("datetime"); auto get = [&](const char *name) { - return reinterpret_cast(m.attr(name).cast().release().ptr()); + return reinterpret_cast(object(m.attr(name)).release().ptr()); }; return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; }(); @@ -53,10 +53,19 @@ inline bool is_date(handle h) { inline bool is_time(handle h) { return PyObject_TypeCheck(h.ptr(), get_datetime_types().time) != 0; } -inline int datetime_field(handle h, const char *name) { return h.attr(name).cast(); } -# define PYBIND11_DATETIME_FIELD(fn, attr) \ - inline int fn(handle h) { return datetime_field(h, #attr); } -# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) +inline int datetime_field(handle h, PyObject *name) { + auto value = reinterpret_steal(PyObject_GetAttr(h.ptr(), name)); + if (!value) { + throw error_already_set(); + } + return value.cast(); +} +// One accessor per field: interned attribute lookup here, the C macro otherwise. +# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ + inline int fn(handle h) { \ + static PyObject *name = interned_name(#attr); \ + return datetime_field(h, name); \ + } inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyObject_CallFunction(reinterpret_cast(get_datetime_types().timedelta), @@ -88,8 +97,7 @@ inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } -# define PYBIND11_DATETIME_FIELD(fn, attr) -# define PYBIND11_DATETIME_FIELD_C(fn, c_macro) \ +# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ inline int fn(handle h) { return c_macro(h.ptr()); } inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyDelta_FromDSU(days, seconds, microseconds); @@ -105,10 +113,6 @@ inline PyObject *datetime_from_timestamp(std::time_t tt) { } #endif -// One accessor per field: attribute lookup under the stable ABI, the C macro otherwise. -#define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ - PYBIND11_DATETIME_FIELD(fn, attr) \ - PYBIND11_DATETIME_FIELD_C(fn, c_macro) PYBIND11_DATETIME_ACCESSOR(timedelta_days, days, PyDateTime_DELTA_GET_DAYS) PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, seconds, PyDateTime_DELTA_GET_SECONDS) PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, microseconds, PyDateTime_DELTA_GET_MICROSECONDS) @@ -120,17 +124,12 @@ PYBIND11_DATETIME_ACCESSOR(datetime_minute, minute, PyDateTime_DATE_GET_MINUTE) PYBIND11_DATETIME_ACCESSOR(datetime_second, second, PyDateTime_DATE_GET_SECOND) PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, microsecond, PyDateTime_DATE_GET_MICROSECOND) PYBIND11_DATETIME_ACCESSOR(datetime_fold, fold, PyDateTime_DATE_GET_FOLD) -PYBIND11_DATETIME_ACCESSOR(date_year, year, PyDateTime_GET_YEAR) -PYBIND11_DATETIME_ACCESSOR(date_month, month, PyDateTime_GET_MONTH) -PYBIND11_DATETIME_ACCESSOR(date_day, day, PyDateTime_GET_DAY) PYBIND11_DATETIME_ACCESSOR(time_hour, hour, PyDateTime_TIME_GET_HOUR) PYBIND11_DATETIME_ACCESSOR(time_minute, minute, PyDateTime_TIME_GET_MINUTE) PYBIND11_DATETIME_ACCESSOR(time_second, second, PyDateTime_TIME_GET_SECOND) PYBIND11_DATETIME_ACCESSOR(time_microsecond, microsecond, PyDateTime_TIME_GET_MICROSECOND) PYBIND11_DATETIME_ACCESSOR(time_fold, fold, PyDateTime_TIME_GET_FOLD) #undef PYBIND11_DATETIME_ACCESSOR -#undef PYBIND11_DATETIME_FIELD -#undef PYBIND11_DATETIME_FIELD_C template class duration_caster { @@ -237,9 +236,9 @@ class type_caster> cal.tm_sec = 0; cal.tm_min = 0; cal.tm_hour = 0; - cal.tm_mday = date_day(src); - cal.tm_mon = date_month(src) - 1; - cal.tm_year = date_year(src) - 1900; + cal.tm_mday = datetime_day(src); + cal.tm_mon = datetime_month(src) - 1; + cal.tm_year = datetime_year(src) - 1900; cal.tm_isdst = -1; msecs = microseconds(0); } else if (is_time(src)) { diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 9b81f98f5e..1896d9c0e1 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -55,26 +55,25 @@ type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, P // the function-local caches make each lookup happen once per module. Below 3.10 (and on PyPy, // where PyType_GetSlot on static types is not reliable) the struct fields are read directly. #if PY_VERSION_HEX >= 0x030A0000 && !defined(PYPY_VERSION) -# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) \ - static const auto cached = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ - return cached; +# define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ + PYBIND11_INLINE slot_type fn() { \ + static const auto cached \ + = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + return cached; \ + } #else -# define PYBIND11_BASE_TYPE_SLOT(type, slot_id, field, slot_type) return type.field; +# define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ + PYBIND11_INLINE slot_type fn() { return type.field; } #endif -PYBIND11_INLINE ternaryfunc type_type_call(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_call, tp_call, ternaryfunc)} PYBIND11_INLINE setattrofunc - type_type_setattro(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_setattro, tp_setattro, setattrofunc)} PYBIND11_INLINE getattrofunc - type_type_getattro(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_getattro, tp_getattro, getattrofunc)} PYBIND11_INLINE destructor - type_type_dealloc(){PYBIND11_BASE_TYPE_SLOT( - PyType_Type, Py_tp_dealloc, tp_dealloc, destructor)} PYBIND11_INLINE descrgetfunc - property_type_descr_get(){PYBIND11_BASE_TYPE_SLOT( - PyProperty_Type, Py_tp_descr_get, tp_descr_get, descrgetfunc)} PYBIND11_INLINE descrsetfunc - property_type_descr_set() { - PYBIND11_BASE_TYPE_SLOT(PyProperty_Type, Py_tp_descr_set, tp_descr_set, descrsetfunc) -} +PYBIND11_BASE_TYPE_SLOT(type_type_call, ternaryfunc, PyType_Type, Py_tp_call, tp_call) +PYBIND11_BASE_TYPE_SLOT(type_type_setattro, setattrofunc, PyType_Type, Py_tp_setattro, tp_setattro) +PYBIND11_BASE_TYPE_SLOT(type_type_getattro, getattrofunc, PyType_Type, Py_tp_getattro, tp_getattro) +PYBIND11_BASE_TYPE_SLOT(type_type_dealloc, destructor, PyType_Type, Py_tp_dealloc, tp_dealloc) +PYBIND11_BASE_TYPE_SLOT( + property_type_descr_get, descrgetfunc, PyProperty_Type, Py_tp_descr_get, tp_descr_get) +PYBIND11_BASE_TYPE_SLOT( + property_type_descr_set, descrsetfunc, PyProperty_Type, Py_tp_descr_set, tp_descr_set) #undef PYBIND11_BASE_TYPE_SLOT @@ -96,9 +95,7 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { // Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two // functions find it through the type's `__dictoffset__`. PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { - static const Py_ssize_t offset = handle(reinterpret_cast(Py_TYPE(self))) - .attr("__dictoffset__") - .cast(); + static const Py_ssize_t offset = type_dictoffset(Py_TYPE(self)); return reinterpret_cast(reinterpret_cast(self) + offset); } extern "C" PYBIND11_INLINE int @@ -132,7 +129,7 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { # else // Append the `__dict__` slot to the property layout. (PEP 697 relative offsets are not // applied to `__dictoffset__` on 3.12, so the offset is absolute.) - const auto dictoffset = type_generic_getattr(&PyProperty_Type, "__basicsize__").cast(); + const auto dictoffset = static_cast(type_basicsize(&PyProperty_Type)); PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}}; PyType_Slot slots[] @@ -226,7 +223,7 @@ extern "C" PYBIND11_INLINE int pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { // Use `type_lookup()` instead of `PyObject_GetAttr()` in order to get the raw // descriptor (`property`) instead of calling `tp_descr_get` (`property.__get__()`). - object descr = type_lookup((PyTypeObject *) obj, name); + auto descr = type_lookup((PyTypeObject *) obj, name); // The following assignment combinations are possible: // 1. `Type.static_prop = value` --> descr_set: `Type.static_prop.__set__(value)` @@ -255,9 +252,9 @@ pybind11_meta_setattro(PyObject *obj, PyObject *name, PyObject *value) { } extern "C" PYBIND11_INLINE PyObject *pybind11_meta_getattro(PyObject *obj, PyObject *name) { - object descr = type_lookup((PyTypeObject *) obj, name); + auto descr = type_lookup((PyTypeObject *) obj, name); if (descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr())) { - return descr.release().ptr(); + return handle(descr).inc_ref().ptr(); } return type_type_getattro()(obj, name); } @@ -500,11 +497,7 @@ struct instancemethod_object { }; PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { -# if defined(PYBIND11_OPAQUE_PYOBJECT) - return static_cast(PyObject_GetTypeData(self, Py_TYPE(self))); -# else - return reinterpret_cast(self); -# endif + return type_data(self); } PYBIND11_INLINE PyTypeObject *get_bound_method_type() { @@ -514,8 +507,7 @@ PYBIND11_INLINE PyTypeObject *get_bound_method_type() { throw error_already_set(); } // Static builtin type: keep the reference forever. - return reinterpret_cast( - types.attr("MethodType").cast().release().ptr()); + return reinterpret_cast(object(types.attr("MethodType")).release().ptr()); }(); return type; } @@ -525,13 +517,7 @@ PYBIND11_INLINE bool is_bound_method(PyObject *obj) { } PYBIND11_INLINE PyObject *bound_method_function(PyObject *obj) { - PyObject *func = PyObject_GetAttrString(obj, "__func__"); - if (func == nullptr) { - PyErr_Clear(); - return nullptr; - } - Py_DECREF(func); // the method object keeps the function alive - return func; + return getattr(obj, "__func__", handle()).ptr(); // the method object keeps the function alive } extern "C" PYBIND11_INLINE PyObject * @@ -566,9 +552,8 @@ extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyO extern "C" PYBIND11_INLINE PyObject *instancemethod_repr(PyObject *self) { PyObject *func = instancemethod_data(self)->func; - auto name = reinterpret_steal(PyObject_GetAttrString(func, "__name__")); + object name = getattr(func, "__name__", handle()); if (!name) { - PyErr_Clear(); return PyUnicode_FromFormat("", self); } return PyUnicode_FromFormat("", name.ptr(), self); @@ -619,12 +604,8 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, slots}; - PyObject *type = PyType_FromSpec(&spec); - if (type == nullptr) { - pybind11_fail("get_instancemethod_type(): failure in PyType_FromSpec(): " - + error_string()); - } - internals.instancemethod_type = reinterpret_cast(type); + internals.instancemethod_type + = type_from_spec("get_instancemethod_type()", nullptr, &spec, nullptr); } return internals.instancemethod_type; }); @@ -697,8 +678,8 @@ PYBIND11_INLINE Py_ssize_t instance_data_offset() { // The same for every pybind11 type: the type data of pybind11_object, whose base is object. static const Py_ssize_t offset = [] { handle base = get_internals().instance_base; - auto basicsize = base.attr("__basicsize__").cast(); - return basicsize - PyType_GetTypeDataSize(reinterpret_cast(base.ptr())); + auto *base_type = reinterpret_cast(base.ptr()); + return type_basicsize(base_type) - PyType_GetTypeDataSize(base_type); }(); return offset; } @@ -1125,8 +1106,8 @@ PYBIND11_INLINE void set_published_type_names(PyTypeObject *type, const type_nam object current_name = reinterpret_borrow(heap_type->ht_name); object current_qualname = reinterpret_borrow(heap_type->ht_qualname); # else - object current_name = type_generic_getattr(type, "__name__"); - object current_qualname = type_generic_getattr(type, "__qualname__"); + object current_name = type_generic_getattr(type, str("__name__")); + object current_qualname = type_generic_getattr(type, str("__qualname__")); # endif if (!names.name.equal(current_name)) { set_name("__name__", names.name); @@ -1215,10 +1196,8 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { int base_basicsize = 0; for (handle b : bases) { auto *b_type = reinterpret_cast(b.ptr()); - base_has_dict - |= type_generic_getattr(b_type, "__dictoffset__").cast() != 0; - base_basicsize = std::max(base_basicsize, - type_generic_getattr(b_type, "__basicsize__").cast()); + base_has_dict |= type_dictoffset(b_type) != 0; + base_basicsize = std::max(base_basicsize, static_cast(type_basicsize(b_type))); } if (!base_has_dict) { dict_member[0].offset = base_basicsize; diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 28fae30189..3058016e76 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -51,6 +51,12 @@ destructor type_type_dealloc(); descrgetfunc property_type_descr_get(); descrsetfunc property_type_descr_set(); +#if PYBIND11_TYPE_CREATION_VIA_SPEC +/// PyType_FromMetaclass(), or pybind11_fail() with the Python error prefixed by `caller`. +PyTypeObject * +type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases); +#endif + #if !defined(PYPY_VERSION) # if defined(Py_LIMITED_API) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index f4c98558dc..cee2df1498 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -34,9 +34,9 @@ # if !defined(PYBIND11_SIMPLE_GIL_MANAGEMENT) # define PYBIND11_SIMPLE_GIL_MANAGEMENT // gil.h reads thread-state internals # endif +// The only type-creation path without struct access. # if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) -# define PYBIND11_TYPE_CREATION_VIA_SPEC \ - 1 // the only type-creation path without struct access +# define PYBIND11_TYPE_CREATION_VIA_SPEC 1 # elif !PYBIND11_TYPE_CREATION_VIA_SPEC # error \ "The legacy type-creation path (PYBIND11_TYPE_CREATION_VIA_SPEC=0) needs struct access." @@ -46,9 +46,6 @@ # elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT # error "Subinterpreter support is not available under Py_LIMITED_API." # endif -# if !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) -# define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET // no Py_TPFLAGS_MANAGED_DICT -# endif #endif // Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative @@ -315,7 +312,6 @@ #endif #include -#include #include #include #include @@ -504,17 +500,16 @@ // A stable-ABI module runs on every CPython from the Py_LIMITED_API version on (abi3). # define PYBIND11_CHECK_PYTHON_VERSION \ { \ - const char *runtime_ver = Py_GetVersion(); \ - long major = std::strtol(runtime_ver, nullptr, 10); \ - const char *dot = std::strchr(runtime_ver, '.'); \ - long minor = dot ? std::strtol(dot + 1, nullptr, 10) : 0; \ - if (major != ((Py_LIMITED_API) >> 24) || minor < (((Py_LIMITED_API) >> 16) & 0xFF)) { \ + const unsigned long want_major = (Py_LIMITED_API) >> 24; \ + const unsigned long want_minor = ((Py_LIMITED_API) >> 16) & 0xFF; \ + if ((Py_Version >> 24) != want_major || ((Py_Version >> 16) & 0xFF) < want_minor) { \ PyErr_Format(PyExc_ImportError, \ - "Python version mismatch: module was compiled for the Python %d.%d " \ - "stable ABI, but the interpreter version is incompatible: %s.", \ - (int) ((Py_LIMITED_API) >> 24), \ - (int) (((Py_LIMITED_API) >> 16) & 0xFF), \ - runtime_ver); \ + "Python version mismatch: module was compiled for the Python " \ + "%lu.%lu stable ABI, but the interpreter version is incompatible: " \ + "%s.", \ + want_major, \ + want_minor, \ + Py_GetVersion()); \ return nullptr; \ } \ } @@ -862,6 +857,17 @@ inline instance *get_instance(PyObject *obj) { return reinterpret_cast(inst); } #endif +/// The C++ data of an object of a pybind11 type with PYBIND11_TYPE_DATA_SIZE(T) as basicsize +/// (not for `instance`: its offset is cached, see get_instance()). +template +T *type_data(PyObject *obj) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); +#else + return reinterpret_cast(obj); +#endif +} + // Some older compilers (e.g. gcc 9.4.0) require // static_assert(always_false::value, "..."); // instead of @@ -1569,9 +1575,8 @@ inline void silence_unused_warnings(Args &&...) {} #endif // CPython 3.11+ provides Py_TPFLAGS_MANAGED_DICT, but PyPy3.11 does not, see PR #5508. (The -// stable ABI does not have it either; see the Py_LIMITED_API block at the top of this file.) -#if (PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION)) \ - && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) +// stable ABI has neither; it records the `__dict__` offset in type_info::dictoffset instead.) +#if PY_VERSION_HEX < 0x030B0000 || defined(PYPY_VERSION) # define PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET #endif diff --git a/include/pybind11/detail/cpp_conduit-inl.h b/include/pybind11/detail/cpp_conduit-inl.h index 733b180cfa..9050917baf 100644 --- a/include/pybind11/detail/cpp_conduit-inl.h +++ b/include/pybind11/detail/cpp_conduit-inl.h @@ -31,8 +31,8 @@ PYBIND11_INLINE bool type_is_managed_by_our_internals(PyTypeObject *type_obj) { } PYBIND11_INLINE bool is_instance_method_of_type(PyTypeObject *type_obj, PyObject *attr_name) { - object descr = type_lookup(type_obj, attr_name); - return descr && PyInstanceMethod_Check(descr.ptr()); + auto descr = type_lookup(type_obj, attr_name); + return descr && PYBIND11_INSTANCE_METHOD_CHECK(descr.ptr()); } PYBIND11_INLINE object try_get_cpp_conduit_method(PyObject *obj) { diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 3f229dca9f..7dfe351763 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -27,11 +27,7 @@ struct function_record_PyObject { }; inline function_record_PyObject *function_record_data(PyObject *obj) { -#if defined(PYBIND11_OPAQUE_PYOBJECT) - return static_cast(PyObject_GetTypeData(obj, Py_TYPE(obj))); -#else - return reinterpret_cast(obj); -#endif + return type_data(obj); } PYBIND11_NAMESPACE_BEGIN(function_record_PyTypeObject_methods) diff --git a/include/pybind11/detail/init.h b/include/pybind11/detail/init.h index 9c7c6ee258..4c341e7d8b 100644 --- a/include/pybind11/detail/init.h +++ b/include/pybind11/detail/init.h @@ -312,7 +312,7 @@ struct constructor { cl.def( "__init__", [](value_and_holder &v_h, Args... args) { - if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { + if (v_h.type_is_exact()) { v_h.value_ptr() = construct_or_initialize>(std::forward(args)...); } else { @@ -399,9 +399,7 @@ struct factory { [func] #endif (value_and_holder &v_h, Args... args) { - construct(v_h, - func(std::forward(args)...), - Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); + construct(v_h, func(std::forward(args)...), !v_h.type_is_exact()); }, is_new_style_constructor(), extra...); @@ -448,7 +446,7 @@ struct factory { [class_func, alias_func] #endif (value_and_holder &v_h, CArgs... args) { - if (Py_TYPE(instance_object(v_h.inst)) == v_h.type->type) { + if (v_h.type_is_exact()) { // If the instance type equals the registered type we don't have inheritance, // so don't need the alias and can construct using the class function: construct(v_h, class_func(std::forward(args)...), false); @@ -536,9 +534,7 @@ struct pickle_factory { [func] #endif (value_and_holder &v_h, ArgState state) { - setstate(v_h, - func(std::forward(state)), - Py_TYPE(instance_object(v_h.inst)) != v_h.type->type); + setstate(v_h, func(std::forward(state)), !v_h.type_is_exact()); }, is_new_style_constructor(), extra...); diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index b4403cf7ba..19aa6f5c51 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -232,7 +232,7 @@ inline PyThreadState *get_thread_state_unchecked() { #elif defined(Py_LIMITED_API) // PyThreadState_GetUnchecked() is not part of the stable ABI. return PyGILState_GetThisThreadState(); -#elif PY_VERSION_HEX < 0x030D0000 && !defined(Py_LIMITED_API) +#elif PY_VERSION_HEX < 0x030D0000 return _PyThreadState_UncheckedGet(); #else return PyThreadState_GetUnchecked(); diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 84c7e27026..ad373dcd60 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -240,16 +240,12 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_type_handle(const std::type_in return handle(type_info ? (reinterpret_cast(type_info->type)) : nullptr); } +#if !defined(PYBIND11_OPAQUE_PYOBJECT) // the registry goes through PyWeakref_GetRef() instead PYBIND11_INLINE bool try_incref(PyObject *obj) { // Tries to increment the reference count of an object if it's not zero. -#if defined(PYBIND11_OPAQUE_PYOBJECT) - // Not used: the registry goes through PyWeakref_GetRef() instead. - assert(Py_REFCNT(obj) > 0); - Py_INCREF(obj); - return true; -#elif defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 +# if defined(Py_GIL_DISABLED) && PY_VERSION_HEX >= 0x030E00A4 return PyUnstable_TryIncRef(obj); -#elif defined(Py_GIL_DISABLED) +# elif defined(Py_GIL_DISABLED) // See // https://github.com/python/cpython/blob/d05140f9f77d7dfc753dd1e5ac3a5962aaa03eff/Include/internal/pycore_object.h#L761 uint32_t local = _Py_atomic_load_uint32_relaxed(&obj->ob_ref_local); @@ -260,9 +256,9 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { } if (_Py_IsOwnedByCurrentThread(obj)) { _Py_atomic_store_uint32_relaxed(&obj->ob_ref_local, local); -# ifdef Py_REF_DEBUG +# ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); -# endif +# endif return true; } Py_ssize_t shared = _Py_atomic_load_ssize_relaxed(&obj->ob_ref_shared); @@ -275,18 +271,19 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { if (_Py_atomic_compare_exchange_ssize( &obj->ob_ref_shared, &shared, shared + (1 << _Py_REF_SHARED_SHIFT))) { -# ifdef Py_REF_DEBUG +# ifdef Py_REF_DEBUG _Py_INCREF_IncRefTotal(); -# endif +# endif return true; } } -#else +# else assert(Py_REFCNT(obj) > 0); Py_INCREF(obj); return true; -#endif +# endif } +#endif PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle find_registered_python_instance(void *src, const detail::type_info *tinfo) { diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 5799b4f95c..4c7691ff8d 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -128,11 +128,17 @@ detail::type_info *get_type_info(const std::type_info &tp, bool throw_if_missing handle get_type_handle(const std::type_info &tp, bool throw_if_missing); +#if !defined(PYBIND11_OPAQUE_PYOBJECT) bool try_incref(PyObject *obj); +#endif // Searches the inheritance graph for a registered Python instance, using all_type_info(). handle find_registered_python_instance(void *src, const detail::type_info *tinfo); +inline bool value_and_holder::type_is_exact() const { + return Py_TYPE(instance_object(inst)) == type->type; +} + // Container for accessing and iterating over an instance's values/holders struct values_and_holders { private: diff --git a/include/pybind11/detail/value_and_holder.h b/include/pybind11/detail/value_and_holder.h index 98ab73c378..df484edb32 100644 --- a/include/pybind11/detail/value_and_holder.h +++ b/include/pybind11/detail/value_and_holder.h @@ -25,6 +25,10 @@ struct value_and_holder { vh{inst->simple_layout ? inst->simple_value_holder : &inst->nonsimple.values_and_holders[vpos]} {} + /// True if the Python object is exactly the bound type, i.e. not a Python subclass that + /// may need the alias (trampoline) type. + bool type_is_exact() const; + // Default constructor (used to signal a value-and-holder not found by get_value_and_holder()) value_and_holder() = default; diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 5a5b92f080..b993a11832 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -508,8 +508,7 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, if (std::string(str(f_code.attr("co_name"))) == name && f_code.attr("co_argcount").cast() > 0) { object self_arg = f_code.attr("co_varnames")[int_(0)]; - auto locals - = reinterpret_steal(PyObject_GetAttrString(frame_obj.ptr(), "f_locals")); + object locals = getattr(frame_obj, "f_locals", handle()); auto self_caller = locals ? reinterpret_steal(PyObject_GetItem(locals.ptr(), self_arg.ptr())) @@ -1330,7 +1329,7 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { tinfo->module_local = rec.module_local; tinfo->holder_enum_v = rec.holder_enum_v; #if defined(Py_LIMITED_API) - tinfo->dictoffset = handle(m_ptr).attr("__dictoffset__").cast(); + tinfo->dictoffset = type_dictoffset(reinterpret_cast(m_ptr)); #endif with_internals([&](internals &internals) { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 0248631ac9..943022f25f 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -2106,10 +2106,8 @@ class enum_ : public class_ { def("__index__", [](Type value) { return (Scalar) value; }, pos_only()); attr("__setstate__") = cpp_function( [](detail::value_and_holder &v_h, Scalar arg) { - detail::initimpl::setstate(v_h, - static_cast(arg), - Py_TYPE(instance_object(v_h.inst)) - != v_h.type->type); + detail::initimpl::setstate( + v_h, static_cast(arg), !v_h.type_is_exact()); }, detail::is_new_style_constructor(), pybind11::name("__setstate__"), diff --git a/include/pybind11/pytypes-inl.h b/include/pybind11/pytypes-inl.h index 90d573e6ca..503f8834df 100644 --- a/include/pybind11/pytypes-inl.h +++ b/include/pybind11/pytypes-inl.h @@ -139,17 +139,10 @@ PYBIND11_INLINE std::string error_fetch_and_normalize::format_value_and_trace() if (m_trace) { #if !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) // Attribute access instead of struct access: the traceback, frame and code object - // layouts are not part of the stable ABI. Returns a null object on failure. - auto get_attr = [](handle obj, const char *name) { - auto value = reinterpret_steal(PyObject_GetAttrString(obj.ptr(), name)); - if (!value) { - // Known flaw: all secondary exceptions raised by this lookup are silently - // discarded, including KeyboardInterrupt and SystemError. - // Reading tb_frame can invoke audit hooks that raise such exceptions. - PyErr_Clear(); - } - return value; - }; + // layouts are not part of the stable ABI. A missing attribute gives a null object. + // Known flaw: all secondary exceptions raised by the lookup are silently discarded, + // including KeyboardInterrupt and SystemError (reading tb_frame can invoke audit hooks). + auto get_attr = [](handle obj, const char *name) { return getattr(obj, name, handle()); }; // Get the deepest trace possible. object tb = m_trace; diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 5fdc7627a4..33493f6921 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -516,6 +517,14 @@ inline std::string get_tp_name(PyTypeObject *type) { return type->tp_name; } inline bool tp_name_equals(PyTypeObject *type, const char *name) { return std::strcmp(type->tp_name, name) == 0; } +inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list names) { + for (const char *name : names) { + if (tp_name_equals(type, name)) { + return true; + } + } + return false; +} #else bool type_is_managed_by_our_internals(PyTypeObject *type_obj); // detail/cpp_conduit.h @@ -546,9 +555,9 @@ inline std::string get_tp_name(PyTypeObject *type) { return module_ + '.' + name; } -inline bool tp_name_equals(PyTypeObject *type, const char *name) { - // Cheap pre-check on the unqualified name before reconstructing the full one. - const char *dot = std::strrchr(name, '.'); +/// True if `get_tp_name(type)` is one of `names`. The unqualified `__name__` is fetched once and +/// compared first; the full name is only reconstructed on a match. +inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list names) { auto type_name = reinterpret_steal(PyType_GetName(type)); const char *type_name_utf8 = type_name ? PyUnicode_AsUTF8AndSize(type_name.ptr(), nullptr) : nullptr; @@ -556,10 +565,16 @@ inline bool tp_name_equals(PyTypeObject *type, const char *name) { PyErr_Clear(); return false; } - if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) != 0) { - return false; + for (const char *name : names) { + const char *dot = std::strrchr(name, '.'); + if (std::strcmp(type_name_utf8, dot ? dot + 1 : name) == 0 && get_tp_name(type) == name) { + return true; + } } - return get_tp_name(type) == name; + return false; +} +inline bool tp_name_equals(PyTypeObject *type, const char *name) { + return tp_name_is_one_of(type, {name}); } #endif @@ -572,7 +587,6 @@ inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { PyTuple_SET_ITEM(tup, i, item); } inline ssize_t list_size(PyObject *lst) { return PyList_GET_SIZE(lst); } -inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GET_ITEM(lst, i); } /// Steals a reference to `item`; `lst` must be a fresh list. inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { PyList_SET_ITEM(lst, i, item); @@ -584,7 +598,6 @@ inline void tuple_set_item(PyObject *tup, ssize_t i, PyObject *item) { PyTuple_SetItem(tup, i, item); } inline ssize_t list_size(PyObject *lst) { return PyList_Size(lst); } -inline PyObject *list_get_item(PyObject *lst, ssize_t i) { return PyList_GetItem(lst, i); } inline void list_set_item(PyObject *lst, ssize_t i, PyObject *item) { PyList_SetItem(lst, i, item); } @@ -2619,7 +2632,6 @@ PYBIND11_MATH_OPERATOR_BINARY_INPLACE(operator>>=, PyNumber_InPlaceRshift) #undef PYBIND11_MATH_OPERATOR_BINARY #undef PYBIND11_MATH_OPERATOR_BINARY_INPLACE -// Meant to return a Python str, but this is not checked. #if !defined(Py_LIMITED_API) /// `type.__bases__` as a tuple (empty if the type has no bases tuple yet). inline tuple get_bases(PyTypeObject *type) { @@ -2633,25 +2645,51 @@ inline tuple get_bases(PyTypeObject *type) { inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } /// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. -/// Returns a null object if the attribute is not found; never raises. -inline object type_lookup(PyTypeObject *type, handle name) { - return reinterpret_borrow(_PyType_Lookup(type, name.ptr())); +/// Returns a null handle if the attribute is not found; never raises. (Borrowed here; the +/// stable-ABI version below returns an owning `object`.) +inline handle type_lookup(PyTypeObject *type, handle name) { + return _PyType_Lookup(type, name.ptr()); } #else +inline PyObject *interned_name(const char *name) { + PyObject *obj = PyUnicode_InternFromString(name); + if (obj == nullptr) { + throw error_already_set(); + } + return obj; // kept alive on purpose +} + // Generic attribute lookup on a type object: resolves the `type.__xxx__` descriptors without // going through the metaclass's tp_getattro (pybind11's own metaclass calls back into here). -inline object type_generic_getattr(PyTypeObject *type, const char *name) { - auto name_obj = reinterpret_steal(PyUnicode_InternFromString(name)); - auto value = name_obj ? reinterpret_steal( - PyObject_GenericGetAttr((PyObject *) type, name_obj.ptr())) - : object(); +// Returns a null object if the attribute is not found; never raises. +inline object type_generic_getattr(PyTypeObject *type, handle name) { + auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name.ptr())); if (!value) { PyErr_Clear(); } return value; } -inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { +inline Py_ssize_t type_ssize_attr(PyTypeObject *type, handle name) { + object value = type_generic_getattr(type, name); + Py_ssize_t result = value ? PyLong_AsSsize_t(value.ptr()) : -1; + if (result == -1 && PyErr_Occurred()) { + throw error_already_set(); + } + return result; +} + +/// `type.__dictoffset__` / `type.__basicsize__` (tp_dictoffset and tp_basicsize). +inline Py_ssize_t type_dictoffset(PyTypeObject *type) { + static PyObject *name = interned_name("__dictoffset__"); + return type_ssize_attr(type, name); +} +inline Py_ssize_t type_basicsize(PyTypeObject *type) { + static PyObject *name = interned_name("__basicsize__"); + return type_ssize_attr(type, name); +} + +inline tuple type_tuple_attr(PyTypeObject *type, handle name) { object value = type_generic_getattr(type, name); if (!value || !PyTuple_Check(value.ptr())) { return tuple(); @@ -2659,24 +2697,14 @@ inline tuple get_type_tuple_attr(PyTypeObject *type, const char *name) { return reinterpret_steal(value.release()); } -inline tuple get_bases(PyTypeObject *type) { return get_type_tuple_attr(type, "__bases__"); } - -inline PyObject *interned_name(const char *name) { - PyObject *obj = PyUnicode_InternFromString(name); - if (obj == nullptr) { - throw error_already_set(); - } - return obj; // kept alive on purpose +inline tuple get_bases(PyTypeObject *type) { + static PyObject *name = interned_name("__bases__"); + return type_tuple_attr(type, name); } inline tuple get_mro(PyTypeObject *type) { static PyObject *name = interned_name("__mro__"); - auto value = reinterpret_steal(PyObject_GenericGetAttr((PyObject *) type, name)); - if (!value || !PyTuple_Check(value.ptr())) { - PyErr_Clear(); - return tuple(); - } - return reinterpret_steal(value.release()); + return type_tuple_attr(type, name); } // No type attribute cache: walks the MRO and looks in each class's `__dict__` (a mappingproxy). @@ -2705,6 +2733,7 @@ inline object type_lookup(PyTypeObject *type, handle name) { } #endif +// Meant to return a Python str, but this is not checked. inline object get_module_name_if_available(handle scope) { if (scope) { if (hasattr(scope, "__module__")) { diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index d1a5eb652d..a31bc36ec8 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -79,12 +79,7 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, if (PyType_Check(obj)) { return false; } - for (const auto *tp_name : tp_names) { - if (tp_name_equals(Py_TYPE(obj), tp_name)) { - return true; - } - } - return false; + return tp_name_is_one_of(Py_TYPE(obj), tp_names); } inline bool object_is_convertible_to_std_vector(const handle &src) { @@ -93,11 +88,6 @@ inline bool object_is_convertible_to_std_vector(const handle &src) { return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); } // Allow generators, set/frozenset and several common iterable types. -#if !defined(Py_LIMITED_API) - if (PyGen_Check(src.ptr()) != 0) { - return true; - } -#endif return (PyAnySet_Check(src.ptr()) != 0) || object_is_instance_with_one_of_tp_names( src.ptr(), {"generator", "dict_keys", "dict_values", "dict_items", "map", "zip"}); diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index 8d204afd39..eeb15b83ef 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -552,11 +552,11 @@ endfunction() # PYBIND11_PRECOMPILE variable requests it, unless NO_PRECOMPILE opts out. function(_pybind11_maybe_precompile target_name precompile no_precompile stable_abi) if((precompile OR PYBIND11_PRECOMPILE) AND NOT no_precompile) + set(_kw "") if(stable_abi) - pybind11_precompile(STABLE_ABI) - else() - pybind11_precompile() + set(_kw STABLE_ABI) endif() + pybind11_precompile(${_kw}) target_link_libraries(${target_name} PRIVATE pybind11::precompiled) endif() endfunction() From f4d6e40350be06954e683748ac53d8cd85e51865 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:38:34 -0400 Subject: [PATCH 22/59] fix(stable-abi): gate 3.13+ API use on Py_LIMITED_API, not the headers A stable-ABI build targeting 3.12 with 3.13+ headers selected PyDict_GetItemStringRef, PyDict_SetDefaultRef, PyEval_GetFrameBuiltins and PyEval_GetFrameGlobals, which the headers hide below their limited API version. Add PYBIND11_API_VERSION_HEX (Py_LIMITED_API when defined, else PY_VERSION_HEX) and use it in those guards. annotations() always takes the 3.14 path under the stable ABI, as the module can run there. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/common.h | 4 ++++ include/pybind11/pybind11-inl.h | 4 ++-- include/pybind11/pytypes.h | 9 +++++---- tools/check_inl_headers.py | 1 + 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index cee2df1498..73e8568e5b 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -46,6 +46,10 @@ # elif PYBIND11_HAS_SUBINTERPRETER_SUPPORT # error "Subinterpreter support is not available under Py_LIMITED_API." # endif +// The C API version the build may use: the limited API target, not the headers' version. +# define PYBIND11_API_VERSION_HEX Py_LIMITED_API +#else +# define PYBIND11_API_VERSION_HEX PY_VERSION_HEX #endif // Heap types declare their extra data as PEP 697 type data when PyObject is opaque: a negative diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index b993a11832..49f7fc7dc5 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -249,7 +249,7 @@ PYBIND11_INLINE PyObject *cached_create_module(PyObject *spec, PyModuleDef *) { PYBIND11_NAMESPACE_END(detail) PYBIND11_INLINE dict globals() { -#if PY_VERSION_HEX >= 0x030d0000 +#if PYBIND11_API_VERSION_HEX >= 0x030d0000 PyObject *p = PyEval_GetFrameGlobals(); return p ? reinterpret_steal(p) : reinterpret_borrow(module_::import("__main__").attr("__dict__").ptr()); @@ -398,7 +398,7 @@ PYBIND11_INLINE void register_local_exception_translator(ExceptionTranslator &&t PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void print(const tuple &args, const dict &kwargs) { -#if PY_VERSION_HEX >= 0x030D0000 +#if PYBIND11_API_VERSION_HEX >= 0x030D0000 auto builtins = reinterpret_steal(PyEval_GetFrameBuiltins()); #else auto builtins = reinterpret_borrow(PyEval_GetBuiltins()); diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 33493f6921..0163d3369d 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -900,7 +900,7 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { // PyDict_GetItemStringRef was added in Python 3.13.0a1. // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { -#if PY_VERSION_HEX >= 0x030D00A1 +#if PYBIND11_API_VERSION_HEX >= 0x030D00A1 PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); @@ -940,7 +940,7 @@ inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *d // PyDict_SetDefaultRef was added in Python 3.13.0a4. // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { -#if PY_VERSION_HEX >= 0x030D00A4 +#if PYBIND11_API_VERSION_HEX >= 0x030D00A4 PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); @@ -2554,8 +2554,9 @@ str_attr_accessor object_api::doc() const { template object object_api::annotations() const { -// This is needed again because of the lazy annotations added in 3.14+ -#if PY_VERSION_HEX < 0x030A0000 || PY_VERSION_HEX >= 0x030E0000 +// This is needed again because of the lazy annotations added in 3.14+ (a stable-ABI module +// built for an older target can still run on 3.14+) +#if PY_VERSION_HEX < 0x030A0000 || PY_VERSION_HEX >= 0x030E0000 || defined(Py_LIMITED_API) // https://docs.python.org/3/howto/annotations.html#accessing-the-annotations-dict-of-an-object-in-python-3-9-and-older if (!hasattr(derived(), "__annotations__")) { setattr(derived(), "__annotations__", dict()); diff --git a/tools/check_inl_headers.py b/tools/check_inl_headers.py index 2f22de63b7..e9399e0797 100755 --- a/tools/check_inl_headers.py +++ b/tools/check_inl_headers.py @@ -29,6 +29,7 @@ "PY_MAJOR_VERSION", "PY_MINOR_VERSION", "PY_VERSION_HEX", + "PYBIND11_API_VERSION_HEX", # Py_LIMITED_API if defined, else PY_VERSION_HEX "Py_REF_DEBUG", "_MSC_VER", "_WIN32", From c4deb0b9333860e17e663b5dc0250217845c2237 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 22:48:08 -0400 Subject: [PATCH 23/59] perf(stable-abi): make instancemethod a vectorcall method descriptor With Py_TPFLAGS_METHOD_DESCRIPTOR and Py_TPFLAGS_HAVE_VECTORCALL (both in the 3.12 stable ABI), `obj.method(...)` calls the instancemethod with `obj` prepended instead of creating a types.MethodType first. `obj.method` without a call is unchanged. Release, CPython 3.12, `d.name()`: 77 ns regular, 45 ns abi3 (was 63). Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 19 ++++++++++++++++++- include/pybind11/detail/class.h | 2 ++ tests/test_methods_and_attributes.py | 9 +++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 1896d9c0e1..0c5e8393b2 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -494,6 +494,7 @@ struct instancemethod_object { PyObject_HEAD # endif PyObject *func; + vectorcallfunc vectorcall; }; PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { @@ -536,6 +537,15 @@ instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs) { return PyObject_Call(instancemethod_data(self)->func, args, kwargs); } +// Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` calls this with `obj` prepended instead of +// creating a bound method. +extern "C" PYBIND11_INLINE PyObject *instancemethod_vectorcall(PyObject *self, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames) { + return PyObject_Vectorcall(instancemethod_data(self)->func, args, nargsf, kwnames); +} + extern "C" PYBIND11_INLINE PyObject *instancemethod_getattro(PyObject *self, PyObject *name) { // Descriptors of the instancemethod type itself (`__func__`, `__class__`, ...) win; // everything else (`__name__`, `__doc__`, `__module__`, ...) comes from the function. @@ -587,6 +597,11 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { offsetof(instancemethod_object, func), Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, nullptr}, + {"__vectorcalloffset__", + Py_T_PYSSIZET, + offsetof(instancemethod_object, vectorcall), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, {nullptr, 0, 0, 0, nullptr}}; static PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, @@ -602,7 +617,8 @@ PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", PYBIND11_TYPE_DATA_SIZE(instancemethod_object), 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_METHOD_DESCRIPTOR | Py_TPFLAGS_HAVE_VECTORCALL, slots}; internals.instancemethod_type = type_from_spec("get_instancemethod_type()", nullptr, &spec, nullptr); @@ -627,6 +643,7 @@ PYBIND11_INLINE PyObject *instancemethod_new(PyObject *func) { } Py_INCREF(func); instancemethod_data(self)->func = func; + instancemethod_data(self)->vectorcall = instancemethod_vectorcall; return self; } diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 3058016e76..c23c9fe8e9 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -143,6 +143,8 @@ struct instancemethod_object; instancemethod_object *instancemethod_data(PyObject *self); extern "C" PyObject *instancemethod_descr_get(PyObject *self, PyObject *obj, PyObject *type); extern "C" PyObject *instancemethod_call(PyObject *self, PyObject *args, PyObject *kwargs); +extern "C" PyObject * +instancemethod_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames); extern "C" PyObject *instancemethod_getattro(PyObject *self, PyObject *name); extern "C" PyObject *instancemethod_repr(PyObject *self); extern "C" int instancemethod_traverse(PyObject *self, visitproc visit, void *arg); diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 8ad08f919a..0f64416bc1 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -106,6 +106,15 @@ class Sub(m.ExampleMandA): Sub.alias = m.ExampleMandA.internal1 assert Sub(9).alias() == 9 + # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method + assert type(unbound).__flags__ & (1 << 17) + bound = m.ExampleMandA(5).overloaded + assert bound.__self__.internal1() == 5 + assert bound.__func__ is m.ExampleMandA.overloaded.__func__ + assert bound(1, 1.0) == "(int, float)" + assert m.ExampleMandA(5).overloaded(1, 1.0) == "(int, float)" + assert m.ExampleMandA.overloaded(m.ExampleMandA(5), 1, 1.0) == "(int, float)" + def test_copy_method(): """Issue #443: calling copied methods fails in Python 3""" From 5510456fae291cfb05791407e086d69264cf938b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 24/59] fix(chrono): import the datetime C API per translation unit `PyDateTimeAPI` is a static per translation unit. The non-limited helpers had external linkage, so the linker could pair one unit's `datetime_import()` with another unit's `is_datetime()`, which then read a null pointer. Give the helpers internal linkage and add a second chrono translation unit to the test module. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/chrono.h | 20 +++++++++++--------- tests/CMakeLists.txt | 1 + tests/test_chrono.py | 8 ++++++++ tests/test_chrono_second_tu.cpp | 23 +++++++++++++++++++++++ 4 files changed, 43 insertions(+), 9 deletions(-) create mode 100644 tests/test_chrono_second_tu.cpp diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index ec62ac9eb5..fb59e071f1 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -88,26 +88,28 @@ inline PyObject *datetime_from_timestamp(std::time_t tt) { return cls.attr("fromtimestamp")(tt).release().ptr(); } #else -inline void datetime_import() { +// `PyDateTimeAPI` is a per-translation-unit static: these helpers get internal linkage so the +// linker cannot pair one unit's import with another unit's (null) pointer. +static inline void datetime_import() { if (!PyDateTimeAPI) { PyDateTime_IMPORT; } } -inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } -inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } -inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } -inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } +static inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } +static inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } +static inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } +static inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } # define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ - inline int fn(handle h) { return c_macro(h.ptr()); } -inline PyObject *make_timedelta(int days, int seconds, int microseconds) { + static inline int fn(handle h) { return c_macro(h.ptr()); } +static inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyDelta_FromDSU(days, seconds, microseconds); } -inline PyObject *make_datetime( +static inline PyObject *make_datetime( int year, int month, int day, int hour, int minute, int second, int microsecond, int fold) { return PyDateTime_FromDateAndTimeAndFold( year, month, day, hour, minute, second, microsecond, fold); } -inline PyObject *datetime_from_timestamp(std::time_t tt) { +static inline PyObject *datetime_from_timestamp(std::time_t tt) { auto args = make_tuple(tt); return PyDateTime_FromTimestamp(args.ptr()); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index de0cfb4a74..2ee4bc5814 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -130,6 +130,7 @@ set(PYBIND11_TEST_FILES test_builtin_casters test_call_policies test_callbacks + test_chrono_second_tu.cpp test_chrono test_class test_class_cross_module_use_after_one_module_dealloc diff --git a/tests/test_chrono.py b/tests/test_chrono.py index db6416fc5f..74bb18c837 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -8,6 +8,7 @@ import env from pybind11_tests import chrono as m +from pybind11_tests import chrono_second_tu as m2 def test_chrono_system_clock(): @@ -31,6 +32,13 @@ def test_chrono_system_clock(): assert diff.seconds <= diff_python.seconds +def test_chrono_second_translation_unit(): + """The datetime C API import must work per translation unit (two-TU crash)""" + delta = datetime.timedelta(seconds=1, microseconds=5) + assert m2.duration_roundtrip(delta) == delta + assert m.test_chrono2(m.test_chrono1()) is not None + + def test_chrono_system_clock_roundtrip(): date1 = datetime.datetime.today() diff --git a/tests/test_chrono_second_tu.cpp b/tests/test_chrono_second_tu.cpp new file mode 100644 index 0000000000..14f18e16df --- /dev/null +++ b/tests/test_chrono_second_tu.cpp @@ -0,0 +1,23 @@ +/* + tests/test_chrono_second_tu.cpp -- chrono casters from a second translation unit + + Only durations are used here. If the datetime helpers have external linkage, the linker can + pair this TU's `datetime_import()` with test_chrono.cpp's `is_datetime()`, which then reads + a null `PyDateTimeAPI`. + + Copyright (c) 2026 Henry Schreiner + + All rights reserved. Use of this source code is governed by a + BSD-style license that can be found in the LICENSE file. +*/ + +#include + +#include "pybind11_tests.h" + +#include + +TEST_SUBMODULE(chrono_second_tu, m) { + m.def("duration_roundtrip", + [](const std::chrono::system_clock::duration &d) { return d; }); +} From 203ee17316a5b7e323ceb486cc3d765d68c49aef Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 25/59] fix(stable-abi): keep instance lookup safe across critical-section suspension Regular free-threaded builds lock the instance-map shards with PyMutex again, which is never suspended. Under abi3t only critical sections are available; `find_registered_python_instance()` now takes strong references under the shard lock and checks the types after releasing it, since `all_type_info()` can suspend the section while waiting for the internals lock. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/internals.h | 6 ++++ .../pybind11/detail/type_caster_base-inl.h | 33 ++++++++++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 19aa6f5c51..a55c0ae391 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -979,7 +979,13 @@ inline auto with_instance_map(const void *ptr, const F &cb) auto idx = static_cast(hash & internals.instance_shards_mask); auto &shard = internals.instance_shards[idx]; +# if defined(PYBIND11_OPAQUE_PYOBJECT) + // A critical section is suspended while `cb` waits for another one (e.g. the internals + // lock): `cb` must not keep iterators or instance pointers across such calls. pycritical_section lock(shard.mutex); +# else + std::unique_lock lock(shard.mutex); +# endif return cb(shard.registered_instances); #else (void) ptr; diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index ad373dcd60..ceaf1b37fd 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -287,29 +287,46 @@ PYBIND11_INLINE bool try_incref(PyObject *obj) { PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle find_registered_python_instance(void *src, const detail::type_info *tinfo) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // all_type_info() takes the internals lock, which can suspend the instance map's critical + // section: take strong references first, then check the types without the map lock. + std::vector candidates = with_instance_map(src, [&](instance_map &instances) { + std::vector found; + auto it_instances = instances.equal_range(src); + for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { + PyObject *wrapper = nullptr; + if (it_i->second->registry_weakref != nullptr + && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { + found.emplace_back(reinterpret_steal(wrapper)); + } + } + return found; + }); + for (auto &wrapper : candidates) { + for (auto *instance_type : detail::all_type_info(Py_TYPE(wrapper.ptr()))) { + if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { + return wrapper.release(); + } + } + } + return handle(); +#else return with_instance_map(src, [&](instance_map &instances) { auto it_instances = instances.equal_range(src); for (auto it_i = it_instances.first; it_i != it_instances.second; ++it_i) { for (auto *instance_type : detail::all_type_info(Py_TYPE(instance_object(it_i->second)))) { if (instance_type && same_type(*instance_type->cpptype, *tinfo->cpptype)) { -#if defined(PYBIND11_OPAQUE_PYOBJECT) - PyObject *wrapper = nullptr; - if (it_i->second->registry_weakref != nullptr - && PyWeakref_GetRef(it_i->second->registry_weakref, &wrapper) == 1) { - return handle(wrapper); - } -#else auto *wrapper = instance_object(it_i->second); if (try_incref(wrapper)) { return handle(wrapper); } -#endif } } } return handle(); }); +#endif } PYBIND11_NOINLINE_ATTR PYBIND11_INLINE value_and_holder From a33955ec0c6214c6e30543e6f85270f2c6060479 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 26/59] test(stable-abi): check the method-descriptor flag only under the limited API CPython's `PyInstanceMethod_Type`, used in regular builds, does not set `Py_TPFLAGS_METHOD_DESCRIPTOR`. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_methods_and_attributes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 0f64416bc1..671c058d3a 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -106,8 +106,10 @@ class Sub(m.ExampleMandA): Sub.alias = m.ExampleMandA.internal1 assert Sub(9).alias() == 9 - # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method - assert type(unbound).__flags__ & (1 << 17) + # Py_TPFLAGS_METHOD_DESCRIPTOR: `obj.method(...)` skips the bound method. Only pybind11's + # own instancemethod (limited API) has it; CPython's PyInstanceMethod_Type does not. + if env.LIMITED_API: + assert type(unbound).__flags__ & (1 << 17) bound = m.ExampleMandA(5).overloaded assert bound.__self__.internal1() == 5 assert bound.__func__ is m.ExampleMandA.overloaded.__func__ From 6bfd92f405d8dd94d02ed7944a35e129ccaf95d0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 27/59] fix(cmake): do not link libpython into SHARED STABLE_ABI libraries `python_add_library(SHARED)` always links `Python::Python`. Create the target with `add_library()` instead so the helper only depends on the stable ABI. Assisted-by: ClaudeCode:claude-fable-5-1 --- tools/pybind11NewTools.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 5618776876..e48fbaa283 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -376,7 +376,10 @@ function(pybind11_add_module target_name) endif() endif() - if("${_Python}" STREQUAL "Python") + if(stable_abi AND lib_type STREQUAL "SHARED") + # python_add_library(SHARED) links the embedding library; a stable-ABI helper must not. + add_library(${target_name} SHARED ${ARG_UNPARSED_ARGUMENTS}) + elseif("${_Python}" STREQUAL "Python") python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") python3_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) From ce9fb4d5730d54d4b191223ca27cb306a682d6b1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:20:22 -0400 Subject: [PATCH 28/59] ci(stable-abi): resolve forward-test ignores relative to build/tests Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 14 ++++++++------ tests/test_chrono_second_tu.cpp | 3 +-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22803214a1..3d90e1a5c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -178,10 +178,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: + # Relative to the working directory of the forward steps, build/tests PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_custom_type_setup.py - --ignore=tests/test_multiple_interpreters.py - --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py steps: - uses: actions/checkout@v7 @@ -242,10 +243,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 env: + # Relative to the working directory of the forward steps, build/tests PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- - --ignore=tests/test_custom_type_setup.py - --ignore=tests/test_multiple_interpreters.py - --ignore=tests/test_class_release_gil_before_calling_cpp_dtor.py + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py steps: - uses: actions/checkout@v7 diff --git a/tests/test_chrono_second_tu.cpp b/tests/test_chrono_second_tu.cpp index 14f18e16df..6f8a2fd4cb 100644 --- a/tests/test_chrono_second_tu.cpp +++ b/tests/test_chrono_second_tu.cpp @@ -18,6 +18,5 @@ #include TEST_SUBMODULE(chrono_second_tu, m) { - m.def("duration_roundtrip", - [](const std::chrono::system_clock::duration &d) { return d; }); + m.def("duration_roundtrip", [](const std::chrono::system_clock::duration &d) { return d; }); } From c5bbfd01335ab50e414b7316451b58d3376cdd13 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:53:03 -0400 Subject: [PATCH 29/59] fix(pypy): use the accessors PyPy provides PyPy has `PyCFunction_GET_SELF` only as a macro and has no `PyThreadState_GetInterpreter()`. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/common.h | 7 +++++++ include/pybind11/functional.h | 2 +- include/pybind11/pybind11-inl.h | 4 ++-- include/pybind11/pybind11.h | 2 +- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 73e8568e5b..baa360caba 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -469,6 +469,13 @@ pybind11::detail::ensure_internals(); \ } +// PyPy has only the macro form. +#if defined(PYPY_VERSION) +# define PYBIND11_PYCFUNCTION_GET_SELF(func) PyCFunction_GET_SELF(func) +#else +# define PYBIND11_PYCFUNCTION_GET_SELF(func) PyCFunction_GetSelf(func) +#endif + #if !defined(GRAALVM_PYTHON) # define PYBIND11_PYCFUNCTION_GET_DOC(func) ((func)->m_ml->ml_doc) # define PYBIND11_PYCFUNCTION_SET_DOC(func, doc) \ diff --git a/include/pybind11/functional.h b/include/pybind11/functional.h index cfa0a88a08..3c169cb842 100644 --- a/include/pybind11/functional.h +++ b/include/pybind11/functional.h @@ -82,7 +82,7 @@ struct type_caster> { captured variables), in which case the roundtrip can be avoided. */ if (auto cfunc = func.cpp_function()) { - auto *cfunc_self = PyCFunction_GetSelf(cfunc.ptr()); + auto *cfunc_self = PYBIND11_PYCFUNCTION_GET_SELF(cfunc.ptr()); if (cfunc_self == nullptr) { PyErr_Clear(); } else { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 49f7fc7dc5..239cbace04 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -635,7 +635,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u detail::function_record *chain = nullptr, *chain_start = rec; if (rec->sibling) { if (PyCFunction_Check(rec->sibling.ptr())) { - auto *self = PyCFunction_GetSelf(rec->sibling.ptr()); + auto *self = PYBIND11_PYCFUNCTION_GET_SELF(rec->sibling.ptr()); if (self == nullptr) { pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GetSelf"); } @@ -697,7 +697,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u // chain. chain_start = rec; rec->next = chain; - auto *py_func_rec = detail::function_record_data(PyCFunction_GetSelf(m_ptr)); + auto *py_func_rec = detail::function_record_data(PYBIND11_PYCFUNCTION_GET_SELF(m_ptr)); py_func_rec->cpp_func_rec = unique_rec.release(); guarded_strdup.release(); } else { diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 943022f25f..f378af6837 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -1903,7 +1903,7 @@ class class_ : public detail::generic_type { return nullptr; } - handle func_self = PyCFunction_GetSelf(h.ptr()); + handle func_self = PYBIND11_PYCFUNCTION_GET_SELF(h.ptr()); if (!func_self) { throw error_already_set(); } From d9fc94b9cf1516be4c8228f9d6acb9e18b26184d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 7 Oct 2026 23:53:03 -0400 Subject: [PATCH 30/59] fix: MSVC C4456 shadowing and clang-tidy macro parentheses Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 4 ++-- include/pybind11/pybind11-inl.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 0c5e8393b2..fbd3f1cc9b 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -58,12 +58,12 @@ type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, P # define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ PYBIND11_INLINE slot_type fn() { \ static const auto cached \ - = reinterpret_cast(PyType_GetSlot(&type, slot_id)); \ + = reinterpret_cast(PyType_GetSlot(&(type), slot_id)); \ return cached; \ } #else # define PYBIND11_BASE_TYPE_SLOT(fn, slot_type, type, slot_id, field) \ - PYBIND11_INLINE slot_type fn() { return type.field; } + PYBIND11_INLINE slot_type fn() { return (type).field; } #endif PYBIND11_BASE_TYPE_SLOT(type_type_call, ternaryfunc, PyType_Type, Py_tp_call, tp_call) diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 239cbace04..7f16ca9088 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -776,13 +776,13 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u #endif if (rec->is_method) { - PyObject *func = m_ptr; + PyObject *cfunc = m_ptr; m_ptr = PYBIND11_INSTANCE_METHOD_NEW(m_ptr, rec->scope.ptr()); if (!m_ptr) { pybind11_fail( "cpp_function::cpp_function(): Could not allocate instance method object"); } - Py_DECREF(func); + Py_DECREF(cfunc); } } From 0e8dc09b217bc07f081a938a0597a50dd4a20260 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 00:50:17 -0400 Subject: [PATCH 31/59] feat: build abi3t from GIL-enabled Python; fix stable ABI CI failures Add PYBIND11_ABI3T (CMake) and py_limited_api="3.15t" (setuptools) to target abi3t with Py_TARGET_ABI3T from GIL-enabled CPython 3.15+, with the .abi3t suffix and python3t.lib on Windows. Document Py_TARGET_ABI3T. Use `auto it` in the tuple/dict iterator tests under Py_LIMITED_API (as on PyPy), and build the cross-module RTTI bindings without the stable ABI so they share internals with the embedding executable. Assisted-by: ClaudeCode:claude-opus-5-5 --- .github/workflows/ci.yml | 71 +++++++++++++++++ docs/advanced/stable_abi.rst | 37 ++++++--- docs/compiling.rst | 7 +- pybind11/setup_helpers.py | 87 ++++++++++++++------- tests/extra_setuptools/test_setuphelper.py | 19 ++++- tests/test_cross_module_rtti/CMakeLists.txt | 5 +- tests/test_pytypes.cpp | 4 +- tools/pybind11Common.cmake | 7 +- tools/pybind11NewTools.cmake | 87 +++++++++++++++++---- 9 files changed, 260 insertions(+), 64 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3d90e1a5c9..0bdcb9bdff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,6 +295,77 @@ jobs: working-directory: build/tests run: python3.15 -m pytest -v ../../tests ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + # PYBIND11_ABI3T builds abi3t from GIL-enabled Python; run the same modules on 3.15t. + stable-abi-abi3t-from-gil: + if: github.event.pull_request.draft == false + strategy: + fail-fast: false + matrix: + runs-on: [ubuntu-latest, windows-latest] + name: 🧱 abi3t from GIL-enabled Python • ${{ matrix.runs-on }} + runs-on: ${{ matrix.runs-on }} + timeout-minutes: 60 + defaults: + run: + shell: bash + env: + # Relative to the working directory of the forward steps, build/tests + PYBIND11_STABLE_ABI_SKIPPED_TESTS: >- + --ignore=../../tests/test_custom_type_setup.py + --ignore=../../tests/test_multiple_interpreters.py + --ignore=../../tests/test_class_release_gil_before_calling_cpp_dtor.py + + steps: + - uses: actions/checkout@v7 + + - name: Setup Python 3.15t (run) + id: py315t + uses: actions/setup-python@v7 + with: + python-version: '3.15t' + allow-prereleases: true + + - name: Setup Python 3.15 (build) + id: py315 + uses: actions/setup-python@v7 + with: + python-version: '3.15' + allow-prereleases: true + + - name: Install uv + uses: astral-sh/setup-uv@v10.0.1 + with: + enable-cache: true + + - name: Prepare env + run: | + for py in "${{ steps.py315t.outputs.python-path }}" "${{ steps.py315.outputs.python-path }}"; do + uv pip install --python="$py" --system -r tests/requirements.txt + done + + - name: Configure (Python 3.15, abi3t) + run: > + cmake -S. -Bbuild -Werror=dev + -DPYBIND11_WERROR=ON + -DDOWNLOAD_CATCH=ON + -DDOWNLOAD_EIGEN=ON + -DPYBIND11_TEST_STABLE_ABI=ON + -DPYBIND11_ABI3T=ON + -DCMAKE_BUILD_TYPE=Release + -DPYTHON_EXECUTABLE="${{ steps.py315.outputs.python-path }}" + + - name: Build + run: cmake --build build --config Release -j 4 + + - name: Python tests (3.15) + run: cmake --build build --config Release --target pytest + + - name: Python tests (3.15t, same binaries) + working-directory: build/tests + run: > + "${{ steps.py315t.outputs.python-path }}" -m pytest -v ../../tests + ${{ env.PYBIND11_STABLE_ABI_SKIPPED_TESTS }} + standard-large: if: github.event.pull_request.draft == false strategy: diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index c794219053..5b179b3b9e 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -8,11 +8,13 @@ built for. pybind11 supports it from CPython 3.12 on: define ``*.abi3.so`` / ``*.pyd`` per platform instead of one module per Python version. -Free-threaded CPython has its own stable ABI from 3.15 on, ``abi3t`` -(:pep:`803`). pybind11 supports it too: build against the free-threaded -headers with ``Py_LIMITED_API`` set to ``0x030F0000`` or newer, and ship one -``*.abi3t.so`` that every free-threaded *and* GIL-enabled CPython from 3.15 -on can load. See `abi3t`_ below for what differs. +CPython 3.15 adds a second stable ABI, ``abi3t`` (:pep:`803`). An ``abi3t`` +module loads on every free-threaded *and* GIL-enabled CPython from 3.15 on, so +one ``*.abi3t.so`` per platform covers both builds. pybind11 supports it too: +define ``Py_TARGET_ABI3T`` to ``0x030F0000`` (or a newer version). Headers of +either build work. With free-threaded headers, ``Py_LIMITED_API`` alone also +selects ``abi3t``, because free-threaded CPython has no ``abi3``. See `abi3t`_ +below for what differs. Enabling it =========== @@ -25,20 +27,31 @@ the default for a build tree: pybind11_add_module(example STABLE_ABI example.cpp) -``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` on -free-threaded Python) selects the targeted version. ``PRECOMPILE`` can be -combined with ``STABLE_ABI``; the precompiled -library is then compiled against the limited API too, and one build tree -cannot mix stable-ABI and regular precompiled modules. +``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` for +``abi3t``) selects the targeted version. On free-threaded Python the modules +are always ``abi3t``. Set ``PYBIND11_ABI3T`` to ``ON`` to also build ``abi3t`` +modules with GIL-enabled Python: + +.. code-block:: bash + + cmake -S . -B build -DPYBIND11_STABLE_ABI=ON -DPYBIND11_ABI3T=ON + +``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled library +is then compiled against the limited API too, and one build tree cannot mix +stable-ABI and regular precompiled modules. With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` -(``*.abi3t.so`` on free-threaded Python) and ``Py_LIMITED_API`` is defined -for you. +and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for +``abi3t``, which is always used on free-threaded Python; ``Py_TARGET_ABI3T`` +is then defined and the module is named ``*.abi3t.so``. With GIL-enabled +Python, use ``build_ext`` from ``pybind11.setup_helpers`` to get that name. With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every translation unit, link no version-specific Python library (``python3.lib`` on Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). +For ``abi3t``, define ``Py_TARGET_ABI3T=0x030F0000`` instead, link +``python3t.lib`` on Windows, and name the module ``.abi3t.so``. pybind11 rejects the combination with PyPy, GraalPy, and free-threaded CPython before 3.15 at compile time: they have no stable ABI. diff --git a/docs/compiling.rst b/docs/compiling.rst index 4f2143172c..ba338dcadf 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -412,9 +412,10 @@ the targeted version on (3.12 by default; set ``PYBIND11_STABLE_ABI_VERSION`` to target a newer one). Set the CMake variable ``PYBIND11_STABLE_ABI`` to make it the default for all ``pybind11_add_module`` calls and use ``NO_STABLE_ABI`` to opt a target out. This needs CMake 3.26+, the FindPython mode -(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers (3.15+ for -free-threaded builds, which produce ``.abi3t`` modules); it is not available -for PyPy or GraalPy. See :doc:`advanced/stable_abi` +(``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers. Free-threaded +builds produce ``.abi3t`` modules and need 3.15+; set ``PYBIND11_ABI3T`` to +also build ``.abi3t`` modules with GIL-enabled CPython 3.15+. It is not +available for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the features that are not available under the stable ABI. .. _precompile-mode: diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 77e02926d7..6e925d0a1b 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -71,6 +71,7 @@ import distutils.ccompiler import distutils.errors +from importlib.machinery import EXTENSION_SUFFIXES if TYPE_CHECKING: from typing_extensions import Self @@ -87,36 +88,40 @@ # directory into your path if it sits beside your setup.py. -def _limited_api_hex(value: Any) -> str: +def _stable_abi_macro(value: Any) -> tuple[str, str]: """ - The Py_LIMITED_API value for ``py_limited_api``: True selects the minimum - pybind11 supports (3.12, or 3.15 on free-threaded Python, where the stable - ABI is abi3t), a string such as "3.13" or "cp313" selects that version, and - an int or hex string is passed through. + The define for ``py_limited_api``, as ``(name, hex value)``. True selects + the minimum pybind11 supports (3.12 for abi3, 3.15 for abi3t), a string + such as "3.13" or "cp313" selects that version, and an int or hex string is + passed through. A "t" suffix ("3.15t", "cp315t") selects abi3t, which is + always used on free-threaded Python. """ - free_threaded = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) - minimum = (3, 15) if free_threaded else (3, 12) - if value is True: - return f"0x{minimum[0]:02X}{minimum[1]:02X}0000" - if isinstance(value, int): - return f"0x{value:08X}" + abi3t = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) text = str(value) - if text.lower().startswith("0x"): - return text - match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) - if not match: - msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" - raise ValueError(msg) - major, minor = (int(part) for part in match.groups()) - if (major, minor) < minimum: + if isinstance(value, str) and text.lower().endswith("t"): + abi3t = True + text = text[:-1] + minimum = (3, 15) if abi3t else (3, 12) + if value is True: + number = (minimum[0] << 24) | (minimum[1] << 16) + elif isinstance(value, int): + number = value + elif text.lower().startswith("0x"): + number = int(text, 16) + else: + match = re.fullmatch(r"(?:cp)?(\d)\.?(\d+)", text) + if not match: + msg = f"py_limited_api must be True or a Python version such as '3.12', got {value!r}" + raise ValueError(msg) + major, minor = (int(part) for part in match.groups()) + number = (major << 24) | (minor << 16) + if (number >> 24, (number >> 16) & 0xFF) < minimum: what = ( - "the free-threaded stable ABI (abi3t)" - if free_threaded - else "the stable ABI" + "the stable ABI for free-threading (abi3t)" if abi3t else "the stable ABI" ) msg = f"pybind11 supports {what} from Python {minimum[0]}.{minimum[1]} on, got {value!r}" raise ValueError(msg) - return f"0x{major:02X}{minor:02X}0000" + return ("Py_TARGET_ABI3T" if abi3t else "Py_LIMITED_API", f"0x{number:08X}") class Pybind11Extension(_Extension): @@ -145,8 +150,12 @@ class Pybind11Extension(_Extension): ``py_limited_api=True`` builds against the Python stable ABI (3.12, or pass a version string such as ``"3.13"``): ``Py_LIMITED_API`` is defined and setuptools names the module ``*.abi3.so``, so one wheel serves every - CPython from that version on. See the pybind11 documentation for the - features that are not available under the stable ABI. + CPython from that version on. A ``"t"`` suffix (``"3.15t"``) selects abi3t + (``Py_TARGET_ABI3T``, ``*.abi3t.so``), which also loads on free-threaded + CPython; it is always used on free-threaded Python. Use the ``build_ext`` + from this module to get the ``*.abi3t.so`` name on GIL-enabled Python. See + the pybind11 documentation for the features that are not available under + the stable ABI. """ # flags are prepended, so that they can be further overridden, e.g. by @@ -172,9 +181,13 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # setuptools only uses the flag for the .abi3 file name; the # define selects the limited API in the headers. kwargs["py_limited_api"] = True + macro = _stable_abi_macro(py_limited_api) + self._abi3t = macro[0] == "Py_TARGET_ABI3T" macros = list(kwargs.get("define_macros", []) or []) - if not any(name == "Py_LIMITED_API" for name, _ in macros): - macros.append(("Py_LIMITED_API", _limited_api_hex(py_limited_api))) + if not any( + name in {"Py_LIMITED_API", "Py_TARGET_ABI3T"} for name, _ in macros + ): + macros.append(macro) kwargs["define_macros"] = macros super().__init__(*args, **kwargs) @@ -334,6 +347,26 @@ def build_extensions(self) -> None: super().build_extensions() + def get_ext_filename(self, fullname: str) -> str: + """ + setuptools picks the first ``.abi3*`` suffix, which is ``.abi3`` on + GIL-enabled Python; abi3t modules need ``.abi3t``. + """ + filename: str = super().get_ext_filename(fullname) # type: ignore[no-untyped-call] + ext = getattr(self, "ext_map", {}).get(fullname) + if getattr(ext, "_abi3t", False) and not WIN: + for suffix in EXTENSION_SUFFIXES: + abi3t_suffix = suffix.replace(".abi3", ".abi3t", 1) + if ( + suffix.startswith(".abi3") + and not suffix.startswith(".abi3t") + and filename.endswith(suffix) + and abi3t_suffix in EXTENSION_SUFFIXES + ): + filename = filename[: -len(suffix)] + abi3t_suffix + break + return filename + def intree_extensions( paths: Iterable[str], package_dir: dict[str, str] | None = None diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index 9e1e86c6f0..c7851a92cd 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -117,7 +117,19 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std): or (sysconfig.get_config_var("Py_GIL_DISABLED") and sys.version_info < (3, 15)), reason="stable ABI needs CPython 3.12+, or 3.15+ when free-threaded (abi3t)", ) -def test_stable_abi_setup_py(monkeypatch, tmpdir): +@pytest.mark.parametrize( + "py_limited_api", + [ + True, + pytest.param( + "3.15t", + marks=pytest.mark.skipif( + sys.version_info < (3, 15), reason="abi3t needs CPython 3.15+" + ), + ), + ], +) +def test_stable_abi_setup_py(monkeypatch, tmpdir, py_limited_api): monkeypatch.chdir(tmpdir) monkeypatch.syspath_prepend(MAIN_DIR) @@ -135,7 +147,7 @@ def test_stable_abi_setup_py(monkeypatch, tmpdir): "stable_abi_setup", ["main.cpp"], cxx_std=17, - py_limited_api=True, + py_limited_api={py_limited_api!r}, ), ] @@ -175,6 +187,9 @@ def test_stable_abi_setup_py(monkeypatch, tmpdir): built = [f for f in tmpdir.listdir() if f.basename.startswith("stable_abi_setup")] assert len(built) == 1 assert ".abi3" in built[0].basename or built[0].ext == ".pyd" + abi3t = py_limited_api == "3.15t" or sysconfig.get_config_var("Py_GIL_DISABLED") + if not WIN: + assert (".abi3t" in built[0].basename) == bool(abi3t) (tmpdir / "test.py").write_text( dedent( diff --git a/tests/test_cross_module_rtti/CMakeLists.txt b/tests/test_cross_module_rtti/CMakeLists.txt index c9b95bfba1..11f6ed3765 100644 --- a/tests/test_cross_module_rtti/CMakeLists.txt +++ b/tests/test_cross_module_rtti/CMakeLists.txt @@ -40,7 +40,10 @@ target_compile_features(test_cross_module_rtti_lib PUBLIC cxx_std_11) generate_export_header(test_cross_module_rtti_lib) -pybind11_add_module(test_cross_module_rtti_bindings SHARED bindings.cpp) +# The embedding executable cannot use the stable ABI, and must share internals with the bindings; +# the precompiled library would be built for the stable ABI too. +pybind11_add_module(test_cross_module_rtti_bindings SHARED NO_STABLE_ABI NO_PRECOMPILE + bindings.cpp) target_link_libraries(test_cross_module_rtti_bindings PUBLIC test_cross_module_rtti_lib::test_cross_module_rtti_lib) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index b500eb7972..e73af5aaf0 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -734,8 +734,8 @@ TEST_SUBMODULE(pytypes, m) { // pytypes.h could be changed to enforce the "most correct" user code below, by removing // `const` from iterator `reference` using type aliases, but that will break existing // user code. -#if (defined(__APPLE__) && defined(__clang__)) || defined(PYPY_VERSION) -// This is "most correct" and enforced on these platforms. +#if (defined(__APPLE__) && defined(__clang__)) || defined(PYPY_VERSION) || defined(Py_LIMITED_API) +// This is "most correct" and enforced on these platforms (and the stable ABI, as on PyPy). # define PYBIND11_AUTO_IT auto it #else // This works on many platforms and is (unfortunately) reflective of existing user code. diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index eeb15b83ef..f8906a8fef 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -525,7 +525,12 @@ function(pybind11_precompile) set_target_properties(pybind11_precompiled PROPERTIES PYBIND11_STABLE_ABI "${ARG_STABLE_ABI}") if(ARG_STABLE_ABI) _pybind11_stable_abi_hex(_sabi_hex) - target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t) + target_compile_definitions(pybind11_precompiled PRIVATE "Py_TARGET_ABI3T=${_sabi_hex}") + else() + target_compile_definitions(pybind11_precompiled PRIVATE "Py_LIMITED_API=${_sabi_hex}") + endif() endif() # pybind11::module (not just pybind11::pybind11): the library must compile with the # interpreter's ABI macros (e.g. Py_GIL_DISABLED, which FindPython attaches to diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index e48fbaa283..d7b4aed999 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -269,6 +269,10 @@ set(PYBIND11_STABLE_ABI_VERSION "3.12" CACHE STRING "Python version whose stable ABI STABLE_ABI modules target (3.12 or newer)") +set(PYBIND11_ABI3T + OFF + CACHE BOOL "STABLE_ABI modules target abi3t (PEP 803), also from GIL-enabled Python 3.15+") + # True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). function(_pybind11_python_is_free_threaded out_var) if(${_Python}_FREE_THREADED @@ -284,11 +288,22 @@ function(_pybind11_python_is_free_threaded out_var) endif() endfunction() -# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 on free-threaded Python (abi3t starts there). +# True if STABLE_ABI modules target abi3t: always on free-threaded Python, else PYBIND11_ABI3T. +function(_pybind11_stable_abi_is_abi3t out_var) + _pybind11_python_is_free_threaded(_abi3t) + if(PYBIND11_ABI3T) + set(_abi3t ON) + endif() + set(${out_var} + ${_abi3t} + PARENT_SCOPE) +endfunction() + +# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 for abi3t (abi3t starts there). function(_pybind11_stable_abi_version out_var) set(_version "${PYBIND11_STABLE_ABI_VERSION}") - _pybind11_python_is_free_threaded(_free_threaded) - if(_free_threaded AND _version VERSION_LESS 3.15) + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND _version VERSION_LESS 3.15) set(_version "3.15") endif() set(${out_var} @@ -308,6 +323,31 @@ function(_pybind11_stable_abi_hex out_var) PARENT_SCOPE) endfunction() +# Add the stable ABI define and import library to a target that FindPython's USE_SABI did not set up. +function(_pybind11_stable_abi_setup target_name) + _pybind11_stable_abi_hex(_hex) + _pybind11_stable_abi_is_abi3t(_abi3t) + _pybind11_python_is_free_threaded(_free_threaded) + if(NOT _abi3t) + target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_hex}") + else() + target_compile_definitions(${target_name} PRIVATE "Py_TARGET_ABI3T=${_hex}") + endif() + if(_abi3t + AND NOT _free_threaded + AND WIN32) + # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same directory. + target_include_directories(${target_name} SYSTEM PRIVATE ${${_Python}_INCLUDE_DIRS}) + find_library( + _pybind11_python3t_lib python3t + PATHS ${${_Python}_SABI_LIBRARY_DIRS} + NO_DEFAULT_PATH REQUIRED) + target_link_libraries(${target_name} PRIVATE "${_pybind11_python3t_lib}") + else() + target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) + endif() +endfunction() + # Fail with an explanation if a STABLE_ABI module cannot be built in this configuration. function(_pybind11_check_stable_abi target_name lib_type) if(CMAKE_VERSION VERSION_LESS 3.26) @@ -328,11 +368,11 @@ function(_pybind11_check_stable_abi target_name lib_type) message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") endif() - _pybind11_python_is_free_threaded(_free_threaded) - if(_free_threaded + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS 3.15) - message(FATAL_ERROR "${target_name}: the free-threaded stable ABI (abi3t) needs Python " + message(FATAL_ERROR "${target_name}: the stable ABI for free-threading (abi3t) needs Python " ">= 3.15, found ${${_Python}_VERSION}.") endif() if(NOT TARGET ${_Python}::SABIModule) @@ -366,19 +406,26 @@ function(pybind11_add_module target_name) # STABLE_ABI keyword, or the PYBIND11_STABLE_ABI variable as the default; NO_STABLE_ABI opts out. set(stable_abi OFF) set(use_sabi "") + set(own_sabi OFF) if((ARG_STABLE_ABI OR PYBIND11_STABLE_ABI) AND NOT ARG_NO_STABLE_ABI) _pybind11_check_stable_abi(${target_name} ${lib_type}) set(stable_abi ON) - if(lib_type STREQUAL "MODULE") + _pybind11_stable_abi_is_abi3t(_abi3t) + _pybind11_python_is_free_threaded(_free_threaded) + if(lib_type STREQUAL "SHARED" OR (_abi3t AND NOT _free_threaded)) + # python_add_library(SHARED) links the embedding library, and USE_SABI only selects abi3t + # on free-threaded Python. + set(own_sabi ON) + elseif(lib_type STREQUAL "MODULE") # Defines Py_LIMITED_API and links Python::SABIModule (python3.lib on Windows). _pybind11_stable_abi_version(_sabi_version) set(use_sabi USE_SABI ${_sabi_version}) endif() endif() - if(stable_abi AND lib_type STREQUAL "SHARED") - # python_add_library(SHARED) links the embedding library; a stable-ABI helper must not. - add_library(${target_name} SHARED ${ARG_UNPARSED_ARGUMENTS}) + if(own_sabi) + add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + _pybind11_stable_abi_setup(${target_name}) elseif("${_Python}" STREQUAL "Python") python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) elseif("${_Python}" STREQUAL "Python3") @@ -390,12 +437,14 @@ function(pybind11_add_module target_name) target_link_libraries(${target_name} PRIVATE pybind11::headers) if(lib_type STREQUAL "MODULE") - target_link_libraries(${target_name} PRIVATE pybind11::module) + if(NOT own_sabi) + target_link_libraries(${target_name} PRIVATE pybind11::module) + else() + # Python::Module would link the version-specific library; take only the module link flags. + target_link_libraries(${target_name} PRIVATE pybind11::python_link_helper) + endif() elseif(stable_abi) # A SHARED helper library that stable-ABI modules link: same ABI, no embedding. - _pybind11_stable_abi_hex(_sabi_hex) - target_compile_definitions(${target_name} PRIVATE "Py_LIMITED_API=${_sabi_hex}") - target_link_libraries(${target_name} PRIVATE ${_Python}::SABIModule) else() target_link_libraries(${target_name} PRIVATE pybind11::embed) endif() @@ -453,7 +502,7 @@ function(pybind11_extension name) SUFFIX "${PYTHON_MODULE_EXTENSION}") endfunction() -# Stable-ABI modules carry the version-independent "abi3" tag (none on Windows). +# Stable-ABI modules carry the version-independent "abi3" or "abi3t" tag (none on Windows). function(pybind11_extension_stable_abi name) if(CMAKE_SYSTEM_NAME STREQUAL "Windows") set(_ext ".pyd") @@ -461,7 +510,13 @@ function(pybind11_extension_stable_abi name) set(_ext "${CMAKE_SHARED_MODULE_SUFFIX}") endif() if(DEFINED ${_Python}_SOSABI AND NOT "${${_Python}_SOSABI}" STREQUAL "") - set(_ext ".${${_Python}_SOSABI}${_ext}") + set(_sosabi "${${_Python}_SOSABI}") + _pybind11_stable_abi_is_abi3t(_abi3t) + if(_abi3t AND _sosabi MATCHES "^abi3(-|$)") + # SOSABI is abi3 on GIL-enabled Python. + string(REGEX REPLACE "^abi3" "abi3t" _sosabi "${_sosabi}") + endif() + set(_ext ".${_sosabi}${_ext}") endif() set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${_ext}") endfunction() From 7387dbbd32f1faa1990a7e2a9ddb867b10ade03b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 09:06:24 -0400 Subject: [PATCH 32/59] fix(stable-abi): Windows abi3t export symbol, MSVC C4702, .pyd name check Assisted-by: ClaudeCode:claude-opus-5-5 --- include/pybind11/cast.h | 4 ++++ pybind11/setup_helpers.py | 12 ++++++++++++ tests/test_cmake_build/test.py | 2 +- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 9ecaaa5123..79e743d9bb 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -925,9 +925,13 @@ struct copyable_holder_caster : public type_caster_base { } } + // The fallback overload always throws; MSVC LTCG then flags this return as unreachable. + PYBIND11_WARNING_PUSH + PYBIND11_WARNING_DISABLE_MSVC(4702) bool set_foreign_holder(handle src) { return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder); } + PYBIND11_WARNING_POP void load_value(value_and_holder &&v_h) { if (v_h.holder_constructed()) { diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 6e925d0a1b..bcc2166508 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -347,6 +347,18 @@ def build_extensions(self) -> None: super().build_extensions() + def get_export_symbols(self, ext: _Extension) -> list[str]: # type: ignore[override] + """ + abi3t modules export the PEP 793 ``PyModExport_*`` hook, not ``PyInit_*``. + """ + symbols: list[str] = super().get_export_symbols(ext) # type: ignore[no-untyped-call] + if getattr(ext, "_abi3t", False): + name = ext.name.split(".")[-1] + symbols = [ + f"PyModExport_{name}" if s == f"PyInit_{name}" else s for s in symbols + ] + return symbols + def get_ext_filename(self, fullname: str) -> str: """ setuptools picks the first ``.abi3*`` suffix, which is ``.abi3`` on diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 9e4af55920..3ea3f0d107 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -10,6 +10,6 @@ assert test_cmake_build.add(1, 2) == 3 expect_abi3 = os.environ.get("PYBIND11_EXPECT_ABI3") -if expect_abi3: +if expect_abi3 and sys.platform != "win32": assert ".abi3" in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") From e7d0de5996b98d4f3305da5fb602466ca1cc1f4b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 09:56:07 -0400 Subject: [PATCH 33/59] docs: describe the scikit-build-core stable ABI setup wheel.py-api only selects the wheel tag; the CMake side must enable STABLE_ABI from SKBUILD_SABI_COMPONENT and SKBUILD_SABI_VERSION. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 14 ++++++++++++++ docs/compiling.rst | 7 +++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 5b179b3b9e..b7f28ba8c3 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -40,6 +40,20 @@ modules with GIL-enabled Python: is then compiled against the limited API too, and one build tree cannot mix stable-ABI and regular precompiled modules. +With scikit-build-core, set ``wheel.py-api = "cp312"`` in ``pyproject.toml`` +(``"cp312.cp315t"`` to also get ``abi3t`` wheels from free-threaded Python; +the pybind11 defaults produce the matching module in both cases). This only +selects the wheel tag, and scikit-build-core ignores it when the build Python +is too old or is not CPython, so enable ``STABLE_ABI`` from the variables it +sets, before ``find_package(pybind11)``: + +.. code-block:: cmake + + if(SKBUILD_SABI_COMPONENT) + set(PYBIND11_STABLE_ABI ON) + set(PYBIND11_STABLE_ABI_VERSION "${SKBUILD_SABI_VERSION}" CACHE STRING "") + endif() + With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for diff --git a/docs/compiling.rst b/docs/compiling.rst index ba338dcadf..094ad88073 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -48,6 +48,8 @@ add a ``pyproject.toml`` file like this: You don't need setuptools files like ``MANIFEST.in``, ``setup.py``, or ``setup.cfg``, as this is not setuptools. See `scikit-build-core`_ for details. +To ship one wheel per platform instead of one per Python version, see +:doc:`advanced/stable_abi`. For projects you plan to upload to PyPI, be sure to fill out the ``[project]`` table with other important metadata as well (see `Writing pyproject.toml`_). @@ -415,8 +417,9 @@ to opt a target out. This needs CMake 3.26+, the FindPython mode (``PYBIND11_FINDPYTHON=ON``) and CPython 3.12+ headers. Free-threaded builds produce ``.abi3t`` modules and need 3.15+; set ``PYBIND11_ABI3T`` to also build ``.abi3t`` modules with GIL-enabled CPython 3.15+. It is not -available for PyPy or GraalPy. See :doc:`advanced/stable_abi` -for the features that are not available under the stable ABI. +available for PyPy or GraalPy. See :doc:`advanced/stable_abi` for the +features that are not available under the stable ABI and for the +scikit-build-core setup (``wheel.py-api`` plus ``SKBUILD_SABI_COMPONENT``). .. _precompile-mode: From 73100337c81f9d472212d2cabc59052f25b74a68 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 8 Oct 2026 10:49:40 -0400 Subject: [PATCH 34/59] fix(stable-abi): gate PyDict_*Ref helpers on the limited API version PyDict_SetDefaultRef is only in the limited API from 3.15, so a Py_LIMITED_API=0x030E0000 build failed to compile. Alpha-suffixed gates also never matched a limited API target, so 3.13 targets did not use PyDict_GetItemStringRef. Add a 3.14-target CI build. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 4 ++++ include/pybind11/pytypes.h | 10 ++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bdcb9bdff..27415f8341 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -149,6 +149,10 @@ jobs: - runs-on: ubuntu-latest python-version: '3.14' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_TEST_PRECOMPILE=ON + # A newer limited API target: gates must follow the limited API, not PY_VERSION_HEX + - runs-on: ubuntu-latest + python-version: '3.14' + cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DPYBIND11_STABLE_ABI_VERSION=3.14 - runs-on: macos-latest python-version: '3.13' cmake-args: -DPYBIND11_TEST_STABLE_ABI=ON -DCMAKE_CXX_STANDARD=17 diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 0163d3369d..c3a4f6843c 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -897,10 +897,11 @@ inline PyObject *dict_getitem(PyObject *v, PyObject *key) { return rv; } -// PyDict_GetItemStringRef was added in Python 3.13.0a1. +// PyDict_GetItemStringRef was added in Python 3.13.0a1 (limited API: 3.13). // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_getitemstringref(PyObject *v, const char *key) { -#if PYBIND11_API_VERSION_HEX >= 0x030D00A1 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API + 0 >= 0x030D0000) \ + || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030D00A1) PyObject *rv = nullptr; if (PyDict_GetItemStringRef(v, key, &rv) < 0) { throw error_already_set(); @@ -937,10 +938,11 @@ inline PyObject *dict_setdefaultstring(PyObject *v, const char *key, PyObject *d return rv; } -// PyDict_SetDefaultRef was added in Python 3.13.0a4. +// PyDict_SetDefaultRef was added in Python 3.13.0a4 (limited API: 3.15). // See also: https://github.com/python/pythoncapi-compat/blob/main/pythoncapi_compat.h inline PyObject *dict_setdefaultstringref(PyObject *v, const char *key, PyObject *defaultobj) { -#if PYBIND11_API_VERSION_HEX >= 0x030D00A4 +#if (defined(Py_LIMITED_API) && Py_LIMITED_API + 0 >= 0x030F0000) \ + || (!defined(Py_LIMITED_API) && PY_VERSION_HEX >= 0x030D00A4) PyObject *kv = PyUnicode_FromString(key); if (kv == nullptr) { throw error_already_set(); From 10f4e8f89da01d9846fbd8c4f9ae9ce3958105ab Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sat, 10 Oct 2026 19:53:32 -0400 Subject: [PATCH 35/59] fix(stable-abi): address review findings - Pin registry candidates before values_and_holders() takes the internals lock in get_object_handle(). - Provide type_dictoffset()/type_basicsize() for non-limited builds so PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET compiles on 3.12+. - Fall back to the legacy type-creation path when a base's metaclass is not compatible with the requested one. - Detect Windows free-threaded SOABI (cp315t-win_amd64). - Read the python3t.lib directory from the Python::SABIModule target. - Include test_chrono_second_tu when PYBIND11_TEST_OVERRIDE selects test_chrono. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 4 ++- .../pybind11/detail/type_caster_base-inl.h | 25 +++++++++++++++++++ include/pybind11/pytypes.h | 4 +++ tests/CMakeLists.txt | 4 +++ tests/test_methods_and_attributes.cpp | 2 +- tests/test_methods_and_attributes.py | 3 ++- tools/pybind11NewTools.cmake | 24 ++++++++++++++---- 7 files changed, 58 insertions(+), 8 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index b7f28ba8c3..46222ff89a 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -101,7 +101,9 @@ available and in how a few things are implemented. * - ``py::metaclass(handle)`` - restricted - The metaclass must not define ``__new__``, and a metaclass less - derived than ``pybind11_type`` resolves to ``pybind11_type``. + derived than ``pybind11_type`` resolves to ``pybind11_type``. The + metaclass of every base must be ``pybind11_type`` or a base of the + requested metaclass (``PyType_FromMetaclass()`` rejects a conflict). * - ``py::custom_type_setup`` - not available - Its callback receives a ``PyHeapTypeObject *``, which is opaque. diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index ceaf1b37fd..c0683293a9 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -427,6 +427,30 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE bool isinstance_generic(handle obj, PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, const detail::type_info *type) { +#if defined(PYBIND11_OPAQUE_PYOBJECT) + // values_and_holders() takes the internals lock, which can suspend the instance map's + // critical section: take strong references first, then inspect them without the map lock. + std::vector candidates = with_instance_map(ptr, [&](instance_map &instances) { + std::vector found; + auto range = instances.equal_range(ptr); + for (auto it = range.first; it != range.second; ++it) { + PyObject *wrapper = nullptr; + if (it->second->registry_weakref != nullptr + && PyWeakref_GetRef(it->second->registry_weakref, &wrapper) == 1) { + found.emplace_back(reinterpret_steal(wrapper)); + } + } + return found; + }); + for (auto &wrapper : candidates) { + for (const auto &vh : values_and_holders(wrapper.ptr())) { + if (vh.type == type) { + return wrapper; + } + } + } + return handle(); +#else return with_instance_map(ptr, [&](instance_map &instances) { auto range = instances.equal_range(ptr); for (auto it = range.first; it != range.second; ++it) { @@ -438,6 +462,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, } return handle(); }); +#endif } PYBIND11_INLINE object cpp_conduit_method(handle self, diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index c3a4f6843c..776ec8a99a 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -2647,6 +2647,10 @@ inline tuple get_bases(PyTypeObject *type) { /// `type.__mro__` as a tuple. inline tuple get_mro(PyTypeObject *type) { return reinterpret_borrow(type->tp_mro); } +/// `type.__dictoffset__` / `type.__basicsize__` (tp_dictoffset and tp_basicsize). +inline Py_ssize_t type_dictoffset(PyTypeObject *type) { return type->tp_dictoffset; } +inline Py_ssize_t type_basicsize(PyTypeObject *type) { return type->tp_basicsize; } + /// Look `name` up along the MRO of `type` without invoking descriptors, like `_PyType_Lookup`. /// Returns a null handle if the attribute is not found; never raises. (Borrowed here; the /// stable-ABI version below returns an owning `object`.) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2ee4bc5814..c8e7dd3aae 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -211,6 +211,10 @@ if(PYBIND11_TEST_OVERRIDE) # Instead of doing a direct override here, we iterate over the overrides without extension and # match them against entries from the PYBIND11_TEST_FILES, anything that not matches goes into the filter list. string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_OVERRIDE_NO_EXT "${PYBIND11_TEST_OVERRIDE};") + # test_chrono.py imports the module built from this second translation unit. + if(test_chrono IN_LIST TEST_OVERRIDE_NO_EXT) + list(APPEND TEST_OVERRIDE_NO_EXT test_chrono_second_tu) + endif() string(REGEX REPLACE "\\.[^.;]*;" ";" TEST_FILES_NO_EXT "${PYBIND11_TEST_FILES};") # This allows the override to be done with extensions, preserving backwards compatibility. foreach(test_name ${TEST_FILES_NO_EXT}) diff --git a/tests/test_methods_and_attributes.cpp b/tests/test_methods_and_attributes.cpp index ddfc00d22e..e80780ca90 100644 --- a/tests/test_methods_and_attributes.cpp +++ b/tests/test_methods_and_attributes.cpp @@ -493,7 +493,7 @@ TEST_SUBMODULE(methods_and_attributes, m) { py::class_(m, "MetaclassOverride", py::metaclass((PyObject *) &PyType_Type)) .def_property_readonly_static("readonly", [](const py::object &) { return 1; }); -#if !defined(GRAALVM_PYTHON) +#if !defined(GRAALVM_PYTHON) && !defined(Py_LIMITED_API) // test_metaclass_base_conflict: a derived class whose base has a metaclass that is neither a // base nor a subtype of pybind11_type (rejected by PyType_FromMetaclass()). GraalPy cannot // create a weak reference to a type whose metaclass was created with type(). diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 671c058d3a..6ee4f5a6cc 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -271,7 +271,8 @@ def test_metaclass_override(): @pytest.mark.skipif( - not hasattr(m, "MetaclassConflictBase"), reason="Not available on GraalPy" + not hasattr(m, "MetaclassConflictBase"), + reason="Not available on GraalPy or under Py_LIMITED_API", ) def test_metaclass_base_conflict(): """A base with an unrelated metaclass does not block binding a derived class""" diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index d7b4aed999..92468fc5a5 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -275,9 +275,10 @@ set(PYBIND11_ABI3T # True if the interpreter is a free-threaded build (needs the abi3t variant of the stable ABI). function(_pybind11_python_is_free_threaded out_var) + # SOABI is cpython-315t-... on POSIX and cp315t-win_amd64 on Windows. if(${_Python}_FREE_THREADED - OR "${${_Python}_SOABI}" MATCHES "^cpython-[0-9]+t" - OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.cpython-[0-9]+t") + OR "${${_Python}_SOABI}" MATCHES "^(cpython-|cp)[0-9]+t" + OR "${PYTHON_MODULE_EXTENSION}" MATCHES "^\\.(cpython-|cp)[0-9]+t") set(${out_var} ON PARENT_SCOPE) @@ -336,11 +337,24 @@ function(_pybind11_stable_abi_setup target_name) if(_abi3t AND NOT _free_threaded AND WIN32) - # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same directory. - target_include_directories(${target_name} SYSTEM PRIVATE ${${_Python}_INCLUDE_DIRS}) + # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same + # directory. Read it from the imported target: FindPython's variables are not visible when + # pybind11 was found in a subdirectory. (The headers come from pybind11::python_headers.) + set(_python3_lib "") + foreach(_prop IMPORTED_IMPLIB_RELEASE IMPORTED_IMPLIB IMPORTED_LOCATION_RELEASE + IMPORTED_LOCATION) + get_property( + _python3_lib + TARGET ${_Python}::SABIModule + PROPERTY ${_prop}) + if(_python3_lib) + break() + endif() + endforeach() + get_filename_component(_python3_lib_dir "${_python3_lib}" DIRECTORY) find_library( _pybind11_python3t_lib python3t - PATHS ${${_Python}_SABI_LIBRARY_DIRS} + PATHS ${_python3_lib_dir} ${${_Python}_SABI_LIBRARY_DIRS} NO_DEFAULT_PATH REQUIRED) target_link_libraries(${target_name} PRIVATE "${_pybind11_python3t_lib}") else() From f0fd84745265cee5f1a358c8d8fb005254589d3b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 29 Oct 2026 09:30:18 -0400 Subject: [PATCH 36/59] fix(stable-abi): repair the rebase onto master - cpp_function::dispatcher() reads its function_record through function_record_data(), which handles the opaque-PyObject layout; the direct cast from #6177 segfaulted every stable-ABI module on 3.15. - test_thread's internals-lock holder (#6215) needs pymutex::lock(), which the stable-ABI critical-section mutex does not have: compile it only for non-limited free-threaded builds and skip the test otherwise. - Link pybind11::python_headers for abi3t modules built from a GIL-enabled Python on Windows (Python.h was not found). Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/function_record_pyobject.h | 2 +- tests/test_thread.cpp | 8 +++++--- tests/test_thread.py | 4 ++++ tools/pybind11NewTools.cmake | 3 ++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index 7dfe351763..da3effa6bf 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -147,7 +147,7 @@ inline function_record *function_record_ptr_from_dispatcher_self(PyObject *self) assert(self != nullptr); assert(PyType_Check(self) == 0); assert(function_record_PyTypeObject_name_matches(Py_TYPE(self))); - return (reinterpret_cast(self))->cpp_func_rec; + return function_record_data(self)->cpp_func_rec; } inline object function_record_PyObject_New() { diff --git a/tests/test_thread.cpp b/tests/test_thread.cpp index 68adc890fc..fcef8bef0f 100644 --- a/tests/test_thread.cpp +++ b/tests/test_thread.cpp @@ -53,7 +53,9 @@ std::shared_ptr get_shared() { return instance; } -#ifdef Py_GIL_DISABLED +// The stable-ABI pymutex is a critical section on a plain object: a native thread without a +// thread state cannot hold it. +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) // Holds detail::internals::mutex on a native thread that never touches the Python C API, so that // a Python-level call can be checked for independence from the internals lock. The lock is a @@ -189,7 +191,7 @@ PyMethodDef internals_lock_methods[] {"take_internals_lock", take_internals_lock, METH_NOARGS, nullptr}, {nullptr, nullptr, 0, nullptr}}; -#endif // Py_GIL_DISABLED +#endif // Py_GIL_DISABLED && !PYBIND11_OPAQUE_PYOBJECT } // namespace @@ -270,7 +272,7 @@ TEST_SUBMODULE(thread, m) { // The call under test for test_dispatch_does_not_need_internals_lock: the smallest possible // bound function, so that the only pybind11 machinery involved is cpp_function::dispatcher(). m.def("dispatch_noop", []() {}); -#ifdef Py_GIL_DISABLED +#if defined(Py_GIL_DISABLED) && !defined(PYBIND11_OPAQUE_PYOBJECT) if (PyModule_AddFunctions(m.ptr(), internals_lock_methods) != 0) { throw py::error_already_set(); } diff --git a/tests/test_thread.py b/tests/test_thread.py index 2be7492927..f5a71a9e6a 100644 --- a/tests/test_thread.py +++ b/tests/test_thread.py @@ -111,6 +111,10 @@ def __del__(self): @pytest.mark.skipif( not env.PY_GIL_DISABLED, reason="The internals lock is a no-op with the GIL" ) +@pytest.mark.skipif( + not hasattr(m, "start_internals_lock_holder"), + reason="The stable-ABI internals lock cannot be held from a native thread", +) def test_dispatch_does_not_need_internals_lock(): """A bound call must not block on the internals mutex while another thread holds it. diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 92468fc5a5..099fd70183 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -339,7 +339,8 @@ function(_pybind11_stable_abi_setup target_name) AND WIN32) # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same # directory. Read it from the imported target: FindPython's variables are not visible when - # pybind11 was found in a subdirectory. (The headers come from pybind11::python_headers.) + # pybind11 was found in a subdirectory. + target_link_libraries(${target_name} PRIVATE pybind11::python_headers) set(_python3_lib "") foreach(_prop IMPORTED_IMPLIB_RELEASE IMPORTED_IMPLIB IMPORTED_LOCATION_RELEASE IMPORTED_LOCATION) From 6e5f221149ad710edc2d62864c7f08ddede4cbd8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 29 Oct 2026 17:41:56 -0400 Subject: [PATCH 37/59] fix(stable-abi): match the C API for datetime fields and __complex__ results Read datetime fields through the base type's descriptors, so subclass overrides do not change the converted value, and reject a __complex__ result that is not a complex object. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- include/pybind11/chrono.h | 62 ++++++++++++++++++++--------------- include/pybind11/complex.h | 3 ++ tests/test_builtin_casters.py | 10 ++++++ tests/test_chrono.py | 15 +++++++++ 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index fb59e071f1..820f007da7 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -53,18 +53,20 @@ inline bool is_date(handle h) { inline bool is_time(handle h) { return PyObject_TypeCheck(h.ptr(), get_datetime_types().time) != 0; } -inline int datetime_field(handle h, PyObject *name) { - auto value = reinterpret_steal(PyObject_GetAttr(h.ptr(), name)); - if (!value) { - throw error_already_set(); - } - return value.cast(); +// Reads the stored field through the base type's descriptor, so a subclass that overrides the +// attribute does not change the converted value (matching the C macros). +// The getter is kept for the lifetime of the process, like the type objects. +inline handle datetime_field_getter(PyTypeObject *type, const char *name) { + return object(reinterpret_borrow(reinterpret_cast(type)) + .attr(name) + .attr("__get__")) + .release(); } -// One accessor per field: interned attribute lookup here, the C macro otherwise. -# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ +// One accessor per field: the base type's descriptor here, the C macro otherwise. +# define PYBIND11_DATETIME_ACCESSOR(fn, type, attr, c_macro) \ inline int fn(handle h) { \ - static PyObject *name = interned_name(#attr); \ - return datetime_field(h, name); \ + static const handle getter = datetime_field_getter(get_datetime_types().type, #attr); \ + return getter(h).cast(); \ } inline PyObject *make_timedelta(int days, int seconds, int microseconds) { @@ -99,7 +101,7 @@ static inline bool is_timedelta(handle h) { return PyDelta_Check(h.ptr()); } static inline bool is_datetime(handle h) { return PyDateTime_Check(h.ptr()); } static inline bool is_date(handle h) { return PyDate_Check(h.ptr()); } static inline bool is_time(handle h) { return PyTime_Check(h.ptr()); } -# define PYBIND11_DATETIME_ACCESSOR(fn, attr, c_macro) \ +# define PYBIND11_DATETIME_ACCESSOR(fn, type, attr, c_macro) \ static inline int fn(handle h) { return c_macro(h.ptr()); } static inline PyObject *make_timedelta(int days, int seconds, int microseconds) { return PyDelta_FromDSU(days, seconds, microseconds); @@ -115,22 +117,28 @@ static inline PyObject *datetime_from_timestamp(std::time_t tt) { } #endif -PYBIND11_DATETIME_ACCESSOR(timedelta_days, days, PyDateTime_DELTA_GET_DAYS) -PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, seconds, PyDateTime_DELTA_GET_SECONDS) -PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, microseconds, PyDateTime_DELTA_GET_MICROSECONDS) -PYBIND11_DATETIME_ACCESSOR(datetime_year, year, PyDateTime_GET_YEAR) -PYBIND11_DATETIME_ACCESSOR(datetime_month, month, PyDateTime_GET_MONTH) -PYBIND11_DATETIME_ACCESSOR(datetime_day, day, PyDateTime_GET_DAY) -PYBIND11_DATETIME_ACCESSOR(datetime_hour, hour, PyDateTime_DATE_GET_HOUR) -PYBIND11_DATETIME_ACCESSOR(datetime_minute, minute, PyDateTime_DATE_GET_MINUTE) -PYBIND11_DATETIME_ACCESSOR(datetime_second, second, PyDateTime_DATE_GET_SECOND) -PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, microsecond, PyDateTime_DATE_GET_MICROSECOND) -PYBIND11_DATETIME_ACCESSOR(datetime_fold, fold, PyDateTime_DATE_GET_FOLD) -PYBIND11_DATETIME_ACCESSOR(time_hour, hour, PyDateTime_TIME_GET_HOUR) -PYBIND11_DATETIME_ACCESSOR(time_minute, minute, PyDateTime_TIME_GET_MINUTE) -PYBIND11_DATETIME_ACCESSOR(time_second, second, PyDateTime_TIME_GET_SECOND) -PYBIND11_DATETIME_ACCESSOR(time_microsecond, microsecond, PyDateTime_TIME_GET_MICROSECOND) -PYBIND11_DATETIME_ACCESSOR(time_fold, fold, PyDateTime_TIME_GET_FOLD) +PYBIND11_DATETIME_ACCESSOR(timedelta_days, timedelta, days, PyDateTime_DELTA_GET_DAYS) +PYBIND11_DATETIME_ACCESSOR(timedelta_seconds, timedelta, seconds, PyDateTime_DELTA_GET_SECONDS) +PYBIND11_DATETIME_ACCESSOR(timedelta_microseconds, + timedelta, + microseconds, + PyDateTime_DELTA_GET_MICROSECONDS) +PYBIND11_DATETIME_ACCESSOR(datetime_year, datetime, year, PyDateTime_GET_YEAR) +PYBIND11_DATETIME_ACCESSOR(datetime_month, datetime, month, PyDateTime_GET_MONTH) +PYBIND11_DATETIME_ACCESSOR(datetime_day, datetime, day, PyDateTime_GET_DAY) +PYBIND11_DATETIME_ACCESSOR(datetime_hour, datetime, hour, PyDateTime_DATE_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(datetime_minute, datetime, minute, PyDateTime_DATE_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(datetime_second, datetime, second, PyDateTime_DATE_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(datetime_microsecond, + datetime, + microsecond, + PyDateTime_DATE_GET_MICROSECOND) +PYBIND11_DATETIME_ACCESSOR(datetime_fold, datetime, fold, PyDateTime_DATE_GET_FOLD) +PYBIND11_DATETIME_ACCESSOR(time_hour, time, hour, PyDateTime_TIME_GET_HOUR) +PYBIND11_DATETIME_ACCESSOR(time_minute, time, minute, PyDateTime_TIME_GET_MINUTE) +PYBIND11_DATETIME_ACCESSOR(time_second, time, second, PyDateTime_TIME_GET_SECOND) +PYBIND11_DATETIME_ACCESSOR(time_microsecond, time, microsecond, PyDateTime_TIME_GET_MICROSECOND) +PYBIND11_DATETIME_ACCESSOR(time_fold, time, fold, PyDateTime_TIME_GET_FOLD) #undef PYBIND11_DATETIME_ACCESSOR template diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 5e2309bedb..750297212b 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -90,6 +90,9 @@ class type_caster> { PyErr_Clear(); return false; } + if (!PyComplex_Check(index.ptr())) { + return false; + } src_or_index = index; } double real = PyComplex_RealAsDouble(src_or_index.ptr()); diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index 474a4a5eb2..7d7bec222b 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -677,6 +677,16 @@ def __index__(self): m.complex_convert(ComplexAndIndex()) +@pytest.mark.parametrize("function", [m.complex_convert, m.complex_noconvert]) +def test_complex_method_wrong_return_type(function): + class BadComplex: + def __complex__(self): + return 2.0 + + with pytest.raises(TypeError): + function(BadComplex()) + + def test_overload_resolution_float_int(): """ Test overload resolution behavior when int can match float (added with PR #5879). diff --git a/tests/test_chrono.py b/tests/test_chrono.py index 74bb18c837..c3f8749534 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -32,6 +32,21 @@ def test_chrono_system_clock(): assert diff.seconds <= diff_python.seconds +def test_chrono_datetime_subclass_overrides_ignored(): + # Conversion must read the stored fields, not attributes a subclass overrides. + class Weird(datetime.datetime): + @property + def year(self): + return 2001 + + @property + def minute(self): + return 59 + + value = Weird(2020, 1, 2, 3, 4, 5) + assert m.test_chrono2(value) == datetime.datetime(2020, 1, 2, 3, 4, 5) + + def test_chrono_second_translation_unit(): """The datetime C API import must work per translation unit (two-TU crash)""" delta = datetime.timedelta(seconds=1, microseconds=5) From 104cd876828aca804a831a878ed28c4652d6651c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 29 Oct 2026 18:27:52 -0400 Subject: [PATCH 38/59] fix(stable-abi): consume WITH_SOABI and look __complex__ up on the type pybind11_add_module forwarded WITH_SOABI to add_library() when it sets up the stable ABI itself (SHARED, or abi3t on GIL-enabled Python). The limited-API complex caster now looks __complex__ up on the type, like Python does, instead of on the instance. Assisted-by: ClaudeCode:claude-fable-5-1 ClaudeCode:claude-fable-5-1 --- include/pybind11/complex.h | 25 +++++++++++-------- tests/test_builtin_casters.py | 11 ++++++++ .../subdirectory_stable_abi/CMakeLists.txt | 5 +++- tools/pybind11NewTools.cmake | 5 +++- 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index 750297212b..a4fc9c5070 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -82,18 +82,21 @@ class type_caster> { value = std::complex((T) result.real, (T) result.imag); #else // Py_complex is not part of the stable ABI. Before 3.13, PyComplex_RealAsDouble() does - // not call __complex__ itself. - if (!PyComplex_Check(src_or_index.ptr()) && hasattr(src_or_index, "__complex__")) { - index = reinterpret_steal( - PyObject_CallMethod(src_or_index.ptr(), "__complex__", nullptr)); - if (!index) { - PyErr_Clear(); - return false; - } - if (!PyComplex_Check(index.ptr())) { - return false; + // not call __complex__ itself. Look the method up on the type, like Python does. + if (!PyComplex_Check(src_or_index.ptr())) { + object method = getattr(type::handle_of(src_or_index), "__complex__", none()); + if (!method.is_none()) { + index = reinterpret_steal( + PyObject_CallFunctionObjArgs(method.ptr(), src_or_index.ptr(), nullptr)); + if (!index) { + PyErr_Clear(); + return false; + } + if (!PyComplex_Check(index.ptr())) { + return false; + } + src_or_index = index; } - src_or_index = index; } double real = PyComplex_RealAsDouble(src_or_index.ptr()); if (real == -1.0 && PyErr_Occurred()) { diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index 7d7bec222b..93bc80c9c3 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -687,6 +687,17 @@ def __complex__(self): function(BadComplex()) +def test_complex_method_instance_shadow_ignored(): + # Special-method lookup reads the type, not an instance attribute. + class Shadowed: + def __complex__(self): + return 1.0 + 2.0j + + value = Shadowed() + value.__complex__ = lambda: 9.0 + 10.0j + assert m.complex_convert(value) == 1.0 + 2.0j + + def test_overload_resolution_float_int(): """ Test overload resolution behavior when int can match float (added with PR #5879). diff --git a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt index d835c7c2ae..a96f3c594c 100644 --- a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt @@ -16,6 +16,9 @@ add_subdirectory("${pybind11_SOURCE_DIR}" pybind11) pybind11_add_module(test_subdirectory_stable_abi STABLE_ABI ../main.cpp) set_target_properties(test_subdirectory_stable_abi PROPERTIES OUTPUT_NAME test_cmake_build) +# WITH_SOABI must be consumed when pybind11 sets up the stable ABI itself (SHARED). +pybind11_add_module(test_subdirectory_stable_abi_shared SHARED STABLE_ABI WITH_SOABI ../main.cpp) + if(DEFINED Python_EXECUTABLE) set(_Python_EXECUTABLE "${Python_EXECUTABLE}") elseif(DEFINED PYTHON_EXECUTABLE) @@ -34,4 +37,4 @@ add_custom_target( ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME} - DEPENDS test_subdirectory_stable_abi) + DEPENDS test_subdirectory_stable_abi test_subdirectory_stable_abi_shared) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 099fd70183..8800a304c3 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -439,6 +439,9 @@ function(pybind11_add_module target_name) endif() if(own_sabi) + # python_add_library would consume WITH_SOABI; here it selects the same abi3/abi3t suffix as + # the default handling below. + list(REMOVE_ITEM ARG_UNPARSED_ARGUMENTS WITH_SOABI) add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) _pybind11_stable_abi_setup(${target_name}) elseif("${_Python}" STREQUAL "Python") @@ -470,7 +473,7 @@ function(pybind11_add_module target_name) _pybind11_default_hidden_visibility(${target_name}) # If we don't pass a WITH_SOABI or WITHOUT_SOABI, use our own default handling of extensions - if(NOT ARG_WITHOUT_SOABI AND NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS) + if(NOT ARG_WITHOUT_SOABI AND (own_sabi OR NOT "WITH_SOABI" IN_LIST ARG_UNPARSED_ARGUMENTS)) if(stable_abi) pybind11_extension_stable_abi(${target_name}) else() From b1fe56dd1d860429d478b18593d8613590c914c5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 29 Oct 2026 19:33:38 -0400 Subject: [PATCH 39/59] fix(stable-abi): one instance layout, header checks in setup_helpers, review cleanups Reserve the `__dict__` slot in every stable-ABI instance so that all pybind11 types have the same size; two `py::dynamic_attr()` bases no longer fail with "multiple bases have instance lay-out conflict", and `type_info::dictoffset` is gone. `Pybind11Extension(py_limited_api=...)` now rejects a build interpreter whose headers do not know the requested ABI instead of producing a version-specific module with a stable-ABI name. Also: create the instancemethod type with the other internals types instead of under the internals lock, match type names on `__module__` and `__name__` so heap types from C extensions compare like static ones, document the abi3t registry weak reference, drop dead PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET branches from the spec path, and fix stale comments. Assisted-by: ClaudeCode:claude-fable-5-1 --- .github/workflows/ci.yml | 4 +- docs/advanced/stable_abi.rst | 11 +- include/pybind11/attr-inl.h | 2 +- include/pybind11/detail/class-inl.h | 124 ++++++++---------- include/pybind11/detail/class.h | 7 + include/pybind11/detail/common.h | 6 + .../detail/function_record_pyobject.h | 2 - include/pybind11/detail/internals.h | 15 ++- include/pybind11/pybind11-inl.h | 11 +- include/pybind11/pytypes.h | 32 ++++- pybind11/setup_helpers.py | 16 ++- tests/extra_setuptools/test_setuphelper.py | 35 +++++ tests/test_multiple_inheritance.cpp | 6 + tests/test_multiple_inheritance.py | 7 +- 14 files changed, 180 insertions(+), 98 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27415f8341..80008f70f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -107,8 +107,8 @@ jobs: python-version: ${{ matrix.python-version }} cmake-args: -DPYBIND11_TEST_PRECOMPILE=ON - # The legacy PyHeapTypeObject type-creation path on CPython 3.12+ - # (PyType_FromMetaclass is the default there). + # PyType_FromMetaclass type creation is the default on CPython 3.12+ (and the only + # path under the stable ABI); this job covers the legacy opt-out. type-creation-legacy: if: github.event.pull_request.draft == false strategy: diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 46222ff89a..d605c0f9bf 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -89,8 +89,9 @@ available and in how a few things are implemented. array fields through NumPy's abi3t accessors). * - ``py::dynamic_attr()`` - supported - - The ``__dict__`` slot is appended to the instance (no - ``Py_TPFLAGS_MANAGED_DICT``). + - Every instance reserves a ``__dict__`` slot (one pointer) that + ``py::dynamic_attr()`` types use; there is no + ``Py_TPFLAGS_MANAGED_DICT`` in the stable ABI. * - Buffer protocol - supported - Through the ``Py_bf_*`` type slots. @@ -134,7 +135,9 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and ``py::module_::create_extension_module()`` is not available; * locks its internals with critical sections on a private object, and keeps one weak reference per registered instance so that the instance registry - can hand out strong references safely (``PyWeakref_GetRef()``). + can hand out strong references safely (``PyWeakref_GetRef()``). This weak + reference is visible from Python: ``weakref.getweakrefcount()`` counts it, + and ``weakref.ref(obj)`` without a callback returns that same object. ``pybind11/numpy.h`` reads NumPy's object fields through the accessors NumPy 2.5 added for abi3t, so older NumPy versions are rejected at run time. The @@ -165,7 +168,7 @@ to specific operations: available as an opt-in for regular builds on CPython 3.12+). * Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances - is found through the type's cached dictionary offset. + is a fixed slot in the instance layout. * Attribute *assignment* on bound classes (``Type.static_prop = value``) walks ``__mro__`` and the class dictionaries instead of using ``_PyType_Lookup()``; attribute reads use the metaclass default and are not affected. diff --git a/include/pybind11/attr-inl.h b/include/pybind11/attr-inl.h index 1782e48e89..43965acf80 100644 --- a/include/pybind11/attr-inl.h +++ b/include/pybind11/attr-inl.h @@ -50,7 +50,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void type_record::add_base(const std::typ bases.append(reinterpret_cast(base_info->type)); #if defined(Py_LIMITED_API) - dynamic_attr |= base_info->dictoffset != 0; + dynamic_attr |= type_dictoffset(base_info->type) != 0; #elif defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) dynamic_attr |= base_info->type->tp_dictoffset != 0; #else diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index fbd3f1cc9b..10f6e07ac2 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -16,8 +16,6 @@ #include "class.h" -#include - PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -589,42 +587,41 @@ extern "C" PYBIND11_INLINE void instancemethod_dealloc(PyObject *self) { Py_DECREF(reinterpret_cast(type)); } +// Called from the internals constructor, like the other internals types. +PYBIND11_INLINE PyTypeObject *make_instancemethod_type() { + static PyMemberDef members[] = {{"__func__", + Py_T_OBJECT_EX, + offsetof(instancemethod_object, func), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, + {"__vectorcalloffset__", + Py_T_PYSSIZET, + offsetof(instancemethod_object, vectorcall), + Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, + nullptr}, + {nullptr, 0, 0, 0, nullptr}}; + static PyType_Slot slots[] + = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, + {Py_tp_call, reinterpret_cast(instancemethod_call)}, + {Py_tp_getattro, reinterpret_cast(instancemethod_getattro)}, + {Py_tp_repr, reinterpret_cast(instancemethod_repr)}, + {Py_tp_traverse, reinterpret_cast(instancemethod_traverse)}, + {Py_tp_clear, reinterpret_cast(instancemethod_clear)}, + {Py_tp_dealloc, reinterpret_cast(instancemethod_dealloc)}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; + static PyType_Spec spec + = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", + PYBIND11_TYPE_DATA_SIZE(instancemethod_object), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_METHOD_DESCRIPTOR | Py_TPFLAGS_HAVE_VECTORCALL, + slots}; + return type_from_spec("make_instancemethod_type()", nullptr, &spec, nullptr); +} + PYBIND11_INLINE PyTypeObject *get_instancemethod_type() { - return with_internals([](internals &internals) { - if (internals.instancemethod_type == nullptr) { - static PyMemberDef members[] = {{"__func__", - Py_T_OBJECT_EX, - offsetof(instancemethod_object, func), - Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, - nullptr}, - {"__vectorcalloffset__", - Py_T_PYSSIZET, - offsetof(instancemethod_object, vectorcall), - Py_READONLY | PYBIND11_MEMBER_OFFSET_FLAGS, - nullptr}, - {nullptr, 0, 0, 0, nullptr}}; - static PyType_Slot slots[] - = {{Py_tp_descr_get, reinterpret_cast(instancemethod_descr_get)}, - {Py_tp_call, reinterpret_cast(instancemethod_call)}, - {Py_tp_getattro, reinterpret_cast(instancemethod_getattro)}, - {Py_tp_repr, reinterpret_cast(instancemethod_repr)}, - {Py_tp_traverse, reinterpret_cast(instancemethod_traverse)}, - {Py_tp_clear, reinterpret_cast(instancemethod_clear)}, - {Py_tp_dealloc, reinterpret_cast(instancemethod_dealloc)}, - {Py_tp_members, reinterpret_cast(members)}, - {0, nullptr}}; - static PyType_Spec spec - = {PYBIND11_DUMMY_MODULE_NAME ".instancemethod", - PYBIND11_TYPE_DATA_SIZE(instancemethod_object), - 0, - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_DISALLOW_INSTANTIATION - | Py_TPFLAGS_METHOD_DESCRIPTOR | Py_TPFLAGS_HAVE_VECTORCALL, - slots}; - internals.instancemethod_type - = type_from_spec("get_instancemethod_type()", nullptr, &spec, nullptr); - } - return internals.instancemethod_type; - }); + return get_internals().instancemethod_type; } PYBIND11_INLINE bool is_instancemethod(PyObject *obj) { @@ -677,19 +674,23 @@ PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { #elif !defined(Py_LIMITED_API) return _PyObject_GetDictPtr(self); #else - // Only `py::dynamic_attr()` types have a `__dict__` slot pybind11 manages (recorded in the - // type_info of the pybind11 type that introduced it); Python subclasses that add a managed - // dict clear it themselves before calling the base tp_dealloc. - for (const auto *tinfo : all_type_info(Py_TYPE(self))) { - if (tinfo->dictoffset > 0) { - return reinterpret_cast(reinterpret_cast(self) - + tinfo->dictoffset); - } - } - return nullptr; + // The slot every pybind11 instance reserves; only `py::dynamic_attr()` types point their + // `__dictoffset__` at it, so it stays null for the others. Python subclasses that add a + // managed dict clear that themselves before calling the base tp_dealloc. + return &get_instance(self)->dict; #endif } +#if defined(Py_LIMITED_API) +PYBIND11_INLINE Py_ssize_t instance_dict_offset() { +# if defined(PYBIND11_OPAQUE_PYOBJECT) + return instance_data_offset() + static_cast(offsetof(instance, dict)); +# else + return static_cast(offsetof(instance, dict)); +# endif +} +#endif + #if defined(PYBIND11_OPAQUE_PYOBJECT) PYBIND11_INLINE Py_ssize_t instance_data_offset() { // The same for every pybind11 type: the type data of pybind11_object, whose base is object. @@ -1196,31 +1197,21 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { if (!rec.is_final) { flags |= Py_TPFLAGS_BASETYPE; } - int basicsize = 0; // inherit the instance layout from the base # if defined(Py_LIMITED_API) - // No Py_TPFLAGS_MANAGED_DICT in the stable ABI: the `__dict__` slot is appended to the base - // layout unless a base has one already. (PEP 697 relative offsets are not applied to - // `__dictoffset__` on 3.12, so the offset is absolute.) - PyMemberDef dict_member[] = {{"__dictoffset__", Py_T_PYSSIZET, 0, Py_READONLY, nullptr}, - {nullptr, 0, 0, 0, nullptr}}; + // No Py_TPFLAGS_MANAGED_DICT in the stable ABI: `__dict__` is the slot every instance + // reserves (instance::dict), so the instance size is the same for all pybind11 types. The + // offset is absolute: the size is inherited from the base, and PEP 697 relative offsets are + // not applied to `__dictoffset__` on 3.12. + PyMemberDef dict_member[] + = {{"__dictoffset__", Py_T_PYSSIZET, instance_dict_offset(), Py_READONLY, nullptr}, + {nullptr, 0, 0, 0, nullptr}}; # endif if (rec.dynamic_attr) { flags |= Py_TPFLAGS_HAVE_GC; # if !defined(Py_LIMITED_API) flags |= Py_TPFLAGS_MANAGED_DICT; # else - bool base_has_dict = false; - int base_basicsize = 0; - for (handle b : bases) { - auto *b_type = reinterpret_cast(b.ptr()); - base_has_dict |= type_dictoffset(b_type) != 0; - base_basicsize = std::max(base_basicsize, static_cast(type_basicsize(b_type))); - } - if (!base_has_dict) { - dict_member[0].offset = base_basicsize; - basicsize = base_basicsize + static_cast(sizeof(PyObject *)); - slots.push_back({Py_tp_members, reinterpret_cast(dict_member)}); - } + slots.push_back({Py_tp_members, reinterpret_cast(dict_member)}); # endif slots.push_back({Py_tp_traverse, reinterpret_cast(pybind11_traverse)}); slots.push_back({Py_tp_clear, reinterpret_cast(pybind11_clear)}); @@ -1232,7 +1223,8 @@ PYBIND11_INLINE PyObject *make_new_python_type(const type_record &rec) { } slots.push_back({0, nullptr}); - PyType_Spec spec = {spec_name.c_str(), basicsize, 0, flags, slots.data()}; + // basicsize 0: inherit the instance layout from the base + PyType_Spec spec = {spec_name.c_str(), 0, 0, flags, slots.data()}; auto *type = type_from_spec(rec.name, metaclass, &spec, base.ptr()); // CPython keeps its own copy of the spec name in _ht_tpname and frees that, not tp_name. set_published_type_names(type, names); diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index c23c9fe8e9..7206eb278d 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -138,6 +138,7 @@ void type_free(PyTypeObject *type, PyObject *self); /// Stand-ins for the instancemethod and method C APIs (see detail/class-inl.h). The /// `is_*`/`*_function` functions are also declared in pytypes.h. PyTypeObject *get_bound_method_type(); +PyTypeObject *make_instancemethod_type(); PyTypeObject *get_instancemethod_type(); struct instancemethod_object; instancemethod_object *instancemethod_data(PyObject *self); @@ -155,6 +156,12 @@ extern "C" void instancemethod_dealloc(PyObject *self); /// Pointer to the `__dict__` slot of an instance, or nullptr if its type has none. PyObject **instance_dict_ptr(PyObject *self); +#if defined(Py_LIMITED_API) +/// Offset of `instance::dict` from the start of the Python object (the `__dictoffset__` of +/// `py::dynamic_attr()` types). +Py_ssize_t instance_dict_offset(); +#endif + /// Instance creation function for all pybind11 types. It allocates the internal instance layout /// for holding C++ objects and holders. Allocation is done lazily (the first time the instance is /// cast to a reference or pointer), and initialization is done by an `__init__` function. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index baa360caba..93ebb6b595 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -832,6 +832,12 @@ struct instance { /// Weak reference used by the instance registry: the limited API has no PyUnstable_TryIncRef. PyObject *registry_weakref; #endif +#if defined(Py_LIMITED_API) + /// The `__dict__` slot of `py::dynamic_attr()` types, null for all others. The stable ABI + /// has no Py_TPFLAGS_MANAGED_DICT; every type reserves the slot so that all pybind11 types + /// keep one instance size (PyType_FromMetaclass() rejects bases of different sizes). + PyObject *dict; +#endif /// Initializes all of the above type/values/holders data (but not the instance values /// themselves) diff --git a/include/pybind11/detail/function_record_pyobject.h b/include/pybind11/detail/function_record_pyobject.h index da3effa6bf..f4ac0d6760 100644 --- a/include/pybind11/detail/function_record_pyobject.h +++ b/include/pybind11/detail/function_record_pyobject.h @@ -109,8 +109,6 @@ inline PyTypeObject *get_function_record_PyTypeObject() { return py_type_obj; } -// This works across extension modules, and does not need the internals lock. -// Note that tp_name is versioned. // This works across extension modules. Note that tp_name is versioned. inline bool function_record_PyTypeObject_name_matches(PyTypeObject *obj_type) { return tp_name_equals(obj_type, function_record_PyTypeObject_methods::tp_qualname_impl) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index a55c0ae391..50063d890a 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -224,6 +224,9 @@ PYBIND11_NAMESPACE_BEGIN(detail) PyTypeObject *make_static_property_type(); PyTypeObject *make_default_metaclass(); PyObject *make_object_base_type(PyTypeObject *metaclass); +#if defined(Py_LIMITED_API) +PyTypeObject *make_instancemethod_type(); +#endif void translate_exception(std::exception_ptr p); inline PyThreadState *get_thread_state_unchecked() { @@ -451,7 +454,11 @@ struct internals { internals() : static_property_type(make_static_property_type()), - default_metaclass(make_default_metaclass()), istate(get_interpreter_state_unchecked()) { + default_metaclass(make_default_metaclass()), +#if defined(Py_LIMITED_API) + instancemethod_type(make_instancemethod_type()), +#endif + istate(get_interpreter_state_unchecked()) { tstate.set(nullptr); // See PR #5870 registered_exception_translators.push_front(&translate_exception); #ifdef Py_GIL_DISABLED @@ -534,12 +541,6 @@ struct type_info { std::forward_list alias_chain; #endif -#if defined(Py_LIMITED_API) - /* Offset of the `__dict__` slot of instances for `py::dynamic_attr()` types (0 if none); - * `_PyObject_GetDictPtr` is not available under the stable ABI. */ - Py_ssize_t dictoffset = 0; -#endif - /* A simple type never occurs as a (direct or indirect) parent * of a class that makes use of multiple inheritance. * A type can be simple even if it has non-simple ancestors as long as it has no descendants. diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 7f16ca9088..4fd96dae5a 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -505,8 +505,12 @@ PYBIND11_INLINE function get_type_override(const void *this_ptr, auto frame_obj = reinterpret_steal(reinterpret_cast(frame)); auto f_code = reinterpret_steal(reinterpret_cast(PyFrame_GetCode(frame))); - if (std::string(str(f_code.attr("co_name"))) == name - && f_code.attr("co_argcount").cast() > 0) { + object co_name = f_code.attr("co_name"); + const char *co_name_utf8 = PyUnicode_AsUTF8AndSize(co_name.ptr(), nullptr); + if (co_name_utf8 == nullptr) { + PyErr_Clear(); + } else if (std::strcmp(co_name_utf8, name) == 0 + && f_code.attr("co_argcount").cast() > 0) { object self_arg = f_code.attr("co_varnames")[int_(0)]; object locals = getattr(frame_obj, "f_locals", handle()); auto self_caller @@ -1328,9 +1332,6 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) { tinfo->simple_ancestors = true; tinfo->module_local = rec.module_local; tinfo->holder_enum_v = rec.holder_enum_v; -#if defined(Py_LIMITED_API) - tinfo->dictoffset = type_dictoffset(reinterpret_cast(m_ptr)); -#endif with_internals([&](internals &internals) { auto tindex = std::type_index(*rec.type); diff --git a/include/pybind11/pytypes.h b/include/pybind11/pytypes.h index 776ec8a99a..bb97979d54 100644 --- a/include/pybind11/pytypes.h +++ b/include/pybind11/pytypes.h @@ -539,9 +539,10 @@ inline std::string utf8_or_empty(const object &o) { return std::string(data, static_cast(size)); } -/// Reconstructs `tp_name`, which is not readable under the stable ABI: `__module__.__name__` -/// for static types outside builtins and for pybind11 types, the bare `__name__` otherwise -/// (builtins and Python heap types). +/// Reconstructs `tp_name`, which is not readable under the stable ABI, for messages: +/// `__module__.__name__` for static types outside builtins and for pybind11 types, the bare +/// `__name__` otherwise (builtins and Python heap types). Heap types of other C extensions come +/// out without their module; compare names with tp_name_is_one_of(), not with this. inline std::string get_tp_name(PyTypeObject *type) { std::string name = utf8_or_empty(reinterpret_steal(PyType_GetName(type))); if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE) && !type_is_managed_by_our_internals(type)) { @@ -555,8 +556,10 @@ inline std::string get_tp_name(PyTypeObject *type) { return module_ + '.' + name; } -/// True if `get_tp_name(type)` is one of `names`. The unqualified `__name__` is fetched once and -/// compared first; the full name is only reconstructed on a match. +/// True if `tp_name` is one of `names`. A dotted name (`numpy.bool`) matches `__module__` and +/// `__name__` separately, whether the type is static or a heap type created from a spec. A bare +/// name matches a builtin or a Python class, whose tp_name has no module. `__name__` is compared +/// first, so a mismatch costs one attribute read. inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list names) { auto type_name = reinterpret_steal(PyType_GetName(type)); const char *type_name_utf8 @@ -565,9 +568,26 @@ inline bool tp_name_is_one_of(PyTypeObject *type, std::initializer_list( + PyObject_GetAttrString(reinterpret_cast(type), "__module__"))); + have_module = true; + } + if (dot != nullptr) { + if (module_.compare(0, std::string::npos, name, static_cast(dot - name)) + == 0) { + return true; + } + } else if (module_.empty() || module_ == PYBIND11_BUILTINS_MODULE + || (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE) + && !type_is_managed_by_our_internals(type))) { return true; } } diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index bcc2166508..207baf58fe 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -115,12 +115,20 @@ def _stable_abi_macro(value: Any) -> tuple[str, str]: raise ValueError(msg) major, minor = (int(part) for part in match.groups()) number = (major << 24) | (minor << 16) - if (number >> 24, (number >> 16) & 0xFF) < minimum: - what = ( - "the stable ABI for free-threading (abi3t)" if abi3t else "the stable ABI" - ) + what = "the stable ABI for free-threading (abi3t)" if abi3t else "the stable ABI" + requested = (number >> 24, (number >> 16) & 0xFF) + if requested < minimum: msg = f"pybind11 supports {what} from Python {minimum[0]}.{minimum[1]} on, got {value!r}" raise ValueError(msg) + # Older headers ignore the define (Py_TARGET_ABI3T is new in 3.15) or lack the API, which + # would silently produce a version-specific module with a stable-ABI name. + building = tuple(sys.version_info[:2]) + if building < max(minimum, requested): + msg = ( + f"building for {what} of Python {requested[0]}.{requested[1]} needs the headers of " + f"that version or newer, but this is Python {building[0]}.{building[1]}" + ) + raise ValueError(msg) return ("Py_TARGET_ABI3T" if abi3t else "Py_LIMITED_API", f"0x{number:08X}") diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index c7851a92cd..f4c9aca1eb 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -205,6 +205,41 @@ def test_stable_abi_setup_py(monkeypatch, tmpdir, py_limited_api): ) +@pytest.mark.parametrize( + ("building", "free_threaded", "value", "expected"), + [ + ((3, 14), False, True, ("Py_LIMITED_API", "0x030C0000")), + ((3, 14), False, "3.13", ("Py_LIMITED_API", "0x030D0000")), + ((3, 14), False, "cp314", ("Py_LIMITED_API", "0x030E0000")), + ((3, 14), False, 0x030C0000, ("Py_LIMITED_API", "0x030C0000")), + ((3, 15), False, "3.15t", ("Py_TARGET_ABI3T", "0x030F0000")), + ((3, 15), True, True, ("Py_TARGET_ABI3T", "0x030F0000")), + ((3, 16), True, "3.15", ("Py_TARGET_ABI3T", "0x030F0000")), + ((3, 14), False, "3.11", ValueError), # below the minimum + ((3, 14), False, "3.15", ValueError), # newer than the headers + # Py_TARGET_ABI3T is unknown to the 3.14 headers + ((3, 14), False, "3.15t", ValueError), + ((3, 14), True, True, ValueError), # abi3t needs 3.15 + ((3, 14), False, "later", ValueError), + ], +) +def test_stable_abi_macro(monkeypatch, building, free_threaded, value, expected): + monkeypatch.syspath_prepend(MAIN_DIR) + from pybind11.setup_helpers import _stable_abi_macro + + monkeypatch.setattr(sys, "version_info", (*building, 0, "final", 0)) + monkeypatch.setattr( + sysconfig, + "get_config_var", + lambda name: int(free_threaded) if name == "Py_GIL_DISABLED" else None, + ) + if expected is ValueError: + with pytest.raises(ValueError): + _stable_abi_macro(value) + else: + assert _stable_abi_macro(value) == expected + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index 5916ae9010..544ad7f462 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -232,6 +232,12 @@ TEST_SUBMODULE(multiple_inheritance, m) { py::class_(m, "WithDict", py::dynamic_attr()).def(py::init<>()); py::class_(m, "VanillaDictMix1").def(py::init<>()); py::class_(m, "VanillaDictMix2").def(py::init<>()); + // Two unrelated bases that both have a `__dict__` slot (the stable ABI has no + // Py_TPFLAGS_MANAGED_DICT, so the slot is part of the instance layout). + struct WithDict2 {}; + struct DictMix12 : WithDict, WithDict2 {}; + py::class_(m, "WithDict2", py::dynamic_attr()).def(py::init<>()); + py::class_(m, "DictMix12").def(py::init<>()); // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance diff --git a/tests/test_multiple_inheritance.py b/tests/test_multiple_inheritance.py index 10932b5dd8..96593c3c3d 100644 --- a/tests/test_multiple_inheritance.py +++ b/tests/test_multiple_inheritance.py @@ -266,7 +266,12 @@ def test_mi_static_properties(): def test_mi_dynamic_attributes(): """Mixing bases with and without dynamic attribute support""" - for cls in (m.VanillaDictMix1, m.VanillaDictMix2): + class PyDictMix12(m.WithDict, m.WithDict2): + def __init__(self): + m.WithDict.__init__(self) + m.WithDict2.__init__(self) + + for cls in (m.VanillaDictMix1, m.VanillaDictMix2, m.DictMix12, PyDictMix12): d = cls() d.dynamic = 1 assert d.dynamic == 1 From b08d061c177001b037c7455939fd83228c8a215c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 29 Oct 2026 19:37:46 -0400 Subject: [PATCH 40/59] test: skip tests whose module is not built under the stable ABI or whose premise is CPython-only CI runs pytest on the whole tests directory, so the stable-ABI jobs collected test_custom_type_setup.py and test_class_release_gil_before_calling_cpp_dtor.py although their C++ is filtered out, and test_multiple_interpreters.py imported modules that are not built there. The first two now use pytest.importorskip, the third skips at module level when the test module was built with Py_LIMITED_API. test_chrono_datetime_subclass_overrides_ignored assumes the datetime C API reads the stored fields; PyPy and GraalPy read the attributes, so the test is CPython-only. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_chrono.py | 4 ++++ tests/test_class.cpp | 2 ++ tests/test_class.py | 4 +++- tests/test_class_release_gil_before_calling_cpp_dtor.py | 3 ++- tests/test_custom_type_setup.py | 4 +++- tests/test_multiple_interpreters.py | 4 ++++ 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tests/test_chrono.py b/tests/test_chrono.py index c3f8749534..8fb0105997 100644 --- a/tests/test_chrono.py +++ b/tests/test_chrono.py @@ -32,6 +32,10 @@ def test_chrono_system_clock(): assert diff.seconds <= diff_python.seconds +@pytest.mark.skipif( + "env.PYPY or env.GRAALPY", + reason="the datetime C API reads the attributes there, not the stored fields", +) def test_chrono_datetime_subclass_overrides_ignored(): # Conversion must read the stored fields, not attributes a subclass overrides. class Weird(datetime.datetime): diff --git a/tests/test_class.cpp b/tests/test_class.cpp index 163a379400..e76d2413da 100644 --- a/tests/test_class.cpp +++ b/tests/test_class.cpp @@ -547,8 +547,10 @@ TEST_SUBMODULE(class_, m) { bind_named_class<5>(names, "a.b[c]"); bind_named_class<6>(base, "a[b.c]"); bind_named_class<7>(dotted, "Child"); +#if !defined(Py_LIMITED_API) auto custom = names.def_submodule("custom"); bind_named_class<8>(custom, "a[b.c]", py::custom_type_setup([](PyHeapTypeObject *) {})); +#endif names.def("bind_with_metaclass", [](py::handle scope, py::handle nested_scope, const py::object &metaclass) { auto module_class diff --git a/tests/test_class.py b/tests/test_class.py index 2063e0f378..500d669426 100644 --- a/tests/test_class.py +++ b/tests/test_class.py @@ -218,11 +218,13 @@ def test_qualname(doc): ) def test_class_names_match_type(request, placement, name): """Bound classes follow Python's naming rules, including literal dots.""" + if placement == "custom" and env.LIMITED_API: + pytest.skip("py::custom_type_setup is not available under Py_LIMITED_API") scopes = { "module": m.class_names, "nested": m.NestBase, "child": getattr(m.class_names, "a[b.c]"), - "custom": m.class_names.custom, + "custom": getattr(m.class_names, "custom", None), } scope = scopes[placement] cls = getattr(scope, name) diff --git a/tests/test_class_release_gil_before_calling_cpp_dtor.py b/tests/test_class_release_gil_before_calling_cpp_dtor.py index 0b1f246bb0..d9dda1c463 100644 --- a/tests/test_class_release_gil_before_calling_cpp_dtor.py +++ b/tests/test_class_release_gil_before_calling_cpp_dtor.py @@ -4,7 +4,8 @@ import pytest -from pybind11_tests import class_release_gil_before_calling_cpp_dtor as m +# Not built under the stable ABI (PyGILState_Check is not part of it) +m = pytest.importorskip("pybind11_tests.class_release_gil_before_calling_cpp_dtor") @pytest.mark.parametrize( diff --git a/tests/test_custom_type_setup.py b/tests/test_custom_type_setup.py index 2216c768f6..12785f57c0 100644 --- a/tests/test_custom_type_setup.py +++ b/tests/test_custom_type_setup.py @@ -8,7 +8,9 @@ import env import pybind11_tests -from pybind11_tests import custom_type_setup as m + +# Not built under the stable ABI (PyHeapTypeObject is opaque there) +m = pytest.importorskip("pybind11_tests.custom_type_setup") @pytest.fixture diff --git a/tests/test_multiple_interpreters.py b/tests/test_multiple_interpreters.py index 6ca4fd8eee..669a44ec85 100644 --- a/tests/test_multiple_interpreters.py +++ b/tests/test_multiple_interpreters.py @@ -13,6 +13,10 @@ if env.IOS: pytest.skip("Subinterpreters not supported on iOS", allow_module_level=True) +if env.LIMITED_API: + pytest.skip( + "Subinterpreters not supported under the stable ABI", allow_module_level=True + ) # 3.14.0b3+, though sys.implementation.supports_isolated_interpreters is being added in b4 # Can be simplified when we drop support for the first three betas From 1a7ffa0341819073f9cc1c5d63183e426ecee76b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 29 Oct 2026 22:00:24 -0700 Subject: [PATCH 41/59] fix(stable-abi): initialize Python type caches without GIL inversion codex GPT-6 on behalf of Ralf: Use GIL-safe once initialization for datetime types, datetime descriptors, and MethodType. Add cold-process concurrent initialization regressions. --- include/pybind11/chrono.h | 28 +++++++----- include/pybind11/detail/class-inl.h | 25 +++++++---- tests/test_cmake_build/main.cpp | 14 ++++++ tests/test_cmake_build/test.py | 67 +++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 19 deletions(-) diff --git a/include/pybind11/chrono.h b/include/pybind11/chrono.h index 820f007da7..393c1e710a 100644 --- a/include/pybind11/chrono.h +++ b/include/pybind11/chrono.h @@ -29,15 +29,18 @@ struct datetime_types { }; inline const datetime_types &get_datetime_types() { - // The type objects are kept for the lifetime of the process. - static const datetime_types types = [] { - module_ m = module_::import("datetime"); - auto get = [&](const char *name) { - return reinterpret_cast(object(m.attr(name)).release().ptr()); - }; - return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; - }(); - return types; + // Importing Python code can release the GIL, so establish the lock order before + // entering the once-only initializer. + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([] { + module_ m = module_::import("datetime"); + auto get = [&](const char *name) { + return reinterpret_cast(object(m.attr(name)).release().ptr()); + }; + return datetime_types{get("datetime"), get("date"), get("time"), get("timedelta")}; + }) + .get_stored(); } inline void datetime_import() { (void) get_datetime_types(); } @@ -65,7 +68,12 @@ inline handle datetime_field_getter(PyTypeObject *type, const char *name) { // One accessor per field: the base type's descriptor here, the C macro otherwise. # define PYBIND11_DATETIME_ACCESSOR(fn, type, attr, c_macro) \ inline int fn(handle h) { \ - static const handle getter = datetime_field_getter(get_datetime_types().type, #attr); \ + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; \ + const handle getter \ + = storage \ + .call_once_and_store_result( \ + [] { return datetime_field_getter(get_datetime_types().type, #attr); }) \ + .get_stored(); \ return getter(h).cast(); \ } diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 10f6e07ac2..f2a2111261 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -16,6 +16,10 @@ #include "class.h" +#if defined(Py_LIMITED_API) +# include +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(detail) @@ -500,15 +504,18 @@ PYBIND11_INLINE instancemethod_object *instancemethod_data(PyObject *self) { } PYBIND11_INLINE PyTypeObject *get_bound_method_type() { - static PyTypeObject *const type = [] { - auto types = reinterpret_steal(PyImport_ImportModule("types")); - if (!types) { - throw error_already_set(); - } - // Static builtin type: keep the reference forever. - return reinterpret_cast(object(types.attr("MethodType")).release().ptr()); - }(); - return type; + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return storage + .call_once_and_store_result([] { + auto types = reinterpret_steal(PyImport_ImportModule("types")); + if (!types) { + throw error_already_set(); + } + // Static builtin type: keep the reference forever. + return reinterpret_cast( + object(types.attr("MethodType")).release().ptr()); + }) + .get_stored(); } PYBIND11_INLINE bool is_bound_method(PyObject *obj) { diff --git a/tests/test_cmake_build/main.cpp b/tests/test_cmake_build/main.cpp index adbaace256..98dd23fe8a 100644 --- a/tests/test_cmake_build/main.cpp +++ b/tests/test_cmake_build/main.cpp @@ -1,3 +1,4 @@ +#include #include #ifdef EXPECT_PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION @@ -13,6 +14,19 @@ namespace py = pybind11; +struct ColdMethodType { + int value() const { return 7; } +}; + PYBIND11_MODULE(test_cmake_build, m, py::mod_gil_not_used()) { m.def("add", [](int i, int j) { return i + j; }); + m.def("chrono_roundtrip", [](std::chrono::milliseconds value) { return value; }); + py::class_(m, "ColdMethodType") + .def(py::init<>()) + .def("value", &ColdMethodType::value); +#if defined(Py_LIMITED_API) + m.attr("limited_api") = true; +#else + m.attr("limited_api") = false; +#endif } diff --git a/tests/test_cmake_build/test.py b/tests/test_cmake_build/test.py index 3ea3f0d107..6c0cb200df 100644 --- a/tests/test_cmake_build/test.py +++ b/tests/test_cmake_build/test.py @@ -1,7 +1,9 @@ from __future__ import annotations import os +import subprocess import sys +import textwrap import test_cmake_build @@ -13,3 +15,68 @@ if expect_abi3 and sys.platform != "win32": assert ".abi3" in expect_abi3, f"expected an abi3 module name, got {expect_abi3}" print(f"{sys.argv[1]} imports, runs, and adds: 1 + 2 = 3") + + +if getattr(test_cmake_build, "limited_api", False): + # Each fresh, small module leaves the selected Python type cache uninitialized. + script = textwrap.dedent( + """ + import builtins + import sys + import threading + import time + + import test_cmake_build as m + + import_name = sys.argv[1] + entered = threading.Event() + second_started = threading.Event() + original_import = builtins.__import__ + + def slow_import(name, *args, **kwargs): + if name == import_name: + entered.set() + assert second_started.wait(10) + time.sleep(0.2) + return original_import(name, *args, **kwargs) + + builtins.__import__ = slow_import + results = [] + errors = [] + if import_name == "types": + instance = m.ColdMethodType() + def call(): + return getattr(instance, "value")() + else: + def call(): + return m.chrono_roundtrip(1.0) + + def read_method(): + try: + results.append(call()) + except BaseException as exc: + errors.append(exc) + + def second_read(): + second_started.set() + read_method() + + first = threading.Thread(target=read_method) + second = threading.Thread(target=second_read) + first.start() + assert entered.wait(10), f"{import_name} was already cached" + second.start() + first.join() + second.join() + assert not errors, errors + assert len(results) == 2 + if import_name == "types": + assert results == [7, 7] + else: + assert all(result.total_seconds() == 1.0 for result in results) + """ + ) + for import_name in ("types", "datetime"): + subprocess.run( + [sys.executable, "-c", script, import_name], check=True, timeout=30 + ) From 1a89de86fc3c8d2aa84598cf6b0b299510c7f205 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 29 Oct 2026 22:00:24 -0700 Subject: [PATCH 42/59] fix(stable-abi): bind complex conversion descriptors correctly codex GPT-6 on behalf of Ralf: Look up the raw complex special method and bind its descriptor before calling, preserving staticmethod, classmethod, custom descriptors, and plain callables. --- include/pybind11/complex.h | 20 +++++++++++++---- tests/test_builtin_casters.py | 41 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/include/pybind11/complex.h b/include/pybind11/complex.h index a4fc9c5070..eb6d9f7a6e 100644 --- a/include/pybind11/complex.h +++ b/include/pybind11/complex.h @@ -84,10 +84,22 @@ class type_caster> { // Py_complex is not part of the stable ABI. Before 3.13, PyComplex_RealAsDouble() does // not call __complex__ itself. Look the method up on the type, like Python does. if (!PyComplex_Check(src_or_index.ptr())) { - object method = getattr(type::handle_of(src_or_index), "__complex__", none()); - if (!method.is_none()) { - index = reinterpret_steal( - PyObject_CallFunctionObjArgs(method.ptr(), src_or_index.ptr(), nullptr)); + auto method + = type_lookup(Py_TYPE(src_or_index.ptr()), handle(interned_name("__complex__"))); + if (method) { + auto descr_get = reinterpret_cast( + PyType_GetSlot(Py_TYPE(method.ptr()), Py_tp_descr_get)); + object bound + = descr_get ? reinterpret_steal(descr_get( + method.ptr(), + src_or_index.ptr(), + reinterpret_cast(Py_TYPE(src_or_index.ptr())))) + : reinterpret_borrow(method); + if (!bound) { + PyErr_Clear(); + return false; + } + index = reinterpret_steal(PyObject_CallNoArgs(bound.ptr())); if (!index) { PyErr_Clear(); return false; diff --git a/tests/test_builtin_casters.py b/tests/test_builtin_casters.py index 93bc80c9c3..6b8be71221 100644 --- a/tests/test_builtin_casters.py +++ b/tests/test_builtin_casters.py @@ -698,6 +698,47 @@ def __complex__(self): assert m.complex_convert(value) == 1.0 + 2.0j +@pytest.mark.parametrize("base", [object, int, float]) +@pytest.mark.parametrize("descriptor", [staticmethod, classmethod]) +def test_complex_method_descriptors(base, descriptor): + class Convertible(base): + @descriptor + def __complex__(*args): + assert args == (() if descriptor is staticmethod else (Convertible,)) + return 1.0 + 2.0j + + value = Convertible() + assert m.complex_convert(value) == complex(value) == 1.0 + 2.0j + if base is not object: + assert m.complex_noconvert(value) == 1.0 + 2.0j + + +def test_complex_callable_without_descriptor(): + class ComplexCallable: + def __call__(self): + return 3.0 + 4.0j + + class Convertible: + __complex__ = ComplexCallable() + + value = Convertible() + assert m.complex_convert(value) == complex(value) == 3.0 + 4.0j + + +def test_complex_custom_descriptor(): + class ComplexDescriptor: + def __get__(self, instance, owner): + return lambda: complex(instance.real, instance.imag) + + class Convertible: + __complex__ = ComplexDescriptor() + real = 5.0 + imag = 6.0 + + value = Convertible() + assert m.complex_convert(value) == complex(value) == 5.0 + 6.0j + + def test_overload_resolution_float_int(): """ Test overload resolution behavior when int can match float (added with PR #5879). From 770d60e6e77005916860a4f24dadd86fc2c28f2b Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 29 Oct 2026 22:00:24 -0700 Subject: [PATCH 43/59] fix(cmake): propagate Python headers to Windows abi3t consumers codex GPT-6 on behalf of Ralf: Propagate Python include requirements through the public pybind11 interface target. Exercise the Windows abi3t consumer path in CI. --- .github/workflows/ci.yml | 3 +++ tests/test_cmake_build/CMakeLists.txt | 2 +- tools/pybind11NewTools.cmake | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80008f70f5..159a77a4fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -361,6 +361,9 @@ jobs: - name: Build run: cmake --build build --config Release -j 4 + - name: CMake stable ABI consumer build + run: cmake --build build --config Release --target test_build_subdirectory_stable_abi + - name: Python tests (3.15) run: cmake --build build --config Release --target pytest diff --git a/tests/test_cmake_build/CMakeLists.txt b/tests/test_cmake_build/CMakeLists.txt index c145122f2e..582452ef55 100644 --- a/tests/test_cmake_build/CMakeLists.txt +++ b/tests/test_cmake_build/CMakeLists.txt @@ -23,7 +23,7 @@ function(pybind11_add_build_test name) # Forward option-driven defines so the nested projects can verify they propagate # through the pybind11_headers usage requirements. foreach(opt IN ITEMS PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION - PYBIND11_SIMPLE_GIL_MANAGEMENT PYBIND11_STABLE_ABI) + PYBIND11_SIMPLE_GIL_MANAGEMENT PYBIND11_STABLE_ABI PYBIND11_ABI3T) if(${opt}) list(APPEND build_options "-D${opt}=ON") endif() diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 8800a304c3..7995d33f90 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -340,7 +340,7 @@ function(_pybind11_stable_abi_setup target_name) # Python::SABIModule is python3.lib here, but abi3t needs python3t.lib from the same # directory. Read it from the imported target: FindPython's variables are not visible when # pybind11 was found in a subdirectory. - target_link_libraries(${target_name} PRIVATE pybind11::python_headers) + target_link_libraries(${target_name} PRIVATE pybind11::pybind11) set(_python3_lib "") foreach(_prop IMPORTED_IMPLIB_RELEASE IMPORTED_IMPLIB IMPORTED_LOCATION_RELEASE IMPORTED_LOCATION) From 19726ebe5778739f0de7a47cbbb97b2441b32418 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Thu, 29 Oct 2026 22:00:24 -0700 Subject: [PATCH 44/59] fix(cmake): preserve Python module names without SOABI codex GPT-6 on behalf of Ralf: Apply Python MODULE prefix and suffix rules even when the ABI tag is omitted, and verify the untagged extension imports. --- .../subdirectory_stable_abi/CMakeLists.txt | 15 ++++++++++++++- tools/pybind11NewTools.cmake | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt index a96f3c594c..a212d5d7a7 100644 --- a/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt +++ b/tests/test_cmake_build/subdirectory_stable_abi/CMakeLists.txt @@ -19,6 +19,14 @@ set_target_properties(test_subdirectory_stable_abi PROPERTIES OUTPUT_NAME test_c # WITH_SOABI must be consumed when pybind11 sets up the stable ABI itself (SHARED). pybind11_add_module(test_subdirectory_stable_abi_shared SHARED STABLE_ABI WITH_SOABI ../main.cpp) +pybind11_add_module(test_subdirectory_stable_abi_without_soabi STABLE_ABI WITHOUT_SOABI + ../main.cpp) +set_target_properties( + test_subdirectory_stable_abi_without_soabi + PROPERTIES OUTPUT_NAME test_cmake_build + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/without_soabi" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/without_soabi") + if(DEFINED Python_EXECUTABLE) set(_Python_EXECUTABLE "${Python_EXECUTABLE}") elseif(DEFINED PYTHON_EXECUTABLE) @@ -37,4 +45,9 @@ add_custom_target( ${_Python_EXECUTABLE} ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME} - DEPENDS test_subdirectory_stable_abi test_subdirectory_stable_abi_shared) + COMMAND + ${CMAKE_COMMAND} -E env + PYTHONPATH=$ ${_Python_EXECUTABLE} + ${PROJECT_SOURCE_DIR}/../test.py ${PROJECT_NAME}_without_soabi + DEPENDS test_subdirectory_stable_abi test_subdirectory_stable_abi_shared + test_subdirectory_stable_abi_without_soabi) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 7995d33f90..923b8a3f83 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -443,6 +443,13 @@ function(pybind11_add_module target_name) # the default handling below. list(REMOVE_ITEM ARG_UNPARSED_ARGUMENTS WITH_SOABI) add_library(${target_name} ${lib_type} ${ARG_UNPARSED_ARGUMENTS}) + # Match python_add_library's importable module name even with WITHOUT_SOABI. + if(lib_type STREQUAL "MODULE") + set_target_properties(${target_name} PROPERTIES PREFIX "") + if(WIN32) + set_target_properties(${target_name} PROPERTIES SUFFIX ".pyd") + endif() + endif() _pybind11_stable_abi_setup(${target_name}) elseif("${_Python}" STREQUAL "Python") python_add_library(${target_name} ${lib_type} ${use_sabi} ${ARG_UNPARSED_ARGUMENTS}) From f408a04824716a9e48f165e0b001a3883aef922b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 11:52:49 -0400 Subject: [PATCH 45/59] fix: use the legacy type path with PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET The spec path uses Py_TPFLAGS_MANAGED_DICT, but a py::custom_type_setup() class on the legacy path places its dict at tp_dictoffset under this macro. Make the macro select the legacy path for all bound classes so a derived class shares the layout of its dynamic_attr base. Derived from cd1fee7d2 by Ralf W. Grosse-Kunstleve (codex GPT-6). Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/advanced/stable_abi.rst | 8 ++++++-- include/pybind11/detail/common.h | 12 ++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index d605c0f9bf..c1d008ee2a 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -164,8 +164,12 @@ where it used to read fields directly. The costs are small and are confined to specific operations: * Type objects are created with ``PyType_FromMetaclass()`` and - ``PyType_Spec`` (the ``PYBIND11_TYPE_CREATION_VIA_SPEC`` path, also - available as an opt-in for regular builds on CPython 3.12+). + ``PyType_Spec``. This is also the default for regular builds on CPython + 3.12+; define ``PYBIND11_TYPE_CREATION_VIA_SPEC=0`` (CMake: + ``-DPYBIND11_TYPE_CREATION_VIA_SPEC=OFF``) to use the previous path, which + fills in the ``PyHeapTypeObject`` fields directly. Defining + ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET`` in a regular build also + selects the previous path, so all bound classes share one dictionary layout. * Instance allocation and deallocation look up ``tp_alloc``/``tp_free`` with ``PyType_GetSlot()``; the ``__dict__`` of ``py::dynamic_attr()`` instances is a fixed slot in the instance layout. diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 93ebb6b595..5f598dd4f8 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -287,11 +287,11 @@ // Create the pybind11 type objects (metaclass, static property, instance base and all bound // classes) through PyType_FromMetaclass() with PyType_Spec instead of filling in raw -// PyHeapTypeObject fields. The default on CPython 3.12+; define PYBIND11_TYPE_CREATION_VIA_SPEC=0 -// to opt out. `py::custom_type_setup()`, metaclasses with a custom `tp_new`, and metaclass -// combinations that PyType_FromMetaclass() rejects still use the legacy path. A user-defined -// PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET also selects the legacy path: the spec path uses -// Py_TPFLAGS_MANAGED_DICT, and all bound classes must share one `__dict__` layout. +// PyHeapTypeObject fields. The default on CPython 3.12+ and the only path under the stable ABI; +// define PYBIND11_TYPE_CREATION_VIA_SPEC=0 to opt out. `py::custom_type_setup()` and +// metaclasses with a custom `tp_new` still use the legacy path. A user-defined +// PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET also selects the legacy path: the spec path +// uses Py_TPFLAGS_MANAGED_DICT, and all bound classes must share one `__dict__` layout. #if !defined(PYBIND11_TYPE_CREATION_VIA_SPEC) # if PY_VERSION_HEX >= 0x030C0000 && !defined(PYPY_VERSION) && !defined(GRAALVM_PYTHON) \ && !defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) @@ -303,7 +303,7 @@ # if PY_VERSION_HEX < 0x030C0000 || defined(PYPY_VERSION) || defined(GRAALVM_PYTHON) # error "PYBIND11_TYPE_CREATION_VIA_SPEC requires CPython 3.12 or newer." # endif -# if defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) && !defined(Py_LIMITED_API) +# if defined(PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET) # error "PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET needs the legacy type creation path" # endif #endif From ce9bbd90440f603bc2cda6183231e4603b5c03a1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 11:53:36 -0400 Subject: [PATCH 46/59] fix(setuptools): derive the abi3t file name and export symbol from explicit macros With py_limited_api=True and an explicit Py_TARGET_ABI3T define on GIL-enabled Python 3.15, the module compiled a PyModExport_* hook but kept the .abi3 file name and requested PyInit_* on Windows. Validate explicit Py_LIMITED_API/Py_TARGET_ABI3T defines and derive _abi3t from them. Derived from 3416a32be by Ralf W. Grosse-Kunstleve (codex GPT-6). Assisted-by: ClaudeCode:claude-fable-5-1 --- pybind11/setup_helpers.py | 26 ++++++---- tests/extra_setuptools/test_setuphelper.py | 56 ++++++++++++++++++++++ 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/pybind11/setup_helpers.py b/pybind11/setup_helpers.py index 207baf58fe..bc8f5ff4b5 100644 --- a/pybind11/setup_helpers.py +++ b/pybind11/setup_helpers.py @@ -88,15 +88,15 @@ # directory into your path if it sits beside your setup.py. -def _stable_abi_macro(value: Any) -> tuple[str, str]: +def _stable_abi_macro(value: Any, *, abi3t: bool = False) -> tuple[str, str]: """ The define for ``py_limited_api``, as ``(name, hex value)``. True selects the minimum pybind11 supports (3.12 for abi3, 3.15 for abi3t), a string such as "3.13" or "cp313" selects that version, and an int or hex string is - passed through. A "t" suffix ("3.15t", "cp315t") selects abi3t, which is - always used on free-threaded Python. + passed through. A "t" suffix ("3.15t", "cp315t") or ``abi3t=True`` selects + abi3t, which is always used on free-threaded Python. """ - abi3t = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + abi3t = abi3t or bool(sysconfig.get_config_var("Py_GIL_DISABLED")) text = str(value) if isinstance(value, str) and text.lower().endswith("t"): abi3t = True @@ -189,13 +189,21 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: # setuptools only uses the flag for the .abi3 file name; the # define selects the limited API in the headers. kwargs["py_limited_api"] = True - macro = _stable_abi_macro(py_limited_api) - self._abi3t = macro[0] == "Py_TARGET_ABI3T" macros = list(kwargs.get("define_macros", []) or []) - if not any( - name in {"Py_LIMITED_API", "Py_TARGET_ABI3T"} for name, _ in macros - ): + abi_macros = [ + (name, value) + for name, value in macros + if name in {"Py_LIMITED_API", "Py_TARGET_ABI3T"} + ] + if abi_macros: + # Explicit defines decide the ABI, the file name, and the export symbol. + abi3t = any(name == "Py_TARGET_ABI3T" for name, _ in abi_macros) + for _, value in abi_macros: + macro = _stable_abi_macro(value, abi3t=abi3t) + else: + macro = _stable_abi_macro(py_limited_api) macros.append(macro) + self._abi3t = macro[0] == "Py_TARGET_ABI3T" kwargs["define_macros"] = macros super().__init__(*args, **kwargs) diff --git a/tests/extra_setuptools/test_setuphelper.py b/tests/extra_setuptools/test_setuphelper.py index f4c9aca1eb..cdecde903a 100644 --- a/tests/extra_setuptools/test_setuphelper.py +++ b/tests/extra_setuptools/test_setuphelper.py @@ -240,6 +240,62 @@ def test_stable_abi_macro(monkeypatch, building, free_threaded, value, expected) assert _stable_abi_macro(value) == expected +@pytest.mark.parametrize( + ("py_limited_api", "define_macros", "expected"), + [ + (True, [("Py_TARGET_ABI3T", "0x030F0000")], "PyModExport_example"), + (True, [("Py_TARGET_ABI3T", 0x030F0000)], "PyModExport_example"), + ("3.15t", [("Py_LIMITED_API", "0x030C0000")], "PyInit_example"), + (True, [("Py_TARGET_ABI3T", "0x03100000")], ValueError), + (True, [("Py_LIMITED_API", "0x030B0000")], ValueError), + ( + True, + [("Py_LIMITED_API", "0x030C0000"), ("Py_TARGET_ABI3T", "0x030F0000")], + ValueError, + ), + ], +) +def test_stable_abi_explicit_macros( + monkeypatch, py_limited_api, define_macros, expected +): + monkeypatch.syspath_prepend(MAIN_DIR) + from setuptools import Distribution + + from pybind11.setup_helpers import Pybind11Extension, build_ext + + get_config_var = sysconfig.get_config_var + with monkeypatch.context() as target: + target.setattr(sys, "version_info", (3, 15, 0, "final", 0)) + target.setattr( + sysconfig, + "get_config_var", + lambda name: 0 if name == "Py_GIL_DISABLED" else get_config_var(name), + ) + if expected is ValueError: + with pytest.raises(ValueError): + Pybind11Extension( + "example", + [], + include_pybind11=False, + cxx_std=17, + py_limited_api=py_limited_api, + define_macros=define_macros, + ) + return + extension = Pybind11Extension( + "example", + [], + include_pybind11=False, + cxx_std=17, + py_limited_api=py_limited_api, + define_macros=define_macros, + ) + + assert extension.define_macros == define_macros + command = build_ext(Distribution({"ext_modules": [extension]})) + assert command.get_export_symbols(extension) == [expected] + + def test_intree_extensions(monkeypatch, tmpdir): monkeypatch.syspath_prepend(MAIN_DIR) From 90f46d77b80c05146fd1b9d7319dad1f74efa9d6 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 11:54:15 -0400 Subject: [PATCH 47/59] fix(stable-abi): encode the limited API version in the precompiled link guard The guard symbol recorded only whether Py_LIMITED_API was defined, so a library precompiled for one stable ABI version linked into a module built for another. Pass the numeric value as an integral_constant parameter so it becomes part of the mangled name, and reject a different stable ABI version or abi3/abi3t target for pybind11::precompiled in one build tree. Derived from 61f504dc7 by Ralf W. Grosse-Kunstleve (codex GPT-6); the CMake-level link test was not adopted. Assisted-by: ClaudeCode:claude-fable-5-1 --- docs/compiling.rst | 8 ++++---- include/pybind11/detail/internals-inl.h | 2 +- include/pybind11/detail/internals.h | 22 ++++++++++++++-------- tools/pybind11Common.cmake | 16 ++++++++++++++++ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/docs/compiling.rst b/docs/compiling.rst index 094ad88073..b7c6bb14a3 100644 --- a/docs/compiling.rst +++ b/docs/compiling.rst @@ -454,7 +454,7 @@ Requirements and caveats: ``PYBIND11_HAS_SUBINTERPRETER_SUPPORT``, ``PYBIND11_BACKWARD_COMPATIBILITY_TP_DICTOFFSET``, ``PYBIND11_TYPE_CREATION_VIA_SPEC``, and - ``Py_LIMITED_API``. A mismatch produces one + ``Py_LIMITED_API`` (including its numeric target version). A mismatch produces one readable undefined symbol at link time referencing ``pybind11_precompiled_config``. * Configuration macros that only change code inside the library (for example @@ -486,9 +486,9 @@ Requirements and caveats: between projects. Each extension module links its own copy, which keeps pybind11's per-module state the same as in header-only mode. * With ``STABLE_ABI`` the library is compiled against the limited API too. - One build tree cannot mix stable-ABI and regular precompiled modules; the - first ``PRECOMPILE`` target decides, and a later mismatch is a configure - error. + One build tree cannot mix stable-ABI and regular precompiled modules, or + different stable ABI versions or abi3/abi3t targets; the first ``PRECOMPILE`` + target decides, and a later mismatch is a configure error. * Not available with ``PYBIND11_NOPYTHON`` (the library needs Python headers). diff --git a/include/pybind11/detail/internals-inl.h b/include/pybind11/detail/internals-inl.h index ec8daf054c..607809f508 100644 --- a/include/pybind11/detail/internals-inl.h +++ b/include/pybind11/detail/internals-inl.h @@ -262,7 +262,7 @@ PYBIND11_INLINE size_t num_registered_instances() { #if defined(PYBIND11_PRECOMPILED) // Link-time configuration guard; see the declaration in internals.h. -PYBIND11_INLINE void PYBIND11_PRECOMPILED_CONFIG_CHECK() {} +PYBIND11_INLINE void PYBIND11_PRECOMPILED_CONFIG_CHECK(precompiled_limited_api) {} #endif PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h index 50063d890a..9828da48f4 100644 --- a/include/pybind11/detail/internals.h +++ b/include/pybind11/detail/internals.h @@ -48,12 +48,12 @@ #if defined(PYBIND11_PRECOMPILED) // PYBIND11_PRECOMPILED_CONFIG_CHECK names a do-nothing function defined in the precompiled -// pybind11 library. The identifier encodes every configuration macro that must match between -// the library and the modules linking it. PYBIND11_MODULE calls it, so a mismatch (or a -// missing library) surfaces as one readable undefined symbol at link time instead of many -// unrelated ones at run time. If you add a configuration macro that changes the code in the -// -inl.h files, encode it here, add it to the list in docs/compiling.rst, and add it to -// tools/check_inl_headers.py. +// pybind11 library. The identifier encodes the configuration flags, and its argument encodes +// the numeric limited API target that must match between the library and the modules linking it. +// PYBIND11_MODULE calls it, so a mismatch (or a missing library) surfaces as one readable +// undefined symbol at link time instead of many unrelated ones at run time. If you add a +// configuration macro that changes the code in the -inl.h files, encode it here, add it to the +// list in docs/compiling.rst, and add it to tools/check_inl_headers.py. # if defined(Py_GIL_DISABLED) # define PYBIND11_PRECOMPILED_CFG_GD 1 # else @@ -104,7 +104,7 @@ PYBIND11_PRECOMPILED_CFG_LA, \ PYBIND11_PRECOMPILED_CFG_TS) # define PYBIND11_PRECOMPILED_CONFIG_GUARD \ - ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK(); + ::pybind11::detail::PYBIND11_PRECOMPILED_CONFIG_CHECK({}); #else # define PYBIND11_PRECOMPILED_CONFIG_GUARD #endif @@ -113,7 +113,13 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) #if defined(PYBIND11_PRECOMPILED) PYBIND11_NAMESPACE_BEGIN(detail) -void PYBIND11_PRECOMPILED_CONFIG_CHECK(); +// Use the numeric value in the signature so equivalent hexadecimal/decimal spellings match. +# if defined(Py_LIMITED_API) +using precompiled_limited_api = std::integral_constant; +# else +using precompiled_limited_api = std::integral_constant; +# endif +void PYBIND11_PRECOMPILED_CONFIG_CHECK(precompiled_limited_api); PYBIND11_NAMESPACE_END(detail) #endif diff --git a/tools/pybind11Common.cmake b/tools/pybind11Common.cmake index f8906a8fef..774f7320b1 100644 --- a/tools/pybind11Common.cmake +++ b/tools/pybind11Common.cmake @@ -501,6 +501,20 @@ function(pybind11_precompile) "pybind11::precompiled was created for STABLE_ABI=${_existing_sabi}; one build tree " "cannot mix stable-ABI and regular precompiled modules.") endif() + if(ARG_STABLE_ABI) + _pybind11_stable_abi_hex(_sabi_hex) + _pybind11_stable_abi_is_abi3t(_abi3t) + get_target_property(_existing_hex pybind11_precompiled PYBIND11_STABLE_ABI_HEX) + get_target_property(_existing_abi3t pybind11_precompiled PYBIND11_ABI3T) + if(NOT "${_existing_hex}" STREQUAL "${_sabi_hex}" OR NOT "${_existing_abi3t}" STREQUAL + "${_abi3t}") + message( + FATAL_ERROR + "pybind11::precompiled must use the same stable ABI target for every " + "module (existing: ${_existing_hex}, abi3t=${_existing_abi3t}; " + "requested: ${_sabi_hex}, abi3t=${_abi3t}).") + endif() + endif() return() endif() @@ -526,6 +540,8 @@ function(pybind11_precompile) if(ARG_STABLE_ABI) _pybind11_stable_abi_hex(_sabi_hex) _pybind11_stable_abi_is_abi3t(_abi3t) + set_target_properties(pybind11_precompiled PROPERTIES PYBIND11_STABLE_ABI_HEX "${_sabi_hex}" + PYBIND11_ABI3T "${_abi3t}") if(_abi3t) target_compile_definitions(pybind11_precompiled PRIVATE "Py_TARGET_ABI3T=${_sabi_hex}") else() From 0dab0c3f291e5794adbb6ef2bb3b68e615d19364 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 15:16:58 -0400 Subject: [PATCH 48/59] fix(stable-abi): reject PYBIND11_PLUGIN under abi3t; document downstream findings PYBIND11_PLUGIN needs the module_(name, doc) constructor and PyInit_, neither of which exists under abi3t. Fail with a clear message instead of an overload error. Document wheel-building, meson-python, and platform-tagged suffix notes found while testing boost-histogram, contourpy, and hnswlib. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/advanced/stable_abi.rst | 36 +++++++++++++++++++++++++++++--- include/pybind11/detail/common.h | 33 +++++++++++++++++------------ 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index c1d008ee2a..405c1782c0 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -30,7 +30,8 @@ the default for a build tree: ``PYBIND11_STABLE_ABI_VERSION`` (default ``3.12``, raised to ``3.15`` for ``abi3t``) selects the targeted version. On free-threaded Python the modules are always ``abi3t``. Set ``PYBIND11_ABI3T`` to ``ON`` to also build ``abi3t`` -modules with GIL-enabled Python: +modules with GIL-enabled Python (plain CMake only; wheel builders tag the +wheel from the build Python, see `Building wheels`_): .. code-block:: bash @@ -58,8 +59,20 @@ With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for ``abi3t``, which is always used on free-threaded Python; ``Py_TARGET_ABI3T`` -is then defined and the module is named ``*.abi3t.so``. With GIL-enabled +is then defined and the module is named ``*.abi3t*.so``. With GIL-enabled Python, use ``build_ext`` from ``pybind11.setup_helpers`` to get that name. +Set the wheel tag with ``options={"bdist_wheel": {"py_limited_api": "cp312"}}`` +in ``setup()``; ``bdist_wheel`` rejects ``"cp315t"``, so pass ``"cp315"`` and +build on free-threaded Python to get the ``abi3.abi3t`` tag. + +With meson-python, set ``limited-api = true`` in ``[tool.meson-python]`` and +pass ``limited_api: '3.12'`` to ``py.extension_module()``. Meson defines +``Py_LIMITED_API`` from that value, so use ``'3.15'`` when the build Python is +free-threaded: + +.. code-block:: meson + + limited_api = py.get_variable('Py_GIL_DISABLED', 0).to_string() == '1' ? '3.15' : '3.12' With any other build system, define ``Py_LIMITED_API=0x030C0000`` for every translation unit, link no version-specific Python library (``python3.lib`` on @@ -67,6 +80,22 @@ Windows), and name the module ``.abi3.so`` (``.pyd`` on Windows). For ``abi3t``, define ``Py_TARGET_ABI3T=0x030F0000`` instead, link ``python3t.lib`` on Windows, and name the module ``.abi3t.so``. +On CPython 3.15+, build tools can also use the platform-tagged suffixes +``.abi3-.so`` and ``.abi3t-.so`` (such as +``.abi3t-darwin.so``). CPython 3.15+ loads both forms, but CPython 3.12 to +3.14 only load ``.abi3.so``. + +Building wheels +--------------- + +* Build ``abi3`` wheels with the CPython version you target. On 3.15+, + setuptools and CMake name the module ``.abi3-.so``, so a module + that targets 3.12 to 3.14 but is built with 3.15+ has a suffix that its + target versions do not load. +* Build ``abi3t`` wheels with free-threaded CPython 3.15+. With GIL-enabled + Python, scikit-build-core and setuptools tag the wheel ``abi3`` even when + the module is ``abi3t``, so the wheel installs where it cannot load. + pybind11 rejects the combination with PyPy, GraalPy, and free-threaded CPython before 3.15 at compile time: they have no stable ABI. @@ -132,7 +161,8 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and (``PyObject_GetTypeData()``) instead of embedding ``PyObject_HEAD``; * exports the module through the :pep:`793` ``PyModExport_`` hook instead of ``PyInit_``, so ``PYBIND11_MODULE`` is unchanged but - ``py::module_::create_extension_module()`` is not available; + ``py::module_::create_extension_module()``, the ``py::module_(name, doc)`` + constructor, and the deprecated ``PYBIND11_PLUGIN`` are not available; * locks its internals with critical sections on a private object, and keeps one weak reference per registered instance so that the instance registry can hand out strong references safely (``PyWeakref_GetRef()``). This weak diff --git a/include/pybind11/detail/common.h b/include/pybind11/detail/common.h index 5f598dd4f8..b35147add7 100644 --- a/include/pybind11/detail/common.h +++ b/include/pybind11/detail/common.h @@ -549,20 +549,27 @@ return m.ptr(); } \endrst */ -#define PYBIND11_PLUGIN(name) \ - PYBIND11_DEPRECATED("PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE") \ - static PyObject *pybind11_init(); \ - PYBIND11_PLUGIN_IMPL(name) { \ - PYBIND11_CHECK_PYTHON_VERSION \ - PYBIND11_PRECOMPILED_CONFIG_GUARD \ - PYBIND11_ENSURE_INTERNALS_READY \ - try { \ - return pybind11_init(); \ +#if defined(PYBIND11_OPAQUE_PYOBJECT) +# define PYBIND11_PLUGIN(name) \ + static_assert(false, \ + "PYBIND11_PLUGIN is not available under abi3t, use PYBIND11_MODULE"); \ + static PyObject *pybind11_init() +#else +# define PYBIND11_PLUGIN(name) \ + PYBIND11_DEPRECATED("PYBIND11_PLUGIN is deprecated, use PYBIND11_MODULE") \ + static PyObject *pybind11_init(); \ + PYBIND11_PLUGIN_IMPL(name) { \ + PYBIND11_CHECK_PYTHON_VERSION \ + PYBIND11_PRECOMPILED_CONFIG_GUARD \ + PYBIND11_ENSURE_INTERNALS_READY \ + try { \ + return pybind11_init(); \ + } \ + PYBIND11_CATCH_INIT_EXCEPTIONS \ + return nullptr; \ } \ - PYBIND11_CATCH_INIT_EXCEPTIONS \ - return nullptr; \ - } \ - PyObject *pybind11_init() + PyObject *pybind11_init() +#endif // this push is for the next several macros PYBIND11_WARNING_PUSH From 2c8f8834c6d9ac14d7414e49402296b04cd99472 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 15:48:26 -0400 Subject: [PATCH 49/59] docs(stable-abi): note find_package(Python), NumPy 2.5, and PyModExport exports Found while testing iminuit, awkward-cpp, and matplotlib. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/advanced/stable_abi.rst | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 405c1782c0..228200b35c 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -55,6 +55,15 @@ sets, before ``find_package(pybind11)``: set(PYBIND11_STABLE_ABI_VERSION "${SKBUILD_SABI_VERSION}" CACHE STRING "") endif() +If your project calls ``find_package(Python)`` itself, add the component that +scikit-build-core selects (``Development.SABIModule``); pybind11 adds it only +when it finds Python itself: + +.. code-block:: cmake + + find_package(Python REQUIRED COMPONENTS Interpreter Development.Module + ${SKBUILD_SABI_COMPONENT}) + With setuptools, pass ``py_limited_api=True`` (or a version such as ``"3.13"``) to ``Pybind11Extension``; the module is named ``*.abi3.so`` and ``Py_LIMITED_API`` is defined for you. Add a ``t`` (``"3.15t"``) for @@ -115,7 +124,8 @@ available and in how a few things are implemented. * - Classes, functions, casters, STL, numpy, Eigen - supported - Under abi3t, ``numpy.h`` needs NumPy 2.5+ at run time (it reads - array fields through NumPy's abi3t accessors). + array fields through NumPy's abi3t accessors). Older NumPy versions + install on 3.15, so require ``numpy>=2.5`` for abi3t wheels. * - ``py::dynamic_attr()`` - supported - Every instance reserves a ``__dict__`` slot (one pointer) that @@ -162,7 +172,10 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and * exports the module through the :pep:`793` ``PyModExport_`` hook instead of ``PyInit_``, so ``PYBIND11_MODULE`` is unchanged but ``py::module_::create_extension_module()``, the ``py::module_(name, doc)`` - constructor, and the deprecated ``PYBIND11_PLUGIN`` are not available; + constructor, and the deprecated ``PYBIND11_PLUGIN`` are not available. If + you restrict exported symbols (a linker version script, + ``-exported_symbols_list``, or a ``.def`` file), export ``PyModExport_*`` + too; * locks its internals with critical sections on a private object, and keeps one weak reference per registered instance so that the instance registry can hand out strong references safely (``PyWeakref_GetRef()``). This weak From 26cbafb7e4a4c6fc7a1cf8d9350cffa5fa0264c8 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 16:05:07 -0400 Subject: [PATCH 50/59] fix(cmake): treat an empty PYBIND11_STABLE_ABI_VERSION as the default scikit-build-core < 0.11 sets SKBUILD_SABI_COMPONENT but not SKBUILD_SABI_VERSION, so the documented snippet cached an empty version and configure failed with "must be 3.12 or newer, got .". Assisted-by: ClaudeCode:claude-opus-5-5 --- tools/pybind11NewTools.cmake | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tools/pybind11NewTools.cmake b/tools/pybind11NewTools.cmake index 923b8a3f83..25f4dd7cb3 100644 --- a/tools/pybind11NewTools.cmake +++ b/tools/pybind11NewTools.cmake @@ -300,9 +300,13 @@ function(_pybind11_stable_abi_is_abi3t out_var) PARENT_SCOPE) endfunction() -# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 for abi3t (abi3t starts there). +# PYBIND11_STABLE_ABI_VERSION, raised to 3.15 for abi3t (abi3t starts there). Empty (such as +# SKBUILD_SABI_VERSION from scikit-build-core < 0.11) means 3.12. function(_pybind11_stable_abi_version out_var) set(_version "${PYBIND11_STABLE_ABI_VERSION}") + if(_version STREQUAL "") + set(_version "3.12") + endif() _pybind11_stable_abi_is_abi3t(_abi3t) if(_abi3t AND _version VERSION_LESS 3.15) set(_version "3.15") @@ -371,7 +375,7 @@ function(_pybind11_check_stable_abi target_name lib_type) if(lib_type STREQUAL "STATIC") message(FATAL_ERROR "${target_name}: STABLE_ABI is not supported for STATIC libraries.") endif() - if(PYBIND11_STABLE_ABI_VERSION VERSION_LESS 3.12) + if(NOT PYBIND11_STABLE_ABI_VERSION STREQUAL "" AND PYBIND11_STABLE_ABI_VERSION VERSION_LESS 3.12) message(FATAL_ERROR "${target_name}: PYBIND11_STABLE_ABI_VERSION must be 3.12 or newer, " "got ${PYBIND11_STABLE_ABI_VERSION}.") endif() @@ -379,9 +383,10 @@ function(_pybind11_check_stable_abi target_name lib_type) message(FATAL_ERROR "${target_name}: STABLE_ABI requires CPython, found " "${${_Python}_INTERPRETER_ID}.") endif() - if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS PYBIND11_STABLE_ABI_VERSION) - message(FATAL_ERROR "${target_name}: STABLE_ABI ${PYBIND11_STABLE_ABI_VERSION} needs Python " - ">= ${PYBIND11_STABLE_ABI_VERSION} headers, found ${${_Python}_VERSION}.") + _pybind11_stable_abi_version(_version) + if(DEFINED ${_Python}_VERSION AND ${_Python}_VERSION VERSION_LESS _version) + message(FATAL_ERROR "${target_name}: STABLE_ABI ${_version} needs Python >= ${_version} " + "headers, found ${${_Python}_VERSION}.") endif() _pybind11_stable_abi_is_abi3t(_abi3t) if(_abi3t From 5838965729fa020ced46585eb8d2f7bb60f727fc Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 16:05:07 -0400 Subject: [PATCH 51/59] docs(stable-abi): note old scikit-build-core and Py_GIL_DISABLED under abi3t Found while testing correctionlib and pikepdf. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/advanced/stable_abi.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index 228200b35c..e3f5cc91e5 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -46,7 +46,9 @@ With scikit-build-core, set ``wheel.py-api = "cp312"`` in ``pyproject.toml`` the pybind11 defaults produce the matching module in both cases). This only selects the wheel tag, and scikit-build-core ignores it when the build Python is too old or is not CPython, so enable ``STABLE_ABI`` from the variables it -sets, before ``find_package(pybind11)``: +sets, before ``find_package(pybind11)`` (scikit-build-core 0.11+ sets +``SKBUILD_SABI_VERSION``; with older versions it is empty and pybind11 uses its +default): .. code-block:: cmake @@ -187,6 +189,11 @@ Under abi3t, ``PyObject`` and ``PyModuleDef`` are incomplete types and internals tag is ``_stable_ft``: an abi3t module and an abi3 module loaded into the same GIL-enabled interpreter do not share internals (see below). +CPython's headers define ``Py_GIL_DISABLED`` for every abi3t build, so the +macro only says that the code must be thread-safe. To know whether the running +interpreter is free-threaded, check at run time, for example with +``sys._is_gil_enabled()`` or ``sysconfig.get_config_var("Py_GIL_DISABLED")``. + ABI isolation ============= From 091b4ca66596e53d010aacc50a28b8dc7f766071 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 30 Oct 2026 16:42:35 -0400 Subject: [PATCH 52/59] docs(stable-abi): note Development.SABIModule for CMake, Python_EXECUTABLE, and mixed wheels Found while testing OpenTimelineIO, pyhepmc, and cvxpy. Assisted-by: ClaudeCode:claude-opus-5-5 --- docs/advanced/stable_abi.rst | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/advanced/stable_abi.rst b/docs/advanced/stable_abi.rst index e3f5cc91e5..c66f634507 100644 --- a/docs/advanced/stable_abi.rst +++ b/docs/advanced/stable_abi.rst @@ -37,6 +37,10 @@ wheel from the build Python, see `Building wheels`_): cmake -S . -B build -DPYBIND11_STABLE_ABI=ON -DPYBIND11_ABI3T=ON +If your project calls ``find_package(Python)`` itself, add the +``Development.SABIModule`` component; pybind11 adds it only when it finds +Python itself. + ``PRECOMPILE`` can be combined with ``STABLE_ABI``; the precompiled library is then compiled against the limited API too, and one build tree cannot mix stable-ABI and regular precompiled modules. @@ -58,8 +62,7 @@ default): endif() If your project calls ``find_package(Python)`` itself, add the component that -scikit-build-core selects (``Development.SABIModule``); pybind11 adds it only -when it finds Python itself: +scikit-build-core selects: .. code-block:: cmake @@ -106,6 +109,11 @@ Building wheels * Build ``abi3t`` wheels with free-threaded CPython 3.15+. With GIL-enabled Python, scikit-build-core and setuptools tag the wheel ``abi3`` even when the module is ``abi3t``, so the wheel installs where it cannot load. +* When a ``setup.py`` runs CMake, pass ``-DPython_EXECUTABLE=``. + FindPython skips free-threaded interpreters by default, so it can find a + GIL-enabled Python and build an ``abi3`` module into an ``abi3t`` wheel. +* Every extension module in an ``abi3`` or ``abi3t`` wheel must use the + stable ABI; one version-specific module makes the wheel tag wrong. pybind11 rejects the combination with PyPy, GraalPy, and free-threaded CPython before 3.15 at compile time: they have no stable ABI. From 6f3e790e2115e4a459edcb39d1e70b47c252b2c2 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 00:17:26 -0500 Subject: [PATCH 53/59] fix(stable-abi): keep the wrapper alive in get_object_handle(); exact generator check get_object_handle() now returns an owning `object`. Under PYBIND11_OPAQUE_PYOBJECT it returned a handle sliced from a local vector of strong references, so the reference from PyWeakref_GetRef was released before the caller used it. Replace the tp_name match on "generator" with an exact type check against types.GeneratorType under Py_LIMITED_API (PyGen_Check() elsewhere), so a user class named `generator` is no longer treated as a std::vector source. ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/type_caster_base-inl.h | 10 +++++----- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/pybind11-inl.h | 2 +- include/pybind11/stl.h | 15 +++++++++++++-- tests/test_stl.py | 8 ++++++++ 5 files changed, 28 insertions(+), 9 deletions(-) diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index c0683293a9..0e41f6fd5e 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -425,7 +425,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE bool isinstance_generic(handle obj, return isinstance(obj, type); } -PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, +PYBIND11_NOINLINE_ATTR PYBIND11_INLINE object get_object_handle(const void *ptr, const detail::type_info *type) { #if defined(PYBIND11_OPAQUE_PYOBJECT) // values_and_holders() takes the internals lock, which can suspend the instance map's @@ -445,22 +445,22 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, for (auto &wrapper : candidates) { for (const auto &vh : values_and_holders(wrapper.ptr())) { if (vh.type == type) { - return wrapper; + return std::move(wrapper); } } } - return handle(); + return object(); #else return with_instance_map(ptr, [&](instance_map &instances) { auto range = instances.equal_range(ptr); for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return handle(instance_object(it->second)); + return reinterpret_borrow(instance_object(it->second)); } } } - return handle(); + return object(); }); #endif } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 4c7691ff8d..984763ae05 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -283,7 +283,7 @@ class old_style_init_scope { bool was_active_ = false; }; -handle get_object_handle(const void *ptr, const detail::type_info *type); +object get_object_handle(const void *ptr, const detail::type_info *type); // Information about how type_caster_generic::cast() can obtain its source object struct cast_sources { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index 4fd96dae5a..db92e88418 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -437,7 +437,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_INLINE function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { - handle self = get_object_handle(this_ptr, this_type); + object self = get_object_handle(this_ptr, this_type); if (!self) { return function(); } diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index a31bc36ec8..ccff467700 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -82,15 +82,26 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, return tp_name_is_one_of(Py_TYPE(obj), tp_names); } +inline bool object_is_generator(PyObject *obj) { +#if defined(Py_LIMITED_API) + // Exact type check, like PyGen_Check(): a user class named `generator` must not match. + static PyTypeObject *const gen_type = reinterpret_cast( + object(module_::import("types").attr("GeneratorType")).release().ptr()); + return Py_TYPE(obj) == gen_type; +#else + return PyGen_Check(obj) != 0; +#endif +} + inline bool object_is_convertible_to_std_vector(const handle &src) { // Allow sequence-like objects, but not (byte-)string-like objects. if (PySequence_Check(src.ptr()) != 0) { return !PyUnicode_Check(src.ptr()) && !PyBytes_Check(src.ptr()); } // Allow generators, set/frozenset and several common iterable types. - return (PyAnySet_Check(src.ptr()) != 0) + return object_is_generator(src.ptr()) || (PyAnySet_Check(src.ptr()) != 0) || object_is_instance_with_one_of_tp_names( - src.ptr(), {"generator", "dict_keys", "dict_values", "dict_items", "map", "zip"}); + src.ptr(), {"dict_keys", "dict_values", "dict_items", "map", "zip"}); } inline bool object_is_convertible_to_std_set(const handle &src, bool convert) { diff --git a/tests/test_stl.py b/tests/test_stl.py index f3f4ccc736..90fcfe08c7 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -627,6 +627,14 @@ def gen_invalid(): assert not tuple(gen_obj) +def test_list_caster_rejects_class_named_generator(): + class generator: + pass + + with pytest.raises(TypeError, match="incompatible function arguments"): + m.pass_std_vector_int(generator()) + + def test_pass_std_set_int(): fn = m.pass_std_set_int assert fn({3, 15}) == 254 From aa32860924da2694de2102b7e40a7e4e2fe2262c Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 00:18:46 -0500 Subject: [PATCH 54/59] refactor(stable-abi): drop duplicate type_from_spec() declaration; fixed static property dict offset The static property tp_traverse/tp_clear read `__dictoffset__` through a Python attribute lookup on first use. Compute the offset once from the property type's size, and use the same value when the type is created. ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 14 +++++++++----- include/pybind11/detail/class.h | 6 ------ 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index f2a2111261..3eeaee5814 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -94,11 +94,15 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { # if PYBIND11_TYPE_CREATION_VIA_SPEC # if defined(Py_LIMITED_API) -// Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. These two -// functions find it through the type's `__dictoffset__`. +// Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. The offset +// is fixed at type creation; tp_traverse/tp_clear must not run Python attribute lookups. +PYBIND11_INLINE Py_ssize_t static_property_dict_offset() { + static const Py_ssize_t offset = type_basicsize(&PyProperty_Type); + return offset; +} PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { - static const Py_ssize_t offset = type_dictoffset(Py_TYPE(self)); - return reinterpret_cast(reinterpret_cast(self) + offset); + return reinterpret_cast(reinterpret_cast(self) + + static_property_dict_offset()); } extern "C" PYBIND11_INLINE int pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg) { @@ -131,7 +135,7 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { # else // Append the `__dict__` slot to the property layout. (PEP 697 relative offsets are not // applied to `__dictoffset__` on 3.12, so the offset is absolute.) - const auto dictoffset = static_cast(type_basicsize(&PyProperty_Type)); + const auto dictoffset = static_cast(static_property_dict_offset()); PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}}; PyType_Slot slots[] diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index 7206eb278d..b90a7562f1 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -51,12 +51,6 @@ destructor type_type_dealloc(); descrgetfunc property_type_descr_get(); descrsetfunc property_type_descr_set(); -#if PYBIND11_TYPE_CREATION_VIA_SPEC -/// PyType_FromMetaclass(), or pybind11_fail() with the Python error prefixed by `caller`. -PyTypeObject * -type_from_spec(const char *caller, PyTypeObject *metaclass, PyType_Spec *spec, PyObject *bases); -#endif - #if !defined(PYPY_VERSION) # if defined(Py_LIMITED_API) From bd38b6994fe58e726dae14cb7fc08081bd9cc175 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 00:20:36 -0500 Subject: [PATCH 55/59] refactor(stable-abi): one __dict__ slot lookup for instances and static properties Under Py_LIMITED_API, instance_dict_ptr() now picks the slot from the layout (instance::dict, or the slot pybind11_static_property appends to property), so the static property type reuses pybind11_traverse/pybind11_clear instead of its own traverse/clear pair. ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/class-inl.h | 56 +++++++++++------------------ include/pybind11/detail/class.h | 2 ++ 2 files changed, 23 insertions(+), 35 deletions(-) diff --git a/include/pybind11/detail/class-inl.h b/include/pybind11/detail/class-inl.h index 3eeaee5814..2a954c6c1d 100644 --- a/include/pybind11/detail/class-inl.h +++ b/include/pybind11/detail/class-inl.h @@ -93,29 +93,6 @@ pybind11_static_set(PyObject *self, PyObject *obj, PyObject *value) { # if PYBIND11_TYPE_CREATION_VIA_SPEC -# if defined(Py_LIMITED_API) -// Without Py_TPFLAGS_MANAGED_DICT the `__dict__` slot follows the property object. The offset -// is fixed at type creation; tp_traverse/tp_clear must not run Python attribute lookups. -PYBIND11_INLINE Py_ssize_t static_property_dict_offset() { - static const Py_ssize_t offset = type_basicsize(&PyProperty_Type); - return offset; -} -PYBIND11_INLINE PyObject **static_property_dict_ptr(PyObject *self) { - return reinterpret_cast(reinterpret_cast(self) - + static_property_dict_offset()); -} -extern "C" PYBIND11_INLINE int -pybind11_static_property_traverse(PyObject *self, visitproc visit, void *arg) { - Py_VISIT(*static_property_dict_ptr(self)); - Py_VISIT(Py_TYPE(self)); - return 0; -} -extern "C" PYBIND11_INLINE int pybind11_static_property_clear(PyObject *self) { - Py_CLEAR(*static_property_dict_ptr(self)); - return 0; -} -# endif - PYBIND11_INLINE PyTypeObject *make_static_property_type() { // Since Python-3.12 property-derived types are required to have dynamic attributes (to set // `__doc__`), hence the GC and dict slots. @@ -138,14 +115,13 @@ PYBIND11_INLINE PyTypeObject *make_static_property_type() { const auto dictoffset = static_cast(static_property_dict_offset()); PyMemberDef members[] = {{"__dictoffset__", Py_T_PYSSIZET, dictoffset, Py_READONLY, nullptr}, {nullptr, 0, 0, 0, nullptr}}; - PyType_Slot slots[] - = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, - {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, - {Py_tp_traverse, reinterpret_cast(pybind11_static_property_traverse)}, - {Py_tp_clear, reinterpret_cast(pybind11_static_property_clear)}, - {Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}, - {Py_tp_members, reinterpret_cast(members)}, - {0, nullptr}}; + PyType_Slot slots[] = {{Py_tp_descr_get, reinterpret_cast(pybind11_static_get)}, + {Py_tp_descr_set, reinterpret_cast(pybind11_static_set)}, + {Py_tp_traverse, reinterpret_cast(pybind11_traverse)}, + {Py_tp_clear, reinterpret_cast(pybind11_clear)}, + {Py_tp_getset, reinterpret_cast(dynamic_attr_getset())}, + {Py_tp_members, reinterpret_cast(members)}, + {0, nullptr}}; PyType_Spec spec = {PYBIND11_DUMMY_MODULE_NAME ".pybind11_static_property", dictoffset + static_cast(sizeof(PyObject *)), 0, @@ -685,10 +661,14 @@ PYBIND11_INLINE PyObject **instance_dict_ptr(PyObject *self) { #elif !defined(Py_LIMITED_API) return _PyObject_GetDictPtr(self); #else - // The slot every pybind11 instance reserves; only `py::dynamic_attr()` types point their - // `__dictoffset__` at it, so it stays null for the others. Python subclasses that add a - // managed dict clear that themselves before calling the base tp_dealloc. - return &get_instance(self)->dict; + // No tp_dictoffset in the stable ABI. pybind11 creates two layouts with a `__dict__` slot: + // instances (instance::dict, which only `py::dynamic_attr()` types point their + // `__dictoffset__` at, so it stays null for the others; Python subclasses that add a managed + // dict clear that themselves before calling the base tp_dealloc) and static properties. + Py_ssize_t offset = PyType_IsSubtype(Py_TYPE(self), &PyProperty_Type) + ? static_property_dict_offset() + : instance_dict_offset(); + return reinterpret_cast(reinterpret_cast(self) + offset); #endif } @@ -700,6 +680,12 @@ PYBIND11_INLINE Py_ssize_t instance_dict_offset() { return static_cast(offsetof(instance, dict)); # endif } + +PYBIND11_INLINE Py_ssize_t static_property_dict_offset() { + // Appended to the property layout; see make_static_property_type(). + static const Py_ssize_t offset = type_basicsize(&PyProperty_Type); + return offset; +} #endif #if defined(PYBIND11_OPAQUE_PYOBJECT) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index b90a7562f1..3e0adaab74 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -154,6 +154,8 @@ PyObject **instance_dict_ptr(PyObject *self); /// Offset of `instance::dict` from the start of the Python object (the `__dictoffset__` of /// `py::dynamic_attr()` types). Py_ssize_t instance_dict_offset(); +/// Offset of the `__dict__` slot that pybind11_static_property appends to the property layout. +Py_ssize_t static_property_dict_offset(); #endif /// Instance creation function for all pybind11 types. It allocates the internal instance layout From 3c6c309a66c9a9d0a80d99aee2493f372e1b7d78 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 00:24:58 -0500 Subject: [PATCH 56/59] fix(stable-abi): get_object_handle() returns a borrowed handle again The opaque-PyObject path borrows from the wrapper like the non-opaque path: the Python owner of the C++ object keeps the wrapper alive for the caller. ClaudeCode:claude-fable-5-1 --- include/pybind11/detail/type_caster_base-inl.h | 12 +++++++----- include/pybind11/detail/type_caster_base.h | 2 +- include/pybind11/pybind11-inl.h | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/include/pybind11/detail/type_caster_base-inl.h b/include/pybind11/detail/type_caster_base-inl.h index 0e41f6fd5e..83049c5b43 100644 --- a/include/pybind11/detail/type_caster_base-inl.h +++ b/include/pybind11/detail/type_caster_base-inl.h @@ -425,7 +425,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE bool isinstance_generic(handle obj, return isinstance(obj, type); } -PYBIND11_NOINLINE_ATTR PYBIND11_INLINE object get_object_handle(const void *ptr, +PYBIND11_NOINLINE_ATTR PYBIND11_INLINE handle get_object_handle(const void *ptr, const detail::type_info *type) { #if defined(PYBIND11_OPAQUE_PYOBJECT) // values_and_holders() takes the internals lock, which can suspend the instance map's @@ -445,22 +445,24 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE object get_object_handle(const void *ptr, for (auto &wrapper : candidates) { for (const auto &vh : values_and_holders(wrapper.ptr())) { if (vh.type == type) { - return std::move(wrapper); + // Borrowed: the Python owner of `ptr` keeps the wrapper alive, as in the + // non-opaque path. + return handle(wrapper); } } } - return object(); + return handle(); #else return with_instance_map(ptr, [&](instance_map &instances) { auto range = instances.equal_range(ptr); for (auto it = range.first; it != range.second; ++it) { for (const auto &vh : values_and_holders(it->second)) { if (vh.type == type) { - return reinterpret_borrow(instance_object(it->second)); + return handle(instance_object(it->second)); } } } - return object(); + return handle(); }); #endif } diff --git a/include/pybind11/detail/type_caster_base.h b/include/pybind11/detail/type_caster_base.h index 984763ae05..4c7691ff8d 100644 --- a/include/pybind11/detail/type_caster_base.h +++ b/include/pybind11/detail/type_caster_base.h @@ -283,7 +283,7 @@ class old_style_init_scope { bool was_active_ = false; }; -object get_object_handle(const void *ptr, const detail::type_info *type); +handle get_object_handle(const void *ptr, const detail::type_info *type); // Information about how type_caster_generic::cast() can obtain its source object struct cast_sources { diff --git a/include/pybind11/pybind11-inl.h b/include/pybind11/pybind11-inl.h index db92e88418..4fd96dae5a 100644 --- a/include/pybind11/pybind11-inl.h +++ b/include/pybind11/pybind11-inl.h @@ -437,7 +437,7 @@ PYBIND11_NAMESPACE_BEGIN(detail) PYBIND11_INLINE function get_type_override(const void *this_ptr, const type_info *this_type, const char *name) { - object self = get_object_handle(this_ptr, this_type); + handle self = get_object_handle(this_ptr, this_type); if (!self) { return function(); } From 9af17629dafdb9a3f2e2f85e0e53c913c1f04ccf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 11:48:56 -0500 Subject: [PATCH 57/59] fix(stable-abi): GIL-safe generator type cache; read numpy void scalars from their own storage object_is_generator() imported `types` inside a function-local static initializer. A cold import can release the GIL while another thread holds it and waits on the once-only guard, so neither thread can continue. Use gil_safe_call_once_and_store like the other limited-API caches. The opaque numpy void scalar converter used PyObject_GetBuffer(), which honors a subclass __buffer__ override and can hand back a temporary that is released before the C++ value is read. Call the base type's buffer slot instead, which exposes the scalar's own storage. Assisted-by: ClaudeCode:claude-fable-5-1 --- include/pybind11/numpy.h | 8 ++++-- include/pybind11/stl.h | 12 ++++++--- tests/test_numpy_dtypes.py | 19 +++++++++++++ tests/test_stl.py | 55 +++++++++++++++++++++++++++++++++++++- 4 files changed, 88 insertions(+), 6 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 34374c1f5b..2ba2614684 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1858,9 +1858,13 @@ struct npy_format_descriptor { if (auto descr = reinterpret_steal(api.PyArray_DescrFromScalar_(obj))) { if (api.PyArray_EquivTypes_(dtype_ptr(), descr.ptr())) { #if defined(PYBIND11_OPAQUE_PYOBJECT) - // The void scalar is opaque; its buffer is the scalar's own storage. + // The scalar struct is opaque. Use the base type's buffer slot: it exposes the + // scalar's own storage, which outlives the view. A subclass __buffer__ + // override may return a temporary, so it must not be used. + auto getbuffer = reinterpret_cast( + PyType_GetSlot(api.PyVoidArrType_Type_, Py_bf_getbuffer)); Py_buffer view; - if (PyObject_GetBuffer(obj, &view, PyBUF_SIMPLE) != 0) { + if (getbuffer == nullptr || getbuffer(obj, &view, PyBUF_SIMPLE) != 0) { PyErr_Clear(); return false; } diff --git a/include/pybind11/stl.h b/include/pybind11/stl.h index ccff467700..ec67b24eb1 100644 --- a/include/pybind11/stl.h +++ b/include/pybind11/stl.h @@ -85,9 +85,15 @@ inline bool object_is_instance_with_one_of_tp_names(PyObject *obj, inline bool object_is_generator(PyObject *obj) { #if defined(Py_LIMITED_API) // Exact type check, like PyGen_Check(): a user class named `generator` must not match. - static PyTypeObject *const gen_type = reinterpret_cast( - object(module_::import("types").attr("GeneratorType")).release().ptr()); - return Py_TYPE(obj) == gen_type; + // The import can release the GIL, so a plain function-local static could deadlock. + PYBIND11_CONSTINIT static gil_safe_call_once_and_store storage; + return Py_TYPE(obj) + == storage + .call_once_and_store_result([] { + return reinterpret_cast( + object(module_::import("types").attr("GeneratorType")).release().ptr()); + }) + .get_stored(); #else return PyGen_Check(obj) != 0; #endif diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index 2e5a67ba8f..a278809cbb 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import sys import pytest @@ -415,6 +416,24 @@ def test_scalar_conversion(): assert "incompatible function arguments" in str(excinfo.value) +@pytest.mark.skipif( + sys.version_info < (3, 12), reason="__buffer__ requires Python 3.12" +) +def test_scalar_subclass_buffer_override(simple_dtype): + """Conversion reads the scalar's own storage, not an overridden __buffer__.""" + arr = np.array([(False, 2, 0.0, 0.0)], dtype=simple_dtype) + + class Scalar(np.void): + def __buffer__(self, flags): + self._tmp = np.array([(False, 7, 0.0, 0.0)], dtype=simple_dtype).tobytes() + return memoryview(self._tmp) + + scalar = arr[0].view(np.dtype((Scalar, simple_dtype))) + assert isinstance(scalar, Scalar) + assert bytes(memoryview(scalar)) == scalar._tmp + assert m.f_simple(scalar) == 20 + + def test_vectorize(): n = 3 array = m.create_rec_simple(n) diff --git a/tests/test_stl.py b/tests/test_stl.py index 90fcfe08c7..f16a5dc743 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -1,10 +1,15 @@ from __future__ import annotations +import os +import subprocess +import sys +import textwrap import weakref import pytest -import env # noqa: F401 +import env +import pybind11_tests from pybind11_tests import ConstructorStats, UserType from pybind11_tests import stl as m @@ -635,6 +640,54 @@ class generator: m.pass_std_vector_int(generator()) +@pytest.mark.skipif(sys.platform.startswith("emscripten"), reason="Requires threads") +@pytest.mark.skipif(env.ANDROID or env.IOS, reason="Requires subprocess support") +def test_generator_check_does_not_deadlock(): + """Two threads doing the first generator conversion at once must both finish. + + The stable-ABI generator check imports `types` once. If the import releases the GIL + while another thread holds it and waits on the once-only guard, neither thread can + proceed. Runs in a subprocess so a regression shows up as a timeout, not a hang. + """ + script = textwrap.dedent( + f""" + import sys + import threading + + sys.path.insert(0, {os.path.dirname(env.__file__)!r}) + sys.path.insert(0, {os.path.dirname(pybind11_tests.__file__)!r}) + + from pybind11_tests import stl as m + + # A cold import runs importlib Python code, which can release the GIL. + del sys.modules["types"] + sys.setswitchinterval(1e-6) + start = threading.Barrier(2) + + def convert(): + start.wait() + for _ in range(100): + assert m.pass_std_vector_int(x for x in (1, 2, 3)) == 112 + + threads = [threading.Thread(target=convert) for _ in range(2)] + for t in threads: + t.start() + for t in threads: + t.join() + print("done") + """ + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=30, + check=False, + ) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "done" + + def test_pass_std_set_int(): fn = m.pass_std_set_int assert fn({3, 15}) == 254 From 6f94025dc2c881794487335c326ff2ed072d81a0 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 3 Nov 2026 15:28:16 -0500 Subject: [PATCH 58/59] test(stable-abi): skip the numpy __buffer__ override test where the override is ignored GraalPy's memoryview() does not call a Python-level __buffer__, so the test cannot set up the dangling-pointer scenario there. Assisted-by: ClaudeCode:claude-fable-5-1 --- tests/test_numpy_dtypes.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index a278809cbb..5479b44090 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -422,15 +422,16 @@ def test_scalar_conversion(): def test_scalar_subclass_buffer_override(simple_dtype): """Conversion reads the scalar's own storage, not an overridden __buffer__.""" arr = np.array([(False, 2, 0.0, 0.0)], dtype=simple_dtype) + other = np.array([(False, 7, 0.0, 0.0)], dtype=simple_dtype).tobytes() class Scalar(np.void): def __buffer__(self, flags): - self._tmp = np.array([(False, 7, 0.0, 0.0)], dtype=simple_dtype).tobytes() - return memoryview(self._tmp) + return memoryview(other) scalar = arr[0].view(np.dtype((Scalar, simple_dtype))) assert isinstance(scalar, Scalar) - assert bytes(memoryview(scalar)) == scalar._tmp + if bytes(memoryview(scalar)) != other: + pytest.skip("__buffer__ override is not honored") # GraalPy assert m.f_simple(scalar) == 20 From bdcd3c34b96cf3f86ff9d9213cac7ee8370adb56 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Fri, 6 Nov 2026 22:41:09 -0800 Subject: [PATCH 59/59] fix(numpy): pair base scalar buffer acquisition and release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit — codex GPT-6 on behalf of Ralf — Use NumPy's base bf_releasebuffer slot after directly acquiring its void scalar buffer, then drop the export's owning reference. Generic PyBuffer_Release dispatches through the scalar subclass and can invoke __release_buffer__ without a corresponding __buffer__ call. Extend the existing scalar-subclass regression to verify that neither buffer hook runs during conversion. Close the capability-check memoryview before clearing the recorded hooks. --- include/pybind11/numpy.h | 8 +++++++- tests/test_numpy_dtypes.py | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/include/pybind11/numpy.h b/include/pybind11/numpy.h index 2ba2614684..99f1e542b5 100644 --- a/include/pybind11/numpy.h +++ b/include/pybind11/numpy.h @@ -1869,7 +1869,13 @@ struct npy_format_descriptor { return false; } value = view.buf; - PyBuffer_Release(&view); + // Match the base getter; PyBuffer_Release would invoke subclass release hooks. + auto releasebuffer = reinterpret_cast( + PyType_GetSlot(api.PyVoidArrType_Type_, Py_bf_releasebuffer)); + if (releasebuffer != nullptr) { + releasebuffer(view.obj, &view); + } + Py_XDECREF(view.obj); #else value = ((PyVoidScalarObject_Proxy *) obj)->obval; #endif diff --git a/tests/test_numpy_dtypes.py b/tests/test_numpy_dtypes.py index 5479b44090..94ff251f6f 100644 --- a/tests/test_numpy_dtypes.py +++ b/tests/test_numpy_dtypes.py @@ -420,19 +420,28 @@ def test_scalar_conversion(): sys.version_info < (3, 12), reason="__buffer__ requires Python 3.12" ) def test_scalar_subclass_buffer_override(simple_dtype): - """Conversion reads the scalar's own storage, not an overridden __buffer__.""" + """Conversion reads the scalar's own storage without invoking subclass buffer hooks.""" arr = np.array([(False, 2, 0.0, 0.0)], dtype=simple_dtype) other = np.array([(False, 7, 0.0, 0.0)], dtype=simple_dtype).tobytes() + calls = [] class Scalar(np.void): def __buffer__(self, flags): + calls.append("acquire") return memoryview(other) + def __release_buffer__(self, view): + calls.append("release") + scalar = arr[0].view(np.dtype((Scalar, simple_dtype))) assert isinstance(scalar, Scalar) - if bytes(memoryview(scalar)) != other: - pytest.skip("__buffer__ override is not honored") # GraalPy + with memoryview(scalar) as view: + if bytes(view) != other: + pytest.skip("__buffer__ override is not honored") # GraalPy + assert calls == ["acquire", "release"] + calls.clear() assert m.f_simple(scalar) == 20 + assert calls == [] def test_vectorize():