From cabb466057c53d1ff065384ff86c8a98719f3bc2 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 20:59:57 -0700 Subject: [PATCH 1/5] Use strong reference APIs. For the free-threaded build, it is not safe use borrowed references. Another thread could deallocate the object and cause the reference to become invalid. Replace API calls that borrow references with strong reference APIs. --- src/dict.cpp | 8 ++++++++ src/object/function_doc_signature.cpp | 15 +++++++++++++-- src/wrapper.cpp | 20 ++++++++++++++------ 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/dict.cpp b/src/dict.cpp index 77d840d455..296bc21e9c 100644 --- a/src/dict.cpp +++ b/src/dict.cpp @@ -68,8 +68,16 @@ object dict_base::get(object_cref k) const { if (check_exact(this)) { +#ifdef Py_GIL_DISABLED + PyObject* result; + if (PyDict_GetItemRef(this->ptr(),k.ptr(),&result) < 0) { + throw_error_already_set(); + } + return object(detail::new_reference(result ? result : Py_None)); +#else PyObject* result = PyDict_GetItem(this->ptr(),k.ptr()); return object(detail::borrowed_reference(result ? result : Py_None)); +#endif } else { diff --git a/src/object/function_doc_signature.cpp b/src/object/function_doc_signature.cpp index 18d458698d..76b620dcb9 100644 --- a/src/object/function_doc_signature.cpp +++ b/src/object/function_doc_signature.cpp @@ -135,7 +135,15 @@ namespace boost { namespace python { namespace objects { str name(get_qualname(py_type)); if ( py_type->tp_flags & Py_TPFLAGS_HEAPTYPE ) { // Qualify the type name if it is defined in a different module. - PyObject *type_module_name = PyDict_GetItemString(py_type->tp_dict, "__module__"); + PyObject *type_module_name; +#if PY_VERSION_HEX >= 0x030D0000 + if (PyDict_GetItemStringRef(py_type->tp_dict, "__module__", &type_module_name) < 0) { + throw_error_already_set(); + } +#else + type_module_name = PyDict_GetItemString(py_type->tp_dict, "__module__"); + Py_XINCREF(type_module_name); +#endif if ( type_module_name && PyObject_RichCompareBool( @@ -144,8 +152,11 @@ namespace boost { namespace python { namespace objects { Py_NE ) != 0 ) { - return str("%s.%s" % make_tuple(handle<>(borrowed(type_module_name)), name)); + str result = str("%s.%s" % make_tuple(handle<>(type_module_name), name)); + return result; } + // Clean up the strong reference if we didn't use it + Py_XDECREF(type_module_name); } return name; } else { diff --git a/src/wrapper.cpp b/src/wrapper.cpp index 8b1b884769..2b053d8311 100644 --- a/src/wrapper.cpp +++ b/src/wrapper.cpp @@ -21,20 +21,28 @@ namespace detail this->m_self, const_cast(name)))) ) { - PyObject* borrowed_f = 0; - + PyObject* class_f = 0; + if ( PyMethod_Check(m.get()) && PyMethod_GET_SELF(m.get()) == this->m_self && class_object->tp_dict != 0 ) { - borrowed_f = ::PyDict_GetItemString( +#if PY_VERSION_HEX >= 0x030D0000 + if (::PyDict_GetItemStringRef( + class_object->tp_dict, const_cast(name), &class_f) < 0) { + throw_error_already_set(); + } +#else + class_f = ::PyDict_GetItemString( class_object->tp_dict, const_cast(name)); - - + Py_XINCREF(class_f); +#endif } - if (borrowed_f != PyMethod_GET_FUNCTION(m.get())) + bool is_override = (class_f != PyMethod_GET_FUNCTION(m.get())); + Py_XDECREF(class_f); + if (is_override) return override(m); } } From 6f5f3b66074b27b44ddc5f1198003d0b01d31a52 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 30 Oct 2025 14:23:37 -0700 Subject: [PATCH 2/5] Add work-around to crash in ~object_base(). For the free-threaded build (and possibly the debug build), it is not safe to call Py_DECREF() if there is no valid Python thread-state. --- include/boost/python/object_core.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/boost/python/object_core.hpp b/include/boost/python/object_core.hpp index 16480d0d89..074360d415 100644 --- a/include/boost/python/object_core.hpp +++ b/include/boost/python/object_core.hpp @@ -419,6 +419,16 @@ inline api::object_base& api::object_base::operator=(api::object_base const& rhs inline api::object_base::~object_base() { +#ifdef Py_GIL_DISABLED + // This is a not very elegant fix for a problem that occurs with the + // free-threaded build of Python. If this is called when the interpreter + // has already been finalized, the thread-state can be null. Unlike the + // GIL-enabled build, Py_DECREF() requires a valid thread-state. This + // causes a memory leak, rather than crash, which seems preferable. + if (PyThreadState_GetUnchecked() == NULL) { + return; + } +#endif assert( Py_REFCNT(m_ptr) > 0 ); Py_DECREF(m_ptr); } From cfbefe893ca6254fd2377c4f6fcdacfd3b851680 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 22:42:42 -0700 Subject: [PATCH 3/5] Use re-entrant mutex to protect global state. Add pymutex.hpp which implements a re-entrant mutex on top of Python's PyMutex. Add BOOST_PYTHON_LOCK_STATE() macro that uses RAII to lock mutable global state as required. --- include/boost/python/detail/pymutex.hpp | 103 ++++++++++++++++++++++++ src/converter/from_python.cpp | 7 +- src/converter/registry.cpp | 9 ++- src/converter/type_id.cpp | 6 +- src/errors.cpp | 22 ++++- src/object/inheritance.cpp | 6 ++ 6 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 include/boost/python/detail/pymutex.hpp diff --git a/include/boost/python/detail/pymutex.hpp b/include/boost/python/detail/pymutex.hpp new file mode 100644 index 0000000000..2d2e2d6266 --- /dev/null +++ b/include/boost/python/detail/pymutex.hpp @@ -0,0 +1,103 @@ +// Copyright 2025 Boost.Python Contributors +// Distributed under the Boost Software License, Version 1.0. (See +// accompanying file LICENSE_1_0.txt or copy at +// http://www.boost.org/LICENSE_1_0.txt) + +#ifndef BOOST_PYTHON_DETAIL_PYMUTEX_HPP +#define BOOST_PYTHON_DETAIL_PYMUTEX_HPP + +#include +#ifdef Py_GIL_DISABLED +// needed for pymutex wrapper +#include +#include +#endif + +namespace boost { namespace python { namespace detail { + +#ifdef Py_GIL_DISABLED + +// Re-entrant wrapper around PyMutex for free-threaded Python +// Similar to _PyRecursiveMutex or threading.RLock +class pymutex { + PyMutex m_mutex; + std::atomic m_owner; + std::size_t m_level; + +public: + pymutex() : m_mutex({}), m_owner(0), m_level(0) {} + + // Non-copyable, non-movable + pymutex(const pymutex&) = delete; + pymutex& operator=(const pymutex&) = delete; + + void lock() { + unsigned long thread = PyThread_get_thread_ident(); + if (m_owner.load(std::memory_order_relaxed) == thread) { + m_level++; + return; + } + PyMutex_Lock(&m_mutex); + m_owner.store(thread, std::memory_order_relaxed); + // m_level should be 0 when we acquire the lock + } + + void unlock() { + unsigned long thread = PyThread_get_thread_ident(); + // Verify current thread owns the lock + if (m_owner.load(std::memory_order_relaxed) != thread) { + // This should never happen - programming error + return; + } + if (m_level > 0) { + m_level--; + return; + } + m_owner.store(0, std::memory_order_relaxed); + PyMutex_Unlock(&m_mutex); + } + + bool is_locked_by_current_thread() const { + unsigned long thread = PyThread_get_thread_ident(); + return m_owner.load(std::memory_order_relaxed) == thread; + } +}; + + +// RAII lock guard for pymutex +class pymutex_guard { + pymutex& m_mutex; + +public: + explicit pymutex_guard(pymutex& mutex) : m_mutex(mutex) { + m_mutex.lock(); + } + + ~pymutex_guard() { + m_mutex.unlock(); + } + + // Non-copyable, non-movable + pymutex_guard(const pymutex_guard&) = delete; + pymutex_guard& operator=(const pymutex_guard&) = delete; +}; + +// Global mutex for protecting all Boost.Python internal state +// Similar to pybind11's internals.mutex +BOOST_PYTHON_DECL pymutex& get_global_mutex(); + +// Macro for acquiring the global lock +// Similar to pybind11's PYBIND11_LOCK_INTERNALS +#define BOOST_PYTHON_LOCK_STATE() \ + ::boost::python::detail::pymutex_guard lock(::boost::python::detail::get_global_mutex()) + +#else + +// No-op macro when not in free-threaded mode +#define BOOST_PYTHON_LOCK_STATE() + +#endif // Py_GIL_DISABLED + +}}} // namespace boost::python::detail + +#endif // BOOST_PYTHON_DETAIL_PYMUTEX_HPP diff --git a/src/converter/from_python.cpp b/src/converter/from_python.cpp index f3989ba77f..53a149fa72 100644 --- a/src/converter/from_python.cpp +++ b/src/converter/from_python.cpp @@ -11,6 +11,7 @@ #include #include +#include #include #include @@ -145,6 +146,8 @@ namespace inline bool visit(rvalue_from_python_chain const* chain) { + BOOST_PYTHON_LOCK_STATE(); + visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain); if (p != visited.end() && *p == chain) return false; @@ -157,9 +160,11 @@ namespace { unvisit(rvalue_from_python_chain const* chain) : chain(chain) {} - + ~unvisit() { + BOOST_PYTHON_LOCK_STATE(); + visited_t::iterator const p = std::lower_bound(visited.begin(), visited.end(), chain); assert(p != visited.end()); visited.erase(p); diff --git a/src/converter/registry.cpp b/src/converter/registry.cpp index aa20c3f685..1b23dbef48 100644 --- a/src/converter/registry.cpp +++ b/src/converter/registry.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -112,9 +113,9 @@ registration::~registration() namespace // { typedef registration entry; - + typedef std::set registry_t; - + #ifndef BOOST_PYTHON_CONVERTER_REGISTRY_APPLE_MACH_WORKAROUND registry_t& entries() { @@ -181,6 +182,8 @@ namespace // entry* get(type_info type, bool is_shared_ptr = false) { + BOOST_PYTHON_LOCK_STATE(); + # ifdef BOOST_PYTHON_TRACE_REGISTRY registry_t::iterator p = entries().find(entry(type)); @@ -293,6 +296,8 @@ namespace registry registration const* query(type_info type) { + BOOST_PYTHON_LOCK_STATE(); + registry_t::iterator p = entries().find(entry(type)); # ifdef BOOST_PYTHON_TRACE_REGISTRY std::cout << "querying " << type diff --git a/src/converter/type_id.cpp b/src/converter/type_id.cpp index c6a8bf7a04..fafb13619c 100644 --- a/src/converter/type_id.cpp +++ b/src/converter/type_id.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include #include @@ -81,7 +82,7 @@ namespace { free_mem(char*p) : p(p) {} - + ~free_mem() { std::free(p); @@ -92,6 +93,7 @@ namespace bool cxxabi_cxa_demangle_is_broken() { + BOOST_PYTHON_LOCK_STATE(); static bool was_tested = false; static bool is_broken = false; if (!was_tested) { @@ -109,6 +111,8 @@ namespace detail { BOOST_PYTHON_DECL char const* gcc_demangle(char const* mangled) { + BOOST_PYTHON_LOCK_STATE(); + typedef std::vector< std::pair > mangling_map; diff --git a/src/errors.cpp b/src/errors.cpp index 34ea22f43e..7f6b1880d5 100644 --- a/src/errors.cpp +++ b/src/errors.cpp @@ -10,9 +10,21 @@ #include #include #include +#include namespace boost { namespace python { +#ifdef Py_GIL_DISABLED +namespace detail { + // Global mutex for protecting all Boost.Python internal state + pymutex& get_global_mutex() + { + static pymutex mutex; + return mutex; + } +} +#endif + error_already_set::~error_already_set() {} // IMPORTANT: this function may only be called from within a catch block! @@ -20,8 +32,13 @@ BOOST_PYTHON_DECL bool handle_exception_impl(function0 f) { try { - if (detail::exception_handler::chain) - return detail::exception_handler::chain->handle(f); + detail::exception_handler* handler_chain = nullptr; + { + BOOST_PYTHON_LOCK_STATE(); + handler_chain = detail::exception_handler::chain; + } + if (handler_chain) + return handler_chain->handle(f); f(); return false; } @@ -80,6 +97,7 @@ exception_handler::exception_handler(handler_function const& impl) : m_impl(impl) , m_next(0) { + BOOST_PYTHON_LOCK_STATE(); if (chain != 0) tail->m_next = this; else diff --git a/src/object/inheritance.cpp b/src/object/inheritance.cpp index a7b3156e41..44062875a4 100644 --- a/src/object/inheritance.cpp +++ b/src/object/inheritance.cpp @@ -4,6 +4,7 @@ // http://www.boost.org/LICENSE_1_0.txt) #include #include +#include #include #if _MSC_FULL_VER >= 13102171 && _MSC_FULL_VER <= 13102179 # include @@ -390,6 +391,8 @@ namespace inline void* convert_type(void* const p, class_id src_t, class_id dst_t, bool polymorphic) { + BOOST_PYTHON_LOCK_STATE(); + // Quickly rule out unregistered types index_entry* src_p = seek_type(src_t); if (src_p == 0) @@ -452,6 +455,8 @@ BOOST_PYTHON_DECL void* find_static_type(void* p, class_id src_t, class_id dst_t BOOST_PYTHON_DECL void add_cast( class_id src_t, class_id dst_t, cast_function cast, bool is_downcast) { + BOOST_PYTHON_LOCK_STATE(); + // adding an edge will invalidate any record of unreachability in // the cache. static std::size_t expected_cache_len = 0; @@ -490,6 +495,7 @@ BOOST_PYTHON_DECL void add_cast( BOOST_PYTHON_DECL void register_dynamic_id_aux( class_id static_id, dynamic_id_function get_dynamic_id) { + BOOST_PYTHON_LOCK_STATE(); tuples::get(*demand_type(static_id)) = get_dynamic_id; } From fc68878e02bb2cb9a0e107a24daf890b5ffc4477 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Mon, 13 Oct 2025 20:59:37 -0700 Subject: [PATCH 4/5] Set the Py_MOD_GIL_NOT_USED flag on modules. This indicates that the free-threaded build of Python can keep the GIL disabled when the module is loaded. Without this module flag, importing the module will cause the GIL to be re-enabled. A warning is emitted if this happens. --- src/module.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/module.cpp b/src/module.cpp index 57675fa2df..707e433941 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -40,8 +40,14 @@ BOOST_PYTHON_DECL void scope_setattr_doc(char const* name, object const& x, char BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, void(*init_function)()) { + PyObject *mod = PyModule_Create(&moduledef); +#ifdef Py_GIL_DISABLED + if (mod != NULL) { + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + } +#endif return init_module_in_scope( - PyModule_Create(&moduledef), + mod, init_function); } From 80731af9ee0541017fbcdbf920a07731388342e0 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 6 Nov 2025 15:45:22 -0800 Subject: [PATCH 5/5] Add "nogil" option for BOOST_PYTHON_MODULE_INIT. Implement optional arguments for BOOST_PYTHON_MODULE_INIT and allow the boost::python::mod_gil_not_used() option. This sets the Py_MOD_GIL_NOT_USED flag for the extension module. To define a module that supports free-threaded Python, define it like this: BOOST_PYTHON_MODULE(my_module, boost::python::mod_gil_not_used()) { ... } --- include/boost/python/module_init.hpp | 75 ++++++++++++++++++++++++++-- src/module.cpp | 5 +- test/fabscript | 1 + test/module_nogil.cpp | 25 ++++++++++ test/module_nogil.py | 29 +++++++++++ 5 files changed, 129 insertions(+), 6 deletions(-) create mode 100644 test/module_nogil.cpp create mode 100644 test/module_nogil.py diff --git a/include/boost/python/module_init.hpp b/include/boost/python/module_init.hpp index 7fe5a1c8a2..bfb4fd9ca8 100644 --- a/include/boost/python/module_init.hpp +++ b/include/boost/python/module_init.hpp @@ -11,11 +11,41 @@ # ifndef BOOST_PYTHON_MODULE_INIT -namespace boost { namespace python { namespace detail { +namespace boost { namespace python { + +#ifdef HAS_CXX11 +// Use to activate the Py_MOD_GIL_NOT_USED flag. +class mod_gil_not_used { +public: + explicit mod_gil_not_used(bool flag = true) : flag_(flag) {} + bool flag() const { return flag_; } + +private: + bool flag_; +}; + +namespace detail { + +inline bool gil_not_used_option() { return false; } +template +bool gil_not_used_option(F &&, O &&...o); +template +inline bool gil_not_used_option(mod_gil_not_used f, O &&...o) { + return f.flag() || gil_not_used_option(o...); +} +template +inline bool gil_not_used_option(F &&, O &&...o) { + return gil_not_used_option(o...); +} + +} +#endif // HAS_CXX11 + +namespace detail { # if PY_VERSION_HEX >= 0x03000000 -BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)()); +BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef&, void(*)(), bool gil_not_used = false); #else @@ -27,7 +57,37 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); # if PY_VERSION_HEX >= 0x03000000 -# define _BOOST_PYTHON_MODULE_INIT(name) \ +# ifdef HAS_CXX11 +# define _BOOST_PYTHON_MODULE_INIT(name, ...) \ + PyObject* BOOST_PP_CAT(PyInit_, name)() \ + { \ + static PyModuleDef_Base initial_m_base = { \ + PyObject_HEAD_INIT(NULL) \ + 0, /* m_init */ \ + 0, /* m_index */ \ + 0 /* m_copy */ }; \ + static PyMethodDef initial_methods[] = { { 0, 0, 0, 0 } }; \ + \ + static struct PyModuleDef moduledef = { \ + initial_m_base, \ + BOOST_PP_STRINGIZE(name), \ + 0, /* m_doc */ \ + -1, /* m_size */ \ + initial_methods, \ + 0, /* m_reload */ \ + 0, /* m_traverse */ \ + 0, /* m_clear */ \ + 0, /* m_free */ \ + }; \ + \ + return boost::python::detail::init_module( \ + moduledef, BOOST_PP_CAT(init_module_, name), \ + boost::python::detail::gil_not_used_option(__VA_ARGS__) ); \ + } \ + void BOOST_PP_CAT(init_module_, name)() + +# else // !HAS_CXX11 +# define _BOOST_PYTHON_MODULE_INIT(name) \ PyObject* BOOST_PP_CAT(PyInit_, name)() \ { \ static PyModuleDef_Base initial_m_base = { \ @@ -53,6 +113,7 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); moduledef, BOOST_PP_CAT(init_module_, name) ); \ } \ void BOOST_PP_CAT(init_module_, name)() +# endif // HAS_CXX11 # else @@ -66,9 +127,15 @@ BOOST_PYTHON_DECL PyObject* init_module(char const* name, void(*)()); # endif -# define BOOST_PYTHON_MODULE_INIT(name) \ +# ifdef HAS_CXX11 +# define BOOST_PYTHON_MODULE_INIT(name, ...) \ + void BOOST_PP_CAT(init_module_,name)(); \ +extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name, __VA_ARGS__) +# else +# define BOOST_PYTHON_MODULE_INIT(name) \ void BOOST_PP_CAT(init_module_,name)(); \ extern "C" BOOST_SYMBOL_EXPORT _BOOST_PYTHON_MODULE_INIT(name) +# endif // HAS_CXX11 # endif diff --git a/src/module.cpp b/src/module.cpp index 707e433941..c32f4187bc 100644 --- a/src/module.cpp +++ b/src/module.cpp @@ -38,11 +38,12 @@ BOOST_PYTHON_DECL void scope_setattr_doc(char const* name, object const& x, char #if PY_VERSION_HEX >= 0x03000000 -BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, void(*init_function)()) +BOOST_PYTHON_DECL PyObject* init_module(PyModuleDef& moduledef, + void(*init_function)(), bool gil_not_used) { PyObject *mod = PyModule_Create(&moduledef); #ifdef Py_GIL_DISABLED - if (mod != NULL) { + if (mod != NULL && gil_not_used) { PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); } #endif diff --git a/test/fabscript b/test/fabscript index a002fb2bf0..7cf22f9c09 100644 --- a/test/fabscript +++ b/test/fabscript @@ -68,6 +68,7 @@ for t in [('injected',), ('raw_ctor',), ('exception_translator',), ('module_init_exception',), + ('module_nogil',), ('test_enum', ['enum_ext']), ('test_cltree', ['cltree']), ('newtest', ['m1', 'm2']), diff --git a/test/module_nogil.cpp b/test/module_nogil.cpp new file mode 100644 index 0000000000..11cba558d3 --- /dev/null +++ b/test/module_nogil.cpp @@ -0,0 +1,25 @@ +// Test for BOOST_PYTHON_MODULE with optional mod_gil_not_used argument + +#include +#include + +// Simple function to export +int get_value() { + return 1234; +} + +#ifdef HAS_CXX11 +// C++11 build: test with mod_gil_not_used option +BOOST_PYTHON_MODULE(module_nogil_ext, boost::python::mod_gil_not_used()) +{ + using namespace boost::python; + def("get_value", get_value); +} +#else +// C++98 build: test without optional arguments +BOOST_PYTHON_MODULE(module_nogil_ext) +{ + using namespace boost::python; + def("get_value", get_value); +} +#endif diff --git a/test/module_nogil.py b/test/module_nogil.py new file mode 100644 index 0000000000..c035436014 --- /dev/null +++ b/test/module_nogil.py @@ -0,0 +1,29 @@ +""" +>>> from module_nogil_ext import * +>>> get_value() +1234 +>>> import sys, sysconfig +>>> Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) +>>> if Py_GIL_DISABLED and sys._is_gil_enabled(): +... print('GIL is enabled and should not be') +... else: +... print('okay') +okay +""" + +from __future__ import print_function + +def run(args = None): + import sys + import doctest + + if args is not None: + sys.argv = args + return doctest.testmod(sys.modules.get(__name__)) + +if __name__ == '__main__': + print("running...") + import sys + status = run()[0] + if (status == 0): print("Done.") + sys.exit(status)