Clone of PR #6066 for experimentation — NOT FOR MERGING

.github/workflows/ci.yml

@@ -48,7 +48,8 @@ jobs:
             python-version: '3.14t'
             cmake-args: -DCMAKE_CXX_STANDARD=17 -DPYBIND11_TEST_SMART_HOLDER=ON
           - runs-on: ubuntu-latest
-            python-version: 'pypy3.11'
+            # Temporarily pinned pending investigation in issue #6049.
+            python-version: 'pypy-3.11-v7.3.21'
             cmake-args: -DCMAKE_CXX_STANDARD=17
           - runs-on: ubuntu-latest
             python-version: 'graalpy-24.2'

include/pybind11/attr.h

@@ -310,6 +310,9 @@ struct type_record {
     /// Function pointer to class_<..>::init_instance
     void (*init_instance)(instance *, const void *) = nullptr;
 
+    /// Function pointer to construct a bound holder from an erased std::shared_ptr.
+    void (*init_instance_from_shared_ptr)(instance *, const std::shared_ptr<void> *) = nullptr;
+
     /// Function pointer to class_<..>::dealloc
     void (*dealloc)(detail::value_and_holder &) = nullptr;
 

include/pybind11/cast.h

@@ -1025,6 +1025,10 @@ struct copyable_holder_caster<
         if (tinfo != nullptr && tinfo->holder_enum_v == holder_enum_t::std_shared_ptr) {
             return type_caster_base<type>::cast_holder(srcs, &src);
         }
+        if (tinfo != nullptr && tinfo->init_instance_from_shared_ptr != nullptr) {
+            return smart_holder_type_caster_support::custom_holder_from_shared_ptr(
+                src, policy, parent, srcs.result);
+        }
 
         if (parent) {
             return type_caster_generic::cast_non_owning(

include/pybind11/detail/internals.h

@@ -39,11 +39,11 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    define PYBIND11_INTERNALS_VERSION 12
+#    define PYBIND11_INTERNALS_VERSION 13
 #endif
 
-#if PYBIND11_INTERNALS_VERSION < 12
-#    error "PYBIND11_INTERNALS_VERSION 12 is the minimum for all platforms for pybind11 v3.1.0"
+#if PYBIND11_INTERNALS_VERSION < 13
+#    error "PYBIND11_INTERNALS_VERSION 13 is the minimum for all platforms for pybind11 v3.1.0"
 #endif
 
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
@@ -402,6 +402,7 @@ struct type_info {
     size_t type_size, type_align, holder_size_in_ptrs;
     void *(*operator_new)(size_t);
     void (*init_instance)(instance *, const void *);
+    void (*init_instance_from_shared_ptr)(instance *, const std::shared_ptr<void> *) = nullptr;
     void (*dealloc)(value_and_holder &v_h);
 
     // Cross-DSO-safe function pointers, to sidestep cross-DSO RTTI issues

include/pybind11/detail/type_caster_base.h

@@ -742,11 +742,29 @@ handle smart_holder_from_unique_ptr(std::unique_ptr<T const, D> &&src,
         cs);
 }
 
+struct shared_ptr_cast_data {
+    object result;
+    object inst;
+    instance *inst_raw_ptr;
+    void *src_raw_void_ptr;
+    const detail::type_info *tinfo;
+
+    explicit shared_ptr_cast_data(object &&result_)
+        : result(std::move(result_)), inst_raw_ptr(nullptr), src_raw_void_ptr(nullptr),
+          tinfo(nullptr) {}
+
+    shared_ptr_cast_data(object &&inst_,
+                         instance *inst_raw_ptr_,
+                         void *src_raw_void_ptr_,
+                         const detail::type_info *tinfo_)
+        : inst(std::move(inst_)), inst_raw_ptr(inst_raw_ptr_), src_raw_void_ptr(src_raw_void_ptr_),
+          tinfo(tinfo_) {}
+};
+
 template <typename T>
-handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
-                                    return_value_policy policy,
-                                    handle parent,
-                                    const cast_sources::resolved_source &cs) {
+shared_ptr_cast_data prepare_shared_ptr_cast(const std::shared_ptr<T> &src,
+                                             return_value_policy policy,
+                                             const cast_sources::resolved_source &cs) {
     switch (policy) {
         case return_value_policy::automatic:
         case return_value_policy::automatic_reference:
@@ -762,7 +780,7 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
             break;
     }
     if (!src) {
-        return none().release();
+        return shared_ptr_cast_data(reinterpret_steal<object>(none().release()));
     }
 
     // cs.cppobj is the subobject pointer appropriate for tinfo (may differ from src.get()
@@ -771,27 +789,43 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
     assert(cs.tinfo != nullptr);
     const detail::type_info *tinfo = cs.tinfo;
     if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) {
-        // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder.
+        // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing holder.
         // PYBIND11:REMINDER: MISSING: keep_alive.
-        return existing_inst;
+        return shared_ptr_cast_data(reinterpret_steal<object>(existing_inst));
     }
 
     auto inst = reinterpret_steal<object>(make_new_instance(tinfo->type));
     auto *inst_raw_ptr = reinterpret_cast<instance *>(inst.ptr());
     inst_raw_ptr->owned = true;
     void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr();
     valueptr = src_raw_void_ptr;
+    return shared_ptr_cast_data(std::move(inst), inst_raw_ptr, src_raw_void_ptr, tinfo);
+}
 
-    auto smhldr = smart_holder::from_shared_ptr(std::shared_ptr<void>(src, src_raw_void_ptr));
-    tinfo->init_instance(inst_raw_ptr, static_cast<const void *>(&smhldr));
-
+inline handle finish_shared_ptr_cast(object &&inst, return_value_policy policy, handle parent) {
     if (policy == return_value_policy::reference_internal) {
         keep_alive_impl(inst, parent);
     }
-
     return inst.release();
 }
 
+template <typename T>
+handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const cast_sources::resolved_source &cs) {
+    auto cast_data = prepare_shared_ptr_cast(src, policy, cs);
+    if (cast_data.result) {
+        return cast_data.result.release();
+    }
+
+    auto smhldr
+        = smart_holder::from_shared_ptr(std::shared_ptr<void>(src, cast_data.src_raw_void_ptr));
+    cast_data.tinfo->init_instance(cast_data.inst_raw_ptr, static_cast<const void *>(&smhldr));
+
+    return finish_shared_ptr_cast(std::move(cast_data.inst), policy, parent);
+}
+
 template <typename T>
 handle smart_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
                                     return_value_policy policy,
@@ -803,6 +837,33 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
                                         cs);
 }
 
+template <typename T>
+handle custom_holder_from_shared_ptr(const std::shared_ptr<T> &src,
+                                     return_value_policy policy,
+                                     handle parent,
+                                     const cast_sources::resolved_source &cs) {
+    auto cast_data = prepare_shared_ptr_cast(src, policy, cs);
+    if (cast_data.result) {
+        return cast_data.result.release();
+    }
+
+    auto erased_shared_ptr = std::shared_ptr<void>(src, cast_data.src_raw_void_ptr);
+    cast_data.tinfo->init_instance_from_shared_ptr(cast_data.inst_raw_ptr, &erased_shared_ptr);
+
+    return finish_shared_ptr_cast(std::move(cast_data.inst), policy, parent);
+}
+
+template <typename T>
+handle custom_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
+                                     return_value_policy policy,
+                                     handle parent,
+                                     const cast_sources::resolved_source &cs) {
+    return custom_holder_from_shared_ptr(std::const_pointer_cast<T>(src), // Const2Mutbl
+                                         policy,
+                                         parent,
+                                         cs);
+}
+
 struct shared_ptr_parent_life_support {
     PyObject *parent;
     explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} {

include/pybind11/pybind11.h

@@ -1824,6 +1824,7 @@ class generic_type : public object {
         tinfo->operator_new = rec.operator_new;
         tinfo->holder_size_in_ptrs = size_in_ptrs(rec.holder_size);
         tinfo->init_instance = rec.init_instance;
+        tinfo->init_instance_from_shared_ptr = rec.init_instance_from_shared_ptr;
         tinfo->dealloc = rec.dealloc;
         tinfo->get_trampoline_self_life_support = rec.get_trampoline_self_life_support;
         tinfo->simple_type = true;
@@ -2361,6 +2362,7 @@ class class_ : public detail::generic_type {
         record.type_align = alignof(conditional_t<has_alias, type_alias, type> &);
         record.holder_size = sizeof(holder_type);
         record.init_instance = init_instance;
+        record.init_instance_from_shared_ptr = get_init_instance_from_shared_ptr();
 
         if (detail::is_instantiation<std::unique_ptr, holder_type>::value) {
             record.holder_enum_v = detail::holder_enum_t::std_unique_ptr;
@@ -2783,6 +2785,35 @@ class class_ : public detail::generic_type {
         }
     }
 
+    template <typename H = holder_type,
+              detail::enable_if_t<std::is_constructible<H, std::shared_ptr<type>>::value, int> = 0>
+    static void init_instance_from_shared_ptr(detail::instance *inst,
+                                              const std::shared_ptr<void> *shared_ptr_void_ptr) {
+        auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type)));
+        if (!v_h.instance_registered()) {
+            register_instance(inst, v_h.value_ptr(), v_h.type);
+            v_h.set_instance_registered();
+        }
+        auto shared_ptr = std::shared_ptr<type>(*shared_ptr_void_ptr, v_h.value_ptr<type>());
+        new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(shared_ptr));
+        v_h.set_holder_constructed();
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<std::is_constructible<H, std::shared_ptr<type>>::value, int> = 0>
+    static constexpr auto get_init_instance_from_shared_ptr()
+        -> void (*)(detail::instance *, const std::shared_ptr<void> *) {
+        return &init_instance_from_shared_ptr<>;
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<!std::is_constructible<H, std::shared_ptr<type>>::value, int>
+              = 0>
+    static constexpr auto get_init_instance_from_shared_ptr()
+        -> void (*)(detail::instance *, const std::shared_ptr<void> *) {
+        return nullptr;
+    }
+
     /// Performs instance initialization including constructing a holder and registering the known
     /// instance.  Should be called as soon as the `type` value_ptr is set for an instance.  Takes
     /// an optional pointer to an existing holder to use; if not specified and the instance is

tests/test_smart_ptr.cpp

@@ -83,6 +83,26 @@ class const_only_shared_ptr {
     // for demonstration purpose only, this imitates smart pointer with a const-only pointer
 };
 
+template <typename T>
+class shared_ptr_as_custom_holder { // Issue #6064
+    std::shared_ptr<T> ptr_;
+
+public:
+    using element_type = T;
+
+    shared_ptr_as_custom_holder() = default;
+
+    explicit shared_ptr_as_custom_holder(T *) {
+        throw std::runtime_error("invalid shared_ptr_as_custom_holder constructor call");
+    }
+
+    explicit shared_ptr_as_custom_holder(std::shared_ptr<T> ptr) : ptr_(std::move(ptr)) {}
+
+    T *get() const { return ptr_.get(); }
+
+    explicit operator std::shared_ptr<T>() const { return ptr_; }
+};
+
 // Custom object with builtin reference counting (see 'object.h' for the implementation)
 class MyObject1 : public Object {
 public:
@@ -239,6 +259,25 @@ struct SharedFromThisRef {
     std::shared_ptr<B> shared = std::make_shared<B>();
 };
 
+class PrivateDtorWithCustomHolder { // Issue #6064
+public:
+    static std::shared_ptr<PrivateDtorWithCustomHolder> create(int value) {
+        return {new PrivateDtorWithCustomHolder(value),
+                [](PrivateDtorWithCustomHolder *ptr) { delete ptr; }};
+    }
+
+    int value = 0;
+
+private:
+    explicit PrivateDtorWithCustomHolder(int value_) : value(value_) {}
+    ~PrivateDtorWithCustomHolder() = default;
+};
+
+std::shared_ptr<PrivateDtorWithCustomHolder> &private_dtor_with_custom_holder_singleton() {
+    static auto singleton = PrivateDtorWithCustomHolder::create(17);
+    return singleton;
+}
+
 // Issue #865: shared_from_this doesn't work with virtual inheritance
 struct SharedFromThisVBase : std::enable_shared_from_this<SharedFromThisVBase> {
     SharedFromThisVBase() = default;
@@ -341,6 +380,7 @@ struct holder_helper<ref<T>> {
 // Make pybind aware of the ref-counted wrapper type (s):
 PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true)
 PYBIND11_DECLARE_HOLDER_TYPE(T, const_only_shared_ptr<T>, true)
+PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_as_custom_holder<T>)
 // The following is not required anymore for std::shared_ptr, but it should compile without error:
 PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>)
 PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr<T>)
@@ -501,6 +541,30 @@ TEST_SUBMODULE(smart_ptr, m) {
         .def(py::init([](const std::string &value) { return MyObject6::createObject(value); }))
         .def_property_readonly("value", &MyObject6::value);
 
+    py::class_<PrivateDtorWithCustomHolder,
+               shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>>(
+        m, "PrivateDtorWithCustomHolder")
+        .def_property(
+            "value",
+            [](const PrivateDtorWithCustomHolder &self) { return self.value; },
+            [](PrivateDtorWithCustomHolder &self, int value) { self.value = value; })
+        .def_static("get_singleton_holder", []() {
+            return shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>(
+                private_dtor_with_custom_holder_singleton());
+        });
+    m.def("get_private_dtor_with_custom_holder_shared_ptr",
+          []() { return private_dtor_with_custom_holder_singleton(); });
+    m.def("get_private_dtor_with_custom_holder_const_shared_ptr", []() {
+        return std::shared_ptr<const PrivateDtorWithCustomHolder>(
+            private_dtor_with_custom_holder_singleton());
+    });
+    m.def("make_private_dtor_with_custom_holder_shared_ptr",
+          []() { return PrivateDtorWithCustomHolder::create(41); });
+    m.def("make_private_dtor_with_custom_holder_const_shared_ptr", []() {
+        return std::shared_ptr<const PrivateDtorWithCustomHolder>(
+            PrivateDtorWithCustomHolder::create(43));
+    });
+
     // test_shared_ptr_and_references
     using A = SharedPtrRef::A;
     py::class_<A, std::shared_ptr<A>>(m, "A");
@@ -559,6 +623,7 @@ TEST_SUBMODULE(smart_ptr, m) {
     py::class_<C, custom_unique_ptr<C>>(m, "TypeWithMoveOnlyHolder")
         .def_static("make", []() { return custom_unique_ptr<C>(new C); })
         .def_static("make_as_object", []() { return py::cast(custom_unique_ptr<C>(new C)); });
+    m.def("get_type_with_move_only_holder_shared_ptr", []() { return std::make_shared<C>(); });
 
     // test_holder_with_addressof_operator
     using HolderWithAddressOf = shared_ptr_with_addressof_operator<TypeForHolderWithAddressOf>;

tests/test_smart_ptr.py

@@ -368,3 +368,35 @@ def test_move_only_holder_caster_shared_ptr_with_smart_holder_support_enabled():
 def test_const_only_holder():
     o = m.MyObject6("my_data")
     assert o.value == "my_data"
+
+
+def test_shared_ptr_cast_for_custom_holder_with_private_dtor():
+    fresh_shared_ptr_obj = m.make_private_dtor_with_custom_holder_shared_ptr()
+    fresh_const_shared_ptr_obj = (
+        m.make_private_dtor_with_custom_holder_const_shared_ptr()
+    )
+
+    assert fresh_shared_ptr_obj.value == 41
+    assert fresh_const_shared_ptr_obj.value == 43
+
+    holder_obj = m.PrivateDtorWithCustomHolder.get_singleton_holder()
+    shared_ptr_obj = m.get_private_dtor_with_custom_holder_shared_ptr()
+    const_shared_ptr_obj = m.get_private_dtor_with_custom_holder_const_shared_ptr()
+
+    holder_obj.value = 23
+
+    assert shared_ptr_obj.value == 23
+    assert const_shared_ptr_obj.value == 23
+    assert m.get_private_dtor_with_custom_holder_shared_ptr().value == 23
+    assert m.get_private_dtor_with_custom_holder_const_shared_ptr().value == 23
+
+
+def test_shared_ptr_cast_for_incompatible_custom_holder_throws():
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Unable to convert std::shared_ptr<T> to Python when the bound type does not use"
+            " std::shared_ptr or py::smart_holder as its holder type"
+        ),
+    ):
+        m.get_type_with_move_only_holder_shared_ptr()