Fix py::cast(std::shared_ptr<T>) for shared_ptr-compatible custom holders

include/pybind11/attr.h

@@ -313,6 +313,9 @@ struct type_record {
     /// Function pointer to class_<..>::dealloc
     void (*dealloc)(detail::value_and_holder &) = nullptr;
 
+    /// Function pointer to construct a bound holder from an erased std::shared_ptr.
+    void (*init_instance_from_shared_ptr)(instance *, const std::shared_ptr<void> *) = nullptr;
+
     /// Function pointer for casting alias class (aka trampoline) pointer to
     /// trampoline_self_life_support pointer. Sidesteps cross-DSO RTTI issues
     /// on platforms like macOS (see PR #5728 for details).

include/pybind11/cast.h

@@ -1025,6 +1025,10 @@ struct copyable_holder_caster<
         if (tinfo != nullptr && tinfo->holder_enum_v == holder_enum_t::std_shared_ptr) {
             return type_caster_base<type>::cast_holder(srcs, &src);
         }
+        if (tinfo != nullptr && tinfo->init_instance_from_shared_ptr != nullptr) {
+            return smart_holder_type_caster_support::custom_holder_from_shared_ptr(
+                src, policy, parent, srcs.result);
+        }
 
         if (parent) {
             return type_caster_generic::cast_non_owning(

include/pybind11/detail/internals.h

@@ -403,6 +403,7 @@ struct type_info {
     void *(*operator_new)(size_t);
     void (*init_instance)(instance *, const void *);
     void (*dealloc)(value_and_holder &v_h);
+    void (*init_instance_from_shared_ptr)(instance *, const std::shared_ptr<void> *) = nullptr;
 
     // Cross-DSO-safe function pointers, to sidestep cross-DSO RTTI issues
     // on platforms like macOS (see PR #5728 for details):

include/pybind11/detail/type_caster_base.h

@@ -742,11 +742,12 @@ handle smart_holder_from_unique_ptr(std::unique_ptr<T const, D> &&src,
         cs);
 }
 
-template <typename T>
-handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
-                                    return_value_policy policy,
-                                    handle parent,
-                                    const cast_sources::resolved_source &cs) {
+template <typename T, typename InitHolder>
+handle cast_shared_ptr_with_holder(const std::shared_ptr<T> &src,
+                                   return_value_policy policy,
+                                   handle parent,
+                                   const cast_sources::resolved_source &cs,
+                                   InitHolder &&init_holder) {
     switch (policy) {
         case return_value_policy::automatic:
         case return_value_policy::automatic_reference:
@@ -771,7 +772,7 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
     assert(cs.tinfo != nullptr);
     const detail::type_info *tinfo = cs.tinfo;
     if (handle existing_inst = find_registered_python_instance(src_raw_void_ptr, tinfo)) {
-        // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing smart_holder.
+        // PYBIND11:REMINDER: MISSING: Enforcement of consistency with existing holder.
         // PYBIND11:REMINDER: MISSING: keep_alive.
         return existing_inst;
     }
@@ -782,16 +783,34 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
     void *&valueptr = values_and_holders(inst_raw_ptr).begin()->value_ptr();
     valueptr = src_raw_void_ptr;
 
-    auto smhldr = smart_holder::from_shared_ptr(std::shared_ptr<void>(src, src_raw_void_ptr));
-    tinfo->init_instance(inst_raw_ptr, static_cast<const void *>(&smhldr));
+    init_holder(tinfo, inst_raw_ptr, src, src_raw_void_ptr);
 
     if (policy == return_value_policy::reference_internal) {
         keep_alive_impl(inst, parent);
     }
-
     return inst.release();
 }
 
+template <typename T>
+handle smart_holder_from_shared_ptr(const std::shared_ptr<T> &src,
+                                    return_value_policy policy,
+                                    handle parent,
+                                    const cast_sources::resolved_source &cs) {
+    return cast_shared_ptr_with_holder(
+        src,
+        policy,
+        parent,
+        cs,
+        [](const detail::type_info *tinfo,
+           instance *inst_raw_ptr,
+           const std::shared_ptr<T> &shared_ptr,
+           void *src_raw_void_ptr) {
+            auto smhldr = smart_holder::from_shared_ptr(
+                std::shared_ptr<void>(shared_ptr, src_raw_void_ptr));
+            tinfo->init_instance(inst_raw_ptr, static_cast<const void *>(&smhldr));
+        });
+}
+
 template <typename T>
 handle smart_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
                                     return_value_policy policy,
@@ -803,6 +822,36 @@ handle smart_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
                                         cs);
 }
 
+template <typename T>
+handle custom_holder_from_shared_ptr(const std::shared_ptr<T> &src,
+                                     return_value_policy policy,
+                                     handle parent,
+                                     const cast_sources::resolved_source &cs) {
+    return cast_shared_ptr_with_holder(
+        src,
+        policy,
+        parent,
+        cs,
+        [](const detail::type_info *tinfo,
+           instance *inst_raw_ptr,
+           const std::shared_ptr<T> &shared_ptr,
+           void *src_raw_void_ptr) {
+            auto erased_shared_ptr = std::shared_ptr<void>(shared_ptr, src_raw_void_ptr);
+            tinfo->init_instance_from_shared_ptr(inst_raw_ptr, &erased_shared_ptr);
+        });
+}
+
+template <typename T>
+handle custom_holder_from_shared_ptr(const std::shared_ptr<T const> &src,
+                                     return_value_policy policy,
+                                     handle parent,
+                                     const cast_sources::resolved_source &cs) {
+    return custom_holder_from_shared_ptr(std::const_pointer_cast<T>(src), // Const2Mutbl
+                                         policy,
+                                         parent,
+                                         cs);
+}
+
 struct shared_ptr_parent_life_support {
     PyObject *parent;
     explicit shared_ptr_parent_life_support(PyObject *parent) : parent{parent} {

include/pybind11/pybind11.h

@@ -1825,6 +1825,7 @@ class generic_type : public object {
         tinfo->holder_size_in_ptrs = size_in_ptrs(rec.holder_size);
         tinfo->init_instance = rec.init_instance;
         tinfo->dealloc = rec.dealloc;
+        tinfo->init_instance_from_shared_ptr = rec.init_instance_from_shared_ptr;
         tinfo->get_trampoline_self_life_support = rec.get_trampoline_self_life_support;
         tinfo->simple_type = true;
         tinfo->simple_ancestors = true;
@@ -2361,6 +2362,7 @@ class class_ : public detail::generic_type {
         record.type_align = alignof(conditional_t<has_alias, type_alias, type> &);
         record.holder_size = sizeof(holder_type);
         record.init_instance = init_instance;
+        record.init_instance_from_shared_ptr = get_init_instance_from_shared_ptr();
 
         if (detail::is_instantiation<std::unique_ptr, holder_type>::value) {
             record.holder_enum_v = detail::holder_enum_t::std_unique_ptr;
@@ -2783,6 +2785,35 @@ class class_ : public detail::generic_type {
         }
     }
 
+    template <typename H = holder_type,
+              detail::enable_if_t<std::is_constructible<H, std::shared_ptr<type>>::value, int> = 0>
+    static void init_instance_from_shared_ptr(detail::instance *inst,
+                                              const std::shared_ptr<void> *shared_ptr_void_ptr) {
+        auto v_h = inst->get_value_and_holder(detail::get_type_info(typeid(type)));
+        if (!v_h.instance_registered()) {
+            register_instance(inst, v_h.value_ptr(), v_h.type);
+            v_h.set_instance_registered();
+        }
+        auto shared_ptr = std::shared_ptr<type>(*shared_ptr_void_ptr, v_h.value_ptr<type>());
+        new (std::addressof(v_h.holder<holder_type>())) holder_type(std::move(shared_ptr));
+        v_h.set_holder_constructed();
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<std::is_constructible<H, std::shared_ptr<type>>::value, int> = 0>
+    static constexpr auto get_init_instance_from_shared_ptr()
+        -> void (*)(detail::instance *, const std::shared_ptr<void> *) {
+        return &init_instance_from_shared_ptr<>;
+    }
+
+    template <typename H = holder_type,
+              detail::enable_if_t<!std::is_constructible<H, std::shared_ptr<type>>::value, int>
+              = 0>
+    static constexpr auto get_init_instance_from_shared_ptr()
+        -> void (*)(detail::instance *, const std::shared_ptr<void> *) {
+        return nullptr;
+    }
+
     /// Performs instance initialization including constructing a holder and registering the known
     /// instance.  Should be called as soon as the `type` value_ptr is set for an instance.  Takes
     /// an optional pointer to an existing holder to use; if not specified and the instance is

tests/test_smart_ptr.cpp

@@ -83,6 +83,26 @@ class const_only_shared_ptr {
     // for demonstration purpose only, this imitates smart pointer with a const-only pointer
 };
 
+template <typename T>
+class shared_ptr_as_custom_holder {
+    std::shared_ptr<T> ptr_;
+
+public:
+    using element_type = T;
+
+    shared_ptr_as_custom_holder() = default;
+
+    explicit shared_ptr_as_custom_holder(T *) {
+        throw std::runtime_error("invalid shared_ptr_as_custom_holder constructor call");
+    }
+
+    explicit shared_ptr_as_custom_holder(std::shared_ptr<T> ptr) : ptr_(std::move(ptr)) {}
+
+    T *get() const { return ptr_.get(); }
+
+    explicit operator std::shared_ptr<T>() const { return ptr_; }
+};
+
 // Custom object with builtin reference counting (see 'object.h' for the implementation)
 class MyObject1 : public Object {
 public:
@@ -239,6 +259,25 @@ struct SharedFromThisRef {
     std::shared_ptr<B> shared = std::make_shared<B>();
 };
 
+class PrivateDtorWithCustomHolder {
+public:
+    static std::shared_ptr<PrivateDtorWithCustomHolder> create(int value) {
+        return {new PrivateDtorWithCustomHolder(value),
+                [](PrivateDtorWithCustomHolder *ptr) { delete ptr; }};
+    }
+
+    int value = 0;
+
+private:
+    explicit PrivateDtorWithCustomHolder(int value_) : value(value_) {}
+    ~PrivateDtorWithCustomHolder() = default;
+};
+
+std::shared_ptr<PrivateDtorWithCustomHolder> &private_dtor_with_custom_holder_singleton() {
+    static auto singleton = PrivateDtorWithCustomHolder::create(17);
+    return singleton;
+}
+
 // Issue #865: shared_from_this doesn't work with virtual inheritance
 struct SharedFromThisVBase : std::enable_shared_from_this<SharedFromThisVBase> {
     SharedFromThisVBase() = default;
@@ -341,6 +380,7 @@ struct holder_helper<ref<T>> {
 // Make pybind aware of the ref-counted wrapper type (s):
 PYBIND11_DECLARE_HOLDER_TYPE(T, ref<T>, true)
 PYBIND11_DECLARE_HOLDER_TYPE(T, const_only_shared_ptr<T>, true)
+PYBIND11_DECLARE_HOLDER_TYPE(T, shared_ptr_as_custom_holder<T>)
 // The following is not required anymore for std::shared_ptr, but it should compile without error:
 PYBIND11_DECLARE_HOLDER_TYPE(T, std::shared_ptr<T>)
 PYBIND11_DECLARE_HOLDER_TYPE(T, huge_unique_ptr<T>)
@@ -501,6 +541,20 @@ TEST_SUBMODULE(smart_ptr, m) {
         .def(py::init([](const std::string &value) { return MyObject6::createObject(value); }))
         .def_property_readonly("value", &MyObject6::value);
 
+    py::class_<PrivateDtorWithCustomHolder,
+               shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>>(
+        m, "PrivateDtorWithCustomHolder")
+        .def_property(
+            "value",
+            [](const PrivateDtorWithCustomHolder &self) { return self.value; },
+            [](PrivateDtorWithCustomHolder &self, int value) { self.value = value; })
+        .def_static("get_singleton_holder", []() {
+            return shared_ptr_as_custom_holder<PrivateDtorWithCustomHolder>(
+                private_dtor_with_custom_holder_singleton());
+        });
+    m.def("get_private_dtor_with_custom_holder_shared_ptr",
+          []() { return private_dtor_with_custom_holder_singleton(); });
+
     // test_shared_ptr_and_references
     using A = SharedPtrRef::A;
     py::class_<A, std::shared_ptr<A>>(m, "A");
@@ -559,6 +613,7 @@ TEST_SUBMODULE(smart_ptr, m) {
     py::class_<C, custom_unique_ptr<C>>(m, "TypeWithMoveOnlyHolder")
         .def_static("make", []() { return custom_unique_ptr<C>(new C); })
         .def_static("make_as_object", []() { return py::cast(custom_unique_ptr<C>(new C)); });
+    m.def("get_type_with_move_only_holder_shared_ptr", []() { return std::make_shared<C>(); });
 
     // test_holder_with_addressof_operator
     using HolderWithAddressOf = shared_ptr_with_addressof_operator<TypeForHolderWithAddressOf>;

tests/test_smart_ptr.py

@@ -368,3 +368,24 @@ def test_move_only_holder_caster_shared_ptr_with_smart_holder_support_enabled():
 def test_const_only_holder():
     o = m.MyObject6("my_data")
     assert o.value == "my_data"
+
+
+def test_shared_ptr_cast_for_custom_holder_with_private_dtor():
+    holder_obj = m.PrivateDtorWithCustomHolder.get_singleton_holder()
+    shared_ptr_obj = m.get_private_dtor_with_custom_holder_shared_ptr()
+
+    holder_obj.value = 23
+
+    assert shared_ptr_obj.value == 23
+    assert m.get_private_dtor_with_custom_holder_shared_ptr().value == 23
+
+
+def test_shared_ptr_cast_for_incompatible_custom_holder_throws():
+    with pytest.raises(
+        RuntimeError,
+        match=(
+            "Unable to convert std::shared_ptr<T> to Python when the bound type does not use"
+            " std::shared_ptr or py::smart_holder as its holder type"
+        ),
+    ):
+        m.get_type_with_move_only_holder_shared_ptr()