Fix #5989: use dynamic_pointer_cast for virtual inheritance in esft downcast

Replace the unconditional static_pointer_cast in set_via_shared_from_this
with a SFINAE-dispatched esft_downcast helper that falls back to
dynamic_pointer_cast when static_cast through a virtual base is ill-formed.

Also add a workaround in the test binding (.def("name") on SftVirtDerived2)
for a separate pre-existing issue with inherited method dispatch through
virtual bases.

Made-with: Cursor

Author: rwgk

include/pybind11/detail/holder_caster_foreign_helpers.h

@@ -31,13 +31,31 @@ struct holder_caster_foreign_helpers {
         PyObject *o;
     };
 
+    // Downcast shared_ptr from the enable_shared_from_this base to the target type.
+    // SFINAE probe: use static_pointer_cast when the static downcast is valid (common case),
+    // fall back to dynamic_pointer_cast when it isn't (virtual inheritance — issue #5989).
+    // We can't use dynamic_pointer_cast unconditionally because it requires polymorphic types;
+    // we can't use is_polymorphic to choose because that's orthogonal to virtual inheritance.
+    // (The implementation uses the "tag dispatch via overload priority" trick.)
+    template <typename type, typename esft_base>
+    static auto esft_downcast(const std::shared_ptr<esft_base> &existing, int /*preferred*/)
+        -> decltype(static_cast<type *>(std::declval<esft_base *>()), std::shared_ptr<type>()) {
+        return std::static_pointer_cast<type>(existing);
+    }
+
+    template <typename type, typename esft_base>
+    static std::shared_ptr<type> esft_downcast(const std::shared_ptr<esft_base> &existing,
+                                               ... /*fallback*/) {
+        return std::dynamic_pointer_cast<type>(existing);
+    }
+
     template <typename type>
     static auto set_via_shared_from_this(type *value, std::shared_ptr<type> *holder_out)
         -> decltype(value->shared_from_this(), bool()) {
         // object derives from enable_shared_from_this;
         // try to reuse an existing shared_ptr if one is known
         if (auto existing = try_get_shared_from_this(value)) {
-            *holder_out = std::static_pointer_cast<type>(existing);
+            *holder_out = esft_downcast<type>(existing, 0);
             return true;
         }
         return false;

tests/test_smart_ptr.cpp

@@ -553,6 +553,10 @@ TEST_SUBMODULE(smart_ptr, m) {
     py::class_<SftVirtDerived2, SftVirtDerived, std::shared_ptr<SftVirtDerived2>>(
         m, "SftVirtDerived2")
         .def(py::init<>(&SftVirtDerived2::create))
+        // TODO: Remove this once inherited methods work through virtual bases.
+        //       Without it, d2.name() segfaults because pybind11 uses an incorrect
+        //       pointer offset when dispatching through the virtual inheritance chain.
+        .def("name", &SftVirtDerived2::name)
         .def("call_name", &SftVirtDerived2::call_name, py::arg("d2"));
 
     // test_move_only_holder