diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h
index 0c41670926..b6f521ac4a 100644
--- a/include/pybind11/attr.h
+++ b/include/pybind11/attr.h
@@ -231,6 +231,7 @@ struct type_record {
 
     // Pointer to RTTI type_info data structure
     const std::type_info *type = nullptr;
+    const std::type_info *holder_type = nullptr;
 
     /// How large is the underlying C++ type?
     size_t type_size = 0;
diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h
index d6e440f787..64bd2fbe5e 100644
--- a/include/pybind11/cast.h
+++ b/include/pybind11/cast.h
@@ -908,8 +908,10 @@ template <typename type> class type_caster_base : public type_caster_generic {
             make_copy_constructor(src), make_move_constructor(src));
     }
 
-    static handle cast_holder(const itype *src, const void *holder) {
+    static handle cast_holder(const itype *src, const void *holder, const std::type_info &holder_type) {
         auto st = src_and_type(src);
+        if (!same_type(*st.second->holder_type, holder_type))
+            throw cast_error(std::string("Unexpected holder type: ") + holder_type.name() + ", expected: " + st.second->holder_type->name());
         return type_caster_generic::cast(
             st.first, return_value_policy::take_ownership, {}, st.second,
             nullptr, nullptr, holder);
@@ -1516,7 +1518,7 @@ struct copyable_holder_caster : public type_caster_base<type> {
 
     static handle cast(const holder_type &src, return_value_policy, handle) {
         const auto *ptr = holder_helper<holder_type>::get(src);
-        return type_caster_base<type>::cast_holder(ptr, &src);
+        return type_caster_base<type>::cast_holder(ptr, std::addressof(src), typeid(holder_type));
     }
 
 protected:
@@ -1528,6 +1530,8 @@ struct copyable_holder_caster : public type_caster_base<type> {
 
     bool load_value(value_and_holder &&v_h) {
         if (v_h.holder_constructed()) {
+            if (!same_type(*typeinfo->holder_type, typeid(holder_type)))
+                throw cast_error(std::string("Unexpected holder type: ") + typeid(holder_type).name() + ", expected: " + typeinfo->holder_type->name());
             value = v_h.value_ptr();
             holder = v_h.template holder<holder_type>();
             return true;
@@ -1574,7 +1578,7 @@ struct move_only_holder_caster {
 
     static handle cast(holder_type &&src, return_value_policy, handle) {
         auto *ptr = holder_helper<holder_type>::get(src);
-        return type_caster_base<type>::cast_holder(ptr, std::addressof(src));
+        return type_caster_base<type>::cast_holder(ptr, std::addressof(src), typeid(holder_type));
     }
     static constexpr auto name = type_caster_base<type>::name;
 };
diff --git a/include/pybind11/detail/internals.h b/include/pybind11/detail/internals.h
index a455715bfd..bbdc2cea3a 100644
--- a/include/pybind11/detail/internals.h
+++ b/include/pybind11/detail/internals.h
@@ -128,6 +128,7 @@ struct internals {
 struct type_info {
     PyTypeObject *type;
     const std::type_info *cpptype;
+    const std::type_info *holder_type = nullptr;
     size_t type_size, type_align, holder_size_in_ptrs;
     void *(*operator_new)(size_t);
     void (*init_instance)(instance *, const void *);
@@ -150,7 +151,7 @@ struct type_info {
 };
 
 /// Tracks the `internals` and `type_info` ABI version independent of the main library version
-#define PYBIND11_INTERNALS_VERSION 4
+#define PYBIND11_INTERNALS_VERSION 5
 
 /// On MSVC, debug and release builds are not ABI-compatible!
 #if defined(_MSC_VER) && defined(_DEBUG)
diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h
index 51fb74a57f..f54acd2d59 100644
--- a/include/pybind11/pybind11.h
+++ b/include/pybind11/pybind11.h
@@ -1042,6 +1042,7 @@ class generic_type : public object {
         auto *tinfo = new detail::type_info();
         tinfo->type = (PyTypeObject *) m_ptr;
         tinfo->cpptype = rec.type;
+        tinfo->holder_type = rec.holder_type;
         tinfo->type_size = rec.type_size;
         tinfo->type_align = rec.type_align;
         tinfo->operator_new = rec.operator_new;
@@ -1225,6 +1226,7 @@ class class_ : public detail::generic_type {
         record.scope = scope;
         record.name = name;
         record.type = &typeid(type);
+        record.holder_type = &typeid(holder_type);
         record.type_size = sizeof(conditional_t<has_alias, type_alias, type>);
         record.type_align = alignof(conditional_t<has_alias, type_alias, type>&);
         record.holder_size = sizeof(holder_type);
diff --git a/tests/test_smart_ptr.cpp b/tests/test_smart_ptr.cpp
index 60c2e692e5..7d0b8df030 100644
--- a/tests/test_smart_ptr.cpp
+++ b/tests/test_smart_ptr.cpp
@@ -85,6 +85,17 @@ class unique_ptr_with_addressof_operator {
 };
 PYBIND11_DECLARE_HOLDER_TYPE(T, unique_ptr_with_addressof_operator<T>);
 
+// Simple custom holder that works like shared_ptr, but has a different memory layout
+template <typename T>
+class huge_shared_ptr {
+    uint64_t padding[10];
+    std::shared_ptr<T> impl;
+public:
+    huge_shared_ptr( ) = default;
+    huge_shared_ptr(T* p) : impl(p) { }
+    T* get() const { return impl.get(); }
+};
+PYBIND11_DECLARE_HOLDER_TYPE(T, huge_shared_ptr<T>);
 
 TEST_SUBMODULE(smart_ptr, m) {
 
@@ -109,7 +120,7 @@ TEST_SUBMODULE(smart_ptr, m) {
     py::implicitly_convertible<py::int_, MyObject1>();
 
     m.def("make_object_1", []() -> Object * { return new MyObject1(1); });
-    m.def("make_object_2", []() -> ref<Object> { return new MyObject1(2); });
+    m.def("make_object_2", []() -> ref<MyObject1> { return new MyObject1(2); });
     m.def("make_myobject1_1", []() -> MyObject1 * { return new MyObject1(4); });
     m.def("make_myobject1_2", []() -> ref<MyObject1> { return new MyObject1(5); });
     m.def("print_object_1", [](const Object *obj) { py::print(obj->toString()); });
@@ -143,6 +154,10 @@ TEST_SUBMODULE(smart_ptr, m) {
     m.def("print_myobject2_2", [](std::shared_ptr<MyObject2> obj) { py::print(obj->toString()); });
     m.def("print_myobject2_3", [](const std::shared_ptr<MyObject2> &obj) { py::print(obj->toString()); });
     m.def("print_myobject2_4", [](const std::shared_ptr<MyObject2> *obj) { py::print((*obj)->toString()); });
+    // Using wrong holder type should raise a cast_error at runtime
+    m.def("make_myobject2_3", []() { return huge_shared_ptr<MyObject2>(new MyObject2(9)); });
+    m.def("print_myobject2_5", [](const huge_shared_ptr<MyObject2> &obj) { py::print(obj.get()->toString()); });
+    m.def("print_myobject2_6", [](const huge_shared_ptr<MyObject2> *obj) { py::print(obj->get()->toString()); });
 
     // Object managed by a std::shared_ptr<>, additionally derives from std::enable_shared_from_this<>
     class MyObject3 : public std::enable_shared_from_this<MyObject3> {
diff --git a/tests/test_smart_ptr.py b/tests/test_smart_ptr.py
index c55bffba07..5f293c03ad 100644
--- a/tests/test_smart_ptr.py
+++ b/tests/test_smart_ptr.py
@@ -59,12 +59,18 @@ def test_smart_ptr(capture):
             m.print_myobject2_3(o)
             m.print_myobject2_4(o)
         assert capture == "MyObject2[{i}]\n".format(i=i) * 4
+        with pytest.raises(RuntimeError):
+            m.print_myobject2_5(o)
+        with pytest.raises(RuntimeError):
+            m.print_myobject2_6(o)
+    with pytest.raises(RuntimeError):
+        m.make_myobject2_3()
 
     cstats = ConstructorStats.get(m.MyObject2)
     assert cstats.alive() == 1
     o = None
     assert cstats.alive() == 0
-    assert cstats.values() == ["MyObject2[8]", "MyObject2[6]", "MyObject2[7]"]
+    assert cstats.values() == ["MyObject2[8]", "MyObject2[6]", "MyObject2[7]", "MyObject2[9]"]
     assert cstats.default_constructions == 0
     assert cstats.copy_constructions == 0
     # assert cstats.move_constructions >= 0 # Doesn't invoke any