private_typeinfo: limit is_dst_type_derived_from_static_type optimization

Patch by Ryan Prichard

If the destination type does not derive from the static type, we can skip
the search_above_dst call, but we still need to run the
!does_dst_type_point_to_our_static_type block of code. That block of code
will increment info->number_to_dst_ptr to 2, and because dest isn't derived
from static, the cast will ultimately fail.

Fixes PR33439

Reviewed as https://reviews.llvm.org/D36447


git-svn-id: https://llvm.org/svn/llvm-project/libcxxabi/trunk@332767 91177308-0d34-0410-b5e6-96231b3b80d8

2 files changed, 84 insertions, 32 deletions

diff --git a/src/private_typeinfo.cpp b/src/private_typeinfo.cpp
index 6da3c77..a0326a1 100644
--- a/src/private_typeinfo.cpp
+++ b/src/private_typeinfo.cpp
@@ -859,13 +859,14 @@ __vmi_class_type_info::search_below_dst(__dynamic_cast_info* info,
             // Record the access path that got us here
             //   If there is more than one dst_type this path doesn't matter.
             info->path_dynamic_ptr_to_dst_ptr = path_below;
+            bool does_dst_type_point_to_our_static_type = false;
             // Only search above here if dst_type derives from static_type, or
             //    if it is unknown if dst_type derives from static_type.
             if (info->is_dst_type_derived_from_static_type != no)
             {
                 // Set up flags to record results from all base classes
                 bool is_dst_type_derived_from_static_type = false;
-                bool does_dst_type_point_to_our_static_type = false;
+
                 // We've found a dst_type with a potentially public path to here.
                 // We have to assume the path is public because it may become
                 //   public later (if we get back to here with a public path).
@@ -909,21 +910,6 @@ __vmi_class_type_info::search_below_dst(__dynamic_cast_info* info,
                         }
                     }
                 }
-                if (!does_dst_type_point_to_our_static_type)
-                {
-                    // We found a dst_type that doesn't point to (static_ptr, static_type)
-                    // So record the address of this dst_ptr and increment the
-                    // count of the number of such dst_types found in the tree.
-                    info->dst_ptr_not_leading_to_static_ptr = current_ptr;
-                    info->number_to_dst_ptr += 1;
-                    // If there exists another dst with a private path to
-                    //    (static_ptr, static_type), then the cast from 
-                    //     (dynamic_ptr, dynamic_type) to dst_type is now ambiguous,
-                    //      so stop search.
-                    if (info->number_to_static_ptr == 1 &&
-                            info->path_dst_ptr_to_static_ptr == not_public_path)
-                        info->search_done = true;
-                }
                 // If we found no static_type,s then dst_type doesn't derive
                 //   from static_type, else it does.  Record this result so that
                 //   next time we hit a dst_type we will know not to search above
@@ -932,7 +918,22 @@ __vmi_class_type_info::search_below_dst(__dynamic_cast_info* info,
                     info->is_dst_type_derived_from_static_type = yes;
                 else
                     info->is_dst_type_derived_from_static_type = no;
-            }
+              }
+              if (!does_dst_type_point_to_our_static_type)
+              {
+                  // We found a dst_type that doesn't point to (static_ptr, static_type)
+                  // So record the address of this dst_ptr and increment the
+                  // count of the number of such dst_types found in the tree.
+                  info->dst_ptr_not_leading_to_static_ptr = current_ptr;
+                  info->number_to_dst_ptr += 1;
+                  // If there exists another dst with a private path to
+                  //    (static_ptr, static_type), then the cast from
+                  //     (dynamic_ptr, dynamic_type) to dst_type is now ambiguous,
+                  //      so stop search.
+                  if (info->number_to_static_ptr == 1 &&
+                          info->path_dst_ptr_to_static_ptr == not_public_path)
+                      info->search_done = true;
+              }
         }
     }
     else
@@ -1030,13 +1031,13 @@ __si_class_type_info::search_below_dst(__dynamic_cast_info* info,
             // Record the access path that got us here
             //   If there is more than one dst_type this path doesn't matter.
             info->path_dynamic_ptr_to_dst_ptr = path_below;
+            bool does_dst_type_point_to_our_static_type = false;
             // Only search above here if dst_type derives from static_type, or
             //    if it is unknown if dst_type derives from static_type.
             if (info->is_dst_type_derived_from_static_type != no)
             {
                 // Set up flags to record results from all base classes
                 bool is_dst_type_derived_from_static_type = false;
-                bool does_dst_type_point_to_our_static_type = false;
                 // Zero out found flags
                 info->found_our_static_ptr = false;
                 info->found_any_static_type = false;
@@ -1047,20 +1048,6 @@ __si_class_type_info::search_below_dst(__dynamic_cast_info* info,
                     if (info->found_our_static_ptr)
                         does_dst_type_point_to_our_static_type = true;
                 }
-                if (!does_dst_type_point_to_our_static_type)
-                {
-                    // We found a dst_type that doesn't point to (static_ptr, static_type)
-                    // So record the address of this dst_ptr and increment the
-                    // count of the number of such dst_types found in the tree.
-                    info->dst_ptr_not_leading_to_static_ptr = current_ptr;
-                    info->number_to_dst_ptr += 1;
-                    // If there exists another dst with a private path to
-                    //    (static_ptr, static_type), then the cast from 
-                    //     (dynamic_ptr, dynamic_type) to dst_type is now ambiguous.
-                    if (info->number_to_static_ptr == 1 &&
-                            info->path_dst_ptr_to_static_ptr == not_public_path)
-                        info->search_done = true;
-                }
                 // If we found no static_type,s then dst_type doesn't derive
                 //   from static_type, else it does.  Record this result so that
                 //   next time we hit a dst_type we will know not to search above
@@ -1070,6 +1057,20 @@ __si_class_type_info::search_below_dst(__dynamic_cast_info* info,
                 else
                     info->is_dst_type_derived_from_static_type = no;
             }
+            if (!does_dst_type_point_to_our_static_type)
+            {
+                // We found a dst_type that doesn't point to (static_ptr, static_type)
+                // So record the address of this dst_ptr and increment the
+                // count of the number of such dst_types found in the tree.
+                info->dst_ptr_not_leading_to_static_ptr = current_ptr;
+                info->number_to_dst_ptr += 1;
+                // If there exists another dst with a private path to
+                //    (static_ptr, static_type), then the cast from
+                //     (dynamic_ptr, dynamic_type) to dst_type is now ambiguous.
+                if (info->number_to_static_ptr == 1 &&
+                        info->path_dst_ptr_to_static_ptr == not_public_path)
+                    info->search_done = true;
+            }
         }
     }
     else
diff --git a/test/dynamic_cast.pass.cpp b/test/dynamic_cast.pass.cpp
index cd24798..5288f7c 100644
--- a/test/dynamic_cast.pass.cpp
+++ b/test/dynamic_cast.pass.cpp
@@ -95,9 +95,60 @@ void test()
 
 }  // t3
 
+namespace t4
+{
+
+// PR33439
+struct C2 { virtual ~C2() {} Pad1 _; };
+struct C3 { virtual ~C3() {} Pad2 _; };
+struct C4 : C3 { Pad3 _; };
+struct C8 : C2, virtual C4 { Pad4 _; };
+struct C9 : C4, C8 { Pad5 _; };
+
+C9 c9;
+C2 *c2 = &c9;
+
+void test()
+{
+    assert(dynamic_cast<C2*>(c2) == static_cast<C2*>(&c9));
+    assert(dynamic_cast<C3*>(c2) == 0);
+    assert(dynamic_cast<C4*>(c2) == 0);
+    assert(dynamic_cast<C8*>(c2) == static_cast<C8*>(&c9));
+    assert(dynamic_cast<C9*>(c2) == static_cast<C9*>(&c9));
+}
+
+}  // t4
+
+namespace t5
+{
+
+// PR33439
+struct Dummy { virtual ~Dummy() {} Pad1 _; };
+struct Src { virtual ~Src() {} Pad2 _; };
+struct Dest : Dummy { Pad3 _; };
+struct A1 : Dest { Pad4 _; };
+struct A2 : Dest { Pad5 _; };
+struct Root : Src, A1, A2 { Pad6 _; };
+
+Root root;
+Src *src = &root;
+
+void test()
+{
+    assert(dynamic_cast<Dummy*>(src) == 0);
+    assert(dynamic_cast<Src*>(src) == static_cast<Src*>(&root));
+    assert(dynamic_cast<Dest*>(src) == 0);
+    assert(dynamic_cast<A1*>(src) == static_cast<A1*>(&root));
+    assert(dynamic_cast<A2*>(src) == static_cast<A2*>(&root));
+}
+
+}  // t5
+
 int main()
 {
     t1::test();
     t2::test();
     t3::test();
+    t4::test();
+    t5::test();
 }