Avoid recursively throwing an exception from a custom operator new.

Allocating an exception object may recursively call our custom operator
new, which would result in an infinite recursion and a crash. So only
initiate the throw when the operator is called non-recursively.
Otherwise, let the memory allocation proceed using malloc.

This should hopefully fix thread constructor test crashes on OpenBSD.

Author: Lastique

test/threads/thread/constr/FArgs_pass.cpp

@@ -26,6 +26,19 @@
 #include <boost/config.hpp>
 
 unsigned throw_one = 0xFFFF;
+unsigned operator_new_recursion_level = 0u;
+
+struct operator_new_recursion_counter
+{
+  operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    ++operator_new_recursion_level;
+  }
+  ~operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    --operator_new_recursion_level;
+  }
+};
 
 #if defined _GLIBCXX_THROW
 void* operator new(std::size_t s) _GLIBCXX_THROW (std::bad_alloc)
@@ -34,10 +47,22 @@ void* operator new(std::size_t s)
 #endif
 {
   //std::cout << __FILE__ << ":" << __LINE__ << std::endl;
-  if (throw_one == 0) throw std::bad_alloc();
-  --throw_one;
+  // Throwing an exception may recursively call operator new. If we're throwing std::bad_alloc,
+  // this may cause infinite recursion and a crash. So only throw if we're at the top recursion level.
+  operator_new_recursion_counter auto_counter;
+  if (operator_new_recursion_level == 1u)
+  {
+    if (throw_one == 0) throw std::bad_alloc();
+    --throw_one;
+  }
   void* p = std::malloc(s);
-  if (!p) throw std::bad_alloc();
+  if (!p)
+  {
+    if (operator_new_recursion_level == 1u)
+      throw std::bad_alloc();
+    else
+      std::abort();
+  }
   return p;
 }
 

test/threads/thread/constr/F_pass.cpp

@@ -26,6 +26,19 @@
 #include <boost/config.hpp>
 
 unsigned throw_one = 0xFFFF;
+unsigned operator_new_recursion_level = 0u;
+
+struct operator_new_recursion_counter
+{
+  operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    ++operator_new_recursion_level;
+  }
+  ~operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    --operator_new_recursion_level;
+  }
+};
 
 #if defined _GLIBCXX_THROW
 void* operator new(std::size_t s) _GLIBCXX_THROW (std::bad_alloc)
@@ -34,10 +47,22 @@ void* operator new(std::size_t s)
 #endif
 {
   //std::cout << __FILE__ << ":" << __LINE__ << std::endl;
-  if (throw_one == 0) throw std::bad_alloc();
-  --throw_one;
+  // Throwing an exception may recursively call operator new. If we're throwing std::bad_alloc,
+  // this may cause infinite recursion and a crash. So only throw if we're at the top recursion level.
+  operator_new_recursion_counter auto_counter;
+  if (operator_new_recursion_level == 1u)
+  {
+    if (throw_one == 0) throw std::bad_alloc();
+    --throw_one;
+  }
   void* p = std::malloc(s);
-  if (!p) throw std::bad_alloc();
+  if (!p)
+  {
+    if (operator_new_recursion_level == 1u)
+      throw std::bad_alloc();
+    else
+      std::abort();
+  }
   return p;
 }
 

test/threads/thread/constr/lambda_pass.cpp

@@ -28,17 +28,42 @@
 #if ! defined BOOST_NO_CXX11_LAMBDAS
 
 unsigned throw_one = 0xFFFF;
+unsigned operator_new_recursion_level = 0u;
+
+struct operator_new_recursion_counter
+{
+  operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    ++operator_new_recursion_level;
+  }
+  ~operator_new_recursion_counter() BOOST_NOEXCEPT_OR_NOTHROW
+  {
+    --operator_new_recursion_level;
+  }
+};
 
 #if defined _GLIBCXX_THROW
 void* operator new(std::size_t s) _GLIBCXX_THROW (std::bad_alloc)
 #else
 void* operator new(std::size_t s)
 #endif
 {
-  if (throw_one == 0) throw std::bad_alloc();
-  --throw_one;
+  // Throwing an exception may recursively call operator new. If we're throwing std::bad_alloc,
+  // this may cause infinite recursion and a crash. So only throw if we're at the top recursion level.
+  operator_new_recursion_counter auto_counter;
+  if (operator_new_recursion_level == 1u)
+  {
+    if (throw_one == 0) throw std::bad_alloc();
+    --throw_one;
+  }
   void* p = std::malloc(s);
-  if (!p) throw std::bad_alloc();
+  if (!p)
+  {
+    if (operator_new_recursion_level == 1u)
+      throw std::bad_alloc();
+    else
+      std::abort();
+  }
   return p;
 }