Register Boost.Python converters for NumPy 2.0 scalar types

Workaround for boostorg/python#511

NumPy 2.0 scalar types (np.float32, np.int32, np.int64, etc.) no longer
subclass Python's built-in float/int, so Boost.Python's rvalue converters
fail to convert them. Register custom from-Python converters that use the
PyNumber_Float/PyNumber_Long protocols instead.

Fixes #1084

Author: dwpaley

scitbx/array_family/boost_python/flex_ext.cpp

@@ -38,6 +38,7 @@ namespace scitbx { namespace af { namespace boost_python {
 #else
   void import_numpy_api_if_available();
 #endif
+  void register_numpy_scalar_converters();
   void wrap_flex_grid();
   void wrap_flex_bool();
   void wrap_flex_size_t();
@@ -464,6 +465,7 @@ namespace {
     using boost::python::arg;
 
     import_numpy_api_if_available();
+    register_numpy_scalar_converters();
 
     register_scitbx_tuple_mappings();
 

scitbx/array_family/boost_python/numpy_bridge.cpp

@@ -1,6 +1,7 @@
 #include <stdint.h>
 #include <boost/python/numpy.hpp>
 #include <boost/python.hpp>
+#include <boost/python/converter/registry.hpp>
 #include <scitbx/array_family/versa.h>
 #include <scitbx/array_family/accessors/flex_grid.h>
 #include <boost_adaptbx/type_id_eq.h>
@@ -230,4 +231,105 @@ namespace scitbx { namespace af { namespace boost_python {
 
 #undef SCITBX_LOC
 
+// Workaround for https://github.com/boostorg/python/issues/511
+// NumPy 2.0 scalar types (e.g. np.float32, np.int32) no longer subclass
+// Python's built-in float/int, so Boost.Python's rvalue converters
+// (which use PyFloat_Check/PyLong_Check) fail to convert them.
+// Register custom converters that use PyNumber_Float/PyNumber_Long instead.
+
+namespace {
+
+  // Converter for numpy scalar -> C++ floating point types
+  template <typename CppType>
+  struct numpy_scalar_to_floating {
+    static void* convertible(PyObject* obj) {
+#if defined(SCITBX_HAVE_NUMPY_INCLUDE)
+      if (PyArray_IsScalar(obj, Number) && !PyArray_IsScalar(obj, ComplexFloating)) {
+        return obj;
+      }
+#endif
+      return nullptr;
+    }
+
+    static void construct(
+        PyObject* obj,
+        boost::python::converter::rvalue_from_python_stage1_data* data)
+    {
+      void* storage = reinterpret_cast<
+        boost::python::converter::rvalue_from_python_storage<CppType>*>(
+          data)->storage.bytes;
+      PyObject* as_float = PyNumber_Float(obj);
+      if (!as_float) boost::python::throw_error_already_set();
+      CppType value = static_cast<CppType>(PyFloat_AsDouble(as_float));
+      if (value == static_cast<CppType>(-1.0) && PyErr_Occurred()) {
+        Py_DECREF(as_float);
+        boost::python::throw_error_already_set();
+      }
+      Py_DECREF(as_float);
+      new (storage) CppType(value);
+      data->convertible = storage;
+    }
+  };
+
+  // Converter for numpy scalar -> C++ integer types
+  template <typename CppType>
+  struct numpy_scalar_to_integer {
+    static void* convertible(PyObject* obj) {
+#if defined(SCITBX_HAVE_NUMPY_INCLUDE)
+      if (PyArray_IsScalar(obj, Integer)) {
+        return obj;
+      }
+#endif
+      return nullptr;
+    }
+
+    static void construct(
+        PyObject* obj,
+        boost::python::converter::rvalue_from_python_stage1_data* data)
+    {
+      void* storage = reinterpret_cast<
+        boost::python::converter::rvalue_from_python_storage<CppType>*>(
+          data)->storage.bytes;
+      PyObject* as_long = PyNumber_Long(obj);
+      if (!as_long) boost::python::throw_error_already_set();
+      CppType value = static_cast<CppType>(PyLong_AsLong(as_long));
+      if (value == static_cast<CppType>(-1) && PyErr_Occurred()) {
+        Py_DECREF(as_long);
+        boost::python::throw_error_already_set();
+      }
+      Py_DECREF(as_long);
+      new (storage) CppType(value);
+      data->convertible = storage;
+    }
+  };
+
+} // anonymous namespace
+
+  void register_numpy_scalar_converters()
+  {
+#if defined(SCITBX_HAVE_NUMPY_INCLUDE)
+    using namespace boost::python;
+
+    // Floating point converters
+    converter::registry::push_back(
+      &numpy_scalar_to_floating<double>::convertible,
+      &numpy_scalar_to_floating<double>::construct,
+      type_id<double>());
+    converter::registry::push_back(
+      &numpy_scalar_to_floating<float>::convertible,
+      &numpy_scalar_to_floating<float>::construct,
+      type_id<float>());
+
+    // Integer converters
+    converter::registry::push_back(
+      &numpy_scalar_to_integer<int>::convertible,
+      &numpy_scalar_to_integer<int>::construct,
+      type_id<int>());
+    converter::registry::push_back(
+      &numpy_scalar_to_integer<long>::convertible,
+      &numpy_scalar_to_integer<long>::construct,
+      type_id<long>());
+#endif
+  }
+
 }}} // namespace scitbx::af::boost_python

scitbx/array_family/boost_python/numpy_bridge.hpp

@@ -34,4 +34,6 @@ namespace scitbx { namespace af { namespace boost_python {
   // SCITBX_LOC(uint64, uint64_t)
 #undef SCITBX_LOC
 
+  void register_numpy_scalar_converters();
+
 }}} // namespace scitbx::af::boost_python

scitbx/array_family/boost_python/tst_numpy_scalar_conversions.py

@@ -0,0 +1,109 @@
+from __future__ import absolute_import, division, print_function
+import sys
+
+def run(args):
+  assert len(args) == 0
+  try:
+    import numpy as np
+  except ImportError:
+    print("numpy not available, skipping")
+    print("OK")
+    return
+  from scitbx.array_family import flex
+
+  exercise_original_reproducer(np, flex)
+  exercise_element_setitem(np, flex)
+  exercise_element_iadd(np, flex)
+  exercise_array_iadd(np, flex)
+  exercise_construction_from_numpy_scalars(np, flex)
+  exercise_value_preservation(np, flex)
+  print("OK")
+
+def exercise_original_reproducer(np, flex):
+  """Reproducer from https://github.com/cctbx/cctbx_project/issues/1084"""
+  arr = flex.double(10)
+  arr[0] += np.array([1,2,3], dtype=np.float32)[0]
+  assert arr[0] == 1.0
+
+def exercise_element_setitem(np, flex):
+  """Test arr[i] = numpy_scalar for various type combinations."""
+  # float scalars -> flex.double
+  a = flex.double(1)
+  for val in [np.float32(3.5), np.float64(3.5)]:
+    a[0] = val
+    assert a[0] == 3.5, (a[0], type(val))
+  # integer scalars -> flex.double (implicit widening)
+  for val in [np.int32(7), np.int64(7)]:
+    a[0] = val
+    assert a[0] == 7.0, (a[0], type(val))
+  # float scalars -> flex.float
+  b = flex.float(1)
+  for val in [np.float32(2.5), np.float64(2.5)]:
+    b[0] = val
+    assert abs(b[0] - 2.5) < 1e-6, (b[0], type(val))
+  # integer scalars -> flex.int
+  c = flex.int(1)
+  for val in [np.int32(42), np.int64(42)]:
+    c[0] = val
+    assert c[0] == 42, (c[0], type(val))
+
+def exercise_element_iadd(np, flex):
+  """Test arr[i] += numpy_scalar for various type combinations."""
+  a = flex.double([10.0])
+  a[0] += np.float32(2.5)
+  assert a[0] == 12.5
+  a[0] += np.float64(1.0)
+  assert a[0] == 13.5
+  a[0] += np.int32(1)
+  assert a[0] == 14.5
+  a[0] += np.int64(1)
+  assert a[0] == 15.5
+
+  b = flex.int([10])
+  b[0] += np.int32(5)
+  assert b[0] == 15
+  b[0] += np.int64(3)
+  assert b[0] == 18
+
+def exercise_array_iadd(np, flex):
+  """Test arr += numpy_scalar (whole-array operations)."""
+  a = flex.double([1.0, 2.0, 3.0])
+  a += np.float32(10.0)
+  assert list(a) == [11.0, 12.0, 13.0]
+  a += np.float64(1.0)
+  assert list(a) == [12.0, 13.0, 14.0]
+  a += np.int32(1)
+  assert list(a) == [13.0, 14.0, 15.0]
+
+  b = flex.int([1, 2, 3])
+  b += np.int32(10)
+  assert list(b) == [11, 12, 13]
+  b += np.int64(1)
+  assert list(b) == [12, 13, 14]
+
+def exercise_construction_from_numpy_scalars(np, flex):
+  """Test constructing flex arrays from lists containing numpy scalars."""
+  a = flex.double([np.float32(1.0), np.float64(2.0), np.float32(3.0)])
+  assert list(a) == [1.0, 2.0, 3.0]
+  b = flex.int([np.int32(1), np.int32(2), np.int32(3)])
+  assert list(b) == [1, 2, 3]
+
+def exercise_value_preservation(np, flex):
+  """Test that values are preserved accurately through conversion."""
+  # float32 has ~7 decimal digits of precision
+  a = flex.double(1)
+  a[0] = np.float32(1.23456789)
+  # float32 truncates, so check against float32 precision
+  assert abs(a[0] - float(np.float32(1.23456789))) < 1e-10
+
+  # float64 should be exact for representable values
+  a[0] = np.float64(1.234567890123456)
+  assert a[0] == 1.234567890123456
+
+  # Large integers
+  b = flex.int(1)
+  b[0] = np.int32(2147483647)  # INT32_MAX
+  assert b[0] == 2147483647
+
+if __name__ == "__main__":
+  run(args=sys.argv[1:])

scitbx/run_tests.py

@@ -44,6 +44,7 @@
   "$D/array_family/boost_python/regression_test.py",
   "$D/array_family/boost_python/tst_flex.py",
   "$D/array_family/boost_python/tst_numpy_bridge.py",
+  "$D/array_family/boost_python/tst_numpy_scalar_conversions.py",
   "$D/array_family/boost_python/tst_smart_selection.py",
   "$D/array_family/boost_python/tst_shared.py",
   "$D/array_family/boost_python/tst_integer_offsets_vs_pointers.py",