[CPyCppyy] Introduce new public CreatePyValueFromMemory function

Add a CPyCppyy API function to build a Python value from a memory
address using a Converter, with the possibility to build also an array
proxy with a given shape.

This is needed to implement the TTree pythonizations without including
private CPyCppyy internals.

In principle, the TTree pythonizations can now also be implemented
completely in Python, which would be the next step.

Author: guitargeek

bindings/pyroot/cppyy/CPyCppyy/include/CPyCppyy/API.h

@@ -127,6 +127,19 @@ CPYCPPYY_EXTERN Converter* CreateConverter(const std::string& name, cdims_t = 0)
 // delete a previously created converter
 CPYCPPYY_EXTERN void DestroyConverter(Converter* p);
 
+// Build a Python value from a memory address using a Converter.
+//
+// If ``ndim`` is non-zero, the value is treated as an array of shape
+// ``dims[0] x dims[1] x ... x dims[ndim-1]`` and ``FromMemory(&address)``
+// is called on the converter (matching the calling convention array
+// converters expect). Otherwise the value is treated as a scalar and
+// ``FromMemory(address)`` is called directly.
+//
+// Returns a new reference on success, or nullptr and sets a Python error if
+// no converter for ``typeName`` could be created.
+CPYCPPYY_EXTERN PyObject*
+CreatePyValueFromMemory(const std::string& typeName, void* address, dim_t ndim = 0, const dim_t* dims = nullptr);
+
 // register a custom converter
 typedef Converter* (*ConverterFactory_t)(cdims_t);
 CPYCPPYY_EXTERN bool RegisterConverter(const std::string& name, ConverterFactory_t);

bindings/pyroot/cppyy/CPyCppyy/src/API.cxx

@@ -1,5 +1,6 @@
 // Bindings
 #include "CPyCppyy.h"
+#include "Dimensions.h"
 #define CPYCPPYY_INTERNAL 1
 #include "CPyCppyy/API.h"
 #undef CPYCPPYY_INTERNAL
@@ -462,3 +463,28 @@ void CPyCppyy::Prompt() {
 // enter i/o interactive mode
     PyRun_InteractiveLoop(stdin, const_cast<char*>("\0"));
 }
+
+//-----------------------------------------------------------------------------
+PyObject* CPyCppyy::CreatePyValueFromMemory(
+    const std::string& typeName, void* address, dim_t ndim, const dim_t* dims)
+{
+// Build a Python value from a memory address using a Converter. See the
+// declaration in CPyCppyy/API.h for the full semantics.
+    Dimensions shape(ndim, dims);
+    Converter* cnv = CreateConverter(typeName, shape);
+    if (!cnv) {
+        PyErr_Format(PyExc_TypeError, "no converter available for type '%s'", typeName.c_str());
+        return nullptr;
+    }
+
+    PyObject* result = nullptr;
+    if (ndim) {
+    // Array converter: FromMemory expects the address of the data pointer.
+        result = cnv->FromMemory(&address);
+    } else {
+    // Scalar converter: FromMemory expects the address of the value.
+        result = cnv->FromMemory(address);
+    }
+    DestroyConverter(cnv);
+    return result;
+}

bindings/pyroot/cppyy/CPyCppyy/src/CPyCppyyModule.cxx

@@ -263,6 +263,11 @@ namespace CPyCppyy {
        static std::map<std::string, std::vector<PyObject*>> pyzMap;
        return pyzMap;
     }
+
+    // Forward-declared from CPyCppyy/API.h (which can't be #include'd here
+    // because it would conflict with the internal Converters.h).
+    PyObject* CreatePyValueFromMemory(
+        const std::string& typeName, void* address, dim_t ndim = 0, const dim_t* dims = nullptr);
 }
 
 
@@ -816,6 +821,54 @@ static PyObject* BindObject(PyObject*, PyObject* args, PyObject* kwds)
     return BindCppObjectNoCast(addr, cast_type);
 }
 
+//----------------------------------------------------------------------------
+static PyObject* CreateValueFromMemoryPy(PyObject*, PyObject* args)
+{
+// Python entry point for CPyCppyy::CreatePyValueFromMemory.
+//
+// Arguments:
+//   type_name (str): C++ type name to convert from, e.g. "double", "int",
+//                    "double[]", "float*", "MyClass*", etc.
+//   address   (int): memory address to read from. For scalar (non-array)
+//                    types this is the address of the value. For array
+//                    types this is the address of the data array itself
+//                    (i.e. the value of the data pointer).
+//   dims (sequence, optional): sequence of integers giving the shape of
+//                    the array. If provided, the value is treated as an
+//                    array (a LowLevelView is returned). If omitted or
+//                    None, the value is treated as a scalar.
+    const char* typeName = nullptr;
+    PyObject* addressArg = nullptr;
+    PyObject* dimsArg = Py_None;
+    if (!PyArg_ParseTuple(args, "sO|O:_create_value_from_memory", &typeName, &addressArg, &dimsArg))
+        return nullptr;
+
+    void* address = PyLong_AsVoidPtr(addressArg);
+    if (PyErr_Occurred())
+        return nullptr;
+
+    std::vector<dim_t> dimsVec;
+    if (dimsArg && dimsArg != Py_None) {
+        if (!PySequence_Check(dimsArg)) {
+            PyErr_SetString(PyExc_TypeError, "_create_value_from_memory: dims must be a sequence");
+            return nullptr;
+        }
+        Py_ssize_t ndim = PySequence_Length(dimsArg);
+        dimsVec.reserve(ndim);
+        for (Py_ssize_t i = 0; i < ndim; ++i) {
+            PyObject* item = PySequence_GetItem(dimsArg, i);
+            if (!item) return nullptr;
+            dim_t d = (dim_t)PyLong_AsLongLong(item);
+            Py_DECREF(item);
+            if (PyErr_Occurred()) return nullptr;
+            dimsVec.push_back(d);
+        }
+    }
+
+    return CPyCppyy::CreatePyValueFromMemory(
+        typeName, address, (dim_t)dimsVec.size(), dimsVec.data());
+}
+
 //----------------------------------------------------------------------------
 static PyObject* Move(PyObject*, PyObject* pyobject)
 {
@@ -996,6 +1049,8 @@ static PyMethodDef gCPyCppyyMethods[] = {
       METH_O, (char*)"Represent an array of objects as raw memory."},
     {(char*)"bind_object", (PyCFunction)BindObject,
       METH_VARARGS | METH_KEYWORDS, (char*) "Create an object of given type, from given address."},
+    {(char*)"_create_value_from_memory", (PyCFunction)CreateValueFromMemoryPy,
+      METH_VARARGS, (char*) "Build a Python value from a memory address using a Converter."},
     {(char*) "move", (PyCFunction)Move,
       METH_O, (char*)"Cast the C++ object to become movable."},
     {(char*) "add_pythonization", (PyCFunction)AddPythonization,

bindings/pyroot/cppyy/CPyCppyy/src/Dimensions.h

@@ -17,7 +17,7 @@ class CPYCPPYY_CLASS_EXPORT Dimensions {
     dim_t* fDims;
 
 public:
-    Dimensions(dim_t ndim = 0, dim_t* dims = nullptr) : fDims(nullptr) {
+    Dimensions(dim_t ndim = 0, const dim_t* dims = nullptr) : fDims(nullptr) {
         if (ndim && ndim != UNKNOWN_SIZE) {
             fDims = new dim_t[ndim+1];
             fDims[0] = ndim;

bindings/pyroot/pythonizations/src/TTreePyz.cxx

@@ -12,18 +12,11 @@
 // Bindings
 #include <Python.h>
 
-// TODO: refactor public CPyCppyy API such that this forward declaration is not
-// needed anymore. Including "CPyCppyy/API.h" should be enough.
-namespace CPyCppyy {
-typedef Py_ssize_t dim_t;
-} // namespace CPyCppyy
+#include "CPyCppyy/API.h"
 
 #include "../../cppyy/CPyCppyy/src/Cppyy.h"
 #include "../../cppyy/CPyCppyy/src/CPPInstance.h"
 #include "../../cppyy/CPyCppyy/src/ProxyWrappers.h"
-#include "../../cppyy/CPyCppyy/src/Dimensions.h"
-
-#include "CPyCppyy/API.h"
 
 #include "PyROOTPythonize.h"
 
@@ -150,30 +143,21 @@ static PyObject *WrapLeaf(TLeaf *leaf)
       if (std::count(title.begin(), title.end(), '[') >= 2) {
          dimsVec = getMultiDims(title);
       }
-      CPyCppyy::Dimensions dims{static_cast<dim_t>(dimsVec.size()), dimsVec.data()};
-      Converter *pcnv = CreateConverter(typeName + (isStatic ? "[]" : "*"), dims);
-
       void *address = 0;
       if (leaf->GetBranch())
          address = (void *)leaf->GetBranch()->GetAddress();
       if (!address)
          address = (void *)leaf->GetValuePointer();
 
-      PyObject *value = pcnv->FromMemory(&address);
-      CPyCppyy::DestroyConverter(pcnv);
-
-      return value;
+      return CPyCppyy::CreatePyValueFromMemory(
+         typeName + (isStatic ? "[]" : "*"), address,
+         static_cast<dim_t>(dimsVec.size()), dimsVec.data());
    } else if (leaf->GetValuePointer()) {
       // value types
-      Converter *pcnv = CreateConverter(leaf->GetTypeName());
-      PyObject *value = 0;
-      if (leaf->IsA() == TLeafElement::Class() || leaf->IsA() == TLeafObject::Class())
-         value = pcnv->FromMemory((void *)*(void **)leaf->GetValuePointer());
-      else
-         value = pcnv->FromMemory((void *)leaf->GetValuePointer());
-      CPyCppyy::DestroyConverter(pcnv);
-
-      return value;
+      void *address = (leaf->IsA() == TLeafElement::Class() || leaf->IsA() == TLeafObject::Class())
+                         ? (void *)*(void **)leaf->GetValuePointer()
+                         : (void *)leaf->GetValuePointer();
+      return CPyCppyy::CreatePyValueFromMemory(leaf->GetTypeName(), address);
    }
 
    return nullptr;