From f0ef80ee5e11e24479920a661d7a377c424e620f Mon Sep 17 00:00:00 2001 From: Jonas Rembser Date: Wed, 10 Jun 2026 20:22:29 +0200 Subject: [PATCH] [tmva][sofie] Drop NumPy C API dependency from SOFIE parsers The PyTorch parser and the parser unit tests included `` and linked against `Python3::NumPy` purely to read tensor shapes and data pointers (`PyArray_NDIM`/`PyArray_DIM`/`PyArray_DATA`). This coupled SOFIE to a specific NumPy ABI at build/link time, even though NumPy is only ever needed at runtime to produce the arrays on the Python side. This commit repaces these uses with the generic CPython API. (cherry picked from commit f799703100c79529076859db2cbdad1428601e67) --- cmake/modules/SearchInstalledSoftware.cmake | 4 +- tmva/sofie/test/CMakeLists.txt | 2 - tmva/sofie/test/TestRModelParserKeras.C | 92 ++++++++----------- tmva/sofie/test/TestRModelParserPyTorch.C | 35 +++---- tmva/sofie_parsers/CMakeLists.txt | 1 - .../src/RModelParser_PyTorch.cxx | 30 +++--- 6 files changed, 71 insertions(+), 93 deletions(-) diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 09225aa7dbd0b..c29bccf691caa 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -413,7 +413,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva)) elseif(tpython OR tmva-pymva) list(APPEND python_components Development) endif() -if(tmva-pymva OR tmva-sofie) +if(tmva-pymva) list(APPEND python_components NumPy) endif() find_package(Python3 3.10 COMPONENTS ${python_components}) @@ -1378,7 +1378,7 @@ if(tmva) endif() endif() endif() - if(tmva-pymva OR tmva-sofie) + if(tmva-pymva) if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND)) message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required" " (python executable: ${Python3_EXECUTABLE})") diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index 120b5800b2f8d..2012cb32b90a0 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -146,7 +146,6 @@ if (tpython AND ROOT_TORCH_FOUND AND ROOT_ONNX_FOUND AND BLAS_FOUND AND NOT brok LIBRARIES ROOTTMVASofie TMVA - Python3::NumPy Python3::Python BLAS::BLAS INCLUDE_DIRS @@ -165,7 +164,6 @@ if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND) ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C LIBRARIES ROOTTMVASofie - Python3::NumPy Python3::Python BLAS::BLAS INCLUDE_DIRS diff --git a/tmva/sofie/test/TestRModelParserKeras.C b/tmva/sofie/test/TestRModelParserKeras.C index e99ac05106ae5..b9b176e3565c1 100644 --- a/tmva/sofie/test/TestRModelParserKeras.C +++ b/tmva/sofie/test/TestRModelParserKeras.C @@ -1,9 +1,8 @@ #include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include #include "gtest/gtest.h" #include +#include #include "TSystem.h" #include "TMVA/RSofieReader.hxx" @@ -21,6 +20,20 @@ const char *PyStringAsString(PyObject *string) return cstring; } +// Read a NumPy array bound in the Python namespace into a std::vector. +// The array is flattened to a Python list and read through the generic CPython +// API, so this avoids any dependency on the NumPy C API (arrayobject.h). +std::vector GetTensorValues(PyObject *globalNS, PyObject *localNS, const char *name) +{ + std::string code = std::string("outputFlat=") + name + ".flatten().tolist()"; + PyRun_String(code.c_str(), Py_single_input, globalNS, localNS); + PyObject *list = PyDict_GetItemString(localNS, "outputFlat"); + std::vector values(PyList_Size(list)); + for (std::size_t i = 0; i < values.size(); ++i) + values[i] = (float)PyFloat_AsDouble(PyList_GetItem(list, i)); + return values; +} + } // namespace void GenerateModels() { @@ -78,15 +91,12 @@ TEST(RModelParser_Keras, SEQUENTIAL) "0.63637339, 0.94483464, 0.11032887, 0.22424818," "0.50972592, 0.04671024, 0.39230661, 0.80500943]).reshape(4,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSequentialSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputSequential=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputSequentialSize=pOutputSequential.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputSequential.size(), pOutputSequentialSize); - PyArrayObject* pSequentialValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputSequential=(float*)PyArray_DATA(pSequentialValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputSequential.size(); ++i) { EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential[i]), TOLERANCE); @@ -124,15 +134,12 @@ TEST(RModelParser_Keras, FUNCTIONAL) PyRun_String("input=numpy.array([0.60828574, 0.50069386, 0.75186709, 0.14968806, 0.7692464 ,0.77027585, 0.75095316, 0.96651197," "0.38536308, 0.95565917, 0.62796356, 0.13818375, 0.65484891,0.89220363, 0.23879365, 0.00635323]).reshape(2,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputFunctionalSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputFunctional=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputFunctionalSize=pOutputFunctional.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputFunctional.size(), pOutputFunctionalSize); - PyArrayObject* pFunctionalValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputFunctional=(float*)PyArray_DATA(pFunctionalValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputFunctional.size(); ++i) { EXPECT_LE(std::abs(outputFunctional[i] - pOutputFunctional[i]), TOLERANCE); @@ -169,15 +176,12 @@ TEST(RModelParser_Keras, BATCH_NORM) PyRun_String("input=numpy.array([0.22308163, 0.95274901, 0.44712538, 0.84640867," "0.69947928, 0.29743695, 0.81379782, 0.39650574]).reshape(2,4)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputBatchNormSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputBatchNorm=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputBatchNormSize=pOutputBatchNorm.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputBatchNorm.size(), pOutputBatchNormSize); - PyArrayObject* pBatchNormValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputBatchNorm=(float*)PyArray_DATA(pBatchNormValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputBatchNorm.size(); ++i) { EXPECT_LE(std::abs(outputBatchNorm[i] - pOutputBatchNorm[i]), TOLERANCE); @@ -218,15 +222,12 @@ TEST(DISABLED_RModelParser_Keras, CONV_VALID) PyRun_String("model=load_model('KerasModelConv2D_Valid.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConv2DValidSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConv2DValid=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputConv2DValidSize=pOutputConv2DValid.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv2D_Valid.size(), pOutputConv2DValidSize); - PyArrayObject* pConv2DValidValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv2DValid=(float*)PyArray_DATA(pConv2DValidValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv2D_Valid.size(); ++i) { EXPECT_LE(std::abs(outputConv2D_Valid[i] - pOutputConv2DValid[i]), TOLERANCE); @@ -268,15 +269,12 @@ TEST(DISABLED_RModelParser_Keras, CONV_SAME) PyRun_String("model=load_model('KerasModelConv2D_Same.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConv2DSameSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConv2DSame=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputConv2DSameSize=pOutputConv2DSame.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv2D_Same.size(), pOutputConv2DSameSize); - PyArrayObject* pConv2DSameValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv2DSame=(float*)PyArray_DATA(pConv2DSameValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv2D_Same.size(); ++i) { EXPECT_LE(std::abs(outputConv2D_Same[i] - pOutputConv2DSame[i]), TOLERANCE); @@ -314,15 +312,12 @@ TEST(RModelParser_Keras, RESHAPE) PyRun_String("model=load_model('KerasModelReshape.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputReshapeSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputReshape=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputReshapeSize=pOutputReshape.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputReshape.size(), pOutputReshapeSize); - PyArrayObject* pReshapeValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputReshape=(float*)PyArray_DATA(pReshapeValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputReshape.size(); ++i) { EXPECT_LE(std::abs(outputReshape[i] - pOutputReshape[i]), TOLERANCE); @@ -359,14 +354,12 @@ TEST(RModelParser_Keras, CONCATENATE) PyRun_String("input_1=numpy.ones((1,2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input_2=numpy.ones((1,2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model([input_1,input_2]).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - long pOutputConcatenateSize=PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConcatenate=GetTensorValues(fGlobalNS,fLocalNS,"output"); + long pOutputConcatenateSize=(long)pOutputConcatenate.size(); //Testing the actual and expected output tensor sizes (can fail if an error eccoured and returns a -1 from Python) EXPECT_EQ((long) outputConcatenate.size(), pOutputConcatenateSize); - PyArrayObject* pConcatenateValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConcatenate=(float*)PyArray_DATA(pConcatenateValues); //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConcatenate.size(); ++i) { @@ -405,14 +398,12 @@ TEST(RModelParser_Keras, BINARY_OP) PyRun_String("input1=numpy.array([[1,2],[3,4]],dtype='float32')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input2=numpy.array([[5,6],[7,8]],dtype='float32')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model([input1,input2]).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - long pOutputBinaryOpSize=PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputBinaryOp=GetTensorValues(fGlobalNS,fLocalNS,"output"); + long pOutputBinaryOpSize=(long)pOutputBinaryOp.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ((long) outputBinaryOp.size(), pOutputBinaryOpSize); - PyArrayObject* pBinaryOpValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputBinaryOp=(float*)PyArray_DATA(pBinaryOpValues); //Testing the actual and expected output tensor values for (size_t i = 0; i < outputBinaryOp.size(); ++i) { @@ -448,15 +439,12 @@ TEST(RModelParser_Keras, ACTIVATIONS) PyRun_String("model=load_model('KerasModelActivations.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,8))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputActivationsSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputActivations=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputActivationsSize=pOutputActivations.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputActivations.size(), pOutputActivationsSize); - PyArrayObject* pActivationsValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputActivations=(float*)PyArray_DATA(pActivationsValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputActivations.size(); ++i) { EXPECT_LE(std::abs(outputActivations[i] - pOutputActivations[i]), TOLERANCE); @@ -491,15 +479,12 @@ TEST(RModelParser_Keras, SWISH) PyRun_String("model=load_model('KerasModelSwish.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,8))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutput=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputSize=pOutput.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(output.size(), pOutputSize); - PyArrayObject* pValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutput=(float*)PyArray_DATA(pValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < output.size(); ++i) { EXPECT_LE(std::abs(output[i] - pOutput[i]), TOLERANCE); @@ -537,8 +522,8 @@ TEST(RModel, CUSTOM_OP) PyRun_String("model.add(Lambda(lambda x: x * 2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.array([1,1,1,1,1,1,1,1]).reshape(1,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputCustomOpSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputCustomOp=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputCustomOpSize=pOutputCustomOp.size(); // get input name for custom (it is output of one before last) PyRun_String("outputName = model.get_layer(index=len(model.layers)-2).output.name",Py_single_input,fGlobalNS,fLocalNS); @@ -557,9 +542,6 @@ TEST(RModel, CUSTOM_OP) //Testing the actual and expected output tensor sizes EXPECT_EQ(outputCustomOp.size(), pOutputCustomOpSize); - PyArrayObject* pCustomOpValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputCustomOp=(float*)PyArray_DATA(pCustomOpValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputCustomOp.size(); ++i) { EXPECT_LE(std::abs(outputCustomOp[i] - pOutputCustomOp[i]), TOLERANCE); diff --git a/tmva/sofie/test/TestRModelParserPyTorch.C b/tmva/sofie/test/TestRModelParserPyTorch.C index 4dbe36b91d983..d136ec9f7088e 100644 --- a/tmva/sofie/test/TestRModelParserPyTorch.C +++ b/tmva/sofie/test/TestRModelParserPyTorch.C @@ -4,8 +4,6 @@ #include #include "TSystem.h" -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include #include "gtest/gtest.h" #include @@ -60,18 +58,17 @@ TEST(RModelParser_PyTorch, SEQUENTIAL_MODEL) "-0.7750, -1.6701," " 0.8171, -0.2858]),(2,4))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(2,6)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSequentialSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pSequentialValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputSequentialSize=(std::size_t)PyList_Size(pSequentialValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputSequential.size(), pOutputSequentialSize); - PyArrayObject* pSequentialValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputSequential=(float*)PyArray_DATA(pSequentialValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputSequential.size(); ++i) { - EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential[i]), TOLERANCE); + float pOutputSequential=(float)PyFloat_AsDouble(PyList_GetItem(pSequentialValues, i)); + EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential), TOLERANCE); } } @@ -114,18 +111,17 @@ TEST(RModelParser_PyTorch, MODULE_MODEL) "-1.8818, 0.4736," " 1.1102, 1.8694]),(2,6))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(2,12)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputModuleSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pModuleValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputModuleSize=(std::size_t)PyList_Size(pModuleValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputModule.size(), pOutputModuleSize); - PyArrayObject* pModuleValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputModule=(float*)PyArray_DATA(pModuleValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputModule.size(); ++i) { - EXPECT_LE(std::abs(outputModule[i] - pOutputModule[i]), TOLERANCE); + float pOutputModule=(float)PyFloat_AsDouble(PyList_GetItem(pModuleValues, i)); + EXPECT_LE(std::abs(outputModule[i] - pOutputModule), TOLERANCE); } } @@ -158,17 +154,16 @@ TEST(RModelParser_PyTorch, CONVOLUTION_MODEL) PyRun_String("model=torch.jit.load('PyTorchModelConvolution.pt')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=torch.arange(1,751,dtype=torch.float).reshape(5,6,5,5)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(5,5,2,2)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConvSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pConvValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputConvSize=(std::size_t)PyList_Size(pConvValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv.size(), pOutputConvSize); - PyArrayObject* pConvValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv=(float*)PyArray_DATA(pConvValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv.size(); ++i) { - EXPECT_LE(std::abs(outputConv[i] - pOutputConv[i]), TOLERANCE); + float pOutputConv=(float)PyFloat_AsDouble(PyList_GetItem(pConvValues, i)); + EXPECT_LE(std::abs(outputConv[i] - pOutputConv), TOLERANCE); } } diff --git a/tmva/sofie_parsers/CMakeLists.txt b/tmva/sofie_parsers/CMakeLists.txt index 4814b62c6ec51..73f9aaf0ad8d5 100644 --- a/tmva/sofie_parsers/CMakeLists.txt +++ b/tmva/sofie_parsers/CMakeLists.txt @@ -94,7 +94,6 @@ ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers src/RModelParser_Keras.cxx src/RModelParser_PyTorch.cxx LIBRARIES - Python3::NumPy Python3::Python ROOTTMVASofie ) diff --git a/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx index 746f136cfb3a1..b19ff5018d8e4 100644 --- a/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx +++ b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx @@ -25,9 +25,6 @@ #include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include - namespace TMVA::Experimental::SOFIE::PyTorch { namespace { @@ -486,32 +483,39 @@ RModel Parse(std::string filename, std::vector> inputShapes, } - //Extracting model weights to add the initialized tensors to the RModel + // Extracting model weights to add the initialized tensors to the RModel + // + // The weight shapes and the raw contiguous bytes are extracted on the Python side + // (NumPy's `shape` attribute and `tobytes()`), so the parser does not depend on the + // NumPy C API and hence is not tied to a specific NumPy ABI/version. PyRunString("weightNames=[k for k in graph[1].keys()]",fGlobalNS,fLocalNS); - PyRunString("weights=[v.numpy() for v in graph[1].values()]",fGlobalNS,fLocalNS); + PyRunString("weightValues=[v.numpy() for v in graph[1].values()]",fGlobalNS,fLocalNS); + PyRunString("weightShapes=[list(v.shape) for v in weightValues]",fGlobalNS,fLocalNS); + PyRunString("weightBytes=[v.tobytes() for v in weightValues]",fGlobalNS,fLocalNS); PyRunString("weightDTypes=[v.type()[6:-6] for v in graph[1].values()]",fGlobalNS,fLocalNS); PyObject* fPWeightNames = PyDict_GetItemString(fLocalNS,"weightNames"); - PyObject* fPWeightTensors = PyDict_GetItemString(fLocalNS,"weights"); + PyObject* fPWeightShapes = PyDict_GetItemString(fLocalNS,"weightShapes"); + PyObject* fPWeightBytes = PyDict_GetItemString(fLocalNS,"weightBytes"); PyObject* fPWeightDTypes = PyDict_GetItemString(fLocalNS,"weightDTypes"); - PyArrayObject* fWeightTensor; std::string fWeightName; ETensorType fWeightDType; std::vector fWeightShape; std::size_t fWeightSize; - for(Py_ssize_t weightIter=0; weightIter fData(malloc(fWeightSize * sizeof(float)), free); std::memcpy(fData.get(),fWeightValue,fWeightSize * sizeof(float)); rmodel.AddInitializedTensor(fWeightName, ETensorType::FLOAT,fWeightShape,fData);