diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 09225aa7dbd0b..c29bccf691caa 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -413,7 +413,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva)) elseif(tpython OR tmva-pymva) list(APPEND python_components Development) endif() -if(tmva-pymva OR tmva-sofie) +if(tmva-pymva) list(APPEND python_components NumPy) endif() find_package(Python3 3.10 COMPONENTS ${python_components}) @@ -1378,7 +1378,7 @@ if(tmva) endif() endif() endif() - if(tmva-pymva OR tmva-sofie) + if(tmva-pymva) if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND)) message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required" " (python executable: ${Python3_EXECUTABLE})") diff --git a/tmva/sofie/test/CMakeLists.txt b/tmva/sofie/test/CMakeLists.txt index 120b5800b2f8d..2012cb32b90a0 100644 --- a/tmva/sofie/test/CMakeLists.txt +++ b/tmva/sofie/test/CMakeLists.txt @@ -146,7 +146,6 @@ if (tpython AND ROOT_TORCH_FOUND AND ROOT_ONNX_FOUND AND BLAS_FOUND AND NOT brok LIBRARIES ROOTTMVASofie TMVA - Python3::NumPy Python3::Python BLAS::BLAS INCLUDE_DIRS @@ -165,7 +164,6 @@ if (tpython AND ROOT_KERAS_FOUND AND BLAS_FOUND) ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C LIBRARIES ROOTTMVASofie - Python3::NumPy Python3::Python BLAS::BLAS INCLUDE_DIRS diff --git a/tmva/sofie/test/TestRModelParserKeras.C b/tmva/sofie/test/TestRModelParserKeras.C index e99ac05106ae5..b9b176e3565c1 100644 --- a/tmva/sofie/test/TestRModelParserKeras.C +++ b/tmva/sofie/test/TestRModelParserKeras.C @@ -1,9 +1,8 @@ #include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include #include "gtest/gtest.h" #include +#include #include "TSystem.h" #include "TMVA/RSofieReader.hxx" @@ -21,6 +20,20 @@ const char *PyStringAsString(PyObject *string) return cstring; } +// Read a NumPy array bound in the Python namespace into a std::vector. +// The array is flattened to a Python list and read through the generic CPython +// API, so this avoids any dependency on the NumPy C API (arrayobject.h). +std::vector GetTensorValues(PyObject *globalNS, PyObject *localNS, const char *name) +{ + std::string code = std::string("outputFlat=") + name + ".flatten().tolist()"; + PyRun_String(code.c_str(), Py_single_input, globalNS, localNS); + PyObject *list = PyDict_GetItemString(localNS, "outputFlat"); + std::vector values(PyList_Size(list)); + for (std::size_t i = 0; i < values.size(); ++i) + values[i] = (float)PyFloat_AsDouble(PyList_GetItem(list, i)); + return values; +} + } // namespace void GenerateModels() { @@ -78,15 +91,12 @@ TEST(RModelParser_Keras, SEQUENTIAL) "0.63637339, 0.94483464, 0.11032887, 0.22424818," "0.50972592, 0.04671024, 0.39230661, 0.80500943]).reshape(4,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSequentialSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputSequential=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputSequentialSize=pOutputSequential.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputSequential.size(), pOutputSequentialSize); - PyArrayObject* pSequentialValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputSequential=(float*)PyArray_DATA(pSequentialValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputSequential.size(); ++i) { EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential[i]), TOLERANCE); @@ -124,15 +134,12 @@ TEST(RModelParser_Keras, FUNCTIONAL) PyRun_String("input=numpy.array([0.60828574, 0.50069386, 0.75186709, 0.14968806, 0.7692464 ,0.77027585, 0.75095316, 0.96651197," "0.38536308, 0.95565917, 0.62796356, 0.13818375, 0.65484891,0.89220363, 0.23879365, 0.00635323]).reshape(2,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputFunctionalSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputFunctional=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputFunctionalSize=pOutputFunctional.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputFunctional.size(), pOutputFunctionalSize); - PyArrayObject* pFunctionalValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputFunctional=(float*)PyArray_DATA(pFunctionalValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputFunctional.size(); ++i) { EXPECT_LE(std::abs(outputFunctional[i] - pOutputFunctional[i]), TOLERANCE); @@ -169,15 +176,12 @@ TEST(RModelParser_Keras, BATCH_NORM) PyRun_String("input=numpy.array([0.22308163, 0.95274901, 0.44712538, 0.84640867," "0.69947928, 0.29743695, 0.81379782, 0.39650574]).reshape(2,4)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputBatchNormSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputBatchNorm=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputBatchNormSize=pOutputBatchNorm.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputBatchNorm.size(), pOutputBatchNormSize); - PyArrayObject* pBatchNormValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputBatchNorm=(float*)PyArray_DATA(pBatchNormValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputBatchNorm.size(); ++i) { EXPECT_LE(std::abs(outputBatchNorm[i] - pOutputBatchNorm[i]), TOLERANCE); @@ -218,15 +222,12 @@ TEST(DISABLED_RModelParser_Keras, CONV_VALID) PyRun_String("model=load_model('KerasModelConv2D_Valid.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConv2DValidSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConv2DValid=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputConv2DValidSize=pOutputConv2DValid.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv2D_Valid.size(), pOutputConv2DValidSize); - PyArrayObject* pConv2DValidValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv2DValid=(float*)PyArray_DATA(pConv2DValidValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv2D_Valid.size(); ++i) { EXPECT_LE(std::abs(outputConv2D_Valid[i] - pOutputConv2DValid[i]), TOLERANCE); @@ -268,15 +269,12 @@ TEST(DISABLED_RModelParser_Keras, CONV_SAME) PyRun_String("model=load_model('KerasModelConv2D_Same.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConv2DSameSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConv2DSame=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputConv2DSameSize=pOutputConv2DSame.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv2D_Same.size(), pOutputConv2DSameSize); - PyArrayObject* pConv2DSameValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv2DSame=(float*)PyArray_DATA(pConv2DSameValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv2D_Same.size(); ++i) { EXPECT_LE(std::abs(outputConv2D_Same[i] - pOutputConv2DSame[i]), TOLERANCE); @@ -314,15 +312,12 @@ TEST(RModelParser_Keras, RESHAPE) PyRun_String("model=load_model('KerasModelReshape.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,4,4,1))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputReshapeSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputReshape=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputReshapeSize=pOutputReshape.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputReshape.size(), pOutputReshapeSize); - PyArrayObject* pReshapeValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputReshape=(float*)PyArray_DATA(pReshapeValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputReshape.size(); ++i) { EXPECT_LE(std::abs(outputReshape[i] - pOutputReshape[i]), TOLERANCE); @@ -359,14 +354,12 @@ TEST(RModelParser_Keras, CONCATENATE) PyRun_String("input_1=numpy.ones((1,2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input_2=numpy.ones((1,2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model([input_1,input_2]).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - long pOutputConcatenateSize=PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputConcatenate=GetTensorValues(fGlobalNS,fLocalNS,"output"); + long pOutputConcatenateSize=(long)pOutputConcatenate.size(); //Testing the actual and expected output tensor sizes (can fail if an error eccoured and returns a -1 from Python) EXPECT_EQ((long) outputConcatenate.size(), pOutputConcatenateSize); - PyArrayObject* pConcatenateValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConcatenate=(float*)PyArray_DATA(pConcatenateValues); //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConcatenate.size(); ++i) { @@ -405,14 +398,12 @@ TEST(RModelParser_Keras, BINARY_OP) PyRun_String("input1=numpy.array([[1,2],[3,4]],dtype='float32')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input2=numpy.array([[5,6],[7,8]],dtype='float32')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model([input1,input2]).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - long pOutputBinaryOpSize=PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputBinaryOp=GetTensorValues(fGlobalNS,fLocalNS,"output"); + long pOutputBinaryOpSize=(long)pOutputBinaryOp.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ((long) outputBinaryOp.size(), pOutputBinaryOpSize); - PyArrayObject* pBinaryOpValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputBinaryOp=(float*)PyArray_DATA(pBinaryOpValues); //Testing the actual and expected output tensor values for (size_t i = 0; i < outputBinaryOp.size(); ++i) { @@ -448,15 +439,12 @@ TEST(RModelParser_Keras, ACTIVATIONS) PyRun_String("model=load_model('KerasModelActivations.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,8))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputActivationsSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputActivations=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputActivationsSize=pOutputActivations.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputActivations.size(), pOutputActivationsSize); - PyArrayObject* pActivationsValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputActivations=(float*)PyArray_DATA(pActivationsValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputActivations.size(); ++i) { EXPECT_LE(std::abs(outputActivations[i] - pOutputActivations[i]), TOLERANCE); @@ -491,15 +479,12 @@ TEST(RModelParser_Keras, SWISH) PyRun_String("model=load_model('KerasModelSwish.keras')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.ones((1,8))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutput=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputSize=pOutput.size(); //Testing the actual and expected output tensor sizes EXPECT_EQ(output.size(), pOutputSize); - PyArrayObject* pValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutput=(float*)PyArray_DATA(pValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < output.size(); ++i) { EXPECT_LE(std::abs(output[i] - pOutput[i]), TOLERANCE); @@ -537,8 +522,8 @@ TEST(RModel, CUSTOM_OP) PyRun_String("model.add(Lambda(lambda x: x * 2))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=numpy.array([1,1,1,1,1,1,1,1]).reshape(1,8)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).numpy()",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputCustomOpSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + std::vector pOutputCustomOp=GetTensorValues(fGlobalNS,fLocalNS,"output"); + std::size_t pOutputCustomOpSize=pOutputCustomOp.size(); // get input name for custom (it is output of one before last) PyRun_String("outputName = model.get_layer(index=len(model.layers)-2).output.name",Py_single_input,fGlobalNS,fLocalNS); @@ -557,9 +542,6 @@ TEST(RModel, CUSTOM_OP) //Testing the actual and expected output tensor sizes EXPECT_EQ(outputCustomOp.size(), pOutputCustomOpSize); - PyArrayObject* pCustomOpValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputCustomOp=(float*)PyArray_DATA(pCustomOpValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputCustomOp.size(); ++i) { EXPECT_LE(std::abs(outputCustomOp[i] - pOutputCustomOp[i]), TOLERANCE); diff --git a/tmva/sofie/test/TestRModelParserPyTorch.C b/tmva/sofie/test/TestRModelParserPyTorch.C index 4dbe36b91d983..d136ec9f7088e 100644 --- a/tmva/sofie/test/TestRModelParserPyTorch.C +++ b/tmva/sofie/test/TestRModelParserPyTorch.C @@ -4,8 +4,6 @@ #include #include "TSystem.h" -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include #include "gtest/gtest.h" #include @@ -60,18 +58,17 @@ TEST(RModelParser_PyTorch, SEQUENTIAL_MODEL) "-0.7750, -1.6701," " 0.8171, -0.2858]),(2,4))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(2,6)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputSequentialSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pSequentialValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputSequentialSize=(std::size_t)PyList_Size(pSequentialValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputSequential.size(), pOutputSequentialSize); - PyArrayObject* pSequentialValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputSequential=(float*)PyArray_DATA(pSequentialValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputSequential.size(); ++i) { - EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential[i]), TOLERANCE); + float pOutputSequential=(float)PyFloat_AsDouble(PyList_GetItem(pSequentialValues, i)); + EXPECT_LE(std::abs(outputSequential[i] - pOutputSequential), TOLERANCE); } } @@ -114,18 +111,17 @@ TEST(RModelParser_PyTorch, MODULE_MODEL) "-1.8818, 0.4736," " 1.1102, 1.8694]),(2,6))",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(2,12)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputModuleSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pModuleValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputModuleSize=(std::size_t)PyList_Size(pModuleValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputModule.size(), pOutputModuleSize); - PyArrayObject* pModuleValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputModule=(float*)PyArray_DATA(pModuleValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputModule.size(); ++i) { - EXPECT_LE(std::abs(outputModule[i] - pOutputModule[i]), TOLERANCE); + float pOutputModule=(float)PyFloat_AsDouble(PyList_GetItem(pModuleValues, i)); + EXPECT_LE(std::abs(outputModule[i] - pOutputModule), TOLERANCE); } } @@ -158,17 +154,16 @@ TEST(RModelParser_PyTorch, CONVOLUTION_MODEL) PyRun_String("model=torch.jit.load('PyTorchModelConvolution.pt')",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("input=torch.arange(1,751,dtype=torch.float).reshape(5,6,5,5)",Py_single_input,fGlobalNS,fLocalNS); PyRun_String("output=model(input).detach().numpy().reshape(5,5,2,2)",Py_single_input,fGlobalNS,fLocalNS); - PyRun_String("outputSize=output.size",Py_single_input,fGlobalNS,fLocalNS); - std::size_t pOutputConvSize=(std::size_t)PyLong_AsLong(PyDict_GetItemString(fLocalNS,"outputSize")); + PyRun_String("outputFlat=output.flatten().tolist()",Py_single_input,fGlobalNS,fLocalNS); + PyObject* pConvValues=PyDict_GetItemString(fLocalNS,"outputFlat"); + std::size_t pOutputConvSize=(std::size_t)PyList_Size(pConvValues); //Testing the actual and expected output tensor sizes EXPECT_EQ(outputConv.size(), pOutputConvSize); - PyArrayObject* pConvValues=(PyArrayObject*)PyDict_GetItemString(fLocalNS,"output"); - float* pOutputConv=(float*)PyArray_DATA(pConvValues); - //Testing the actual and expected output tensor values for (size_t i = 0; i < outputConv.size(); ++i) { - EXPECT_LE(std::abs(outputConv[i] - pOutputConv[i]), TOLERANCE); + float pOutputConv=(float)PyFloat_AsDouble(PyList_GetItem(pConvValues, i)); + EXPECT_LE(std::abs(outputConv[i] - pOutputConv), TOLERANCE); } } diff --git a/tmva/sofie_parsers/CMakeLists.txt b/tmva/sofie_parsers/CMakeLists.txt index 4814b62c6ec51..73f9aaf0ad8d5 100644 --- a/tmva/sofie_parsers/CMakeLists.txt +++ b/tmva/sofie_parsers/CMakeLists.txt @@ -94,7 +94,6 @@ ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers src/RModelParser_Keras.cxx src/RModelParser_PyTorch.cxx LIBRARIES - Python3::NumPy Python3::Python ROOTTMVASofie ) diff --git a/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx index 746f136cfb3a1..b19ff5018d8e4 100644 --- a/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx +++ b/tmva/sofie_parsers/src/RModelParser_PyTorch.cxx @@ -25,9 +25,6 @@ #include -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include - namespace TMVA::Experimental::SOFIE::PyTorch { namespace { @@ -486,32 +483,39 @@ RModel Parse(std::string filename, std::vector> inputShapes, } - //Extracting model weights to add the initialized tensors to the RModel + // Extracting model weights to add the initialized tensors to the RModel + // + // The weight shapes and the raw contiguous bytes are extracted on the Python side + // (NumPy's `shape` attribute and `tobytes()`), so the parser does not depend on the + // NumPy C API and hence is not tied to a specific NumPy ABI/version. PyRunString("weightNames=[k for k in graph[1].keys()]",fGlobalNS,fLocalNS); - PyRunString("weights=[v.numpy() for v in graph[1].values()]",fGlobalNS,fLocalNS); + PyRunString("weightValues=[v.numpy() for v in graph[1].values()]",fGlobalNS,fLocalNS); + PyRunString("weightShapes=[list(v.shape) for v in weightValues]",fGlobalNS,fLocalNS); + PyRunString("weightBytes=[v.tobytes() for v in weightValues]",fGlobalNS,fLocalNS); PyRunString("weightDTypes=[v.type()[6:-6] for v in graph[1].values()]",fGlobalNS,fLocalNS); PyObject* fPWeightNames = PyDict_GetItemString(fLocalNS,"weightNames"); - PyObject* fPWeightTensors = PyDict_GetItemString(fLocalNS,"weights"); + PyObject* fPWeightShapes = PyDict_GetItemString(fLocalNS,"weightShapes"); + PyObject* fPWeightBytes = PyDict_GetItemString(fLocalNS,"weightBytes"); PyObject* fPWeightDTypes = PyDict_GetItemString(fLocalNS,"weightDTypes"); - PyArrayObject* fWeightTensor; std::string fWeightName; ETensorType fWeightDType; std::vector fWeightShape; std::size_t fWeightSize; - for(Py_ssize_t weightIter=0; weightIter fData(malloc(fWeightSize * sizeof(float)), free); std::memcpy(fData.get(),fWeightValue,fWeightSize * sizeof(float)); rmodel.AddInitializedTensor(fWeightName, ETensorType::FLOAT,fWeightShape,fData);