[tmva][sofie] Generate ONNX models and refs instead of binaries in repo

The SOFIE ONNX unit tests relied on 145 binary .onnx files checked into
tmva/sofie/test/input_models, plus ~108 frozen reference headers
(input_models/references/*.ref.hxx) whose expected outputs were computed
once from the exact weights inside those binaries. This made it impossible
to tell from the repository what the models contain, and impossible to
regenerate them (some were exported with pytorch versions as old as 1.5).

Replace both with a single script, tmva/sofie/test/generate_input_models.py:

 * Each model is built with the onnx helper API in a make_<Name>()
   function, so graph structure, attributes, shapes and initializers are
   readable and reviewable. Large weight tensors that used to be opaque
   random blobs are seeded-random via _random_tensor(), with pytorch-like
   1/sqrt(fan_in) scaling.
 * The inputs for the value-based tests are defined in TEST_INPUTS, and
   the expected outputs are computed with onnx's ReferenceEvaluator and
   written to references/<Name>.ref next to the generated models. Where
   the evaluator is wrong or unimplemented (MaxPool with asymmetric
   padding, Mean with multidirectional broadcasting, bidirectional or
   batchwise RNN/LSTM/GRU), numpy fallbacks implement the ONNX operator
   definitions directly.
 * The script runs as the new SofieGenerateModels_ONNX ctest, which the
   other SOFIE ONNX tests require as a fixture. CMake gets the model list
   at configure time from `generate_input_models.py --list`, which works
   without the onnx module.

TestCustomModelsFromONNX.cxx and TestCladAutodiff.cxx read the inputs and
expected outputs at runtime through a small reader in test_helpers.h
instead of including frozen headers. SOFIE's results are thus checked
against an independent implementation instead of a snapshot of its own
past output.

While migrating, all computed references were validated to reproduce the
frozen .ref.hxx values with the original weights before switching to
seeded-random weights.

The ONNX model tests now require the onnx python package; they are
disabled with a warning if it is missing or has the broken version 1.19.0.
The TMVA_SOFIE_ONNX.C tutorial input is likewise generated at configure
time instead of being copied from the test sources.

🤖 Done with the help of AI.

Author: guitargeek

tmva/sofie/test/CMakeLists.txt

@@ -10,56 +10,107 @@
 ############################################################################
 
 
+# Look for needed Python modules
+ROOT_FIND_PYTHON_MODULE(torch)
+
+# onnx 1.19.0 has a bug that makes this version unusable:
+# https://github.com/onnx/onnx/issues/7249
+# 1.19.1 instead could be used
+# In that case, we have to disable the "TestSofieModels" test,
+# which imports onnx indirectly via torch.onnx
+ROOT_FIND_PYTHON_MODULE(onnx)
+if (ROOT_ONNX_FOUND AND DEFINED ROOT_ONNX_VERSION)
+  if(ROOT_ONNX_VERSION VERSION_EQUAL "1.19.0")
+    message(WARNING "Found broken onnx version ${ROOT_ONNX_VERSION} (see https://github.com/onnx/onnx/issues/7249). Some TMVA SOFIE tests will be disabled.")
+    set(broken_onnx TRUE)
+  endif()
+endif()
+
+# The ONNX input models are not checked into the repository: they are created
+# by generate_input_models.py, which runs as the SofieGenerateModels_ONNX test
+# that the other ONNX tests depend on. Set ONNX_MODELS_DIR to use pre-existing
+# models from a custom directory instead.
 if (NOT ONNX_MODELS_DIR)
-  set(ONNX_MODELS_DIR input_models)
+  if (ROOT_ONNX_FOUND AND NOT broken_onnx)
+    set(ONNX_MODELS_DIR ${CMAKE_CURRENT_BINARY_DIR}/input_models)
+    set(generate_onnx_models TRUE)
+    # Get the model names from the generator script (--list does not require
+    # the onnx module, only a Python interpreter).
+    execute_process(
+      COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_input_models.py --list
+      OUTPUT_VARIABLE _onnx_model_names
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      RESULT_VARIABLE _onnx_list_status)
+    if (NOT _onnx_list_status EQUAL 0)
+      message(FATAL_ERROR "Failed to list the SOFIE ONNX test models with generate_input_models.py")
+    endif()
+    string(REPLACE "\n" ";" ONNX_MODEL_NAMES "${_onnx_model_names}")
+  else()
+    message(WARNING "The onnx Python module is not usable: the ONNX input models for the TMVA SOFIE tests cannot be generated and the corresponding tests are disabled.")
+  endif()
+else()
+  get_filename_component(ONNX_MODELS_DIR ${ONNX_MODELS_DIR} ABSOLUTE)
+  file(GLOB ONNX_FILES "${ONNX_MODELS_DIR}/*.onnx")
+  set(ONNX_MODEL_NAMES "")
+  foreach(onnx_file ${ONNX_FILES})
+    get_filename_component(fname ${onnx_file} NAME_WE)
+    list(APPEND ONNX_MODEL_NAMES ${fname})
+  endforeach()
 endif()
 
-# Finding .onnx files to be parsed and creating the appropriate code to
-# parse all file. It is much faster to combine all parsing in a single executable
-# which will avoid initialization time (especially when using ROOT)
-set(CAPTURE_STR "EmitModel( \"@1\", \"@2\");")
-set(ALL_CAPTURES "")
-# Finding .onnx files to be parsed and creating the appropriate command
-file(GLOB ONNX_FILES "${ONNX_MODELS_DIR}/*.onnx")
-foreach(onnx_file ${ONNX_FILES})
-  get_filename_component(fname ${onnx_file} NAME_WE)
-  get_filename_component(fdir ${onnx_file} DIRECTORY)
-  string(REPLACE "@1" ${onnx_file} cap ${CAPTURE_STR})
-  string(REPLACE "@2" ${fname} cap ${cap})
-  list(APPEND ALL_CAPTURES ${cap})
-endforeach()
-string(REPLACE ";" ";\n" EMIT_CAPTURES "${ALL_CAPTURES}")
-configure_file(EmitFromONNX.cxx.in EmitFromONNX_all.cxx @ONLY)
-
-ROOTTEST_GENERATE_EXECUTABLE(emitFromONNX EmitFromONNX_all.cxx
-                             LIBRARIES protobuf::libprotobuf ROOTTMVASofie ROOTTMVASofieParser
-                             FIXTURES_SETUP sofie-compile-models-onnx-build)
-
-# silence protobuf warnings seen in version 3.0 and 3.6. Not needed from protobuf version 3.17
-target_compile_options(emitFromONNX PRIVATE -Wno-unused-parameter -Wno-array-bounds)
-
-ROOTTEST_ADD_TEST(SofieCompileModels_ONNX
-  COMMAND ${CMAKE_COMMAND} -E env ROOTIGNOREPREFIX=1 ./emitFromONNX ${onnx_file} ${CMAKE_CURRENT_BINARY_DIR}/${fname}
-  FIXTURES_REQUIRED sofie-compile-models-onnx-build
-  FIXTURES_SETUP sofie-compile-models-onnx
-)
-
-# Creating a Google Test
-if (BLAS_FOUND)  # we need BLAS for compiling the models
-  ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx
-    LIBRARIES Core GTest::gtest GTest::gtest_main
+if (ONNX_MODELS_DIR)
+  # Creating the appropriate code to parse all ONNX model files. It is much
+  # faster to combine all parsing in a single executable which will avoid
+  # initialization time (especially when using ROOT)
+  set(CAPTURE_STR "EmitModel( \"@1\", \"@2\");")
+  set(ALL_CAPTURES "")
+  foreach(fname ${ONNX_MODEL_NAMES})
+    string(REPLACE "@1" ${ONNX_MODELS_DIR}/${fname}.onnx cap ${CAPTURE_STR})
+    string(REPLACE "@2" ${fname} cap ${cap})
+    list(APPEND ALL_CAPTURES ${cap})
+  endforeach()
+  string(REPLACE ";" ";\n" EMIT_CAPTURES "${ALL_CAPTURES}")
+  configure_file(EmitFromONNX.cxx.in EmitFromONNX_all.cxx @ONLY)
+
+  ROOTTEST_GENERATE_EXECUTABLE(emitFromONNX EmitFromONNX_all.cxx
+                               LIBRARIES protobuf::libprotobuf ROOTTMVASofie ROOTTMVASofieParser
+                               FIXTURES_SETUP sofie-compile-models-onnx-build)
+
+  # silence protobuf warnings seen in version 3.0 and 3.6. Not needed from protobuf version 3.17
+  target_compile_options(emitFromONNX PRIVATE -Wno-unused-parameter -Wno-array-bounds)
+
+  set(compile_models_fixtures sofie-compile-models-onnx-build)
+  if (generate_onnx_models)
+    ROOTTEST_ADD_TEST(SofieGenerateModels_ONNX
+      COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/generate_input_models.py --outdir ${ONNX_MODELS_DIR}
+      FIXTURES_SETUP sofie-generate-models-onnx
+    )
+    list(APPEND compile_models_fixtures sofie-generate-models-onnx)
+  endif()
+
+  ROOTTEST_ADD_TEST(SofieCompileModels_ONNX
+    COMMAND ${CMAKE_COMMAND} -E env ROOTIGNOREPREFIX=1 ./emitFromONNX
+    FIXTURES_REQUIRED ${compile_models_fixtures}
+    FIXTURES_SETUP sofie-compile-models-onnx
   )
-  ROOTTEST_ADD_TEST(TestCustomModelsFromONNX
-                    EXEC ./TestCustomModelsFromONNX
-                    FIXTURES_REQUIRED sofie-compile-models-onnx)
 
-  if (clad)
-    ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx
+  # Creating a Google Test
+  if (BLAS_FOUND)  # we need BLAS for compiling the models
+    ROOT_EXECUTABLE(TestCustomModelsFromONNX TestCustomModelsFromONNX.cxx
       LIBRARIES Core GTest::gtest GTest::gtest_main
     )
-    ROOTTEST_ADD_TEST(TestCladAutodiff
-                      EXEC ./TestCladAutodiff
+    ROOTTEST_ADD_TEST(TestCustomModelsFromONNX
+                      EXEC ./TestCustomModelsFromONNX
                       FIXTURES_REQUIRED sofie-compile-models-onnx)
+
+    if (clad)
+      ROOT_EXECUTABLE(TestCladAutodiff TestCladAutodiff.cxx
+        LIBRARIES Core GTest::gtest GTest::gtest_main
+      )
+      ROOTTEST_ADD_TEST(TestCladAutodiff
+                        EXEC ./TestCladAutodiff
+                        FIXTURES_REQUIRED sofie-compile-models-onnx)
+    endif()
   endif()
 endif()
 
@@ -70,22 +121,6 @@ if (BLAS_FOUND)
   endif()
 endif()
 
-# Look for needed Python modules
-ROOT_FIND_PYTHON_MODULE(torch)
-
-# onnx 1.19.0 has a bug that makes this version unusable:
-# https://github.com/onnx/onnx/issues/7249
-# 1.19.1 instead could be used
-# In that case, we have to disable the "TestSofieModels" test,
-# which imports onnx indirectly via torch.onnx
-ROOT_FIND_PYTHON_MODULE(onnx)
-if (ROOT_ONNX_FOUND AND DEFINED ROOT_ONNX_VERSION)
-  if(ROOT_ONNX_VERSION VERSION_EQUAL "1.19.0")
-    message(WARNING "Found broken onnx version ${ROOT_ONNX_VERSION} (see https://github.com/onnx/onnx/issues/7249). Some TMVA SOFIE tests will be disabled.")
-    set(broken_onnx TRUE)
-  endif()
-endif()
-
 if (ROOT_TORCH_FOUND AND ROOT_ONNX_FOUND AND NOT broken_onnx)
   configure_file(Conv1dModelGenerator.py  Conv1dModelGenerator.py COPYONLY)
   configure_file(Conv2dModelGenerator.py  Conv2dModelGenerator.py COPYONLY)

tmva/sofie/test/TestCladAutodiff.cxx

@@ -2,8 +2,6 @@ constexpr auto modelHeaderSuffix = "_FromONNX_unoptimized.hxx";
 constexpr auto modelDataSuffix = "_FromONNX_unoptimized.dat";
 #include "test_helpers.h"
 
-#include "input_models/references/Linear_16.ref.hxx"
-
 #include "gtest/gtest.h"
 
 // Test differentiating a fully-connected neural network with Clad.
@@ -12,9 +10,9 @@ TEST(ONNXClad, Linear16)
 {
    constexpr float TOLERANCE = DEFAULT_TOLERANCE;
 
-   // Preparing the standard all-ones input
-   std::vector<float> input(1600);
-   std::fill_n(input.data(), input.size(), 1.0f);
+   SofieReference ref = readReference("Linear_16");
+   // Mutable copy: the numeric differentiation below perturbs the input values
+   std::vector<float> input = ref.f32("input0");
 
    ASSERT_INCLUDE_AND_RUN(std::vector<float>, "Linear_16", input);
 
@@ -135,13 +133,5 @@ float Linear_16_wrapper_num_diff(TMVA_SOFIE_Linear_16::Session const &session, f
       ADD_FAILURE() << "Further mismatches suppressed (total mismatches: " << mismatchCount << ")";
    }
 
-   // Checking output size
-   EXPECT_EQ(output.size(), sizeof(Linear_16_ExpectedOutput::all_ones) / sizeof(float));
-
-   float *correct = Linear_16_ExpectedOutput::all_ones;
-
-   // Checking every output value, one by one
-   for (size_t i = 0; i < output.size(); ++i) {
-      EXPECT_LE(std::abs(output[i] - correct[i]), TOLERANCE);
-   }
+   expectNear(output, ref.f32("output0"), TOLERANCE);
 }