Make BundleIO ATOL/RTOL configurable per-test via runtime args

Previously tolerances were compile-time constants (ET_ATOL=5.0, ET_RTOL=1.0) baked into the executor runner binary, applied uniformly to all models. This masked numerical regressions on precise models.

Add -atol and -rtol CLI args to the semihosting runner so tolerances can be set per-model at runtime. Tighten defaults to 1e-3 and move per-model overrides to a lookup table in test_cortex_m_e2e.sh.

Resolves #18424

Author: BryanBradfo

.ci/scripts/test_cortex_m_e2e.sh

@@ -14,6 +14,15 @@
 set -eux
 
 MODEL=$1
+
+# Per-model BundleIO tolerances. Int8 quantized models need relaxed bounds
+# due to quantization error accumulating through layers. Models not listed
+# use the tight default (1e-3); override here for models that need it.
+declare -A MODEL_ATOL=( [mv2]="5.0" [mv3]="5.0" )
+declare -A MODEL_RTOL=( [mv2]="2.5" [mv3]="2.5" )
+ATOL="${MODEL_ATOL[$MODEL]:-1e-3}"
+RTOL="${MODEL_RTOL[$MODEL]:-1e-3}"
+
 mkdir -p "./cortex_m_e2e/${MODEL}"
 WORK_DIR=$(realpath "./cortex_m_e2e/${MODEL}")
 
@@ -51,7 +60,7 @@ FVP_Corstone_SSE-300_Ethos-U55 \
     -C cpu0.semihosting-heap_limit=0 \
     -C "cpu0.semihosting-cwd=${WORK_DIR}" \
     -C "ethosu.extra_args='--fast'" \
-    -C "cpu0.semihosting-cmd_line='executor_runner -m ${MODEL}.bpte -i dummy.bin -o out'" \
+    -C "cpu0.semihosting-cmd_line='executor_runner -m ${MODEL}.bpte -i dummy.bin -o out -atol ${ATOL} -rtol ${RTOL}'" \
     -a "${ELF}" \
     --timelimit 300 2>&1 | tee "${LOG_FILE}" || true
 

backends/cortex_m/test/build_test_runner.sh

@@ -31,4 +31,4 @@ aten::unsqueeze_copy.out,\
 aten::select_copy.int_out,\
 aten::amax.out"
 
-${build_executor_runner} --pte=semihosting --bundleio --target=ethos-u55-128 --output="${build_root_test_dir}" --select_ops_list="${select_ops_list}" --extra_build_flags="-DET_ATOL=5.0 -DET_RTOL=1.0"
+${build_executor_runner} --pte=semihosting --bundleio --target=ethos-u55-128 --output="${build_root_test_dir}" --select_ops_list="${select_ops_list}"

examples/arm/executor_runner/CMakeLists.txt

@@ -22,12 +22,12 @@ option(ET_LOG_DUMP_OUTPUT "Dump output in log" ON)
 
 option(ET_BUNDLE_IO "Set to compile in BundleIO support" OFF)
 set(ET_ATOL
-    "0.01"
+    "1e-3"
     CACHE STRING "Set atol to use for BundleIO testing (Requires ET_BUNDLE_IO)"
 )
 set(ET_RTOL
-    "0.01"
-    CACHE STRING "Set atol to use for BundleIO testing (Requires ET_BUNDLE_IO)"
+    "1e-3"
+    CACHE STRING "Set rtol to use for BundleIO testing (Requires ET_BUNDLE_IO)"
 )
 
 option(
@@ -417,11 +417,11 @@ if(ET_BUNDLE_IO)
   target_compile_definitions(arm_executor_runner PUBLIC -DET_BUNDLE_IO)
 endif()
 
-if(ET_ATOL)
+if(DEFINED ET_ATOL)
   target_compile_definitions(arm_executor_runner PUBLIC ET_ATOL=${ET_ATOL})
 endif()
 
-if(ET_RTOL)
+if(DEFINED ET_RTOL)
   target_compile_definitions(arm_executor_runner PUBLIC ET_RTOL=${ET_RTOL})
 endif()
 

examples/arm/executor_runner/arm_executor_runner.cpp

@@ -31,9 +31,12 @@
  *                      use bpte with bundled input and output refdata to
  *                      compare output.
  *                      See also ET_ATOL and ET_RTOL
- *   ET_ATOL              - The atol used to compare the output and ref data
- * when using ET_BUNDLE_IO ET_RTOL              - The rtol used to compare the
- * output and ref data when using ET_BUNDLE_IO
+ *   ET_ATOL              - The absolute tolerance used to compare output and
+ *                          ref data when using ET_BUNDLE_IO.
+ *                          Can be overridden at runtime with -atol <value>.
+ *   ET_RTOL              - The relative tolerance used to compare output and
+ *                          ref data when using ET_BUNDLE_IO.
+ *                          Can be overridden at runtime with -rtol <value>.
  *
  * Devtools ETDump: Speed and dumping output
  *
@@ -221,15 +224,15 @@ unsigned char __attribute__((
 const size_t testset_idx = 0; // BundleIO test indexes to test if used
 
 #if defined(ET_ATOL)
-const float et_atol = ET_ATOL;
+float et_atol = ET_ATOL;
 #else
-const float et_atol = 0.01;
+float et_atol = 1e-3f;
 #endif
 
 #if defined(ET_RTOL)
-const float et_rtol = ET_RTOL;
+float et_rtol = ET_RTOL;
 #else
-const float et_rtol = 0.01;
+float et_rtol = 1e-3f;
 #endif
 
 #endif
@@ -1192,13 +1195,16 @@ int main(int argc, const char* argv[]) {
     ET_LOG(Fatal, "Not right number of parameters!");
     ET_LOG(
         Fatal,
+#if defined(ET_BUNDLE_IO)
+        "app -m model.pte -i input.bin [-i input2.bin] -o output_basename [-atol 0.001] [-rtol 0.001]");
+#else
         "app -m model.pte -i input.bin [-i input2.bin] -o output_basename");
+#endif
     ET_LOG(Fatal, "Exiting!");
     _exit(1);
   }
-  ET_LOG(Info, "   %s", argv[0]);
-  for (int i = 1; i < argc; i++) {
-    ET_LOG(Info, "   %s %s", argv[i], argv[++i]);
+  for (int i = 0; i < argc; i++) {
+    ET_LOG(Info, "   %s", argv[i]);
   }
 #else
   (void)argc;
@@ -1265,6 +1271,22 @@ int main(int argc, const char* argv[]) {
     } else if (std::strcmp(argv[i], "-o") == 0) {
       // store the base filename to write output to.
       ctx.output_basename = argv[++i];
+#if defined(ET_BUNDLE_IO)
+    } else if (std::strcmp(argv[i], "-atol") == 0) {
+      if (i + 1 < argc && sscanf(argv[++i], "%f", &et_atol) == 1) {
+        ET_LOG(Info, "Setting atol to %f", et_atol);
+      } else {
+        ET_LOG(Fatal, "Invalid or missing value for -atol");
+        _exit(1);
+      }
+    } else if (std::strcmp(argv[i], "-rtol") == 0) {
+      if (i + 1 < argc && sscanf(argv[++i], "%f", &et_rtol) == 1) {
+        ET_LOG(Info, "Setting rtol to %f", et_rtol);
+      } else {
+        ET_LOG(Fatal, "Invalid or missing value for -rtol");
+        _exit(1);
+      }
+#endif
     }
   }
 #endif