diff --git a/.ci/scripts/test_cortex_m_e2e.sh b/.ci/scripts/test_cortex_m_e2e.sh index 231fc68a349..6e5144511f0 100755 --- a/.ci/scripts/test_cortex_m_e2e.sh +++ b/.ci/scripts/test_cortex_m_e2e.sh @@ -14,6 +14,15 @@ set -eux MODEL=$1 + +# Per-model BundleIO tolerances. Int8 quantized models need relaxed bounds +# due to quantization error accumulating through layers. Models not listed +# use the tight default (1e-3); override here for models that need it. +declare -A MODEL_ATOL=( [mv2]="5.0" [mv3]="5.0" ) +declare -A MODEL_RTOL=( [mv2]="2.5" [mv3]="2.5" ) +ATOL="${MODEL_ATOL[$MODEL]:-1e-3}" +RTOL="${MODEL_RTOL[$MODEL]:-1e-3}" + mkdir -p "./cortex_m_e2e/${MODEL}" WORK_DIR=$(realpath "./cortex_m_e2e/${MODEL}") @@ -51,7 +60,7 @@ FVP_Corstone_SSE-300_Ethos-U55 \ -C cpu0.semihosting-heap_limit=0 \ -C "cpu0.semihosting-cwd=${WORK_DIR}" \ -C "ethosu.extra_args='--fast'" \ - -C "cpu0.semihosting-cmd_line='executor_runner -m ${MODEL}.bpte -i dummy.bin -o out'" \ + -C "cpu0.semihosting-cmd_line='executor_runner -m ${MODEL}.bpte -i dummy.bin -o out -atol ${ATOL} -rtol ${RTOL}'" \ -a "${ELF}" \ --timelimit 300 2>&1 | tee "${LOG_FILE}" || true diff --git a/backends/cortex_m/test/build_test_runner.sh b/backends/cortex_m/test/build_test_runner.sh index 6ac9aa55e73..730d3b2dd4f 100755 --- a/backends/cortex_m/test/build_test_runner.sh +++ b/backends/cortex_m/test/build_test_runner.sh @@ -31,4 +31,4 @@ aten::unsqueeze_copy.out,\ aten::select_copy.int_out,\ aten::amax.out" -${build_executor_runner} --pte=semihosting --bundleio --target=ethos-u55-128 --output="${build_root_test_dir}" --select_ops_list="${select_ops_list}" --extra_build_flags="-DET_ATOL=5.0 -DET_RTOL=1.0" +${build_executor_runner} --pte=semihosting --bundleio --target=ethos-u55-128 --output="${build_root_test_dir}" --select_ops_list="${select_ops_list}" diff --git a/examples/arm/executor_runner/CMakeLists.txt b/examples/arm/executor_runner/CMakeLists.txt index c169f5d447a..c38a00766b0 100644 --- a/examples/arm/executor_runner/CMakeLists.txt +++ b/examples/arm/executor_runner/CMakeLists.txt @@ -22,12 +22,12 @@ option(ET_LOG_DUMP_OUTPUT "Dump output in log" ON) option(ET_BUNDLE_IO "Set to compile in BundleIO support" OFF) set(ET_ATOL - "0.01" + "1e-3" CACHE STRING "Set atol to use for BundleIO testing (Requires ET_BUNDLE_IO)" ) set(ET_RTOL - "0.01" - CACHE STRING "Set atol to use for BundleIO testing (Requires ET_BUNDLE_IO)" + "1e-3" + CACHE STRING "Set rtol to use for BundleIO testing (Requires ET_BUNDLE_IO)" ) option( @@ -415,14 +415,12 @@ endif() if(ET_BUNDLE_IO) target_compile_definitions(arm_executor_runner PUBLIC -DET_BUNDLE_IO) -endif() - -if(ET_ATOL) - target_compile_definitions(arm_executor_runner PUBLIC ET_ATOL=${ET_ATOL}) -endif() - -if(ET_RTOL) - target_compile_definitions(arm_executor_runner PUBLIC ET_RTOL=${ET_RTOL}) + if(DEFINED ET_ATOL) + target_compile_definitions(arm_executor_runner PUBLIC ET_ATOL=${ET_ATOL}) + endif() + if(DEFINED ET_RTOL) + target_compile_definitions(arm_executor_runner PUBLIC ET_RTOL=${ET_RTOL}) + endif() endif() # Devtools ETDump: Speed and dumping output diff --git a/examples/arm/executor_runner/arm_executor_runner.cpp b/examples/arm/executor_runner/arm_executor_runner.cpp index 753d229596f..57a134b3823 100644 --- a/examples/arm/executor_runner/arm_executor_runner.cpp +++ b/examples/arm/executor_runner/arm_executor_runner.cpp @@ -31,9 +31,12 @@ * use bpte with bundled input and output refdata to * compare output. * See also ET_ATOL and ET_RTOL - * ET_ATOL - The atol used to compare the output and ref data - * when using ET_BUNDLE_IO ET_RTOL - The rtol used to compare the - * output and ref data when using ET_BUNDLE_IO + * ET_ATOL - The absolute tolerance used to compare output and + * ref data when using ET_BUNDLE_IO. + * Can be overridden at runtime with -atol . + * ET_RTOL - The relative tolerance used to compare output and + * ref data when using ET_BUNDLE_IO. + * Can be overridden at runtime with -rtol . * * Devtools ETDump: Speed and dumping output * @@ -221,15 +224,15 @@ unsigned char __attribute__(( const size_t testset_idx = 0; // BundleIO test indexes to test if used #if defined(ET_ATOL) -const float et_atol = ET_ATOL; +float et_atol = ET_ATOL; #else -const float et_atol = 0.01; +float et_atol = 1e-3f; #endif #if defined(ET_RTOL) -const float et_rtol = ET_RTOL; +float et_rtol = ET_RTOL; #else -const float et_rtol = 0.01; +float et_rtol = 1e-3f; #endif #endif @@ -1192,13 +1195,16 @@ int main(int argc, const char* argv[]) { ET_LOG(Fatal, "Not right number of parameters!"); ET_LOG( Fatal, +#if defined(ET_BUNDLE_IO) + "app -m model.pte -i input.bin [-i input2.bin] -o output_basename [-atol 0.001] [-rtol 0.001]"); +#else "app -m model.pte -i input.bin [-i input2.bin] -o output_basename"); +#endif ET_LOG(Fatal, "Exiting!"); _exit(1); } - ET_LOG(Info, " %s", argv[0]); - for (int i = 1; i < argc; i++) { - ET_LOG(Info, " %s %s", argv[i], argv[++i]); + for (int i = 0; i < argc; i++) { + ET_LOG(Info, " %s", argv[i]); } #else (void)argc; @@ -1265,6 +1271,22 @@ int main(int argc, const char* argv[]) { } else if (std::strcmp(argv[i], "-o") == 0) { // store the base filename to write output to. ctx.output_basename = argv[++i]; +#if defined(ET_BUNDLE_IO) + } else if (std::strcmp(argv[i], "-atol") == 0) { + if (i + 1 < argc && sscanf(argv[++i], "%f", &et_atol) == 1) { + ET_LOG(Info, "Setting atol to %f", et_atol); + } else { + ET_LOG(Fatal, "Invalid or missing value for -atol"); + _exit(1); + } + } else if (std::strcmp(argv[i], "-rtol") == 0) { + if (i + 1 < argc && sscanf(argv[++i], "%f", &et_rtol) == 1) { + ET_LOG(Info, "Setting rtol to %f", et_rtol); + } else { + ET_LOG(Fatal, "Invalid or missing value for -rtol"); + _exit(1); + } +#endif } } #endif