diff --git a/ci/run_single_test.sh b/ci/run_single_test.sh index 0ebc86966e31..d22cd2dc2e62 100755 --- a/ci/run_single_test.sh +++ b/ci/run_single_test.sh @@ -101,6 +101,7 @@ case ${TEST_TYPE} in echo "Creating temporary virtualenv for import profile..." python3 -m venv .venv-profiler source .venv-profiler/bin/activate + export PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 python -m pip install --upgrade pip setuptools PACKAGE_NAME=$(basename $(pwd)) @@ -139,9 +140,9 @@ case ${TEST_TYPE} in pip install -e . if [ -f "${BASELINE_CSV}" ]; then - python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000 --diff-baseline "${BASELINE_CSV}" --diff-threshold 100 + python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 20000 --diff-baseline "${BASELINE_CSV}" --diff-threshold 100 else - python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 5000 + python ${PROFILER_SCRIPT} --package ${PACKAGE_NAME} --iterations 11 --fail-threshold 20000 fi retval=$? deactivate diff --git a/scripts/import_profiler/profiler.py b/scripts/import_profiler/profiler.py index d2734dadce88..a0898bba9c87 100644 --- a/scripts/import_profiler/profiler.py +++ b/scripts/import_profiler/profiler.py @@ -266,10 +266,9 @@ def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True relative_diff_threshold = 0.15 * baseline_p50 if diff > diff_threshold and diff > relative_diff_threshold: final_messages.append( - f"FAILURE: Import time regression of {diff:.2f} ms exceeds both the absolute threshold ({diff_threshold} ms) " + f"WARNING: Import time regression of {diff:.2f} ms exceeds both the absolute threshold ({diff_threshold} ms) " f"and the relative threshold ({relative_diff_threshold:.2f} ms, 15% of baseline Median)." ) - exit_code = 1 else: if diff > diff_threshold: final_messages.append(f"SUCCESS: Import time regression of {diff:.2f} ms exceeds absolute threshold ({diff_threshold} ms) but is within relative threshold ({relative_diff_threshold:.2f} ms, 15%).") @@ -281,10 +280,9 @@ def run_master(iterations, target_module, cpu=0, csv_path=None, clear_cache=True if fail_threshold is not None: if p50_time > fail_threshold: if baseline_p50 is not None and baseline_p50 > fail_threshold: - final_messages.append(f"WARNING: Median import time ({p50_time:.2f} ms) exceeds the absolute failure threshold ({fail_threshold} ms), but the baseline ({baseline_p50:.2f} ms) also exceeded it. Bypassing absolute backstop failure.") + final_messages.append(f"WARNING: Median import time ({p50_time:.2f} ms) exceeds the absolute threshold ({fail_threshold} ms), and the baseline ({baseline_p50:.2f} ms) also exceeded it.") else: - final_messages.append(f"FAILURE: Median import time ({p50_time:.2f} ms) exceeds the failure threshold ({fail_threshold} ms).") - exit_code = 1 + final_messages.append(f"WARNING: Median import time ({p50_time:.2f} ms) exceeds the threshold ({fail_threshold} ms).") else: final_messages.append(f"SUCCESS: Median import time ({p50_time:.2f} ms) is within the failure threshold ({fail_threshold} ms).")