Fix bash scripts to fail correctly for ROCm jobs (#5564)

Summary:

- Fix bash scripts to fail correctly for ROCm jobs

Differential Revision: D98951961

Author: q10

.github/scripts/fbgemm_gpu_test.bash

@@ -64,8 +64,18 @@ run_python_test () {
   # shellcheck disable=SC2155
   local start=$(date +%s)
 
+  # Run pytest directly (not through eval) so that the exit code from conda run
+  # is captured reliably.  This is defense-in-depth against conda run + eval
+  # silently swallowing non-zero exit codes.
   # shellcheck disable=SC2086
-  if print_exec conda run --no-capture-output ${env_prefix} python -m pytest "${pytest_args[@]}" --cache-clear  "${python_test_file}"; then
+  echo "+ conda run --no-capture-output ${env_prefix} python -m pytest ${pytest_args[*]} --cache-clear ${python_test_file}"
+  echo ""
+  # shellcheck disable=SC2086
+  conda run --no-capture-output ${env_prefix} python -m pytest "${pytest_args[@]}" --cache-clear "${python_test_file}"
+  local retcode=$?
+  echo ""
+
+  if [ $retcode -eq 0 ]; then
     echo "[TEST] Python test suite PASSED: ${python_test_file}"
     local test_time=$(($(date +%s)-start))
     echo "[TEST] Python test time for ${python_test_file}: ${test_time} seconds"
@@ -75,7 +85,7 @@ run_python_test () {
     return 0
   fi
 
-  echo "[TEST] Some tests FAILED.  Re-attempting only FAILED tests: ${python_test_file}"
+  echo "[TEST] Some tests FAILED (exit code: ${retcode}).  Re-attempting only FAILED tests: ${python_test_file}"
   echo ""
   echo ""
 
@@ -84,8 +94,28 @@ run_python_test () {
   # suites, we only run tests that have failed in the previous round.  This is
   # enabled by using the pytest cache and the --lf flag.
 
-  # shellcheck disable=SC2086
-  if exec_with_retries 2 conda run --no-capture-output ${env_prefix} python -m pytest "${pytest_args[@]}" --lf --last-failed-no-failures none "${python_test_file}"; then
+  local max_retries=2
+  local retry_retcode=0
+  for i in $(seq 0 ${max_retries}); do
+    # shellcheck disable=SC2086
+    echo "[EXEC] [ATTEMPT ${i}/${max_retries}]    + conda run --no-capture-output ${env_prefix} python -m pytest ${pytest_args[*]} --lf --last-failed-no-failures none ${python_test_file}"
+    # shellcheck disable=SC2086
+    conda run --no-capture-output ${env_prefix} python -m pytest "${pytest_args[@]}" --lf --last-failed-no-failures none "${python_test_file}"
+    retry_retcode=$?
+    echo ""
+
+    if [ $retry_retcode -eq 0 ]; then
+      break
+    fi
+
+    echo "[EXEC] [ATTEMPT ${i}/${max_retries}] Command attempt failed (exit code: ${retry_retcode})."
+    echo ""
+    if [ "$i" -ne "$max_retries" ]; then
+      sleep 2
+    fi
+  done
+
+  if [ $retry_retcode -eq 0 ]; then
     echo "[TEST] Python test suite PASSED with retries: ${python_test_file}"
     local test_time=$(($(date +%s)-start))
     echo "[TEST] Python test time with retries for ${python_test_file}: ${test_time} seconds"

.github/scripts/nova_postscript.bash

@@ -47,7 +47,7 @@ echo "[NOVA] Time taken to install wheel: ${runtime} seconds"
 # Test with PyTest
 $CONDA_RUN python3 -c "import torch; print('cuda.is_available() ', torch.cuda.is_available()); print ('device_count() ',torch.cuda.device_count());"
 cd "${FBGEMM_REPO}" || { echo "[NOVA] Failed to cd to ${FBGEMM_REPO} from $(pwd)"; };
-test_all_fbgemm_gpu_modules "${BUILD_ENV_NAME}"
+test_all_fbgemm_gpu_modules "${BUILD_ENV_NAME}" || exit 1
 end_time=$(date +%s)
 runtime=$((end_time-start_time))
 start_time=${end_time}