Limit ARM retries to operator tests (pytorch#19343)

Jobs been timing out since the first attempt

Author: JacobSzwejbka

.ci/scripts/test_backend.sh

@@ -58,7 +58,9 @@ if [[ "$FLOW" == *vulkan* ]]; then
 fi
 
 if [[ "$FLOW" == *arm* ]]; then
-    PYTEST_RETRY_ARGS=(--reruns 2 --reruns-delay 1)
+    if [[ "$SUITE" == "operators" ]]; then
+        PYTEST_RETRY_ARGS=(--reruns 2 --reruns-delay 1)
+    fi
 
     # Setup ARM deps.
     if [[ "$FLOW" == *vgf* ]]; then