Retry hostap tests up to 2 times to reduce flakyness

Author: Frauschi

.github/workflows/hostap-vm.yml

@@ -345,12 +345,18 @@ jobs:
             TESTS=$(printf '%s\n' "${ary[@]}" | tr '\n' ' ')
             HWSIM_RES=0 # Not set when command succeeds
             ./vm/parallel-vm.py ${{ env.hostap_debug_flags }} --nocurses $(nproc) $TESTS  || HWSIM_RES=$?
-            if [ "$HWSIM_RES" -ne "0" ]; then
-              # Let's re-run the failing tests. We gather the failed tests from the log file.
-              FAILED_TESTS=$(grep 'failed tests' /tmp/hwsim-test-logs/*-parallel.log | sed 's/failed tests: //' | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
-              printf 'failed tests: %s\n' "$FAILED_TESTS"
-              ./vm/parallel-vm.py ${{ env.hostap_debug_flags }} --nocurses $(nproc) $FAILED_TESTS
-            fi
+            # Retry failing tests up to 2 times to mitigate flakiness in the
+            # upstream hostap tests themselves.
+            for i in 1 2; do
+              if [ "$HWSIM_RES" -ne "0" ]; then
+                # Let's re-run the failing tests. We gather the failed tests from the log file.
+                FAILED_TESTS=$(grep 'failed tests' /tmp/hwsim-test-logs/*-parallel.log | sed 's/failed tests: //' | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
+                printf 'failed tests (retry %d): %s\n' "$i" "$FAILED_TESTS"
+                rm -r /tmp/hwsim-test-logs
+                HWSIM_RES=0
+                ./vm/parallel-vm.py ${{ env.hostap_debug_flags }} --nocurses $(nproc) $FAILED_TESTS || HWSIM_RES=$?
+              fi
+            done
             rm -r /tmp/hwsim-test-logs
           done < $GITHUB_WORKSPACE/wolfssl/.github/workflows/hostap-files/configs/${{ matrix.config.hostap_ref }}/tests