Adds retry logic for timeout tests (isaac-sim#5448)

# Description

Some tests arbitrarily times out due to inconsistent CI runs. This
change adds a logic to rerun tests that have timed out in an attempt to
reduce flaky timeout issues.

## Type of change

<!-- As you go through the list, delete the ones that are not
applicable. -->

- Bug fix (non-breaking change which fixes an issue)

## Checklist

- [x] I have read and understood the [contribution
guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html)
- [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [x] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [ ] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

Author: kellyguo11

source/isaaclab/test/performance/test_robot_load_performance.py

@@ -36,8 +36,8 @@
         ({"name": "Cartpole", "robot_cfg": CARTPOLE_CFG, "expected_load_time": 15.0}, "cuda:0"),
         ({"name": "Cartpole", "robot_cfg": CARTPOLE_CFG, "expected_load_time": 15.0}, "cpu"),
         # TODO: regression - this used to be 40
-        ({"name": "Anymal_D", "robot_cfg": ANYMAL_D_CFG, "expected_load_time": 55.0}, "cuda:0"),
-        ({"name": "Anymal_D", "robot_cfg": ANYMAL_D_CFG, "expected_load_time": 55.0}, "cpu"),
+        ({"name": "Anymal_D", "robot_cfg": ANYMAL_D_CFG, "expected_load_time": 60.0}, "cuda:0"),
+        ({"name": "Anymal_D", "robot_cfg": ANYMAL_D_CFG, "expected_load_time": 60.0}, "cpu"),
     ],
 )
 def test_robot_load_performance(test_config, device):

tools/conftest.py

@@ -48,6 +48,9 @@ def pytest_ignore_collect(collection_path, config):
 STARTUP_HANG_RETRIES = 2
 """Number of times to retry a test that hangs during startup before giving up."""
 
+TIMEOUT_RETRIES = 2
+"""Number of times to retry a test that reaches its hard timeout before giving up."""
+
 SHUTDOWN_GRACE_PERIOD = 30
 """Seconds to wait for clean exit after the JUnit XML report file appears.
 
@@ -352,10 +355,12 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci):
 
         report_file = f"tests/test-reports-{str(file_name)}.xml"
 
-        # -- Run with retry on startup hang --------------------------------
+        # -- Run with retry on startup hang or hard timeout -----------------
         returncode, stdout_data, stderr_data, kill_reason = -1, b"", b"", ""
         wall_time, pre_kill_diag = 0.0, ""
-        for attempt in range(STARTUP_HANG_RETRIES + 1):
+        startup_hang_attempts = 0
+        timeout_attempts = 0
+        while True:
             with contextlib.suppress(FileNotFoundError):
                 os.remove(report_file)
 
@@ -365,11 +370,32 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci):
                 )
             )
 
-            if kill_reason == "startup_hang" and attempt < STARTUP_HANG_RETRIES:
+            has_report = os.path.exists(report_file)
+
+            if kill_reason == "startup_hang" and startup_hang_attempts < STARTUP_HANG_RETRIES:
+                startup_hang_attempts += 1
                 print(
                     f"⚠️  {test_file}: startup hang detected after {startup_deadline}s"
-                    f" (attempt {attempt + 1}/{STARTUP_HANG_RETRIES + 1}), retrying..."
+                    f" (attempt {startup_hang_attempts}/{STARTUP_HANG_RETRIES + 1}), retrying..."
+                )
+                if stderr_data:
+                    print("=== STDERR (last 5000 chars) ===")
+                    print(stderr_data.decode("utf-8", errors="replace")[-5000:])
+                diag = pre_kill_diag or _capture_system_diagnostics()
+                if len(diag) > 10000:
+                    diag = diag[:10000] + "\n... (truncated)"
+                print(diag)
+                continue
+
+            if kill_reason == "timeout" and not has_report and timeout_attempts < TIMEOUT_RETRIES:
+                timeout_attempts += 1
+                print(
+                    f"⚠️  {test_file}: timeout detected after {timeout}s"
+                    f" (attempt {timeout_attempts}/{TIMEOUT_RETRIES + 1}), retrying..."
                 )
+                if stdout_data:
+                    print("=== STDOUT (last 5000 chars) ===")
+                    print(stdout_data.decode("utf-8", errors="replace")[-5000:])
                 if stderr_data:
                     print("=== STDERR (last 5000 chars) ===")
                     print(stderr_data.decode("utf-8", errors="replace")[-5000:])
@@ -417,7 +443,7 @@ def run_individual_tests(test_files, workspace_root, isaacsim_ci):
             print(f"Test {test_file} timed out after {timeout} seconds...")
             print(diag)
 
-            msg = f"Timeout after {timeout} seconds"
+            msg = f"Timeout after {timeout} seconds (retried {timeout_attempts} time(s))"
             details = f"{msg}\n\n=== SYSTEM DIAGNOSTICS ===\n{diag}\n\n"
             if stdout_data:
                 details += "=== STDOUT (last 5000 chars) ===\n"

tools/test_settings.py

@@ -17,7 +17,7 @@
 
 
 PER_TEST_TIMEOUTS = {
-    "test_articulation.py": 1000,
+    "test_articulation.py": 1500,
     "test_stage_in_memory.py": 1000,
     "test_imu.py": 1000,
     "test_environments.py": 10000,  # This test runs through all the environments for 100 steps each