Reduce rendering test flakiness (#5475)

huidongc · web-flow · commit 5afbca4f9eeb · 2026-05-05T08:37:00.000+08:00
# Description - In the Dexsuite env the success and failure markers are placed exactly at the same location. If both markers are visible, the rendering order will determine which one is visible in the camera output. Hide both markers to avoid this nondeterministic behavior. - Ordering of test cases appear to affect camera outputs for some reason, therefore I move the `newton_renderer` test cases after `isaacsim_rtx_renderer` test cases. Fixes # (issue)  ## Type of change  - Bug fix (non-breaking change which fixes an issue) - New feature (non-breaking change which adds functionality) - Breaking change (existing functionality will not work without user modification) - Documentation update ## Screenshots Please attach before and after screenshots of the change if applicable.  ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have updated the changelog and the corresponding version in the extension's `config/extension.toml` file - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there
diff --git a/source/isaaclab_tasks/changelog.d/rendering-test-flakiness.skip b/source/isaaclab_tasks/changelog.d/rendering-test-flakiness.skip
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-albedo.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-albedo.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:97d7f37390b32f99bdaba7c0e06be5b1385518096960d1eaaa5e747f1787f74d
-size 3130
+oid sha256:ec727622d2e85c742051daac4c8ad9ce56af4b2d4d10766810a696b60cef82da
+size 2579
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-depth.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e69349cb0c92ba3aa41b925b40cfcffad78f01671e19c812c2310ab2a6756ac1
-size 538
+oid sha256:c003b10810539f538464992860c74ee3bf531b8b4e9b6e0ebe84041d42dba643
+size 532
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-rgb.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e5772c11bcff5935e7442ca167380478dada58c7a3c71af1b208aa9921e9f0d
-size 18393
+oid sha256:8e199c01ed54ce1f9d3430017b73e5793c09340054288efddde04ea031ff2c19
+size 17594
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-rgba.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e952109fdcf84c7ecd6e0c2a0d1f724415e6e2ab54fc8e224ea6e65c0794327c
-size 20912
+oid sha256:63a63059efc82372e87f8c117e94cd2c5690dd982c65bae0002845a704741182
+size 19970
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-semantic_segmentation.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93570a548512d61f5090af8cfda2d885da9686f405d1181f21c1acfdfa3fb8ae
-size 702
+oid sha256:0b8729d722d1780272b24f0804e517ea024ec2b93f4a0fa3e26c82f222f5c9eb
+size 700
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:250bd7b5e958080c66f68cbdab73eef4683e500d69b38587d7cf250423543aed
-size 1562
+oid sha256:8e1d94f0c6ae2e40a1b0ff9cf27a0f2f9b756ebcebd9ddf27b0da31c89e3a57f
+size 1485
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0c71d8adf9b6ce38d35f6595d87cbf77935f149e68b3df13c1bb76e90e72a62a
-size 4111
+oid sha256:058466b74a49061695567d84edeffb21ad9e88d8c83125d6b2b59cb70bdd5a70
+size 3701
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_full_mdl.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-isaacsim_rtx_renderer-simple_shading_full_mdl.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:221fa320e5cc59fd37449a2acaeeca18c0a7b0aba26ad2bedf1cfda674ff3f7c
-size 4336
+oid sha256:1bf8a42b1b3c652ea9f8d79d89a02865ea2c1b347bf61dee810e01890aa00563
+size 4248
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-newton_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-newton_renderer-depth.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47f7ad39d12f9b9e86d5d717f0b1637a909e629352c3c607a09f034f4fd1f665
-size 1768
+oid sha256:e12023f4f5314fe7214e4cc23368c7e24aef6997d2bfdcdf5a8654e03ea80c3c
+size 1064
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-albedo.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-albedo.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4979bdc7bb0f3786f5bb08e9e2333a36f2f008f3f7bc303dffa88319eef28e20
-size 3463
+oid sha256:7cf76622f5f5cc7e7889fe6032ba4cda22516248fa7bb5e957f181c92c86b42b
+size 3054
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-rgb.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f8f2305b02d61096172a83b3729c0fbc94f135d50e2dbfe2f7b3acb655855de
-size 14781
+oid sha256:3445142682c88dc5ce11c5d749c7754bf2d54bf0f1aab6420513a733bf3cf645
+size 14919
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-rgba.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86ee5f3822aae350436ae3fcd1edb7bdbdd3472c080059c573cab5d97cf4160e
-size 17683
+oid sha256:179a5acba0a763fcc2317cd784f8c347ef48f0d81f8028a1a64996ade67f8706
+size 17836
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-simple_shading_diffuse_mdl.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/newton-ovrtx_renderer-simple_shading_diffuse_mdl.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbac25af4087687f2ae5434770a724ef1453ec988c39e46d4b53afa22421f5c7
-size 4044
+oid sha256:83ca3d8f55f971d473409c73e77582175906670f3962b56723cccd28d062a868
+size 3513
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-albedo.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-albedo.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42629303589629819727b1103ce9f2962df1918cf69942673c6bc84674c1dbc0
-size 3104
+oid sha256:fddc33b267fe9810973419babf140faf39b964466bfb6f50193abc8f278cfce4
+size 3052
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-depth.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:78c4026191543acd93b5ad40193608609e6e9e82419df3595769f1c337d52906
-size 538
+oid sha256:66a908918302e713a2cc82a091c4e82db3da46a06cf08311b3818e93714b9132
+size 532
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-rgb.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c58d4f31b94ece0fa2c046e8425ca3e9062fe2958d73e18ccb024f16c37eafd7
-size 18208
+oid sha256:d132e5a1d1dbfbeb250345f19e1389019ce79a388d51e94a8e954abc2825029a
+size 17348
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-rgba.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:518f27da2aa3fe7892d977373aaa857c363f4ad55c5f0e046cbd5b3a77235544
-size 20719
+oid sha256:164ec6a8afffc5706573dfcad5747f818f059cd476b65ce9bbfed97c53ed0a83
+size 19729
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-semantic_segmentation.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-semantic_segmentation.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e2119a3294737e13549e40ae6685545985a6df609dd1bd73adbe31aa1a6d5c0
-size 690
+oid sha256:f77680ebb1f5c9dedc3419d526b65feb82e38013ce402d7805c75b0ad3be8cd6
+size 696
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_constant_diffuse.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0d7829371c05612226bb4a3567f009c885bdaeeb5d1616201c3e6a018864033
-size 1565
+oid sha256:70cc200aa9d558e309d0e69c618b8235de0af11d6cf8b524bb7e7777f5d950cc
+size 1492
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_diffuse_mdl.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91cdd65e9c067c8448405e3d3b587f1f1478309168ac5c90ceacd8a12eff1c7d
-size 3798
+oid sha256:14f277009292a7cc2e21fdb4f2b5e77a0fb1025435badcb0ccc96871bd8c322d
+size 3698
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_full_mdl.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-isaacsim_rtx_renderer-simple_shading_full_mdl.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f62edc212a53998fb4e74cb40ffa7a889a6e1936103d623429fcdbea5440c040
-size 4315
+oid sha256:5e9155c484a7ab6131add9adce634f37d6c8205cb1696108f83acc96f6d19ea6
+size 4241
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-depth.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-depth.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31dc6e250e53ffec0e2d2839cfc73284bf5e12d725d7212fa0eeb4f420297b82
-size 1060
+oid sha256:2eb315a195ddddaabb31529efe051c5543d27f9b603bf1eb910b2ad426df22fa
+size 1061
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-rgb.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-rgb.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2792cc0ee8b489d286dd4e376b5d57f8abd90d1d245975b6f91ec0f68e13d483
-size 1629
+oid sha256:8f280f00e8cace33fe5ccf03ca0767954690285ee1908395bff9e8dbf92994e3
+size 1618
diff --git a/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-rgba.png b/source/isaaclab_tasks/test/golden_images/dexsuite_kuka/physx-newton_renderer-rgba.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec3854d8483d64e9e51f41e40d0bec2f6ff7c3db69b7889247c2d6056ddc1f4a
-size 2693
+oid sha256:da6cf857f7248d6c46d9c7b9a742609ac9c81eaeb99c9eeec36714bc34aacd17
+size 2670
diff --git a/source/isaaclab_tasks/test/rendering_test_utils.py b/source/isaaclab_tasks/test/rendering_test_utils.py
@@ -28,15 +28,15 @@
 # The max percentage of pixels allowed to differ. If the percentage exceeds this value, the test will fail.
 # The value is set case by case based on the screen space taken up by the env in camera output images. It
 # needs to be large enough to tolerate minor rendering noise while small enough to catch unexpected changes.
-_MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME = {
+MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME = {
     "cartpole": 1.0,
     # Shadow-hand renderings (incl. ``Isaac-Repose-Cube-Shadow-Vision-Direct-v0``) show up to
-    # ~3.28 % per-pixel diff from anti-aliasing noise along the many finger/cube edges. 7.0 gives
+    # ~3.28 % per-pixel diff from anti-aliasing noise along the many finger/cube edges. 5.0 gives
     # headroom above that without masking real regressions, which the SSIM gate still catches.
-    "shadow_hand": 7.0,
-    "dexsuite_kuka": 10.0,  # texture aliasing artifacts on the ground (ticket has been filed for OVRTX)
+    "shadow_hand": 5.0,
+    # Texture aliasing artifacts on the ground (NVBUG#6116767)
+    "dexsuite_kuka": 8.0,
 }
-MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME = _MAX_DIFFERENT_PIXELS_PERCENTAGE_BY_ENV_NAME
 
 # Minimum SSIM score below which two images are considered structurally different. SSIM is a perceptual metric
 # robust to uniform per-pixel noise that penalises structural changes (geometry shifts, swapped colours, missing
@@ -47,7 +47,7 @@
 # Per-env SSIM overrides. Envs not listed fall back to ``_SSIM_THRESHOLD``. Loosened individually
 # (not globally) to keep the strict gate active everywhere it already passes.
 _SSIM_THRESHOLD_BY_ENV_NAME = {
-    # Texture aliasing artifacts on the ground (ticket has been filed for OVRTX)
+    # Texture aliasing artifacts on the ground (NVBUG#6116767)
     "dexsuite_kuka": 0.95,
 }
 
@@ -117,19 +117,6 @@
         "semantic_segmentation",
         id="physx-isaacsim_rtx-semantic_segmentation",
     ),
-    # physx + newton_renderer (warp)
-    pytest.param(
-        "physx",
-        "newton_renderer",
-        "rgb",
-        id="physx-newton_warp-rgb",
-    ),
-    pytest.param(
-        "physx",
-        "newton_renderer",
-        "depth",
-        id="physx-newton_warp-depth",
-    ),
     # newton + isaacsim_rtx_renderer
     pytest.param(
         "newton",
@@ -173,6 +160,19 @@
         "semantic_segmentation",
         id="newton-isaacsim_rtx-semantic_segmentation",
     ),
+    # physx + newton_renderer (warp)
+    pytest.param(
+        "physx",
+        "newton_renderer",
+        "rgb",
+        id="physx-newton_warp-rgb",
+    ),
+    pytest.param(
+        "physx",
+        "newton_renderer",
+        "depth",
+        id="physx-newton_warp-depth",
+    ),
 ]
 
 KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS = [
@@ -760,6 +760,12 @@ def rendering_test_dexsuite_kuka(
     if point_cloud_term is not None:
         point_cloud_term.params["visualize"] = False
 
+    # The success and failure markers are placed exactly at the same location. If both markers are
+    # visible, the rendering order will determine which one is visible in the camera output. Hide
+    # both markers to avoid this nondeterministic behavior.
+    for marker_cfg in env_cfg.commands.object_pose.success_visualizer_cfg.markers.values():
+        marker_cfg.visible = False
+
     env = None
 
     try:
