Skip to content

Commit bf646bb

Browse files
huidongcgreptile-apps[bot]
huidongc
and
greptile-apps[bot]
authored
mark all rtx-based rendering tests flaky for now (#5508)
# Description Mark all RTX-based rendering test cases flaky until they can produce deterministic low-res camera outputs that pass golden image testing on every CI run. Fixes # (issue) <!-- As a practice, it is recommended to open an issue to have discussions on the proposed pull request. This makes it easier for the community to keep track of what is being developed or added, and if a given feature is demanded by more than one party. --> ## Type of change - Test change ## Screenshots Please attach before and after screenshots of the change if applicable. <!-- Example: | Before | After | | ------ | ----- | | _gif/png before_ | _gif/png after_ | To upload images to a PR -- simply drag and drop an image while in edit mode and it should upload the image directly. You can then paste that source into the above before/after sections. --> ## Checklist - [x] I have read and understood the [contribution guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html) - [x] I have run the [`pre-commit` checks](https://pre-commit.com/) with `./isaaclab.sh --format` - [x] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have updated the changelog and the corresponding version in the extension's `config/extension.toml` file - [x] I have added my name to the `CONTRIBUTORS.md` or my name already exists there <!-- As you go through the checklist above, you can mark something as done by putting an x character in it For example, - [x] I have done this task - [ ] I have not done this task --> --------- Signed-off-by: HuiDong Chen <huidongc@nvidia.com> Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com>
1 parent 2d52d62 commit bf646bb

4 files changed

Lines changed: 30 additions & 14 deletions

File tree

‎source/isaaclab_tasks/changelog.d/huidongc-flaky-mark.skip‎

Whitespace-only changes.

‎source/isaaclab_tasks/test/rendering_test_utils.py‎

Lines changed: 30 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -66,99 +66,117 @@
6666
# Parametrization: (physics_backend, renderer, data_type)
6767
# ---------------------------------------------------------------------------
6868

69-
# OVRTX kitless paths can segfault on GitHub Actions runners; keep warp/Kit paths in CI.
70-
_SKIP_ON_GITHUB_ACTIONS = os.environ.get("GITHUB_ACTIONS") == "true"
71-
_SKIP_ON_GITHUB_ACTIONS_MARK = pytest.mark.skipif(
72-
_SKIP_ON_GITHUB_ACTIONS,
73-
reason="Skipped on GitHub Actions until the test can run on GitHub Actions.",
69+
# OVRTX kitless paths can segfault on CI runners; keep warp/Kit paths in CI.
70+
_SKIP_ON_CI = any(os.environ.get(name) == "true" for name in ("CI", "GITHUB_ACTIONS", "GITLAB_CI"))
71+
_SKIP_ON_CI_MARK = pytest.mark.skipif(
72+
_SKIP_ON_CI,
73+
reason="Skipped on CI runners until the test can run on CI runners.",
7474
)
7575

76+
# Let's just accept the fact that low-resolution camera outputs from RTX renderers are not deterministic enough to pass
77+
# golden image testing on every CI run.
78+
_FLAKY_MARK = pytest.mark.flaky(max_runs=3, min_passes=1)
79+
7680
PHYSICS_RENDERER_AOV_COMBINATIONS = [
7781
# physx + isaacsim_rtx_renderer
7882
pytest.param(
7983
"physx",
8084
"isaacsim_rtx_renderer",
8185
"rgb",
8286
id="physx-isaacsim_rtx-rgb",
87+
marks=_FLAKY_MARK,
8388
),
8489
pytest.param(
8590
"physx",
8691
"isaacsim_rtx_renderer",
8792
"albedo",
8893
id="physx-isaacsim_rtx-albedo",
94+
marks=_FLAKY_MARK,
8995
),
9096
pytest.param(
9197
"physx",
9298
"isaacsim_rtx_renderer",
9399
"depth",
94100
id="physx-isaacsim_rtx-depth",
101+
marks=_FLAKY_MARK,
95102
),
96103
pytest.param(
97104
"physx",
98105
"isaacsim_rtx_renderer",
99106
"simple_shading_constant_diffuse",
100107
id="physx-isaacsim_rtx-simple_shading_constant_diffuse",
108+
marks=_FLAKY_MARK,
101109
),
102110
pytest.param(
103111
"physx",
104112
"isaacsim_rtx_renderer",
105113
"simple_shading_diffuse_mdl",
106114
id="physx-isaacsim_rtx-simple_shading_diffuse_mdl",
115+
marks=_FLAKY_MARK,
107116
),
108117
pytest.param(
109118
"physx",
110119
"isaacsim_rtx_renderer",
111120
"simple_shading_full_mdl",
112121
id="physx-isaacsim_rtx-simple_shading_full_mdl",
122+
marks=_FLAKY_MARK,
113123
),
114124
pytest.param(
115125
"physx",
116126
"isaacsim_rtx_renderer",
117127
"semantic_segmentation",
118128
id="physx-isaacsim_rtx-semantic_segmentation",
129+
marks=_FLAKY_MARK,
119130
),
120131
# newton + isaacsim_rtx_renderer
121132
pytest.param(
122133
"newton",
123134
"isaacsim_rtx_renderer",
124135
"rgb",
125136
id="newton-isaacsim_rtx-rgb",
137+
marks=_FLAKY_MARK,
126138
),
127139
pytest.param(
128140
"newton",
129141
"isaacsim_rtx_renderer",
130142
"albedo",
131143
id="newton-isaacsim_rtx-albedo",
144+
marks=_FLAKY_MARK,
132145
),
133146
pytest.param(
134147
"newton",
135148
"isaacsim_rtx_renderer",
136149
"depth",
137150
id="newton-isaacsim_rtx-depth",
151+
marks=_FLAKY_MARK,
138152
),
139153
pytest.param(
140154
"newton",
141155
"isaacsim_rtx_renderer",
142156
"simple_shading_constant_diffuse",
143157
id="newton-isaacsim_rtx-simple_shading_constant_diffuse",
158+
marks=_FLAKY_MARK,
144159
),
145160
pytest.param(
146161
"newton",
147162
"isaacsim_rtx_renderer",
148163
"simple_shading_diffuse_mdl",
149164
id="newton-isaacsim_rtx-simple_shading_diffuse_mdl",
165+
marks=_FLAKY_MARK,
150166
),
151167
pytest.param(
152168
"newton",
153169
"isaacsim_rtx_renderer",
154170
"simple_shading_full_mdl",
155171
id="newton-isaacsim_rtx-simple_shading_full_mdl",
172+
marks=_FLAKY_MARK,
156173
),
157174
pytest.param(
158175
"newton",
159176
"isaacsim_rtx_renderer",
160177
"semantic_segmentation",
161178
id="newton-isaacsim_rtx-semantic_segmentation",
179+
marks=_FLAKY_MARK,
162180
),
163181
# physx + newton_renderer (warp)
164182
pytest.param(
@@ -182,49 +200,49 @@
182200
"ovrtx_renderer",
183201
"rgb",
184202
id="newton-ovrtx-rgb",
185-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
203+
marks=_SKIP_ON_CI_MARK,
186204
),
187205
pytest.param(
188206
"newton",
189207
"ovrtx_renderer",
190208
"albedo",
191209
id="newton-ovrtx-albedo",
192-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
210+
marks=_SKIP_ON_CI_MARK,
193211
),
194212
pytest.param(
195213
"newton",
196214
"ovrtx_renderer",
197215
"depth",
198216
id="newton-ovrtx-depth",
199-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
217+
marks=_SKIP_ON_CI_MARK,
200218
),
201219
pytest.param(
202220
"newton",
203221
"ovrtx_renderer",
204222
"simple_shading_constant_diffuse",
205223
id="newton-ovrtx-simple_shading_constant_diffuse",
206-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
224+
marks=_SKIP_ON_CI_MARK,
207225
),
208226
pytest.param(
209227
"newton",
210228
"ovrtx_renderer",
211229
"simple_shading_diffuse_mdl",
212230
id="newton-ovrtx-simple_shading_diffuse_mdl",
213-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
231+
marks=_SKIP_ON_CI_MARK,
214232
),
215233
pytest.param(
216234
"newton",
217235
"ovrtx_renderer",
218236
"simple_shading_full_mdl",
219237
id="newton-ovrtx-simple_shading_full_mdl",
220-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
238+
marks=_SKIP_ON_CI_MARK,
221239
),
222240
pytest.param(
223241
"newton",
224242
"ovrtx_renderer",
225243
"semantic_segmentation",
226244
id="newton-ovrtx-semantic_segmentation",
227-
marks=_SKIP_ON_GITHUB_ACTIONS_MARK,
245+
marks=_SKIP_ON_CI_MARK,
228246
),
229247
# newton + newton_renderer (warp)
230248
pytest.param(

‎source/isaaclab_tasks/test/test_rendering_dexsuite_kuka.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
_attach_comparison_properties_fixture = make_attach_comparison_properties_fixture(_COMPARISON_SCORES)
3232

3333

34-
@pytest.mark.flaky(max_runs=3, min_passes=1)
3534
@pytest.mark.parametrize("physics_backend,renderer,data_type", PHYSICS_RENDERER_AOV_COMBINATIONS)
3635
def test_rendering_dexsuite_kuka(physics_backend, renderer, data_type):
3736
"""Test dexsuite kuka allegro lift environment rendering correctness."""

‎source/isaaclab_tasks/test/test_rendering_dexsuite_kuka_kitless.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@
2727
_require_ovrtx_install_fixture = make_require_ovrtx_install_fixture()
2828

2929

30-
@pytest.mark.flaky(max_runs=3, min_passes=1)
3130
@pytest.mark.parametrize("physics_backend,renderer,data_type", KITLESS_PHYSICS_RENDERER_AOV_COMBINATIONS)
3231
def test_rendering_dexsuite_kuka_kitless(physics_backend, renderer, data_type):
3332
"""Camera output must match golden images (Dexsuite Kuka-Allegro Lift, single camera)."""

0 commit comments

Comments
 (0)