Patch newton model reqs (isaac-sim#5398)

# Description

<!--
Thank you for your interest in sending a pull request. Please make sure
to check the contribution guidelines.

Link:
https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html

💡 Please try to keep PRs small and focused. Large PRs are harder to
review and merge.
-->

Patch newton model reqs issue where in some caes env configs were
constructed in a way where newton model reqs were not correctly
determined, leading to downstream issues.

Also, revert the revert to physx scene data provider, since the fix
above no longer requires the revert, which is non ideal since building
newton model from usd fallback is slow.

<!-- As a practice, it is recommended to open an issue to have
discussions on the proposed pull request.
This makes it easier for the community to keep track of what is being
developed or added, and if a given feature
is demanded by more than one party. -->

## Type of change

<!-- As you go through the list, delete the ones that are not
applicable. -->

- Bug fix (non-breaking change which fixes an issue)


## Screenshots

Please attach before and after screenshots of the change if applicable.

<!--
Example:

| Before | After |
| ------ | ----- |
| _gif/png before_ | _gif/png after_ |

To upload images to a PR -- simply drag and drop an image while in edit
mode and it should upload the image directly. You can then paste that
source into the above before/after sections.
-->

## Checklist

- [ ] I have read and understood the [contribution
guidelines](https://isaac-sim.github.io/IsaacLab/main/source/refs/contributing.html)
- [ ] I have run the [`pre-commit` checks](https://pre-commit.com/) with
`./isaaclab.sh --format`
- [ ] I have made corresponding changes to the documentation
- [ ] My changes generate no new warnings
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have updated the changelog and the corresponding version in the
extension's `config/extension.toml` file
- [ ] I have added my name to the `CONTRIBUTORS.md` or my name already
exists there

<!--
As you go through the checklist above, you can mark something as done by
putting an x character in it

For example,
- [x] I have done this task
- [ ] I have not done this task
-->

---------

Signed-off-by: matthewtrepte <mtrepte@nvidia.com>
Co-authored-by: Kelly Guo <kellyg@nvidia.com>
Co-authored-by: HuiDong Chen <huidongc@nvidia.com>
Co-authored-by: myurasov-nv <168484206+myurasov-nv@users.noreply.github.com>
Co-authored-by: hujc <leo.hu.sh@gmail.com>
Co-authored-by: Antoine Richard <antoiner@nvidia.com>
Co-authored-by: peterd-NV <peterd@nvidia.com>

Author: 7 people

source/isaaclab/isaaclab/scene/interactive_scene.py

@@ -28,7 +28,7 @@
     RigidObjectCollection,
     RigidObjectCollectionCfg,
 )
-from isaaclab.physics.scene_data_requirements import resolve_scene_data_requirements
+from isaaclab.physics.scene_data_requirements import aggregate_requirements, resolve_scene_data_requirements
 from isaaclab.sensors import ContactSensorCfg, FrameTransformerCfg, SensorBase, SensorBaseCfg
 from isaaclab.sim import SimulationContext
 from isaaclab.sim.utils.stage import get_current_stage, get_current_stage_id
@@ -140,7 +140,6 @@ def __init__(self, cfg: InteractiveSceneCfg):
         self.stage_id = get_current_stage_id()
         self.sim.clear_scene_data_visualizer_prebuilt_artifact()
         self.physics_backend = self.sim.physics_manager.__name__.lower()
-        visualizer_clone_fn = None
         requested_viz_types = set(self.sim.resolve_visualizer_types())
         if self.physics_backend.startswith("ovphysx"):
             from isaaclab_ovphysx.cloner import ovphysx_replicate
@@ -200,26 +199,7 @@ def __init__(self, cfg: InteractiveSceneCfg):
         if has_scene_cfg_entities:
             self._add_entities_from_cfg()
 
-        requirements = resolve_scene_data_requirements(
-            visualizer_types=requested_viz_types,
-            renderer_types=self._sensor_renderer_types(),
-        )
-        self.sim.update_scene_data_requirements(requirements)
-        visualizer_clone_fn = cloner.resolve_visualizer_clone_fn(
-            physics_backend=self.physics_backend,
-            requirements=requirements,
-            stage=self.stage,
-            set_visualizer_artifact=self.sim.set_scene_data_visualizer_prebuilt_artifact,
-        )
-        if visualizer_clone_fn is not None:
-            logger.debug(
-                "Enabling visualizer artifact prebuild for clone path "
-                "(backend=%s, requires_newton_model=%s, requires_usd_stage=%s).",
-                self.physics_backend,
-                requirements.requires_newton_model,
-                requirements.requires_usd_stage,
-            )
-            self.cloner_cfg.visualizer_clone_fn = visualizer_clone_fn
+        self._refresh_visualizer_clone_fn_from_requirements(requested_viz_types)
 
         if has_scene_cfg_entities:
             self.clone_environments(copy_from_source=(not self.cfg.replicate_physics))
@@ -236,6 +216,8 @@ def clone_environments(self, copy_from_source: bool = False):
             If True, clones are independent copies of the source prim and won't reflect its changes (start-up time
             may increase). Defaults to False.
         """
+        self._refresh_visualizer_clone_fn_from_requirements()
+
         # PhysX-only: set env id bit count for replicated physics. Newton handles env separation in its own API.
         # Intentionally matches both physx and ovphysx (both are PhysX-based)
         if self.cfg.replicate_physics and "physx" in self.physics_backend:
@@ -267,6 +249,33 @@ def clone_environments(self, copy_from_source: bool = False):
                 if self.cloner_cfg.clone_usd:
                     cloner.usd_replicate(self.stage, *replicate_args)
 
+    def _refresh_visualizer_clone_fn_from_requirements(self, visualizer_types=()) -> None:
+        """Refresh clone-time visualizer prebuild hook from current scene-data requirements."""
+        discovered_req = resolve_scene_data_requirements(
+            visualizer_types=visualizer_types,
+            renderer_types=self._sensor_renderer_types(),
+        )
+        current_req = self.sim.get_scene_data_requirements()
+        requirements = aggregate_requirements((current_req, discovered_req))
+        if requirements != current_req:
+            self.sim.update_scene_data_requirements(requirements)
+
+        visualizer_clone_fn = cloner.resolve_visualizer_clone_fn(
+            physics_backend=self.physics_backend,
+            requirements=requirements,
+            stage=self.stage,
+            set_visualizer_artifact=self.sim.set_scene_data_visualizer_prebuilt_artifact,
+        )
+        if visualizer_clone_fn is not None:
+            logger.debug(
+                "Enabling visualizer artifact prebuild for clone path "
+                "(backend=%s, requires_newton_model=%s, requires_usd_stage=%s).",
+                self.physics_backend,
+                requirements.requires_newton_model,
+                requirements.requires_usd_stage,
+            )
+            self.cloner_cfg.visualizer_clone_fn = visualizer_clone_fn
+
     def _sensor_renderer_types(self) -> list[str]:
         """Return renderer type names used by scene sensors."""
         renderer_types: list[str] = []

source/isaaclab/isaaclab/sensors/camera/camera.py

@@ -106,6 +106,7 @@ def __init__(self, cfg: CameraCfg):
         self._check_supported_data_types(cfg)
         # initialize base class
         super().__init__(cfg)
+        self._register_renderer_scene_data_requirements()
 
         # TODO(follow-up PR): move this flag flip out of Camera. The cleanest path is
         # an apply_pre_reset_settings() hook on RendererCfg (default no-op) that
@@ -133,6 +134,30 @@ def __init__(self, cfg: CameraCfg):
         self._renderer: BaseRenderer | None = None
         self._render_data = None
 
+    def _register_renderer_scene_data_requirements(self) -> None:
+        """Register renderer requirements early enough for clone-time prebuilds."""
+        renderer_type = getattr(getattr(self.cfg, "renderer_cfg", None), "renderer_type", None)
+        if renderer_type is None:
+            return
+
+        from isaaclab.physics.scene_data_requirements import aggregate_requirements, requirement_for_renderer_type
+        from isaaclab.sim import SimulationContext
+
+        sim = SimulationContext.instance()
+        if sim is None:
+            logger.debug("SimulationContext not available; deferring renderer requirements registration.")
+            return
+
+        try:
+            renderer_req = requirement_for_renderer_type(renderer_type)
+        except ValueError:
+            return
+
+        current_req = sim.get_scene_data_requirements()
+        merged_req = aggregate_requirements((current_req, renderer_req))
+        if merged_req != current_req:
+            sim.update_scene_data_requirements(merged_req)
+
     def __del__(self):
         """Unsubscribes from callbacks and cleans up renderer resources."""
         # unsubscribe callbacks

source/isaaclab/isaaclab/visualizers/visualizer_cfg.py

@@ -40,7 +40,7 @@ class VisualizerCfg:
     lookat: tuple[float, float, float] = (0.0, 0.0, 0.0)
     """Initial camera look-at point (x, y, z) in world coordinates."""
 
-    cam_source: Literal["cfg", "prim_path"] = "cfg"
+    cam_source: Literal["cfg", "prim_path"] = "prim_path"
     """Camera source mode: 'cfg' uses eye/lookat, 'prim_path' follows a camera prim."""
 
     cam_prim_path: str = "/World/envs/env_0/Camera"

source/isaaclab/test/scene/test_interactive_scene.py

@@ -21,6 +21,7 @@
 import isaaclab.sim as sim_utils
 from isaaclab.actuators import ImplicitActuatorCfg
 from isaaclab.assets import ArticulationCfg, RigidObjectCfg
+from isaaclab.physics.scene_data_requirements import SceneDataRequirement
 from isaaclab.scene import InteractiveScene, InteractiveSceneCfg
 from isaaclab.sim import build_simulation_context
 from isaaclab.utils import configclass
@@ -134,6 +135,13 @@ def test_clone_environments_non_cfg_invokes_visualizer_clone_fn(monkeypatch: pyt
     scene = object.__new__(InteractiveScene)
     scene.cfg = SimpleNamespace(replicate_physics=False, num_envs=3)
     scene.stage = object()
+    scene.physics_backend = "physx"
+    scene._sensors = {}
+    scene.sim = SimpleNamespace(
+        get_scene_data_requirements=lambda: SceneDataRequirement(),
+        update_scene_data_requirements=lambda requirements: None,
+        set_scene_data_visualizer_prebuilt_artifact=lambda artifact: None,
+    )
     scene.env_fmt = "/World/envs/env_{}"
     scene._ALL_INDICES = torch.arange(3, dtype=torch.long)
     scene._default_env_origins = torch.zeros((3, 3), dtype=torch.float32)
@@ -189,6 +197,38 @@ def _usd_replicate(stage, *args, **kwargs):
     assert len(usd_calls) == 1
 
 
+def test_refresh_visualizer_clone_fn_uses_registered_requirements(monkeypatch: pytest.MonkeyPatch):
+    """Clone-time prebuild hook should be installed from requirements registered after scene init."""
+    scene = object.__new__(InteractiveScene)
+    scene.physics_backend = "physx"
+    scene.stage = object()
+    scene._sensors = {}
+    scene.cloner_cfg = SimpleNamespace(visualizer_clone_fn=None)
+
+    requirements = SceneDataRequirement(requires_newton_model=True)
+    scene.sim = SimpleNamespace(
+        get_scene_data_requirements=lambda: requirements,
+        update_scene_data_requirements=lambda requirements: None,
+        set_scene_data_visualizer_prebuilt_artifact=lambda artifact: None,
+    )
+
+    captured = {}
+
+    def _resolve_visualizer_clone_fn(**kwargs):
+        captured.update(kwargs)
+        return "visualizer-clone-fn"
+
+    monkeypatch.setattr(
+        "isaaclab.scene.interactive_scene.cloner.resolve_visualizer_clone_fn",
+        _resolve_visualizer_clone_fn,
+    )
+
+    scene._refresh_visualizer_clone_fn_from_requirements()
+
+    assert captured["requirements"].requires_newton_model
+    assert scene.cloner_cfg.visualizer_clone_fn == "visualizer-clone-fn"
+
+
 def assert_state_equal(s1: dict, s2: dict, path=""):
     """
     Recursively assert that s1 and s2 have the same nested keys

source/isaaclab/test/sensors/test_camera.py

@@ -17,6 +17,7 @@
 
 import copy
 import random
+from types import SimpleNamespace
 
 import numpy as np
 import pytest
@@ -27,7 +28,9 @@
 from pxr import Gf, Usd, UsdGeom
 
 import isaaclab.sim as sim_utils
+from isaaclab.physics.scene_data_requirements import SceneDataRequirement
 from isaaclab.sensors.camera import Camera, CameraCfg
+from isaaclab.sim import SimulationContext
 
 pytestmark = pytest.mark.isaacsim_ci
 
@@ -45,6 +48,23 @@
 WIDTH = 320
 
 
+def test_camera_registers_renderer_scene_data_requirements(monkeypatch: pytest.MonkeyPatch):
+    """Camera creation path should register renderer-driven scene-data requirements."""
+    camera = object.__new__(Camera)
+    camera.cfg = SimpleNamespace(renderer_cfg=SimpleNamespace(renderer_type="newton_warp"))
+    updates = []
+    sim = SimpleNamespace(
+        get_scene_data_requirements=lambda: SceneDataRequirement(),
+        update_scene_data_requirements=updates.append,
+    )
+
+    monkeypatch.setattr(SimulationContext, "instance", staticmethod(lambda: sim))
+
+    camera._register_renderer_scene_data_requirements()
+
+    assert updates == [SceneDataRequirement(requires_newton_model=True)]
+
+
 def setup() -> tuple[sim_utils.SimulationContext, CameraCfg, float]:
     camera_cfg = CameraCfg(
         height=HEIGHT,

source/isaaclab/test/sim/test_physx_scene_data_provider_visualizer_contract.py

@@ -67,68 +67,11 @@ def test_load_prebuilt_artifact_populates_provider_state():
     assert provider._xform_mask_buf is None
 
 
-def test_load_prebuilt_artifact_missing_falls_back_to_usd_build():
-    """When no artifact is registered, the USD-traversal fallback is invoked."""
+def test_load_prebuilt_artifact_missing_sets_error_state():
+    """When no artifact is registered, model/state stay unset."""
     provider = _make_provider()
-    fallback_artifact = VisualizerPrebuiltArtifacts(
-        model="usd-built-model",
-        state="usd-built-state",
-        rigid_body_paths=["/World/envs/env_0/A"],
-        articulation_paths=[],
-        num_envs=2,
-    )
-    stored: list[VisualizerPrebuiltArtifacts] = []
-    provider._simulation_context = SimpleNamespace(
-        get_scene_data_visualizer_prebuilt_artifact=lambda: None,
-        set_scene_data_visualizer_prebuilt_artifact=stored.append,
-    )
-    provider._stage = None
-    provider._xform_views = {}
-    provider._view_body_index_map = {}
-    provider._view_order_tensors = {}
-    provider._pose_buf_num_bodies = 0
-    provider._positions_buf = None
-    provider._orientations_buf = None
-    provider._covered_buf = None
-    provider._xform_mask_buf = None
-
-    with (
-        patch.object(
-            PhysxSceneDataProvider,
-            "_build_newton_artifact_from_usd_fallback",
-            autospec=True,
-            return_value=fallback_artifact,
-        ),
-        patch(
-            "isaaclab_physx.scene_data_providers.physx_scene_data_provider.replace_newton_shape_colors",
-            lambda m, s: None,
-        ),
-    ):
-        provider._load_newton_model_from_prebuilt_artifact()
-
-    assert provider._last_newton_model_build_source == "usd_fallback"
-    assert provider._newton_model == "usd-built-model"
-    assert provider._newton_state == "usd-built-state"
-    assert provider._rigid_body_paths == ["/World/envs/env_0/A"]
-    assert provider._num_envs_at_last_newton_build == 2
-    # The fallback artifact is cached on the simulation context so subsequent providers see it.
-    assert stored == [fallback_artifact]
-
-
-def test_load_prebuilt_artifact_missing_and_fallback_failed_sets_missing_state():
-    """When both the prebuilt artifact and the USD-traversal fallback fail, model/state stay unset."""
-    provider = _make_provider()
-    provider._simulation_context = SimpleNamespace(
-        get_scene_data_visualizer_prebuilt_artifact=lambda: None,
-        set_scene_data_visualizer_prebuilt_artifact=lambda artifact: None,
-    )
-    with patch.object(
-        PhysxSceneDataProvider,
-        "_build_newton_artifact_from_usd_fallback",
-        autospec=True,
-        return_value=None,
-    ):
-        provider._load_newton_model_from_prebuilt_artifact()
+    provider._simulation_context = SimpleNamespace(get_scene_data_visualizer_prebuilt_artifact=lambda: None)
+    provider._load_newton_model_from_prebuilt_artifact()
     assert provider._last_newton_model_build_source == "missing"
     assert provider._newton_model is None
     assert provider._newton_state is None

source/isaaclab/test/sim/test_simulation_context_visualizers.py

@@ -183,6 +183,9 @@ def get_newton_state(self):
         self.state_calls.append(None)
         return {"state_call": len(self.state_calls)}
 
+    def get_camera_transforms(self):
+        return {}
+
 
 class _DummyViserViewer:
     def __init__(self):
@@ -354,6 +357,9 @@ def get_newton_model(self):
         def get_newton_state(self):
             return {"ok": True}
 
+        def get_camera_transforms(self):
+            return {}
+
     monkeypatch.setattr(rerun_visualizer, "NewtonViewerRerun", _FakeNewtonViewerRerun)
     monkeypatch.setattr(
         rerun_visualizer, "_ensure_rerun_server", lambda **kwargs: ("rerun+http://127.0.0.1:9876/proxy", False)