Pre and post physics renderer initialization

source/isaaclab/changelog.d/scene-initialize-renderers.minor.rst

@@ -0,0 +1,31 @@
+Added
+^^^^^
+
+* Added :meth:`~isaaclab.scene.InteractiveScene.initialize_renderers` to
+  pre-create renderer backends for all scene sensors with a
+  ``renderer_cfg`` against the shared
+  :class:`~isaaclab.renderers.render_context.RenderContext`. The method is
+  idempotent and is now invoked from
+  :class:`~isaaclab.envs.DirectRLEnv`,
+  :class:`~isaaclab.envs.DirectMARLEnv`,
+  :class:`~isaaclab.envs.ManagerBasedEnv`, and
+  :class:`~isaaclab.envs.LeappDeploymentEnv` after scene construction so
+  that renderer backend creation order is deterministic and front-loaded
+  before the first :meth:`~isaaclab.sim.SimulationContext.reset`.
+* Added :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.initialize`
+  post-physics lifecycle hook (default no-op) that runs once per backend
+  after :meth:`~isaaclab.sim.SimulationContext.reset` builds physics
+  models. ``__init__`` now defines the pre-physics phase (eagerly invoked
+  by :meth:`~isaaclab.scene.InteractiveScene.initialize_renderers`) and
+  ``initialize`` defines the post-physics phase, letting backends whose
+  setup needs scene data (e.g. a built Newton model) defer that work
+  cleanly. Driven by
+  :meth:`~isaaclab.renderers.render_context.RenderContext.ensure_initialize`,
+  registered on
+  :class:`~isaaclab.physics.physics_manager.PhysicsEvent` ``PHYSICS_READY``
+  by :class:`~isaaclab.sim.SimulationContext` at ``order=5`` so it fires
+  before sensor/asset callbacks (``order=10``). This decouples renderer
+  post-physics setup from camera initialization. Backends created lazily
+  after PHYSICS_READY are eagerly initialized at
+  :meth:`~isaaclab.renderers.render_context.RenderContext.get_renderer`
+  time.

source/isaaclab/isaaclab/envs/direct_marl_env.py

@@ -145,6 +145,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs):
             with use_stage(self.sim.stage):
                 self.scene = InteractiveScene(self.cfg.scene)
                 self._setup_scene()
+                self.scene.initialize_renderers()
         print("[INFO]: Scene manager: ", self.scene)
 
         # set up camera viewport controller

source/isaaclab/isaaclab/envs/direct_rl_env.py

@@ -150,6 +150,7 @@ def _init_sim(self, render_mode: str | None = None, **kwargs):
             with use_stage(self.sim.stage):
                 self.scene = InteractiveScene(self.cfg.scene)
                 self._setup_scene()
+                self.scene.initialize_renderers()
         print("[INFO]: Scene manager: ", self.scene)
 
         # set up camera viewport controller

source/isaaclab/isaaclab/envs/leapp_deployment_env.py

@@ -183,6 +183,7 @@ def __init__(self, cfg: Any, leapp_yaml_path: str):
 
         with use_stage(self.sim.stage):
             self.scene = InteractiveScene(cfg.scene)
+            self.scene.initialize_renderers()
         with use_stage(self.sim.stage):
             self.sim.reset()
         self.scene.update(dt=self.physics_dt)

source/isaaclab/isaaclab/envs/manager_based_env.py

@@ -169,6 +169,7 @@ def _init_sim(self):
             # set the stage context for scene creation steps which use the stage
             with use_stage(self.sim.stage):
                 self.scene = InteractiveScene(self.cfg.scene)
+                self.scene.initialize_renderers()
         print("[INFO]: Scene manager: ", self.scene)
 
         # set up camera viewport controller

source/isaaclab/isaaclab/renderers/base_renderer.py

@@ -22,6 +22,10 @@
 class BaseRenderer(ABC):
     """Abstract base class for renderer implementations."""
 
+    def initialize(self) -> None:
+        """Post-physics one-time initialization hook. Called only once."""
+        return
+
     @abstractmethod
     def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]:
         """Per-output layout (channels + dtype) this renderer can produce.

source/isaaclab/isaaclab/renderers/render_context.py

@@ -33,13 +33,15 @@ class RenderContext:
 
     __slots__ = (
         "_renderer_entries",
+        "_physics_initialized",
         "_prepared_renderer_ids",
         "_prepared_num_envs",
         "_last_transforms_step",
     )
 
     def __init__(self) -> None:
         self._renderer_entries: list[tuple[RendererCfg, BaseRenderer]] = []
+        self._physics_initialized: bool = False  # Set to True after the first PHYSICS_READY callback fires.
         self._prepared_renderer_ids: set[int] = set()
         self._prepared_num_envs: int | None = None
         self._last_transforms_step: int | None = None
@@ -65,8 +67,18 @@ def get_renderer(self, cfg: RendererCfg) -> BaseRenderer:
             "Created new renderer for simulation: %s",
             type(new_renderer).__name__,
         )
+        if self._physics_initialized:
+            new_renderer.initialize()
         return new_renderer
 
+    def ensure_initialize(self) -> None:
+        """Idempotent call fired after PHYSICS_READY callback."""
+        if self._physics_initialized:
+            return
+        self._physics_initialized = True
+        for _cfg, renderer in self._renderer_entries:
+            renderer.initialize()
+
     def ensure_prepare_stage(self, stage: Any, num_envs: int) -> None:
         """Call :meth:`BaseRenderer.prepare_stage` for each registered backend (once per backend).
 

source/isaaclab/isaaclab/scene/interactive_scene.py

@@ -12,6 +12,8 @@
 if TYPE_CHECKING:
     from isaaclab_physx.assets import DeformableObject, SurfaceGripper
 
+    from isaaclab.renderers.base_renderer import BaseRenderer
+
 import torch
 import warp as wp
 
@@ -364,6 +366,42 @@ def _sensor_renderer_types(self) -> list[str]:
             if (rcfg := getattr(getattr(s, "cfg", None), "renderer_cfg", None)) is not None
         ]
 
+    def initialize_renderers(self) -> list[BaseRenderer]:
+        """Pre-create renderer backends for all scene sensors with a ``renderer_cfg``.
+
+        Walks the constructed sensors and registers each unique
+        :class:`~isaaclab.renderers.renderer_cfg.RendererCfg` with the
+        simulation-scoped :class:`~isaaclab.renderers.render_context.RenderContext`.
+        Configs that compare equal share a single backend (see
+        :meth:`~isaaclab.renderers.render_context.RenderContext.get_renderer`), so
+        calling this method is idempotent and safe to invoke before
+        :meth:`~isaaclab.sim.SimulationContext.reset`.
+
+        Pre-creating backends here makes the order of renderer construction
+        deterministic (matches sensor registration order) and front-loads logging
+        instead of trickling out during the first :meth:`Camera._initialize_impl`.
+        :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.prepare_stage` is
+        intentionally not invoked here; it runs on first camera initialization
+        with the correct ``num_envs`` and final stage.
+
+        Returns:
+            The list of unique renderer backends now registered on the
+            shared :class:`~isaaclab.renderers.render_context.RenderContext`,
+            in sensor registration order.
+        """
+        ctx = self.sim.render_context
+        backends: list[BaseRenderer] = []
+        seen: set[int] = set()
+        for sensor in self._sensors.values():
+            rcfg = getattr(getattr(sensor, "cfg", None), "renderer_cfg", None)
+            if rcfg is None:
+                continue
+            backend = ctx.get_renderer(rcfg)
+            if id(backend) not in seen:
+                seen.add(id(backend))
+                backends.append(backend)
+        return backends
+
     def filter_collisions(self, global_prim_paths: list[str] | None = None):
         """Filter environments collisions.
 

source/isaaclab/isaaclab/sim/simulation_context.py

@@ -23,7 +23,7 @@
 from isaaclab.app.settings_manager import SettingsManager
 from isaaclab.envs.utils.recording_hooks import run_recording_hooks_after_visualizers
 from isaaclab.markers.vis_marker_registry import VisMarkerRegistry
-from isaaclab.physics import BaseSceneDataProvider, PhysicsManager, SceneDataProvider
+from isaaclab.physics import BaseSceneDataProvider, PhysicsEvent, PhysicsManager, SceneDataProvider
 from isaaclab.physics.scene_data_requirements import (
     SceneDataRequirement,
     resolve_scene_data_requirements,
@@ -207,6 +207,13 @@ def __init__(self, cfg: SimulationCfg | None = None):
         # Shared renderers for all Camera sensors (compatible renderer_cfg only).
         self._render_context = RenderContext()
 
+        # Run renderer post-physics setup.
+        self.physics_manager.register_callback(
+            lambda _payload: self._render_context.ensure_initialize(),
+            PhysicsEvent.PHYSICS_READY,
+            order=5,
+        )
+
         type(self)._instance = self  # Mark as valid singleton only after successful init
 
     def _apply_render_cfg_settings(self) -> None:

source/isaaclab_experimental/changelog.d/scene-initialize-renderers.rst

@@ -0,0 +1,10 @@
+Changed
+^^^^^^^
+
+* Pre-create renderer backends in
+  :class:`~isaaclab_experimental.envs.ManagerBasedEnvWarp` and
+  :class:`~isaaclab_experimental.envs.DirectRLEnvWarp` by invoking
+  :meth:`~isaaclab.scene.InteractiveScene.initialize_renderers` after scene
+  construction so that renderer backend creation order is deterministic and
+  front-loaded before the first
+  :meth:`~isaaclab.sim.SimulationContext.reset`.

source/isaaclab_experimental/isaaclab_experimental/envs/direct_rl_env_warp.py

@@ -164,6 +164,7 @@ def __init__(self, cfg: DirectRLEnvCfg, render_mode: str | None = None, **kwargs
             with use_stage(self.sim.stage):
                 self.scene = InteractiveSceneWarp(self.cfg.scene)
                 self._setup_scene()
+                self.scene.initialize_renderers()
                 # attach_stage_to_usd_context()
         print("[INFO]: Scene manager: ", self.scene)
 

source/isaaclab_experimental/isaaclab_experimental/envs/manager_based_env_warp.py

@@ -132,6 +132,7 @@ def __init__(self, cfg: ManagerBasedEnvCfg):
             with use_stage(self.sim.stage):
                 self.scene = InteractiveScene(self.cfg.scene)
                 # attach_stage_to_usd_context()
+                self.scene.initialize_renderers()
         print("[INFO]: Scene manager: ", self.scene)
 
         # Shared per-env Warp RNG state (accessible to all managers/terms via `env`).

source/isaaclab_newton/changelog.d/scene-initialize-renderers.rst

@@ -0,0 +1,9 @@
+Changed
+^^^^^^^
+
+* Split :class:`~isaaclab_newton.renderers.NewtonWarpRenderer` construction
+  into a pre-physics ``__init__`` (stores cfg and registers the Newton-Warp
+  scene-data requirement on
+  :class:`~isaaclab.sim.SimulationContext`) and a post-physics
+  :meth:`~isaaclab_newton.renderers.NewtonWarpRenderer.initialize` (reads
+  the built Newton model.

source/isaaclab_newton/isaaclab_newton/renderers/newton_warp_renderer.py

@@ -139,19 +139,24 @@ class NewtonWarpRenderer(BaseRenderer):
     RenderData = RenderData
 
     def __init__(self, cfg: NewtonWarpRendererCfg):
+        """Pre-physics initialization."""
         from isaaclab.physics.scene_data_requirements import (
             aggregate_requirements,
             requirement_for_renderer_type,
         )
 
         self.cfg = cfg
+        self.newton_sensor: newton.sensors.SensorTiledCamera | None = None
+
         sim = SimulationContext.instance()
         current_req = sim.get_scene_data_requirements()
         renderer_req = requirement_for_renderer_type("newton_warp")
         merged = aggregate_requirements([current_req, renderer_req])
         if merged != current_req:
             sim.update_scene_data_requirements(merged)
 
+    def initialize(self) -> None:
+        """Post-physics setup: read the built Newton model and construct the sensor."""
         newton_model = self.get_scene_data_provider().get_newton_model()
         if newton_model is None:
             raise RuntimeError(
@@ -164,11 +169,11 @@ def __init__(self, cfg: NewtonWarpRendererCfg):
         self.newton_sensor = newton.sensors.SensorTiledCamera(
             newton_model,
             config=newton.sensors.SensorTiledCamera.RenderConfig(
-                enable_textures=cfg.enable_textures,
-                enable_shadows=cfg.enable_shadows,
-                enable_ambient_lighting=cfg.enable_ambient_lighting,
-                enable_backface_culling=cfg.enable_backface_culling,
-                max_distance=cfg.max_distance,
+                enable_textures=self.cfg.enable_textures,
+                enable_shadows=self.cfg.enable_shadows,
+                enable_ambient_lighting=self.cfg.enable_ambient_lighting,
+                enable_backface_culling=self.cfg.enable_backface_culling,
+                max_distance=self.cfg.max_distance,
             ),
         )
 
@@ -180,8 +185,8 @@ def __init__(self, cfg: NewtonWarpRendererCfg):
         if newton_model.shape_count > 0 and newton_model.bvh_shapes is None:
             newton.geometry.build_bvh_shape(newton_model, newton_model.state())
 
-        if cfg.create_default_light:
-            self.newton_sensor.utils.create_default_light(enable_shadows=cfg.enable_shadows)
+        if self.cfg.create_default_light:
+            self.newton_sensor.utils.create_default_light(enable_shadows=self.cfg.enable_shadows)
 
     def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]:
         """Publish the per-output layout this Newton Warp backend writes.

source/isaaclab_ov/changelog.d/scene-initialize-renderers.rst

@@ -0,0 +1,20 @@
+Changed
+^^^^^^^
+
+* Construct the underlying OVRTX ``Renderer`` in
+  :class:`~isaaclab_ov.renderers.OVRTXRenderer` ``__init__`` instead of
+  during :meth:`~isaaclab_ov.renderers.OVRTXRenderer.prepare_stage`. This
+  pairs with the new pre-physics ``__init__`` /
+  post-physics :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.initialize`
+  lifecycle: when invoked eagerly via
+  :meth:`~isaaclab.scene.InteractiveScene.initialize_renderers`, the OVRTX
+  ``Renderer`` is created before
+  :meth:`~isaaclab.sim.SimulationContext.reset` (and therefore before
+  ovphysx initialises), which OVRTX 0.3 requires.
+* Replaced an ``assert`` on the OVRTX ``Renderer`` construction with an
+  explicit :class:`RuntimeError` so the failure is reported even when
+  Python is run with ``-O``.
+* Renamed the internal ``OVRTXRenderer.initialize(spec)`` helper to
+  ``_initialize_from_spec(spec)`` to avoid shadowing the new
+  no-arg :meth:`~isaaclab.renderers.base_renderer.BaseRenderer.initialize`
+  lifecycle hook.

source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py

@@ -159,6 +159,21 @@ def __init__(self, cfg: OVRTXRendererCfg):
         self._camera_rel_path: str | None = None
         self._output_semantic_color_buffer: wp.array | None = None
 
+        logger.info("Creating OVRTX renderer...")
+        OVRTX_CONFIG = RendererConfig(
+            log_file_path=self.cfg.log_file_path,
+            log_level=self.cfg.log_level,
+            read_gpu_transforms=_IS_OVRTX_0_3_0_OR_NEWER,
+            keep_system_alive=True,
+        )
+        self._renderer = Renderer(OVRTX_CONFIG)
+        if not self._renderer:
+            raise RuntimeError(
+                "Failed to create OVRTX Renderer; the underlying ovrtx.Renderer constructor returned a falsy"
+                " value. Check that ovrtx is installed correctly and its native dependencies are available."
+            )
+        logger.info("OVRTX renderer created successfully")
+
     def prepare_stage(self, stage: Any, num_envs: int) -> None:
         """Export the USD stage for OVRTX before create_render_data.
 
@@ -178,7 +193,7 @@ def prepare_stage(self, stage: Any, num_envs: int) -> None:
         self._exported_usd_path = export_path
         logger.info("Exported to %s", export_path)
 
-    def initialize(self, spec: CameraRenderSpec):
+    def _initialize_from_spec(self, spec: CameraRenderSpec):
         """Initialize the OVRTX renderer with internal environment cloning.
 
         Args:
@@ -198,17 +213,6 @@ def initialize(self, spec: CameraRenderSpec):
         usd_scene_path = self._exported_usd_path
         use_cloning = self.cfg.use_cloning
 
-        logger.info("Creating OVRTX renderer...")
-        OVRTX_CONFIG = RendererConfig(
-            log_file_path=self.cfg.log_file_path,
-            log_level=self.cfg.log_level,
-            read_gpu_transforms=_IS_OVRTX_0_3_0_OR_NEWER,
-            keep_system_alive=True,
-        )
-        self._renderer = Renderer(OVRTX_CONFIG)
-        assert self._renderer, "Renderer should be valid after creation"
-        logger.info("OVRTX renderer created successfully")
-
         if usd_scene_path is not None:
             logger.info("Injecting camera definitions...")
 
@@ -367,7 +371,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXRenderData:
         matching the interface of Isaac RTX and Newton Warp which need no separate initialize().
         """
         if not self._initialized_scene:
-            self.initialize(spec)
+            self._initialize_from_spec(spec)
         return OVRTXRenderData(spec, DEVICE)
 
     # Map torch dtypes to their warp counterparts for zero-copy wrapping.