wip

Author: matthewtrepte

source/isaaclab/isaaclab/app/app_launcher.py

@@ -190,7 +190,6 @@ def __init__(self, launcher_args: argparse.Namespace | dict | None = None, **kwa
         self._sim_experience_file: str  # Experience file to load
         self._visualizer_max_worlds: int | None  # Optional max worlds override for Newton-based visualizers
         self._video_enabled: bool  # Whether --video recording is enabled
-        self._video_auto_start_kit: bool  # Whether headless video should auto-inject Kit visualizer
 
         # Exposed to train scripts
         self.device_id: int  # device ID for GPU simulation (defaults to 0)
@@ -869,19 +868,6 @@ def _resolve_viewport_settings(self, launcher_args: dict):
         if self._headless and not self._livestream and not self._video_enabled:
             self._render_viewport = False
 
-        # Auto-start a headless Kit visualizer when recording video without an explicit
-        # visualizer selection. This ensures app.update() is pumped and camera updates
-        # can be forwarded from viewer settings.
-        has_explicit_kit = self._cli_visualizer_explicit and "kit" in set(self._cli_visualizer_types)
-        self._video_auto_start_kit = (
-            self._video_enabled
-            and self._headless
-            and not self._livestream
-            and not self._xr
-            and not self._cli_visualizer_disable_all
-            and not has_explicit_kit
-        )
-
         # hide_ui flag
         launcher_args["hide_ui"] = False
         if self._headless and not self._livestream:
@@ -1101,9 +1087,8 @@ def _load_extensions(self):
         # (no Kit GUI) the AR profile must be enabled programmatically so that
         # the OpenXR session starts without user interaction
         settings.set_bool("/isaaclab/xr/auto_start", self._headless and self._xr)
-        # set setting to indicate video recording mode and optional Kit auto-start
+        # set setting to indicate video recording mode
         settings.set_bool("/isaaclab/video/enabled", self._video_enabled)
-        settings.set_bool("/isaaclab/video/auto_start_kit", self._video_auto_start_kit)
 
         # set setting to indicate no RTX sensors are used (set to True when RTX sensor is created)
         settings.set_bool("/isaaclab/render/rtx_sensors", False)

source/isaaclab/isaaclab/sim/simulation_context.py

@@ -180,7 +180,6 @@ def __init__(self, cfg: SimulationCfg | None = None):
         self._has_gui = bool(self.get_setting("/isaaclab/has_gui"))
         self._has_offscreen_render = bool(self.get_setting("/isaaclab/render/offscreen"))
         self._xr_enabled = bool(self.get_setting("/isaaclab/xr/enabled"))
-        self._video_auto_start_kit = bool(self.get_setting("/isaaclab/video/auto_start_kit"))
         # Note: has_rtx_sensors is NOT cached because it changes when Camera sensors are created
         self._pending_camera_view: tuple[tuple[float, float, float], tuple[float, float, float]] | None = None
 
@@ -345,9 +344,7 @@ def has_offscreen_render(self) -> bool:
 
     def has_active_visualizers(self) -> bool:
         """Return whether any visualizer path is active for rendering/camera control."""
-        return bool(self.get_setting("/isaaclab/visualizer/types")) or bool(
-            self.get_setting("/isaaclab/video/auto_start_kit")
-        )
+        return bool(self.get_setting("/isaaclab/visualizer/types"))
 
     def can_render_rgb_array(self) -> bool:
         """Return whether rgb-array rendering is currently available."""
@@ -555,26 +552,6 @@ def _resolve_visualizer_cfgs(self) -> list[Any]:
                         exc,
                     )
 
-        # Headless video auto-start: inject a KitVisualizer when needed so that
-        # app.update() is pumped and viewer camera updates can be applied in
-        # rgb-array recording flows.
-        if self._video_auto_start_kit and not cli_disable_all:
-            has_kit = any(getattr(cfg, "visualizer_type", None) == "kit" for cfg in resolved)
-            if not has_kit:
-                try:
-                    import importlib
-
-                    mod = importlib.import_module("isaaclab_visualizers.kit")
-                    kit_cfg_cls = getattr(mod, "KitVisualizerCfg")
-                    resolved.append(kit_cfg_cls(headless=True))
-                    logger.info("[SimulationContext] Auto-injecting KitVisualizer for headless video recording.")
-                except (ImportError, ModuleNotFoundError, AttributeError, TypeError) as exc:
-                    logger.warning(
-                        "[SimulationContext] Headless video could not auto-inject a KitVisualizer: %s. "
-                        "Install isaaclab_visualizers[kit] or pass --visualizer kit.",
-                        exc,
-                    )
-
         return resolved
 
     def initialize_visualizers(self) -> None:
@@ -744,6 +721,7 @@ def render(self, mode: int | None = None) -> None:
         """
         self.physics_manager.pre_render()
         self.update_visualizers(self.get_rendering_dt())
+        self._pump_kit_app_if_needed()
         self._render_generation += 1
 
         # Call render callbacks
@@ -769,9 +747,6 @@ def update_visualizers(self, dt: float) -> None:
                     visualizers_to_remove.append(viz)
                     continue
                 if viz.is_rendering_paused():
-                    # Keep visualizer event loops responsive while rendering is paused
-                    # so UI controls (e.g. "Resume Rendering") remain interactive.
-                    viz.step(0.0)
                     continue
                 while viz.is_training_paused() and viz.is_running():
                     viz.step(0.0)
@@ -788,6 +763,21 @@ def update_visualizers(self, dt: float) -> None:
             except Exception as exc:
                 logger.error("Error closing visualizer: %s", exc)
 
+    def _pump_kit_app_if_needed(self) -> None:
+        """Pump Kit app-loop for headless rgb-array rendering when needed."""
+        if not bool(self.get_setting("/isaaclab/video/enabled")):
+            return
+        if not has_kit():
+            return
+        if any(viz.pumps_app_update() for viz in self._visualizers):
+            return
+        try:
+            from isaaclab_physx.renderers.isaac_rtx_renderer_utils import ensure_isaac_rtx_render_update
+
+            ensure_isaac_rtx_render_update()
+        except Exception as exc:
+            logger.debug("[SimulationContext] Skipping Kit app-loop pump in render(): %s", exc)
+
     def update_scene_data_provider(self, force_require_forward: bool = False):
         if force_require_forward or self._should_forward_before_visualizer_update():
             self.physics_manager.forward()

source/isaaclab/setup.py

@@ -22,7 +22,6 @@
     "torch>=2.10",
     "onnx>=1.18.0",  # 1.16.2 throws access violation on Windows
     "prettytable==3.3.0",
-    "munch",
     "toml",
     # devices
     "hidapi==0.14.0.post2",

source/isaaclab/test/sim/test_simulation_context_visualizers.py

@@ -136,22 +136,11 @@ def test_update_visualizers_removes_closed_nonrunning_and_failed(caplog):
     assert stopped_viz.close_calls == 1
     assert failing_viz.close_calls == 1
     assert paused_viz.close_calls == 0
-    assert paused_viz.step_calls == [0.0]
+    assert paused_viz.step_calls == []
     assert healthy_viz.step_calls == [0.1]
     assert any("Error stepping visualizer" in r.message for r in caplog.records)
 
 
-def test_update_visualizers_keeps_rendering_paused_visualizer_ui_responsive():
-    provider = _FakeProvider()
-    paused_viz = _FakeVisualizer(rendering_paused=True)
-    ctx = _make_context([paused_viz], provider=provider)
-
-    ctx.update_visualizers(0.3)
-
-    # Rendering-paused visualizers still receive zero-dt ticks to keep UI callbacks alive.
-    assert paused_viz.step_calls == [0.0]
-
-
 def test_update_visualizers_handles_training_pause_loop():
     provider = _FakeProvider()
     viz = _FakeVisualizer(training_paused_steps=1)

source/isaaclab_mimic/isaaclab_mimic/datagen/generation.py

@@ -5,19 +5,16 @@
 
 import asyncio
 import contextlib
-import os
 import sys
 import traceback
 from typing import Any
 
 import torch
-import warp as wp
 
 from isaaclab.envs import ManagerBasedRLMimicEnv
 from isaaclab.envs.mdp.recorders.recorders_cfg import ActionStateRecorderManagerCfg
 from isaaclab.managers import DatasetExportMode, TerminationTermCfg
 from isaaclab.managers.recorder_manager import RecorderManagerBaseCfg
-from isaaclab.utils.math import subtract_frame_transforms
 
 from isaaclab_mimic.datagen.data_generator import DataGenerator
 from isaaclab_mimic.datagen.datagen_info_pool import DataGenInfoPool
@@ -30,59 +27,6 @@
 num_attempts = 0
 
 
-def _debug_wrist_cam_pose(env: ManagerBasedRLMimicEnv, env_id: int, tag: str, hand_body_idx: int | None):
-    """Print wrist camera and hand-link transforms for debugging camera alignment."""
-    try:
-        if "wrist_cam" not in env.scene.keys() or "robot" not in env.scene.keys():
-            print(f"[WRIST_CAM_DEBUG] {tag} env={env_id} missing wrist_cam or robot in scene")
-            return
-
-        wrist_cam = env.scene["wrist_cam"]
-        robot = env.scene["robot"]
-        cam_pos = wrist_cam.data.pos_w[env_id].detach().cpu()
-        cam_quat = wrist_cam.data.quat_w_world[env_id].detach().cpu()
-        cam_frame = int(wrist_cam.frame[env_id].item()) if wrist_cam.frame.numel() > env_id else -1
-        live_pos_all, live_quat_all = wrist_cam._view.get_world_poses([env_id])
-        live_cam_pos = live_pos_all[0].detach().cpu()
-        live_cam_quat = live_quat_all[0].detach().cpu()
-
-        if hand_body_idx is None:
-            print(
-                f"[WRIST_CAM_DEBUG] {tag} env={env_id} cam_frame={cam_frame} "
-                f"cam_pos={cam_pos.tolist()} cam_quat_xyzw={cam_quat.tolist()} "
-                f"live_cam_pos={live_cam_pos.tolist()} live_cam_quat_xyzw={live_cam_quat.tolist()} "
-                f"hand_body_idx=None"
-            )
-            return
-
-        hand_pos = wp.to_torch(robot.data.body_pos_w)[env_id, hand_body_idx].detach().cpu()
-        hand_quat = wp.to_torch(robot.data.body_quat_w)[env_id, hand_body_idx].detach().cpu()
-        rel_pos, rel_quat = subtract_frame_transforms(
-            hand_pos.unsqueeze(0),
-            hand_quat.unsqueeze(0),
-            cam_pos.unsqueeze(0),
-            cam_quat.unsqueeze(0),
-        )
-        live_rel_pos, live_rel_quat = subtract_frame_transforms(
-            hand_pos.unsqueeze(0),
-            hand_quat.unsqueeze(0),
-            live_cam_pos.unsqueeze(0),
-            live_cam_quat.unsqueeze(0),
-        )
-        print(
-            f"[WRIST_CAM_DEBUG] {tag} env={env_id} cam_frame={cam_frame} "
-            f"cam_pos={cam_pos.tolist()} cam_quat_xyzw={cam_quat.tolist()} "
-            f"live_cam_pos={live_cam_pos.tolist()} live_cam_quat_xyzw={live_cam_quat.tolist()} "
-            f"hand_pos={hand_pos.tolist()} hand_quat_xyzw={hand_quat.tolist()} "
-            f"cam_rel_hand_pos={rel_pos[0].detach().cpu().tolist()} "
-            f"cam_rel_hand_quat_xyzw={rel_quat[0].detach().cpu().tolist()} "
-            f"live_cam_rel_hand_pos={live_rel_pos[0].detach().cpu().tolist()} "
-            f"live_cam_rel_hand_quat_xyzw={live_rel_quat[0].detach().cpu().tolist()}"
-        )
-    except Exception as exc:
-        print(f"[WRIST_CAM_DEBUG] {tag} env={env_id} logging_failed={exc}")
-
-
 async def run_data_generator(
     env: ManagerBasedRLMimicEnv,
     env_id: int,
@@ -150,18 +94,6 @@ def env_loop(
     global num_success, num_failures, num_attempts
     env_id_tensor = torch.tensor([0], dtype=torch.int64, device=env.device)
     prev_num_attempts = 0
-    debug_wrist_cam = os.environ.get("ISAACLAB_DEBUG_WRIST_CAM", "0") == "1"
-    post_reset_step_counters = [0 for _ in range(env.num_envs)]
-    hand_body_idx = None
-    if debug_wrist_cam:
-        try:
-            if "robot" in env.scene.keys():
-                body_ids, _ = env.scene["robot"].find_bodies("panda_hand")
-                hand_body_idx = int(body_ids[0]) if len(body_ids) > 0 else None
-            print(f"[WRIST_CAM_DEBUG] enabled=1 hand_body_idx={hand_body_idx}")
-        except Exception as exc:
-            print(f"[WRIST_CAM_DEBUG] enabled=1 hand_body_idx_lookup_failed={exc}")
-
     # simulate environment -- run everything in inference mode
     with contextlib.suppress(KeyboardInterrupt) and torch.inference_mode():
         while True:
@@ -176,11 +108,6 @@ def env_loop(
                 while not env_reset_queue.empty():
                     env_id_tensor[0] = env_reset_queue.get_nowait()
                     env.reset(env_ids=env_id_tensor)
-                    reset_env_id = int(env_id_tensor[0].item())
-                    if 0 <= reset_env_id < env.num_envs:
-                        post_reset_step_counters[reset_env_id] = 0
-                    if debug_wrist_cam:
-                        _debug_wrist_cam_pose(env, reset_env_id, tag="after_reset", hand_body_idx=hand_body_idx)
                     env_reset_queue.task_done()
 
             actions = torch.zeros(env.action_space.shape)
@@ -191,16 +118,6 @@ def env_loop(
 
             # perform action on environment
             env.step(actions)
-            if debug_wrist_cam:
-                for env_id in range(env.num_envs):
-                    if post_reset_step_counters[env_id] <= 1:
-                        _debug_wrist_cam_pose(
-                            env,
-                            env_id,
-                            tag=f"after_step_{post_reset_step_counters[env_id]}",
-                            hand_body_idx=hand_body_idx,
-                        )
-                        post_reset_step_counters[env_id] += 1
 
             # mark done so the data generators can continue with the step results
             for i in range(env.num_envs):

source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py

@@ -11,6 +11,7 @@
 import logging
 from typing import TYPE_CHECKING
 
+import carb
 from pxr import UsdGeom
 
 from isaaclab.visualizers.base_visualizer import BaseVisualizer
@@ -109,7 +110,10 @@ def step(self, dt: float) -> None:
 
             app = omni.kit.app.get_app()
             if app is not None and app.is_running():
+                settings = carb.settings.get_settings()
+                settings.set_bool("/app/player/playSimulations", False)
                 app.update()
+                settings.set_bool("/app/player/playSimulations", True)
         except (ImportError, AttributeError) as exc:
             logger.debug("[KitVisualizer] App update skipped: %s", exc)
 
@@ -143,15 +147,11 @@ def is_running(self) -> bool:
             return False
 
     def is_training_paused(self) -> bool:
-        """Return whether Kit timeline transport is paused."""
+        """Return whether simulation play flag is paused in Kit settings."""
         try:
-            import omni.timeline
-
-            timeline = omni.timeline.get_timeline_interface()
-            if timeline is None:
-                return False
-            # Pause is transport state that is neither playing nor stopped.
-            return (not timeline.is_playing()) and (not timeline.is_stopped())
+            settings = carb.settings.get_settings()
+            play_flag = settings.get("/app/player/playSimulations")
+            return play_flag is not None and not bool(play_flag)
         except Exception:
             return False