Enable pipelined IsaacTeleop retargeting

Default IsaacTeleop to deadline-paced pipelined retargeting.

Pass the execution config directly into TeleopSessionConfig.

Document why the 25 ms pacing margin staggers IsaacTeleop Python work.

It lets native rendering overlap instead of competing for the GIL.

Author: hougantc-nvda

docs/source/features/isaac_teleop.rst

@@ -686,10 +686,13 @@ Key ``IsaacTeleopCfg`` fields:
 * ``plugins`` -- list of Isaac Teleop plugin configurations (e.g. Manus).
 * ``sim_device`` -- torch device string (default ``"cuda:0"``).
 * ``retargeting_execution`` -- IsaacTeleop retargeting execution settings.
-  Defaults to ``RetargetingExecutionConfig(mode="pipelined")`` so retargeting
-  can run on the IsaacTeleop worker instead of blocking the simulation loop
-  when the installed IsaacTeleop version supports that API. Older IsaacTeleop
-  versions ignore this setting and use their historical execution behavior.
+  Defaults to ``RetargetingExecutionConfig(mode="pipelined")`` with
+  ``DeadlinePacingConfig(safety_margin_s=0.025)`` so retargeting can run on
+  the IsaacTeleop worker instead of blocking the simulation loop.
+  The 25 ms safety margin staggers IsaacTeleop's Python work behind Isaac
+  Lab's step Python, giving native work such as rendering time to overlap
+  instead of having both Python stacks contend for the GIL at the start of
+  the step.
 
 .. warning::
 

source/isaaclab_teleop/changelog.d/hougantc-pipelined-retargeting.rst

@@ -8,10 +8,10 @@ Changed
 ^^^^^^^
 
 * Changed :class:`~isaaclab_teleop.IsaacTeleopCfg` to enable IsaacTeleop
-  deadline-paced pipelined retargeting by default when supported by the
-  installed IsaacTeleop version. This returns the latest completed retargeting
-  output while the current frame is submitted, using
-  ``DeadlinePacingConfig(safety_margin_s=0.025)`` to sample close to
-  the next simulation consumption point. Set
+  deadline-paced pipelined retargeting by default. This returns the latest
+  completed retargeting output while the current frame is submitted, using
+  ``DeadlinePacingConfig(safety_margin_s=0.025)`` to sample close to the next
+  simulation consumption point and stagger IsaacTeleop's Python work behind
+  Isaac Lab's step Python. Set
   ``retargeting_execution=RetargetingExecutionConfig(mode="sync")`` to restore
   exact current-frame retargeting.

source/isaaclab_teleop/docs/README.md

@@ -120,10 +120,14 @@ rendering without blocking.
 | `retargeters_to_tune` | `Callable[[], list[BaseRetargeter]] \| None` | `None` | Retargeters to expose in the tuning UI |
 | `plugins` | `list[PluginConfig]` | `[]` | IsaacTeleop plugin configurations |
 | `sim_device` | `str` | `"cuda:0"` | Torch device for output action tensors |
-| `retargeting_execution` | `RetargetingExecutionConfig \| None` | `mode="pipelined", pacing=DeadlinePacingConfig(safety_margin_s=0.025)` when supported | IsaacTeleop retargeting execution settings |
+| `retargeting_execution` | `RetargetingExecutionConfig` | `mode="pipelined", pacing=DeadlinePacingConfig(safety_margin_s=0.025)` | IsaacTeleop retargeting execution settings |
 | `teleoperation_active_default` | `bool` | `False` | Whether teleoperation is active on session start |
 | `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the IsaacTeleop session |
 
+The 25 ms `DeadlinePacingConfig` safety margin staggers IsaacTeleop's Python work behind Isaac Lab's
+step Python, giving native work such as rendering time to overlap instead of having both Python stacks
+contend for the GIL at the start of the step.
+
 ### `XrCfg`
 
 | Field | Type | Default | Description |

source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py

@@ -10,7 +10,9 @@
 from collections.abc import Callable
 from dataclasses import MISSING, field
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
+
+from isaacteleop.teleop_session_manager import DeadlinePacingConfig, RetargetingExecutionConfig
 
 from isaaclab.utils import configclass
 
@@ -30,33 +32,6 @@
     from isaacteleop.teleop_session_manager import PluginConfig
 
 
-def _retargeting_execution_is_supported(teleop_session_manager: Any | None) -> bool:
-    """Check whether IsaacTeleop exposes retargeting execution configuration."""
-    return (
-        teleop_session_manager is not None
-        and hasattr(teleop_session_manager, "RetargetingExecutionConfig")
-        and hasattr(teleop_session_manager, "DeadlinePacingConfig")
-    )
-
-
-try:
-    import isaacteleop.teleop_session_manager as _tsm
-except ImportError:
-    _tsm = None
-
-_RETARGETING_EXECUTION_SUPPORTED = _retargeting_execution_is_supported(_tsm)
-
-
-def _default_retargeting_execution_config() -> Any | None:
-    """Build Isaac Lab's default IsaacTeleop retargeting execution config."""
-    if not _RETARGETING_EXECUTION_SUPPORTED:
-        return None
-    return _tsm.RetargetingExecutionConfig(
-        mode="pipelined",
-        pacing=_tsm.DeadlinePacingConfig(safety_margin_s=0.025),
-    )
-
-
 @configclass
 class IsaacTeleopCfg:
     """Configuration for IsaacTeleop-based teleoperation.
@@ -122,14 +97,17 @@ def build_pipeline():
     sim_device: str = "cuda:0"
     """Torch device string for placing output action tensors."""
 
-    retargeting_execution: Any | None = field(default_factory=_default_retargeting_execution_config)
+    retargeting_execution: RetargetingExecutionConfig = field(
+        default_factory=lambda: RetargetingExecutionConfig(
+            mode="pipelined",
+            pacing=DeadlinePacingConfig(safety_margin_s=0.025),
+        )
+    )
     """IsaacTeleop retargeting execution settings.
 
-    Isaac Lab opts into IsaacTeleop's pipelined execution by default when the
-    installed IsaacTeleop package exposes ``RetargetingExecutionConfig``.
-    Older IsaacTeleop releases leave this as ``None`` and use their historical
-    execution behavior. Set this to ``RetargetingExecutionConfig(mode="sync")``
-    for exact current-frame retargeting while debugging or comparing behavior.
+    Isaac Lab opts into IsaacTeleop's pipelined execution by default. Set this
+    to ``RetargetingExecutionConfig(mode="sync")`` for exact current-frame
+    retargeting while debugging or comparing behavior.
     """
 
     teleoperation_active_default: bool = False

source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py

@@ -23,7 +23,7 @@
     from isaacteleop.teleop_session_manager import TeleopSession
 
 from .control_events import _NO_OP_EVENTS, ControlEvents
-from .isaac_teleop_cfg import _RETARGETING_EXECUTION_SUPPORTED, IsaacTeleopCfg
+from .isaac_teleop_cfg import IsaacTeleopCfg
 from .teleop_message_processor import TeleopMessageProcessor
 
 
@@ -495,24 +495,14 @@ def _try_start_session(self) -> bool:
                 self._session_start_deferred_logged = True
             return False
 
-        extra_kwargs: dict[str, Any] = {}
-        if self._cfg.retargeting_execution is not None:
-            if _RETARGETING_EXECUTION_SUPPORTED:
-                extra_kwargs["retargeting_execution"] = self._cfg.retargeting_execution
-            else:
-                logger.warning(
-                    "Installed IsaacTeleop does not support retargeting_execution; "
-                    "using its default retargeting behavior."
-                )
-
         session_config = TeleopSessionConfig(
             app_name=self._cfg.app_name,
             trackers=[],
             pipeline=self._pipeline,
             teleop_control_pipeline=self._teleop_control_pipeline,
             plugins=self._cfg.plugins,
             oxr_handles=oxr_handles,
-            **extra_kwargs,
+            retargeting_execution=self._cfg.retargeting_execution,
         )
 
         # Create and enter the TeleopSession

source/isaaclab_teleop/test/test_cloudxr_lifecycle.py

@@ -150,24 +150,6 @@ def _make_lifecycle(
     )
 
 
-def _make_supported_tsm_module() -> ModuleType:
-    """Build a fake ``teleop_session_manager`` with the new execution config API."""
-
-    @dataclass
-    class DeadlinePacingConfig:
-        safety_margin_s: float = 0.025
-
-    @dataclass
-    class RetargetingExecutionConfig:
-        mode: str = "sync"
-        pacing: DeadlinePacingConfig | None = None
-
-    tsm = ModuleType("isaacteleop.teleop_session_manager")
-    tsm.DeadlinePacingConfig = DeadlinePacingConfig  # type: ignore[attr-defined]
-    tsm.RetargetingExecutionConfig = RetargetingExecutionConfig  # type: ignore[attr-defined]
-    return tsm
-
-
 # ============================================================================
 # Shipped .env profile paths
 # ============================================================================
@@ -206,48 +188,14 @@ def test_profiles_are_in_same_directory(self):
 class TestRetargetingExecutionConfig:
     """Tests for Isaac Lab's IsaacTeleop retargeting execution defaults."""
 
-    def test_cfg_defaults_to_deadline_paced_pipelined_retargeting(self):
-        """Isaac Lab defaults to deadline-paced pipelined retargeting."""
-        import isaaclab_teleop.isaac_teleop_cfg as cfg_module
-
-        with (
-            patch.object(cfg_module, "_RETARGETING_EXECUTION_SUPPORTED", True),
-            patch.object(cfg_module, "_tsm", _make_supported_tsm_module()),
-        ):
-            cfg = _make_cfg()
+    def test_session_config_receives_deadline_paced_pipelined_retargeting(self):
+        """The default retargeting execution config is passed into TeleopSession."""
+        cfg = _make_cfg()
 
         assert cfg.retargeting_execution.mode == "pipelined"
         assert cfg.retargeting_execution.pacing.safety_margin_s == 0.025
 
-    def test_cfg_defaults_to_none_with_legacy_isaacteleop(self):
-        """Older IsaacTeleop releases can still construct IsaacTeleopCfg."""
-        import isaaclab_teleop.isaac_teleop_cfg as cfg_module
-
-        with patch.object(cfg_module, "_RETARGETING_EXECUTION_SUPPORTED", False):
-            cfg = _make_cfg()
-
-        assert cfg.retargeting_execution is None
-
-    def test_retargeting_execution_support_requires_new_config_classes(self):
-        """IsaacTeleop must expose both execution config classes to enable the default."""
-        import isaaclab_teleop.isaac_teleop_cfg as cfg_module
-
-        legacy_tsm = ModuleType("isaacteleop.teleop_session_manager")
-        assert not cfg_module._retargeting_execution_is_supported(legacy_tsm)
-
-        legacy_tsm.RetargetingExecutionConfig = object
-        assert not cfg_module._retargeting_execution_is_supported(legacy_tsm)
-
-        legacy_tsm.DeadlinePacingConfig = object
-        assert cfg_module._retargeting_execution_is_supported(legacy_tsm)
-
-    def test_session_config_receives_cfg_retargeting_execution(self):
-        """The configured IsaacTeleop execution mode is passed into TeleopSession."""
-        import isaaclab_teleop.session_lifecycle as lifecycle_module
-
-        cfg = _make_cfg()
-        sentinel_execution = object()
-        cfg.retargeting_execution = sentinel_execution
+        sentinel_execution = cfg.retargeting_execution
 
         lifecycle = TeleopSessionLifecycle(cfg)
         lifecycle._pipeline = MagicMock()
@@ -258,7 +206,6 @@ def test_session_config_receives_cfg_retargeting_execution(self):
         fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"]
 
         with (
-            patch.object(lifecycle_module, "_RETARGETING_EXECUTION_SUPPORTED", True),
             patch.object(fake_tsm_module, "TeleopSessionConfig", session_config_cls),
             patch.object(fake_tsm_module, "TeleopSession", session_cls),
             patch.object(lifecycle, "_ensure_xr_ar_profile_enabled"),
@@ -268,64 +215,6 @@ def test_session_config_receives_cfg_retargeting_execution(self):
 
         assert session_config_cls.call_args.kwargs["retargeting_execution"] is sentinel_execution
 
-    def test_session_config_omits_retargeting_execution_for_legacy_isaacteleop(self, caplog):
-        """When IsaacTeleop lacks the new API, the kwarg is never passed."""
-        import isaaclab_teleop.isaac_teleop_cfg as cfg_module
-        import isaaclab_teleop.session_lifecycle as lifecycle_module
-
-        with patch.object(cfg_module, "_RETARGETING_EXECUTION_SUPPORTED", False):
-            cfg = _make_cfg()
-
-        cfg.retargeting_execution = object()
-
-        lifecycle = TeleopSessionLifecycle(cfg)
-        lifecycle._pipeline = MagicMock()
-        lifecycle._teleop_control_pipeline = None
-
-        session_config_cls = MagicMock(return_value=MagicMock())
-        session_cls = MagicMock()
-        fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"]
-
-        with (
-            patch.object(lifecycle_module, "_RETARGETING_EXECUTION_SUPPORTED", False),
-            patch.object(fake_tsm_module, "TeleopSessionConfig", session_config_cls),
-            patch.object(fake_tsm_module, "TeleopSession", session_cls),
-            patch.object(lifecycle, "_ensure_xr_ar_profile_enabled"),
-            patch.object(lifecycle, "_acquire_kit_oxr_handles", return_value=object()),
-            caplog.at_level("WARNING", logger="isaaclab_teleop.session_lifecycle"),
-        ):
-            assert lifecycle.try_start_session() is True
-
-        assert "retargeting_execution" not in session_config_cls.call_args.kwargs
-        assert session_config_cls.call_args.kwargs["app_name"] == cfg.app_name
-        assert "does not support retargeting_execution" in caplog.text
-
-    def test_session_config_propagates_typeerror_from_supported_isaacteleop(self):
-        """A TypeError from a supported IsaacTeleop must surface, not be swallowed."""
-        import isaaclab_teleop.session_lifecycle as lifecycle_module
-
-        cfg = _make_cfg()
-        cfg.retargeting_execution = object()
-
-        lifecycle = TeleopSessionLifecycle(cfg)
-        lifecycle._pipeline = MagicMock()
-        lifecycle._teleop_control_pipeline = None
-
-        def raising_session_config(**kwargs):
-            raise TypeError("invalid value for retargeting_execution")
-
-        fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"]
-
-        with (
-            patch.object(lifecycle_module, "_RETARGETING_EXECUTION_SUPPORTED", True),
-            patch.object(fake_tsm_module, "TeleopSessionConfig", raising_session_config),
-            patch.object(fake_tsm_module, "TeleopSession", MagicMock()),
-            patch.object(lifecycle, "_ensure_xr_ar_profile_enabled"),
-            patch.object(lifecycle, "_acquire_kit_oxr_handles", return_value=object()),
-        ):
-            with pytest.raises(TypeError, match="retargeting_execution"):
-                lifecycle.try_start_session()
-
 
 # ============================================================================
 # _ensure_cloudxr_runtime