diff --git a/docs/conf.py b/docs/conf.py index bd62975a2e54..da4c74c9eea0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -294,6 +294,7 @@ def _read_pinned_versions() -> dict: "ipywidgets", "mpl_toolkits", "isaacteleop", + "isaaccapture", "scipy", "hydra", "hydra.core", diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index a95671d22e14..16b0848e5cb2 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -99,11 +99,11 @@ ["IsaacContrib-Open-Drawer-Franka-IK-Abs", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-Franka-IK-Rel", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-OpenArm", "rl_games,rsl_rl", "", "", "", "environment-previews/openarm_uni_open_drawer.jpg"], - ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], + ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "rgb", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "||rgb", "|isaacsim_rtx|", "|isaacsim_rtx|rgb", "|newton_renderer|", "|newton_renderer|rgb", "isaacsim_physx||", "isaacsim_physx||rgb", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|isaacsim_rtx|rgb", "isaacsim_physx|newton_renderer|", "isaacsim_physx|newton_renderer|rgb", "newton_mjwarp||", "newton_mjwarp||rgb", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|isaacsim_rtx|rgb", "newton_mjwarp|newton_renderer|", "newton_mjwarp|newton_renderer|rgb", "newton_mjwarp|ovrtx|", "newton_mjwarp|ovrtx|rgb", "ovphysx|newton_renderer|", "ovphysx|newton_renderer|rgb", "ovphysx|ovrtx|", "ovphysx|ovrtx|rgb"]], ["IsaacContrib-PickPlace-G1-InspireFTP-Abs", "", "", "", "", "environment-previews/g1_pick_place.jpg"], - ["IsaacContrib-PickPlace-GR1T2-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/gr_1_pick_place.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], - ["IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "", "", "", "", "environment-previews/gr_1_pick_place_waist.jpg"], - ["IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/g1_pick_place_locomanipulation.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], + ["IsaacContrib-PickPlace-GR1T2-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/gr_1_pick_place.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], + ["IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/gr_1_pick_place_waist.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], + ["IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/g1_pick_place_locomanipulation.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], ["IsaacContrib-Place-Mug-Agibot-Left-Arm-RmpFlow", "", "isaacsim_physx", "", "", "environment-previews/agibot_place_mug.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Place-Toy2Box-Agibot-Right-Arm-RmpFlow", "", "isaacsim_physx", "", "", "environment-previews/agibot_place_toy.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Reach-OpenArm", "rl_games,rsl_rl,skrl", "", "", "", "environment-previews/openarm_uni_reach.jpg"], @@ -130,7 +130,7 @@ ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka-IK-Rel", "", "", "", "", "environment-previews/franka_instance_randomize_stack.jpg"], ["IsaacContrib-Stack-Cube-RedGreen-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-RedGreenBlue-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], - ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], + ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "ovphysx||"]], ["IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-SO101-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/ur10_stack_surface_gripper.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], diff --git a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst index de123825d152..84518c38e1d6 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst @@ -23,6 +23,10 @@ both OVPhysX and Newton. OVPhysX also retains native physics cloning. ``0`` selects native OVRTX cloning. Set the flag before creating the simulation. Reset retains the selected route. This flag alone never enables physics/rendering stage sharing. +An active :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` makes OVStage rendering the +unset default for every OVRTX renderer in that simulation, and rejects +``ISAAC_LAB_OVRTX_USE_OVSTAGE=0`` and ``ISAAC_LAB_SHARE_OVSTAGE=1``. + To share one OVStage with OVPhysX and OVRTX, set ``ISAAC_LAB_SHARE_OVSTAGE=1``. Sharing requires OVPhysX and selects OVStage rendering regardless of ``ISAAC_LAB_OVRTX_USE_OVSTAGE``. diff --git a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst index 4909893f624c..4c779b1a74f6 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst @@ -16,9 +16,12 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. OVRTXRenderer OVRTXRendererCfg OVRTXBackendCfg + OvLibrariesRenderLease + OvLibrariesScenePublisher .. autoclass:: OVRTXRenderer :show-inheritance: + :members: acquire_ov_libraries_render_lease .. autoclass:: OVRTXRendererCfg :show-inheritance: @@ -26,6 +29,23 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. .. autoclass:: OVRTXBackendCfg :show-inheritance: +Renderer leases +--------------- + +An external consumer, such as the OVXR teleoperation adapter, can borrow one exact OVRTX +renderer and its private OVStage with :meth:`OVRTXRenderer.acquire_ov_libraries_render_lease` +before clone preparation. While the lease is active, ordinary camera rendering, scene writes and +renderer/stage destruction are rejected; eager scene updates are left to the lease's scene +publisher, which :meth:`OvLibrariesRenderLease.capture_scene_publisher` returns. A failed leased +publication poisons publication only; the borrower can still retire the pair, after which the +renderer must be recreated. + +.. autoclass:: OvLibrariesRenderLease + :members: + +.. autoclass:: OvLibrariesScenePublisher + :members: + Simulation-owned stages ----------------------- diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 26ceeec6f5b9..ca5d0b3ba060 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -22,6 +22,9 @@ isaaclab_teleop XrCfg XrAnchorRotationMode XrAnchorSynchronizer + OvLibrariesXrCfg + OvLibrariesXrCameraCfg + OvLibrariesXrSession .. rubric:: Functions @@ -61,6 +64,18 @@ XR Camera Feedback .. autoclass:: XrCameraFeedSession :members: +Kitless OVXR Session +-------------------- + +.. autoclass:: OvLibrariesXrCfg + :members: + +.. autoclass:: OvLibrariesXrCameraCfg + :members: + +.. autoclass:: OvLibrariesXrSession + :members: + Device ------ diff --git a/docs/source/concepts/physics-backends/ovphysx.rst b/docs/source/concepts/physics-backends/ovphysx.rst index 064002501037..1d609549f8e8 100644 --- a/docs/source/concepts/physics-backends/ovphysx.rst +++ b/docs/source/concepts/physics-backends/ovphysx.rst @@ -15,6 +15,10 @@ OvPhysX OvPhysX runs PhysX without Omniverse Kit. It reads scene-level solver settings from the USD ``PhysicsScene`` prim. :class:`~isaaclab_ov.physics.OvPhysxCfg` holds GPU buffer settings that USD does not represent. +It can also author selected contact settings onto that prim: +``bounce_threshold_velocity``, ``friction_correlation_distance`` and +``solve_articulation_contact_last``. Contact settings left at ``None`` keep the +stage value. OvPhysX is selected through :class:`~isaaclab_ov.physics.OvPhysxCfg`: diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 5d1fc11eb33b..abc7b75f0bfc 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1344,6 +1344,31 @@ through this variable -- later runs no longer prompt. The variable applies to ev that launches the runtime, including the process-scoped launcher in ``teleop_replay_agent.py``. +Replay MCAP with a Kitless OVXR Viewer +-------------------------------------- + +The MCAP replay entrypoint can present recorded input through an independent +OVXR viewer. Follow the :ref:`kitless XR guide ` to install Isaac +Capture with its OVXR extra and select the compatible rendering dependencies. + +.. code-block:: bash + + uv run --extra ovxr python \ + scripts/environments/teleoperation/teleop_replay_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs --physics ovphysx \ + --device cpu --num_envs 1 --ovxr \ + --replay_file /absolute/path/to/session.mcap + +Use ``--physics newton_mjwarp`` for Newton. MCAP supplies tracker and control +input; the connected client supplies the view. A recorded STOP, task success +or failure, or the replay duration limit ends the run. The separate +``isaaclab teleop replay`` command replays HDF5 demonstrations through Kit. + +Add ``--ovxr_pip_camera robot_pov_cam --ovxr_pip_head_locked`` for a camera +panel. OVXR live and MCAP replay open a desktop mirror by default; add +``--headless`` to disable that window while keeping XR and PiP rendering. +See the kitless XR guide for installation, camera selection and display requirements. + .. _isaac-teleop-xr-anchor: Configure the XR Anchor @@ -1419,8 +1444,10 @@ vertical, and grid layouts. The following registered tasks enable PiP by default * ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` Both G1 tasks present the calibrated head camera in a ``head_locked`` panel that follows headset -position and orientation. Locomanipulation retains the camera as a recorded policy observation; -fixed-base G1 uses it only for PiP and keeps its policy observation schema unchanged. +position and orientation. Fixed-base G1 also shows its left and right wrist cameras in panels +0.43 m to either side of the head panel. Locomanipulation retains the head camera as a recorded +policy observation. Fixed-base G1 keeps a state-only policy observation schema by default; add +``presets=rgb`` to record its head and wrist images as policy observations. To run fixed-base G1 upper-body teleoperation with its default head-locked PiP: @@ -1437,9 +1464,9 @@ use the GPU. ``teleop_se3_agent.py`` and ``record_demos.py`` show every enabled feed when an Isaac Capture-enabled environment runs with ``--xr``. PiP is absent unless the task explicitly selects an existing -``CameraCfg`` through ``xr_camera_feeds``. Except for fixed-base G1, the selected -``robot_pov_cam`` is also a policy image observation, so the normal demonstration recorder stores -the same view shown to the operator. Each camera is parented to a physical robot body link, so the +``CameraCfg`` through ``xr_camera_feeds``. Except for fixed-base G1 without ``presets=rgb``, the +selected cameras are also policy image observations, so the normal demonstration recorder stores +the same views shown to the operator. Each camera is parented to a physical robot body link, so the camera view follows robot motion: .. figure:: ../_static/teleop/xr-camera-pip.jpg @@ -1485,7 +1512,7 @@ unchanged. For either G1 task, set ``env.isaac_teleop.xr_camera_feeds=[]`` to di the camera sensor and any configured camera observations. The reference feeds request render-product-local DLSS Ray Reconstruction and ``quality`` execution -mode through :class:`~isaaclab_teleop.XrCameraFeedCfg`. The private PiP adapter authors those two +mode through :class:`~isaaclab_teleop.XrCameraFeedCfg`. The Kit Scene UI presenter authors those two attributes only on the selected camera render product. Ray Reconstruction also requires the process-global responsive-denoising setting, which the teleoperation and recording scripts enable before environment construction when a selected feed requests Ray Reconstruction. @@ -1596,7 +1623,7 @@ Kit Scene UI presentation PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until an enabled feed is requested. If the Scene UI modules cannot be imported, the scripts log a warning and continue without PiP; task-owned cameras and recording observations remain unchanged. This keeps -the camera selection configuration usable when a future kitless entry point no longer provides +the camera selection configuration usable by the :ref:`kitless XR path `, whose OpenXR quad layers need no Scene UI. Render-product tuning is also best-effort: backends without a compatible render product keep the Camera-buffer path, and schema-authoring failures warn while PiP continues. Other configuration, camera-buffer, and panel-initialization errors still fail during startup. @@ -2033,6 +2060,7 @@ Optimize XR Performance The ``--headless`` CLI flag was removed in Isaac Lab 3.0. Headless is now the **default** (omit ``--visualizer``), or set ``HEADLESS=1`` in the environment. + Kitless ``--ovxr`` accepts ``--headless`` to disable its default desktop mirror. .. dropdown:: Configure retargeting execution :open: diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index 066f2f4f01d0..2a41ddd0fe8f 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -150,6 +150,10 @@ to capture demonstrations, and ``replay`` to play a dataset back. :ref:`The next bundled Isaac Sim pins ``packaging==26.0`` while those runtimes require ``<24``. Install the OV runtimes separately when you need them. +For XR teleoperation without Isaac Sim, follow the :ref:`kitless XR guide `. It +explains how to select a public Isaac Capture wheel and its optional OVXR dependencies. The Kit +setup below uses the published ``teleop`` extra. + Complete these steps first: #. Install the system libraries required by the CloudXR runtime: diff --git a/docs/source/how-to/index.rst b/docs/source/how-to/index.rst index fa71e2db08c2..805f7d7a1881 100644 --- a/docs/source/how-to/index.rst +++ b/docs/source/how-to/index.rst @@ -510,6 +510,12 @@ How-to Guides :guide-tag:`PhysX` :guide-tag:`Isaac Sim` + .. container:: guide-entry + + :doc:`Kitless XR with Isaac Capture ` + + Install Isaac Capture and OVXR packages for live teleoperation and MCAP replay. + .. container:: guide-entry :doc:`Setting up Haply teleoperation ` @@ -556,3 +562,9 @@ How-to Guides This collection is a work in progress. If a question is not answered here, open an issue on the `Isaac Lab GitHub repository `_. + +.. toctree:: + :hidden: + + kitless_xr + kitless_xr_source diff --git a/docs/source/how-to/kitless_xr.md b/docs/source/how-to/kitless_xr.md new file mode 100644 index 000000000000..bcf93859f773 --- /dev/null +++ b/docs/source/how-to/kitless_xr.md @@ -0,0 +1,15 @@ +--- +myst: + heading_anchors: 2 +--- + + + +(kitless-xr)= + +```{include} ../../../source/isaaclab_teleop/docs/kitless_xr.md +``` diff --git a/docs/source/how-to/kitless_xr_source.md b/docs/source/how-to/kitless_xr_source.md new file mode 100644 index 000000000000..7903c9ebc592 --- /dev/null +++ b/docs/source/how-to/kitless_xr_source.md @@ -0,0 +1,8 @@ + + +```{include} ../../../source/isaaclab_teleop/docs/kitless_xr_source.md +``` diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst new file mode 100644 index 000000000000..3b4e265d5cf5 --- /dev/null +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst @@ -0,0 +1,2 @@ +* Added the ``cleanup_context`` keyword to :func:`~isaaclab.app.launch_simulation` so an + asynchronous XR owner can drain before the simulation context is released. diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst new file mode 100644 index 000000000000..6df62986cc73 --- /dev/null +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst @@ -0,0 +1,2 @@ +* Fixed ``wp.to_torch`` failing on empty CPU Warp arrays, whose null data pointer NumPy rejects. + They now convert through DLPack. diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.minor b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab/changelog.d/ovxr-camera-panels.added.rst new file mode 100644 index 000000000000..e6ed703f720e --- /dev/null +++ b/source/isaaclab/changelog.d/ovxr-camera-panels.added.rst @@ -0,0 +1,3 @@ +* Added :meth:`~isaaclab.sensors.camera.Camera.refresh_world_poses` to pass the latest camera poses + to the data buffers and the renderer without rendering, for renderers presented outside + :meth:`~isaaclab.sensors.camera.Camera.update`. diff --git a/source/isaaclab/changelog.d/ovxr-camera-panels.minor b/source/isaaclab/changelog.d/ovxr-camera-panels.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst new file mode 100644 index 000000000000..d8d9c86fb406 --- /dev/null +++ b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst @@ -0,0 +1,2 @@ +* Fixed Pink hand targets being applied to the wrong joints when the articulation's joint order differs + from ``hand_joint_names`` (for example, Newton's G1 order). diff --git a/source/isaaclab/isaaclab/app/sim_launcher.py b/source/isaaclab/isaaclab/app/sim_launcher.py index d832f83b1d0c..bf8d08fd98aa 100644 --- a/source/isaaclab/isaaclab/app/sim_launcher.py +++ b/source/isaaclab/isaaclab/app/sim_launcher.py @@ -412,12 +412,19 @@ def validate_launch( # noqa: C901 def launch_simulation( cfg, launcher_args: argparse.Namespace | dict | None = None, + *, + cleanup_context: bool = True, ) -> Generator[PhysicsCfg | None, None, None]: """Context manager that launches the appropriate simulation runtime for *cfg*. Resolves configured consumers before starting their required runtimes. Each runtime starts once and closes on exit. Cameras are auto-enabled for Kit-renderer sensors. + On exit, stops and releases a simulation context created inside the scope before closing + its runtimes. An existing context owned by the caller is preserved. Applications + with asynchronous resource owners can set ``cleanup_context=False`` and take + responsibility for context cleanup, including failed construction and drain. + Resolve viewer and physics choices on the same config used to construct the simulation:: sim_cfg = SimulationCfg() @@ -437,6 +444,10 @@ def launch_simulation( a tool that reaches a Kit-only extension API. This is additive -- it can only turn a kitless launch into a Kit one, never the reverse, so a config that already needs Kit still launches it when the key is absent or ``False``. + cleanup_context: Automatically release a context created inside this scope. + Disable only when the application owns its cleanup and must retain it + if asynchronous consumers cannot prove they have drained. This controls + context release only; configured runtime launchers still close on exit. """ launcher_args = {} if launcher_args is None else launcher_args launcher_args = vars(launcher_args) if isinstance(launcher_args, argparse.Namespace) else launcher_args @@ -500,7 +511,7 @@ def launch_simulation( finally: # Release runtime consumers before a launcher can terminate the process. context = SimulationContext.instance() - if context is not None and context is not previous_context: + if cleanup_context and context is not None and context is not previous_context: SimulationContext.clear_instance() except KeyboardInterrupt: exit_code = 130 diff --git a/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py b/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py index b5910b64d0f6..f17a11253ee8 100644 --- a/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py +++ b/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py @@ -29,7 +29,10 @@ class PinkInverseKinematicsActionCfg(ActionTermCfg): """List of joint names or regular expression patterns that specify the joints controlled by pink IK.""" hand_joint_names: list[str] = MISSING - """List of joint names or regular expression patterns that specify the joints controlled by hand retargeting.""" + """List of joint names or regular expression patterns that specify the joints controlled by hand retargeting. + + Hand targets in the action are applied in this order. + """ controller: PinkIKControllerCfg = MISSING """Configuration for the Pink IK controller that will be used to solve the inverse kinematics.""" diff --git a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py index ebd129bb6f62..56b73acfbc38 100644 --- a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py +++ b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py @@ -73,7 +73,12 @@ def _initialize_joint_info(self) -> None: self.cfg.controller.all_joint_names = self._asset.data.joint_names # Resolve hand joints - self._hand_joint_ids, self._hand_joint_names = self._asset.find_joints(self.cfg.hand_joint_names) + # Hand targets occupy the action tensor in configuration order. Newton's + # articulation order groups these joints differently, so preserve the + # declared order when resolving ids instead of silently permuting targets. + self._hand_joint_ids, self._hand_joint_names = self._asset.find_joints( + self.cfg.hand_joint_names, preserve_order=True + ) # Combine all joint information self._controlled_joint_ids = self._isaaclab_controlled_joint_ids + self._hand_joint_ids diff --git a/source/isaaclab/isaaclab/sensors/camera/camera.py b/source/isaaclab/isaaclab/sensors/camera/camera.py index d052dd59aa73..70eb9e0b3cc6 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera.py @@ -534,6 +534,21 @@ def get_world_poses( ) return positions.torch, orientations + def refresh_world_poses(self, env_ids: Sequence[int] | None = None) -> None: + """Re-read camera poses into the data buffers and pass them to the renderer, without rendering. + + Use this when an external presenter renders the camera outside :meth:`update`, so that the + rendered view follows the latest simulation or reset kinematics. Frame counters and image + outputs are left unchanged, regardless of :attr:`CameraCfg.update_latest_camera_pose`. + + Args: + env_ids: Sensor indices to refresh. None selects all sensors. + + Raises: + RuntimeError: If the camera is not initialized. + """ + self._update_poses(env_ids=env_ids) + def set_world_poses( self, positions: torch.Tensor | None = None, diff --git a/source/isaaclab/isaaclab/utils/warp/__init__.py b/source/isaaclab/isaaclab/utils/warp/__init__.py index 2b2eef4ecef1..334be0153c32 100644 --- a/source/isaaclab/isaaclab/utils/warp/__init__.py +++ b/source/isaaclab/isaaclab/utils/warp/__init__.py @@ -22,6 +22,8 @@ from .proxy_array import ProxyArray # noqa: E402 _WP_TO_TORCH_ORIGINAL = wp.to_torch +# Bound at import so a later ``wp.array`` patch cannot break the empty-array check. +_WP_ARRAY = wp.array _WP_TO_TORCH_WARNED = False @@ -37,8 +39,9 @@ def _wp_to_torch_with_proxyarray(a, requires_grad=None): The shim routes :class:`ProxyArray` arguments to their cached ``.torch`` view (a zero-copy :class:`torch.Tensor` of the same underlying memory) and - emits a one-shot :class:`DeprecationWarning`. For any other input type, - the original :func:`warp.to_torch` handles the call as before. + emits a one-shot :class:`DeprecationWarning`. Empty CPU arrays use DLPack + because NumPy rejects their null-pointer array interface. Other native + arrays keep the original :func:`warp.to_torch` conversion. """ global _WP_TO_TORCH_WARNED if isinstance(a, ProxyArray): @@ -51,6 +54,19 @@ def _wp_to_torch_with_proxyarray(a, requires_grad=None): stacklevel=2, ) return a.torch + if isinstance(a, _WP_ARRAY) and a.size == 0 and a.device is not None and a.device.is_cpu: + # Preserve Warp's structured-array rejection before the DLPack fallback. + if isinstance(a.dtype, wp._src.codegen.Struct): # noqa: SLF001 + return _WP_TO_TORCH_ORIGINAL(a, requires_grad=requires_grad) + import torch + + if requires_grad is None: + requires_grad = a.requires_grad + tensor = torch.from_dlpack(wp.to_dlpack(a)) + tensor.requires_grad = requires_grad + if requires_grad and a.requires_grad: + tensor.grad = torch.from_dlpack(wp.to_dlpack(a.grad)) + return tensor return _WP_TO_TORCH_ORIGINAL(a, requires_grad=requires_grad) diff --git a/source/isaaclab/test/app/test_launch_simulation_require_kit.py b/source/isaaclab/test/app/test_launch_simulation_require_kit.py index 32d754ba16ec..e8f44054ce19 100644 --- a/source/isaaclab/test/app/test_launch_simulation_require_kit.py +++ b/source/isaaclab/test/app/test_launch_simulation_require_kit.py @@ -256,3 +256,40 @@ def test_legacy_recording_reuses_the_display_producer(kit_branch_taken, visualiz assert cfg.sim.visualizer_cfgs[0].headless == (not visualizers) assert args.get("enable_cameras", False) == (source == "kit") assert kit_branch_taken == [True] * (source == "kit") + + +@pytest.mark.parametrize( + "application_owns_cleanup,drain_failed", [(False, False), (False, True), (True, False), (True, True)] +) +def test_context_cleanup_respects_application_drain_ownership(monkeypatch, application_owns_cleanup, drain_failed): + """An outer launcher must not release a context retained by a failed native drain.""" + from isaaclab.sim import SimulationContext + + state = {"context": None} + released = [] + + def clear(): + released.append(state["context"]) + state["context"] = None + + monkeypatch.setattr(SimulationContext, "instance", lambda: state["context"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear) + context = object() + + def run(): + with launch_simulation(PhysicsCfg(), cleanup_context=not application_owns_cleanup): + state["context"] = context + if drain_failed: + raise RuntimeError("consumer drain unproved") + if application_owns_cleanup: + # The already-owning environment/worker performs normal release. + SimulationContext.clear_instance() + + if drain_failed: + with pytest.raises(RuntimeError, match="consumer drain unproved"): + run() + else: + run() + retained = application_owns_cleanup and drain_failed + assert released == ([] if retained else [context]) + assert state["context"] is (context if retained else None) diff --git a/source/isaaclab/test/cli/test_teleop_entrypoints.py b/source/isaaclab/test/cli/test_teleop_entrypoints.py index 210e42c92dab..e80d41b9bc1c 100644 --- a/source/isaaclab/test/cli/test_teleop_entrypoints.py +++ b/source/isaaclab/test/cli/test_teleop_entrypoints.py @@ -57,6 +57,36 @@ def test_teleop_workflow_help_exposes_task_preset_selectors(source_checkout_root assert "physics=NAME" in result.stdout assert "renderer=NAME" in result.stdout assert "presets=NAME[,NAME,...]" in result.stdout + if command == "run": + assert re.search(r"^\s+--ovxr(?:\s|$)", result.stdout, re.MULTILINE) + assert "--ovxr_pip_camera" in result.stdout + + +def test_ovxr_omitted_launch_options_match_documented_defaults(source_checkout_root, monkeypatch): + """The shortened user command retains the profile, launch policy and visualizers.""" + from isaaclab.app import validate_launch + + monkeypatch.delenv("LIVESTREAM", raising=False) + script = source_checkout_root / "scripts/environments/teleoperation/teleop_se3_agent.py" + + def parse(arguments): + monkeypatch.setattr(sys, "argv", [str(script), *arguments]) + return importlib.import_module("isaaclab_teleop.entrypoints.teleop_se3_agent")._parse_args() + + implicit = parse(["--ovxr"]) + explicit = parse(["--ovxr", "--viz", "none", "--auto_launch_cloudxr", "--cloudxr_env", "cloudxrjs"]) + assert implicit.auto_launch_cloudxr is explicit.auto_launch_cloudxr is True + assert implicit.cloudxr_env == explicit.cloudxr_env + assert implicit.cloudxr_env.endswith(".env") + left, right = vars(implicit).copy(), vars(explicit).copy() + from isaaclab.sim import SimulationCfg + + left_cfg, right_cfg = SimulationCfg(), SimulationCfg() + validate_launch(left_cfg, left) + validate_launch(right_cfg, right) + assert left_cfg.visualizer_cfgs == right_cfg.visualizer_cfgs == [] + assert parse(["--ovxr", "--no-auto_launch_cloudxr"]).auto_launch_cloudxr is False + assert parse(["--ovxr", "--cloudxr_env", "none"]).cloudxr_env is None @pytest.mark.parametrize(("command", "script_parts"), TELEOP_WORKFLOWS.items()) diff --git a/source/isaaclab/test/envs/test_pink_hand_joint_order.py b/source/isaaclab/test/envs/test_pink_hand_joint_order.py new file mode 100644 index 000000000000..dfebf344dcdc --- /dev/null +++ b/source/isaaclab/test/envs/test_pink_hand_joint_order.py @@ -0,0 +1,46 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit coverage for Pink hand-target ordering across articulation backends.""" + +from types import SimpleNamespace + +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction +from isaaclab.utils.string import resolve_matching_names + + +def test_pink_hand_commands_reach_the_configured_joints(): + """A non-self-inverse hand permutation must preserve physical joint targets.""" + + class _Asset: + data = SimpleNamespace(joint_names=["arm", "hand_a", "hand_b", "hand_c"]) + + def __init__(self): + self.joint_targets = torch.zeros((1, 4)) + + def find_joints(self, expressions, preserve_order=False): + return resolve_matching_names(expressions, self.data.joint_names, preserve_order=preserve_order) + + def set_joint_position_target_index(self, target, joint_ids): + self.joint_targets[:, joint_ids] = target + + action = PinkInverseKinematicsAction.__new__(PinkInverseKinematicsAction) + action._asset = _Asset() + action.cfg = SimpleNamespace( + pink_controlled_joint_names=["arm"], + hand_joint_names=["hand_c", "hand_a", "hand_b"], + controller=SimpleNamespace(), + enable_gravity_compensation=False, + ) + + action._initialize_joint_info() + + action._compute_ik_solutions = lambda: torch.tensor([[9.0]]) + action._target_hand_joint_positions = torch.tensor([[30.0, 10.0, 20.0]]) + action.apply_actions() + + torch.testing.assert_close(action._asset.joint_targets, torch.tensor([[9.0, 10.0, 20.0, 30.0]])) diff --git a/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py new file mode 100644 index 000000000000..56258850567a --- /dev/null +++ b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py @@ -0,0 +1,96 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Camera pose refresh for renderers presented outside :meth:`Camera.update`.""" + +from types import SimpleNamespace + +import pytest +import torch +import warp as wp + +from isaaclab.sensors.camera import Camera +from isaaclab.sensors.camera.camera_data import CameraData +from isaaclab.utils.warp import ProxyArray + +pytestmark = pytest.mark.unit + +_NUM_CAMERAS = 2 + + +class _RecordingRenderer: + """Record the camera poses each renderer notification carries.""" + + def __init__(self): + self.poses = [] + + def update_camera(self, render_data, positions, orientations, intrinsics): + self.poses.append((render_data, positions.torch.clone(), orientations.torch.clone())) + + def cleanup(self, render_data): + pass + + +def _assert_same_rotations(actual: torch.Tensor, expected: torch.Tensor) -> None: + """Compare unit quaternions up to sign, since ``q`` and ``-q`` describe one rotation.""" + torch.testing.assert_close((actual * expected).sum(dim=-1).abs(), torch.ones(actual.shape[0])) + + +class _CpuCamera(Camera): + """A CPU camera whose pose source is an in-memory world pose.""" + + def __init__(self): + self.cfg = SimpleNamespace(update_period=0.0, update_latest_camera_pose=False) + self._device = "cpu" + # No capture is pending, so reading ``data`` returns the buffers without rendering. + self._data_dirty = False + self._sensor_prims = [object()] * _NUM_CAMERAS + self._renderer = _RecordingRenderer() + self._render_data = object() + self._data = CameraData() + self._data.create_buffers(_NUM_CAMERAS, "cpu") + self._frame = ProxyArray(wp.zeros(_NUM_CAMERAS, dtype=wp.int64, device="cpu")) + self._ALL_INDICES = wp.array(range(_NUM_CAMERAS), dtype=wp.int32, device="cpu") + self._ALL_ENV_MASK = wp.ones(_NUM_CAMERAS, dtype=wp.bool, device="cpu") + self.positions = torch.zeros(_NUM_CAMERAS, 3) + self.orientations = torch.tensor([[0.0, 0.0, 0.0, 1.0]] * _NUM_CAMERAS) + self._view = SimpleNamespace(count=_NUM_CAMERAS, get_world_poses=self._world_poses, close=lambda: None) + + def __del__(self): + pass + + def _world_poses(self, env_ids): + indices = slice(None) if env_ids is None else env_ids.numpy() + positions = wp.from_torch(self.positions[indices].contiguous(), dtype=wp.vec3f) + orientations = wp.from_torch(self.orientations[indices].contiguous(), dtype=wp.quatf) + return ProxyArray(positions), ProxyArray(orientations) + + +def test_refresh_world_poses_updates_data_and_renderer_without_capturing(): + """A refresh exposes moved poses to the data and the renderer, but captures no frame.""" + camera = _CpuCamera() + camera.positions = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + camera.orientations = torch.tensor([[0.0, 0.0, 1.0, 0.0], [0.0, 1.0, 0.0, 0.0]]) + + camera.refresh_world_poses() + + # The data and renderer carry the world-convention orientation that a direct pose query reports. + _, world_orientations = camera.get_world_poses(convention="world") + torch.testing.assert_close(camera.data.pos_w.torch, camera.positions) + _assert_same_rotations(camera.data.quat_w_world.torch, world_orientations) + [(render_data, positions, orientations)] = camera._renderer.poses + assert render_data is camera._render_data + torch.testing.assert_close(positions, camera.positions) + _assert_same_rotations(orientations, world_orientations) + assert camera.frame.torch.tolist() == [0] * _NUM_CAMERAS + + # A partial refresh leaves the other camera's cached pose unchanged. + moved = torch.tensor([[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]) + camera.positions = moved + camera.refresh_world_poses(env_ids=[1]) + + torch.testing.assert_close(camera.data.pos_w.torch, torch.tensor([[1.0, 2.0, 3.0], [10.0, 11.0, 12.0]])) + assert len(camera._renderer.poses) == 2 + assert camera.frame.torch.tolist() == [0] * _NUM_CAMERAS diff --git a/source/isaaclab/test/utils/warp/test_proxy_array.py b/source/isaaclab/test/utils/warp/test_proxy_array.py index 247644674b71..d9a0870c6d62 100644 --- a/source/isaaclab/test/utils/warp/test_proxy_array.py +++ b/source/isaaclab/test/utils/warp/test_proxy_array.py @@ -469,3 +469,36 @@ def test_requires_grad_forwarded_to_raw_wp_array(self): assert isinstance(t, torch.Tensor) assert t.shape == (2,) assert t.requires_grad is False + + def test_empty_cpu_arrays_convert_through_dlpack(self): + """Empty CPU views preserve dtype, shape and gradient selection.""" + import isaaclab.utils.warp # noqa: F401 # ensure shim is installed + + empty_ids = wp.from_torch(torch.arange(4, dtype=torch.int32)[:0]) + ids = wp.to_torch(empty_ids) + assert ids.shape == (0,) and ids.dtype == torch.int32 + assert ids.data_ptr() == 0 and ids.stride() == (1,) + + for inherited, override, expected, has_grad in ( + (False, None, False, False), + (True, None, True, True), + (True, False, False, False), + (False, True, True, False), + ): + empty = wp.zeros((0, 3), dtype=wp.float32, device="cpu", requires_grad=inherited) + view = wp.to_torch(empty, requires_grad=override) + assert view.shape == (0, 3) and view.dtype == torch.float32 + assert view.requires_grad is expected + assert (view.grad is not None) is has_grad + if has_grad: + assert view.grad.shape == (0, 3) and view.grad.dtype == torch.float32 + + def test_conversion_survives_patched_array_constructor(self, monkeypatch): + """Tests may replace ``wp.array`` with a factory; conversion must not depend on it.""" + import isaaclab.utils.warp # noqa: F401 # ensure shim is installed + + empty = wp.zeros(0, dtype=wp.float32, device="cpu") + full = wp.array([1.0, 2.0], dtype=wp.float32, device="cpu") + monkeypatch.setattr(wp, "array", lambda value, dtype=None, device=None: value) + assert wp.to_torch(empty).shape == (0,) + assert wp.to_torch(full).tolist() == [1.0, 2.0] diff --git a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst new file mode 100644 index 000000000000..0463675e9735 --- /dev/null +++ b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst @@ -0,0 +1,4 @@ +* Kept the GR1T2 and G1 locomanipulation Mimic tasks and the G1 locomanipulation SDG task + (``Isaac-G1-SteeringWheel-Locomanipulation``) on Isaac Sim PhysX without the ``ovphysx`` and + ``newton_mjwarp`` selectors of their base teleoperation tasks, because their environments do not run + the kitless backend preparation. diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py index 17bfc63f68be..5721a05d811c 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import ( @@ -16,6 +19,9 @@ class LocomanipulationG1MimicEnvCfg(LocomanipulationG1EnvCfg, MimicEnvCfg): """Configuration for G1 Locomanipulation Mimic environment.""" class_type: type | str = "{DIR}.locomanipulation_g1_mimic_env:LocomanipulationG1MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Call parent post-init diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py index d1daf6d240bf..60d47b766767 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg @@ -14,6 +17,9 @@ class PickPlaceGR1T2MimicEnvCfg(PickPlaceGR1T2EnvCfg, MimicEnvCfg): """Configuration for GR1T2 Pick Place Mimic environment.""" class_type: type | str = "{DIR}.pickplace_gr1t2_mimic_env:PickPlaceGR1T2MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Calling post init of parents diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py index ff326c66e41e..c9636a3e8856 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import ( @@ -16,6 +19,9 @@ class PickPlaceGR1T2WaistEnabledMimicEnvCfg(PickPlaceGR1T2WaistEnabledEnvCfg, Mi """Configuration for GR1T2 Pick Place Waist Enabled Mimic environment.""" class_type: type | str = "{DIR}.pickplace_gr1t2_mimic_env:PickPlaceGR1T2MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Calling post init of parents diff --git a/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py index fbe822afda29..55e78fe2ccc4 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg import isaaclab.sim as sim_utils @@ -125,6 +126,11 @@ class PolicyCfg(ObservationsCfg.PolicyCfg): class G1LocomanipulationSDGEnvCfg(LocomanipulationG1EnvCfg, LocomanipulationSDGEnvCfg): """Configuration for the G1 29DoF environment.""" + class_type: type | str = "{DIR}.g1_locomanipulation_sdg_env:G1LocomanipulationSDGEnv" + sim: sim_utils.SimulationCfg = sim_utils.SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the SDG environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" + # Scene settings scene: G1LocomanipulationSDGSceneCfg = G1LocomanipulationSDGSceneCfg(num_envs=1, env_spacing=2.5) recorders: LocomanipulationSDGRecorderManagerCfg = LocomanipulationSDGRecorderManagerCfg() diff --git a/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py new file mode 100644 index 000000000000..a617eca7e9c3 --- /dev/null +++ b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py @@ -0,0 +1,33 @@ +# Copyright (c) 2024-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Humanoid Mimic and SDG tasks keep Isaac Sim PhysX; their environments skip kitless preparation.""" + +import pytest + + +@pytest.mark.parametrize( + "task", + [ + "Isaac-PickPlace-GR1T2-Abs-Mimic-v0", + "Isaac-PickPlace-GR1T2-WaistEnabled-Abs-Mimic-v0", + "Isaac-Locomanipulation-G1-Abs-Mimic-v0", + "Isaac-G1-SteeringWheel-Locomanipulation", + ], +) +def test_humanoid_mimic_tasks_offer_only_isaac_sim_physx(task): + from isaaclab_physx.physics import PhysxCfg + + import isaaclab_mimic.envs # noqa: F401 -- registers the Mimic task IDs + import isaaclab_mimic.locomanipulation_sdg.envs # noqa: F401 -- registers the SDG task ID + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task, device="cpu", num_envs=1) + assert type(cfg.sim.physics) is PhysxCfg + for selector in ("physics=ovphysx", "physics=newton_mjwarp"): + with pytest.raises(ValueError, match="did not match a selectable preset"): + parse_env_cfg(task, device="cpu", num_envs=1, overrides=[selector]) diff --git a/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst b/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst new file mode 100644 index 000000000000..10743010474e --- /dev/null +++ b/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst @@ -0,0 +1 @@ +* Recorded Newton CUDA graphs on a nonblocking stream to support concurrent renderer and input uploads. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py index 920f682da35d..567d111807c7 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py @@ -640,11 +640,12 @@ def capture_graph(device: wp.DeviceLike, fn: Callable[[], None], *, relaxed: boo Returns: The captured graph, replayed with :func:`warp.capture_launch`. """ - stream = wp.get_stream(device) + # A blocking capture stream forbids concurrent legacy-stream transfers, + # including renderer or input uploads. ScopedStream retains the incoming + # dependency; graph replay still uses the caller's stream. + stream = wp.stream_from_torch(torch.cuda.Stream(device=wp.device_to_torch(device))) mode = wp.CaptureMode.THREAD_LOCAL if relaxed: - # RTX uses the legacy CUDA stream. A nonblocking stream avoids implicit synchronization. - stream = wp.stream_from_torch(torch.cuda.Stream(device=wp.device_to_torch(device))) mode = wp.CaptureMode.RELAXED with _paused_gc(), wp.ScopedStream(stream): with wp.ScopedCapture(stream=stream, capture_mode=mode) as capture: diff --git a/source/isaaclab_newton/test/physics/test_newton_manager.py b/source/isaaclab_newton/test/physics/test_newton_manager.py index a37544233624..3349813152ac 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager.py @@ -10,6 +10,7 @@ import gc import logging +import threading from functools import partial as P from types import SimpleNamespace @@ -17,6 +18,7 @@ import newton import numpy as np import pytest +import torch import warp as wp from isaaclab_newton.physics import ( FeatherstoneSolverCfg, @@ -485,6 +487,52 @@ def test_step_runs_eagerly_once_then_replays_a_graph_only_when_the_solver_suppor assert [event for event in events if event[0] == "check"] == [("check", supported)] * 2 +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) +def test_graph_capture_allows_concurrent_legacy_stream_copy(device): + """A render/UI copy must not invalidate physics capture on the same GPU.""" + source = wp.full(4, 7.0, device=device) + result = wp.zeros(4, device=device) + image = torch.full((4,), 23.0, device=device) + copied = torch.zeros_like(image) + # Setup is complete before capture. No allocations or implicit dependency + # on captured physics work are needed by the independent image consumer. + wp.synchronize_device(device) + torch.cuda.synchronize(device) + entered = threading.Event() + finished = threading.Event() + errors = [] + + def copy_image(): + try: + assert entered.wait(5), "capture did not start" + with torch.cuda.device(device), torch.cuda.stream(torch.cuda.default_stream(device)): + copied.copy_(image) + torch.cuda.default_stream(device).synchronize() + except BaseException as error: + errors.append(error) + finally: + finished.set() + + def record_physics(): + wp.copy(result, source) + entered.set() + assert finished.wait(5), "concurrent image copy did not complete" + + worker = threading.Thread(target=copy_image) + worker.start() + try: + graph = newton_backend.capture_graph(device, record_physics) + finally: + entered.set() + worker.join(5) + assert not errors, errors + assert not worker.is_alive() + np.testing.assert_array_equal(result.numpy(), [0.0] * 4) # Recording did not advance physics. + wp.capture_launch(graph) + np.testing.assert_array_equal(result.numpy(), [7.0] * 4) + np.testing.assert_array_equal(copied.cpu().numpy(), [23.0] * 4) + + # ----- Reset masks ------------------------------------------------------------------------- diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst new file mode 100644 index 000000000000..9584406eff2e --- /dev/null +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst @@ -0,0 +1,4 @@ +* Added leased camera render products and independent camera-panel scheduling to external consumers. +* Added CPU simulation-device support to OVRTX cameras. Render products leave GPU selection automatic, + and camera poses, intrinsics and image reads use host memory without a CUDA stream, so kitless OVXR + camera panels can run with CPU physics. diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst new file mode 100644 index 000000000000..5343e2f25c71 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst @@ -0,0 +1,3 @@ +* Changed leased render-stage exports to preserve an authored ``showAllPartitionsByDefault`` value + instead of forcing it on, so the OVXR scene partition can hide excluded meshes. Exports without an + active render lease still force every partition visible. diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.minor b/source/isaaclab_ov/changelog.d/leased-camera-products.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst new file mode 100644 index 000000000000..e2a512611bec --- /dev/null +++ b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst @@ -0,0 +1,5 @@ +* Added ``bounce_threshold_velocity``, ``friction_correlation_distance`` and + ``solve_articulation_contact_last`` to :class:`~isaaclab_ov.physics.OvPhysxCfg`. Unset values keep + the stage's scene settings. +* Added CPU simulation-device support to OVRTX scene transform and geometry publication, so kitless + OVXR can present CPU physics. diff --git a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.minor b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst new file mode 100644 index 000000000000..7478cad04aa5 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst @@ -0,0 +1,6 @@ +* Added :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` and + :meth:`~isaaclab_ov.renderers.OVRTXRenderer.acquire_ov_libraries_render_lease` for application-owned + OVRTX/OVStage borrowing by external consumers, with private-stage selection, capability-checked + render-thread ownership transfer, deferred Stage materialization, scene publication through + :class:`~isaaclab_ov.renderers.OvLibrariesScenePublisher`, and ordered cleanup. +* Published dirty visual materials with leased scene updates, including material-only changes. diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.minor b/source/isaaclab_ov/changelog.d/renderer-lease.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py index f350bd6b2626..387d1531621b 100644 --- a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py +++ b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py @@ -177,6 +177,7 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderers = sim.get_backends(OVRTXRendererCfg) if not renderers: return + leased = [renderer for renderer in renderers if renderer._render_lease.active is not None] initialized = next((renderer for renderer in renderers if renderer.backend is not None), None) if initialized is not None: use_ovstage = initialized._use_ovstage @@ -185,13 +186,16 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], # Default to native OVRTX cloning to preserve camera-task throughput on both physics backends. # Keep sharing opt-in until OVPhysX cold binding and shared-stage update performance are fixed. with_ovphysx = sim.backend_manager.clone_context_type is OvPhysxReplicateContext - value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0").strip() + # An explicit lease selects its private OVStage without changing process globals. + value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1" if leased else "0").strip() if value not in {"0", "1"}: raise ValueError(f"Invalid ISAAC_LAB_OVRTX_USE_OVSTAGE: {value!r}. Expected 0 or 1.") share_ovstage = os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1" use_ovstage = share_ovstage or value == "1" if share_ovstage and not with_ovphysx: raise ValueError("Shared OVStage cloning requires OVPhysX.") + if leased and (not use_ovstage or share_ovstage): + raise RuntimeError("The render lease requires a private rendering OVStage") if share_ovstage and sim.backend_manager.backend is not None and sim.backend_manager.backend.owns_scene: raise RuntimeError("Configure OVRTX cameras before the first OVPhysX reset to share its OVStage.") if use_ovstage: @@ -230,6 +234,8 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderer.backend = sim.get_or_create_backend(backend_cfg) renderer.scene = scene if use_ovstage else renderer.backend renderer._use_ovstage = use_ovstage + if renderer in leased: + renderer._render_lease.reserve_pair() if renderer.cfg.async_rendering and use_ovstage: logger.warning("Asynchronous OVRTX rendering is unsupported with ovstage; rendering synchronously.") diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py index 73cd1f00ec45..67ce84512939 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py @@ -811,8 +811,8 @@ def _configure_physx_scene_prim(self, scene_prim, cfg, device: str) -> None: """Apply the OVPhysX scene configuration to the simulation's physics scene prim. The schema, dynamics device and broadphase (see :func:`_set_scene_dynamics_device`), - scene-query-support, and solver-determinism/accuracy attributes are applied regardless of - device. The GPU buffer-capacity attributes are applied only to GPU scenes. + scene-query-support, solver-determinism/accuracy and selected contact attributes are applied + regardless of device. The GPU buffer-capacity attributes are applied only to GPU scenes. Args: scene_prim: The /World/PhysicsScene prim to configure. @@ -838,6 +838,13 @@ def _configure_physx_scene_prim(self, scene_prim, cfg, device: str) -> None: scene_prim.CreateAttribute("physxScene:enableExternalForcesEveryIteration", Sdf.ValueTypeNames.Bool).Set( cfg.enable_external_forces_every_iteration ) + for attr, value_type, value in ( + ("bounceThreshold", Sdf.ValueTypeNames.Float, cfg.bounce_threshold_velocity), + ("frictionCorrelationDistance", Sdf.ValueTypeNames.Float, cfg.friction_correlation_distance), + ("solveArticulationContactLast", Sdf.ValueTypeNames.Bool, cfg.solve_articulation_contact_last), + ): + if value is not None: + scene_prim.CreateAttribute(f"physxScene:{attr}", value_type).Set(value) if device == "gpu" and cfg is not None: for attr, val in [ diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py index c9c4a0d3274d..f5306f58e7ab 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py @@ -44,8 +44,10 @@ class OvPhysxCfg(PhysicsCfg): """Configuration for the ovphysx physics manager. PhysX scene-level parameters (solver iterations, GPU buffer sizes, etc.) are - read from the USD PhysicsScene prim. Only ovphysx-specific settings that are - not captured in USD live here. + read from the USD PhysicsScene prim. Before loading, the manager authors the + settings held here onto that prim, including the optional contact parameters + :attr:`bounce_threshold_velocity`, :attr:`friction_correlation_distance` and + :attr:`solve_articulation_contact_last`, which keep the stage value when ``None``. """ class_type = "{DIR}.ovphysx_manager:OvPhysxManager" @@ -89,6 +91,29 @@ class OvPhysxCfg(PhysicsCfg): setting can affect simulated velocities and contact forces. """ + bounce_threshold_velocity: float | None = None + """Relative contact velocity below which contacts do not bounce [m/s]. + + ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is 0, which sets no speed + threshold, so contact restitution alone decides whether a contact bounces. + Authored as ``physxScene:bounceThreshold``. + """ + + friction_correlation_distance: float | None = None + """Distance below which contact points merge into one friction anchor [m]. + + ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is ``0.025 * tolerancesLength``, + which is 0.025 m on a meter-based stage. Authored as ``physxScene:frictionCorrelationDistance``. + """ + + solve_articulation_contact_last: bool | None = None + """Whether the solver resolves articulation contacts after other constraints. + + Enable it so object contacts can stall position-driven grippers before they tunnel through a + grasp. ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is off. Authored as + ``physxScene:solveArticulationContactLast``. + """ + gpu_max_rigid_contact_count: int = 2**23 """Size of the GPU rigid-body contact buffer.""" diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi index d430a333224a..71a097ec2cbd 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi +++ b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi @@ -7,9 +7,12 @@ __all__ = [ "OVRTXRenderer", "OVRTXRendererCfg", "OVRTXBackendCfg", + "OvLibrariesRenderLease", + "OvLibrariesScenePublisher", "Renderer", ] from .ovrtx_renderer import OVRTXRenderer from .ovrtx_renderer import OVRTXRenderer as Renderer from .ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg +from .render_lease import OvLibrariesRenderLease, OvLibrariesScenePublisher diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index d8c82f22c128..adbe46d8234f 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -14,6 +14,8 @@ its own pending ``step_async`` render operation and publishes it at the next ``read_output``. The ovrtx binding write operations retain their input buffers until ovrtx has consumed them. +- **render_lease.py**: External borrowing, ownership transfer and scene publication ordering. + - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. - **ovrtx_usd.py**: USD helpers for OVRTX: render var config, camera injection, etc. @@ -108,6 +110,8 @@ xform_tensor_from_warp, ) +from .render_lease import OvLibrariesRenderLease, RenderLeaseController + if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, FrameOutput, Operation, PendingFetch, RenderVarOutput @@ -214,6 +218,7 @@ class OVRTXBackend: def __init__(self, cfg: OVRTXBackendCfg): self.cfg = cfg self.attached = False + self._render_lease_owner: OVRTXRenderer | None = None self.render_products: set[str] = set() # Resolve the wheel's native dependency here, including callers without a viewer. dependency = Path(ovstage.__file__).parent / "bin/plugins/libosdCPU.so.3.6.0" @@ -243,13 +248,25 @@ def __init__(self, cfg: OVRTXBackendCfg): self.renderer = Renderer(native_cfg) def detach(self) -> None: - """Release the borrowed stage before it is rebuilt or destroyed.""" + """Release the borrowed stage before it is rebuilt or destroyed. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot detach a backend with an active render lease") if self.attached: self.renderer.detach_ovstage() self.attached = False def close(self) -> None: - """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward.""" + """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy a backend with an active render lease") if self.renderer is not None: # The SDK destroys bindings and detaches only if attached, including partial initialization. self.renderer.destroy() @@ -335,6 +352,8 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): super().__init__(cfg, sim=sim) + self._closed = False + self._render_lease = RenderLeaseController(self) self._warp_device = wp.get_device(self._sim.device) self._device = str(self._warp_device) self._camera_render_data: list[OVRTXCameraRenderData] = [] @@ -367,17 +386,52 @@ def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): @property def visual_material_writer(self): - """Return a writer factory once this renderer has a scene consumer.""" - return self._create_visual_material_writer if self._initialized_scene else None + """Return a writer factory once this renderer has a scene consumer. + + A leased pair registers its writer before the borrower binds the Stage, because + environment reset finalizes material consumers first; addresses compile on the + first publication after population. + + The writer requires material buffers on one CUDA device. On a CPU simulation device, + an environment with :class:`~isaaclab.assets.VisualMaterial` assets therefore fails at + reset, with or without a lease. + """ + return ( + self._create_visual_material_writer + if self._initialized_scene or self._render_lease.active is not None + else None + ) def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> OVRTXVisualMaterialWriter: - """Create the detached-scene material writer after scene population.""" + """Retain material buffers; deferred scenes compile addresses after population.""" writer = OVRTXVisualMaterialWriter(self, batches) self._visual_material_writer_ref = weakref.ref(writer) return writer + @contextlib.contextmanager + def _publish_visual_materials(self) -> Iterator[None]: + """Publish dirty materials and complete their reads before buffers may change.""" + material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None + try: + if material_writer is not None: + material_writer.publish() + yield + finally: + if material_writer is not None: + # When another exception is already propagating, log the drain failure instead of + # replacing it. Losing it silently would hide a failed material write. + primary_active = sys.exc_info()[0] is not None + try: + material_writer.drain() + except Exception as e: + if not primary_active: + raise + logger.warning("Error draining material writes after a failed scene update: %s", e, exc_info=True) + def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" + if not self._render_lease.prepare_stage(num_envs): + return logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -390,6 +444,7 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: preserve_references=not ( self._use_ovstage and self.scene.cfg.population_domains & ovstage.PopulationDomain.PHYSICS ), + preserve_partition_visibility=self._render_lease.active is not None, ) # fmt: skip if not self._use_ovstage and _OVRTX_VERSION is not None and _OVRTX_VERSION.release[:2] == (0, 7): # Author the camera mode before loading the stage: changing it later @@ -407,7 +462,7 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: if self.cfg.temp_usd_dir is not None: _write_file(Path(self.cfg.temp_usd_dir), "ovrtx_renderer_stage.usda", usd) if self._use_ovstage: - self.scene.populate(usd) + self._render_lease.populate_stage(usd) else: self.backend.renderer.open_usd_from_string(usd) ovrtx_replicate(self.backend.renderer, self.scene.clone_copies, env_paths, self.scene.clone_positions) @@ -450,6 +505,11 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: self.scene.resources.callback(self.close) self._initialized_scene = True self._needs_transform_initialization = not self._use_ovstage and self.backend.cfg.read_gpu_transforms + self._render_lease.bind_stage() + + def acquire_ov_libraries_render_lease(self) -> OvLibrariesRenderLease: + """Reserve the native renderer/scene pair before stage preparation.""" + return self._render_lease.acquire() def _setup_xform_bindings(self, stage: Usd.Stage) -> None: """Bind SDP body paths and preserve their authored scales when writing world poses.""" @@ -558,26 +618,48 @@ def _setup_geometry_bindings(self) -> None: self._bindings.callback(self._geometry_points_binding.unbind) def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: - """Register a camera's product and bindings in the prepared scene.""" + """Register an ordinary camera, or reserve an authorized external camera until handoff.""" + auxiliary = self._render_lease.accept_camera_spec(spec) + if not auxiliary: + self._render_lease.reject_ordinary_operation("create ordinary camera render data") if str(wp.get_device(spec.device)) != self._device: raise ValueError("Cameras sharing an OVRTX renderer must use the simulation device.") + camera = self._render_lease.camera_for_spec(spec) if auxiliary else None + if camera is not None and camera.render_data is not None: + raise RuntimeError("Auxiliary camera render data already exists") camera_paths = list(spec.camera_prim_paths) env_partitions = {Sdf.Path(path): f"env_{i}" for i, path in enumerate(self.scene.clone_env_paths)} source_root = next((root for root in Sdf.Path(camera_paths[0]).GetPrefixes() if root in env_partitions), None) + if source_root is not None and len(camera_paths) < spec.num_instances: + # Native cloning may leave only the source camera in USD. Complete paths retain their row order. + suffix = camera_paths[0][len(str(source_root)) :] + camera_paths = [path + suffix for path in self.scene.clone_env_paths] + data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) + if not auxiliary or not self._render_lease.stage_deferred: + self._register_camera(spec, data) + self.scene.next_camera_id += 1 + self._camera_render_data.append(data) + if camera is not None: + camera.render_data = data + return data + + def _register_camera(self, spec: CameraRenderSpec, render_data: OVRTXCameraRenderData) -> None: + """Bind one product in the already populated simulation-owned scene.""" + camera_paths = render_data.camera_paths + env_partitions = {Sdf.Path(path): f"env_{i}" for i, path in enumerate(self.scene.clone_env_paths)} + source_root = next((root for root in Sdf.Path(camera_paths[0]).GetPrefixes() if root in env_partitions), None) camera_partitions = [] if source_root is not None: - # Native cloning may leave only the source camera in USD. Complete paths retain their row order. - if len(camera_paths) < spec.num_instances: - suffix = camera_paths[0][len(str(source_root)) :] - camera_paths = [path + suffix for path in self.scene.clone_env_paths] camera_partitions = [ next(env_partitions[root] for root in Sdf.Path(path).GetPrefixes() if root in env_partitions) for path in camera_paths ] - render_data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) scope, product_path = render_data.render_scope_name, render_data.render_product_path usd = build_render_product_as_string( - spec, render_data, device_id=self._warp_device.ordinal, enable_shadows=self.cfg.enable_shadows + spec, + render_data, + device_id=self._warp_device.ordinal if self._warp_device.is_cuda else None, + enable_shadows=self.cfg.enable_shadows, ) try: # Registration mutates the shared scene; finish renders using its existing products first. @@ -667,11 +749,8 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: except Exception: render_data.cleanup() raise - self.scene.next_camera_id += 1 - self._camera_render_data.append(render_data) self.backend.render_products.add(render_data.render_product_path) render_data.resources.callback(self.backend.render_products.discard, render_data.render_product_path) - return render_data def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, ProxyArray]) -> None: """Register pre-allocated warp output buffers for rendering. @@ -749,7 +828,7 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Wait for rendering on the consuming stream, or on the host on Linux unless :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` is ``1``. Consume the zero-copy view inside the context. Unmapping records the consuming stream; native release waits for its queued - reads and the last view to be dropped. + reads and the last view to be dropped. CPU consumers wait on the host and release without a CUDA stream. Args: render_var: OVRTX ``RenderVarOutput`` to map (looked up from ``frame.render_vars``). @@ -757,6 +836,14 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Yields: The render var's contents as a Warp array, valid for the duration of the context. """ + if not self._warp_device.is_cuda: + mapping = render_var.map(device=Device.CPU) + try: + mapping.wait() + yield wp.from_dlpack(mapping) + finally: + mapping.unmap() + return gpu_side_sync = _gpu_side_render_var_sync_enabled() # OVRTX uses 0 for no synchronization and 1 for Torch's legacy default stream (CUDA handle 0). stream = self._warp_device.stream.cuda_stream or 1 @@ -990,6 +1077,17 @@ def _process_render_products( ) def update_transforms(self) -> None: + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes, so eager scene + updates (for example ``lazy_sensor_update=False``) return without writing. + """ + if self._render_lease.owns_scene_updates(): + return + self._render_lease.reject_ordinary_operation("update ordinary transforms") + self._update_transforms() + + def _update_transforms(self) -> None: """Write changed SDP transforms to OVRTX.""" binding = self._object_xform_binding if binding is None: @@ -997,7 +1095,7 @@ def update_transforms(self) -> None: timestamp = self._sdp.backend.transforms_timestamp if self._transforms_timestamp == timestamp: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage output = self._transform_writes.acquire(stream) if not self._sdp.get_transforms(output, allow_passthrough=not asynchronous, scales=self._object_scales): @@ -1013,15 +1111,29 @@ def update_transforms(self) -> None: tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._transform_writes.submit(binding, matrices, stream) else: - binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + matrices, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._transforms_timestamp = timestamp def update_geometries(self) -> None: + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes. + """ + if self._render_lease.owns_scene_updates(): + return + self._render_lease.reject_ordinary_operation("update ordinary geometries") + self._update_geometries() + + def _update_geometries(self) -> None: """Write changed SDP geometry to OVRTX.""" for callback in self.geometry_callbacks.values(): callback(self) @@ -1040,7 +1152,7 @@ def update_geometries(self) -> None: binding = self._geometry_points_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage if asynchronous: output = self._geometry_writes.acquire(stream) @@ -1061,12 +1173,16 @@ def update_geometries(self) -> None: tensors=[points_tensor_from_warp(array) for array in points], is_array=True, semantic=ovstage.AttributeSemantic.POINT, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._geometry_writes.submit(binding, points, stream) else: - binding.write(points, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + points, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._geometry_timestamp = timestamp def update_camera( @@ -1075,12 +1191,25 @@ def update_camera( positions: ProxyArray, orientations: ProxyArray, intrinsics: ProxyArray, + ) -> None: + """Update an ordinary camera or queue a sample for an external borrower.""" + if self._render_lease.queue_camera_update(render_data, positions, orientations, intrinsics): + return + self._render_lease.reject_ordinary_operation("update ordinary cameras") + self._update_camera(render_data, positions, orientations, intrinsics) + + def _update_camera( + self, + render_data: OVRTXCameraRenderData, + positions: ProxyArray, + orientations: ProxyArray, + intrinsics: ProxyArray, ) -> None: """Update camera poses using the camera's reusable conversion buffers.""" binding = render_data.camera_xform_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None matrices = render_data.camera_writes.acquire(stream) wp.launch( create_camera_transforms_kernel, len(matrices), [positions, orientations, matrices], device=self._device @@ -1093,7 +1222,7 @@ def update_camera( tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() else: operation = render_data.camera_writes.submit(binding, matrices, stream) @@ -1101,13 +1230,14 @@ def update_camera( operation.wait() def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsics: wp.array, parameters: wp.array): - """Publish calibration columns from GPU memory into the renderer-owned scene.""" + """Publish calibration columns from their producing device into the renderer-owned scene.""" # A runtime calibration change is a scene write for this camera's bindings, so its # in-flight renders must finish first. Other cameras keep their pipelines. + self._render_lease.reject_ordinary_operation("update ordinary camera intrinsics") errors = self.drain_pending_renders((render_data,)) if errors: raise ExceptionGroup("OVRTX renders failed before calibration update", errors) - stream = wp.get_stream(parameters.device).cuda_stream or 1 + stream = (wp.get_stream(parameters.device).cuda_stream or 1) if parameters.device.is_cuda else None if self._use_ovstage: self.scene.stage.write_attributes( render_data.camera_xform_binding, @@ -1134,6 +1264,7 @@ def render(self, render_data: OVRTXCameraRenderData) -> None: def log_mesh(self, name: str, *args, **kwargs) -> None: """Publish a runtime mesh through the same scene used by every camera.""" + self._render_lease.reject_ordinary_operation("log runtime meshes") if name not in self.meshes: self.meshes[name] = OVRTXMesh(self, f"/IsaacLabMeshes/{name.lstrip('/')}") self.meshes[name].update(*args, **kwargs) @@ -1155,15 +1286,13 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: """ if not render_data: return + self._render_lease.reject_ordinary_operation("render an ordinary camera batch") if not self._initialized_scene: raise RuntimeError("Scene not initialized. Call initialize() first.") if self.backend.renderer is None or not self._camera_render_data: return products = self.backend.render_products.copy() - material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None - try: - if material_writer is not None: - material_writer.publish() + with self._publish_visual_materials(): if self._needs_transform_initialization: # OVRTX 0.6 initializes GPU world transforms on its first step using # authored poses. Flush that initialization before capturing runtime @@ -1185,18 +1314,6 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: else: result = self.backend.renderer.step(render_products=products, delta_time=_RENDER_DELTA_TIME) self._process_render_products(render_data, result) - finally: - if material_writer is not None: - # When another exception is already propagating, log the drain failure instead of - # replacing it. Losing it silently would hide a failed material write. - primary_active = sys.exc_info()[0] is not None - try: - material_writer.drain() - except Exception as e: - if not primary_active: - raise - logger.warning("Error draining material writes after a failed render: %s", e, exc_info=True) - if self._use_ovstage: # Stage writes and material reads finish before rendering. Consume only after advancing # the ordinal so a failed readback cannot leave later writes at the sealed floor. @@ -1210,9 +1327,15 @@ def cleanup(self, render_data: OVRTXCameraRenderData | None) -> None: Each camera owns its product and pose binding. Scene and physics bindings remain alive until :meth:`close`, so other cameras can continue rendering. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + Releasing a product removes it from the borrowed Stage, so the camera keeps its + resources until the lease is released; :meth:`close` releases them afterwards. """ if render_data is None: return + self._render_lease.reject_ordinary_operation("release camera render data", retiring=True) try: self.reset(render_data) finally: @@ -1228,7 +1351,12 @@ def _remove_usd_reference(self, reference: int) -> None: ovstage.population.apply_usd_changes(self.scene.stage, ordinal=self.scene.ordinal) def close(self) -> None: - """Complete borrowed-buffer reads and release bindings; the registry owns the native engine.""" + """Complete borrowed-buffer reads and release bindings; the registry owns the native engine. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + self._render_lease.reject_ordinary_operation("close the renderer", retiring=True) try: with contextlib.ExitStack() as resources: resources.callback(self._bindings.close) @@ -1253,6 +1381,7 @@ def close(self) -> None: self.meshes.clear() self.markers.clear() self.geometry_callbacks.clear() + self._closed = True class OVRTXMesh: @@ -1591,27 +1720,29 @@ def close(self) -> None: class _AsyncWriteBuffers(Generic[_BufferT]): - """Retain GPU buffers until native writes finish; reuse them in submission order.""" + """Retain buffers until native writes finish; reuse them in submission order.""" def __init__(self, buffers: Iterable[_BufferT] = ()): self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( (buffer, None, None) for buffer in buffers ) - def acquire(self, stream: wp.Stream) -> _BufferT: + def acquire(self, stream: wp.Stream | None) -> _BufferT: """Wait until the next buffer is writable on the given stream.""" buffer, operation, producer = self._writes[0] if operation is not None: operation.wait() - if producer != stream: + if stream is not None and producer is not None and producer != stream: stream.wait_stream(producer) return buffer def submit( - self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream + self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream | None ) -> Operation[bool]: """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" - operation = binding.write_async(values, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + operation = binding.write_async( + values, data_access=DataAccess.ASYNC, cuda_stream=(stream.cuda_stream or 1) if stream is not None else None + ) self._writes[0] = (self._writes[0][0], operation, stream) self._writes.rotate(-1) return operation @@ -1622,5 +1753,6 @@ def close(self) -> None: writes.callback(self._writes.clear) for _, operation, producer in self._writes: if operation is not None: - writes.callback(wp.synchronize_stream, producer) + if producer is not None: + writes.callback(wp.synchronize_stream, producer) writes.callback(operation.wait) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 85032bcbd42a..30094ae61f29 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -144,7 +144,7 @@ def build_render_product_as_string( spec: CameraRenderSpec, render_data: OVRTXCameraRenderData, *, - device_id: int, + device_id: int | None = None, enable_shadows: bool = False, ) -> str: """Build a complete render product USD layer as a string. @@ -160,6 +160,7 @@ def build_render_product_as_string( render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. + When None, leave native GPU selection automatic for CPU consumers. enable_shadows: Whether lights cast shadows. Defaults to False. Only honored for the ``simple_shading_*`` data types, which are the ones that select RTX Minimal mode. @@ -210,6 +211,8 @@ def build_render_product_as_string( for _, name, source in render_var_configs ) + device_ids_line = "" if device_id is None else f"uint[] deviceIds = [{device_id}]" + layer = f'''#usda 1.0 (defaultPrim = "{render_data.render_scope_name}") @@ -219,7 +222,7 @@ def RenderProduct "RenderProduct" ( prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] ) {{ rel camera = [<{render_data.camera_paths[0]}>] - uint[] deviceIds = [{device_id}] + {device_ids_line} {bg_type_line} {render_mode_block} token[] omni:rtx:waitForEvents = ["AllLoadingFinished", "OnlyOnFirstRequest"] diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py new file mode 100644 index 000000000000..3d4009f2a5f8 --- /dev/null +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -0,0 +1,936 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Exclusive borrowing, thread handoff and publication ordering for OVRTX/OVStage.""" + +from __future__ import annotations + +import contextlib +import math +import os +import re +import threading +from collections.abc import Hashable, Iterator, Sequence +from dataclasses import dataclass +from enum import Enum, auto +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +import ovstage +import torch + +from pxr import Gf, Sdf, Usd + +from isaaclab.cloner.cloner_cfg import expand_env_regex_ns +from isaaclab.renderers.camera_render_spec import CameraRenderSpec + +if TYPE_CHECKING: + from ovrtx import Renderer + from ovstage import Stage as OvStage + + from .ovrtx_renderer import OVRTXCameraRenderData, OVRTXRenderer +_NO_PHYSICS_STEP_ID = object() + + +class _OvLibrariesRenderLeaseState(Enum): + """Internal lifecycle state for a renderer/scene pair lease.""" + + UNBOUND = auto() + BOUND = auto() + RELEASED = auto() + + +@dataclass +class _AuxiliaryCamera: + cfg: Any + max_fps: float | None + spec: CameraRenderSpec | None = None + render_data: OVRTXCameraRenderData | None = None + state: tuple[Any, ...] | None = None + product_path: str | None = None + + +class OvLibrariesRenderLease: + """Exclusive non-owning lease over one exact attached OVRTX/OVStage pair. + + The lease is acquired before stage preparation. Stage preparation then + materializes the renderer-owned scene without requiring a camera sensor and + binds this object to the exact live Python ``Renderer`` and ``Stage`` + instances. External consumers must release the lease before the renderer is + closed; they must never detach or destroy either borrowed object. + """ + + _controller: RenderLeaseController + """Renderer-owned controller that created this lease.""" + + def __init__(self) -> None: + raise RuntimeError("OvLibrariesRenderLease must be acquired from OVRTXRenderer") + + def __enter__(self) -> OvLibrariesRenderLease: + self._validate_active() + return self + + def __exit__(self, *_exc: object) -> None: + self.release() + + @property + def auxiliary_render_product_paths(self) -> tuple[str, ...]: + """RenderProduct paths of the selected auxiliary cameras, in configuration order. + + Raises: + RuntimeError: If the lease is not bound and live, or a selected camera did not + materialize exactly one product. + """ + self.validate_live() + return self._controller.product_paths(self) + + @property + def ovrtx_renderer(self) -> Renderer: + """Return the exact retained application renderer.""" + self.validate_live() + return cast("Renderer", self._renderer) + + @property + def ovstage(self) -> OvStage: + """Return the exact retained renderer-attached OVStage.""" + self.validate_live() + return cast("OvStage", self._stage) + + def configure_auxiliary_camera(self, camera_cfg: Any, *, max_fps: float | None = None) -> None: + """Authorize an exact camera before Stage materialization. + + The leased renderer then accepts the one camera sensor whose specification matches + ``camera_cfg`` exactly: a single camera view with the same width, height and ordered + ``data_types``, at ``prim_path`` (``{ENV_REGEX_NS}`` matches any environment) or its + ``camera`` child prim. Every other camera on the leased renderer remains rejected. + + Args: + camera_cfg: Configuration of the camera sensor to authorize, such as a + :class:`~isaaclab.sensors.camera.CameraCfg`. Its ``prim_path`` must be unique among + the lease's cameras. + max_fps: Maximum native rate of the camera's frames [Hz], authored on its prim in the + exported Stage. ``None`` authors no rate, so the camera renders with every frame of + its render product. + + Raises: + RuntimeError: If the lease is released or already bound to a Stage, or ``prim_path`` is + already configured. + ValueError: If ``camera_cfg`` is ``None``, or ``max_fps`` is not finite and positive. + """ + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Auxiliary camera configuration must precede Stage materialization") + if camera_cfg is None: + raise ValueError("Auxiliary camera configuration cannot be None") + if any(camera.cfg.prim_path == camera_cfg.prim_path for camera in self._auxiliary_cameras): + raise RuntimeError("This auxiliary camera is already configured") + if max_fps is not None and (not math.isfinite(max_fps) or max_fps <= 0.0): + raise ValueError("Auxiliary camera max_fps must be finite and positive [Hz]") + self._auxiliary_cameras.append(_AuxiliaryCamera(camera_cfg, max_fps)) + + def offer_owner_transfer(self) -> object: + """Freeze one unbound lease and return its one-shot owner capability.""" + owner = self._controller + return owner.offer_owner_transfer(self) + + def claim_owner_transfer(self, capability: object) -> None: + """Make the calling thread the render owner using the offered capability. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If the capability is stale, already claimed, or the lease is no longer unbound. + """ + self._controller.claim_owner_transfer(self, capability) + + def cancel_owner_transfer(self, capability: object) -> None: + """Withdraw an unclaimed offer on the original owner thread. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If called off the original owner thread or with a stale capability. + """ + self._controller.cancel_owner_transfer(self, capability) + + def defer_stage_until_owner_transfer(self) -> None: + """Defer Stage materialization requested before the owner transfer is claimed. + + Camera initialization calls ``prepare_stage`` for every registered renderer on the + application thread. With deferral, that call only records the request; the claimed + render owner completes it with :meth:`materialize_deferred_stage`. + """ + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Stage deferral must be selected before Stage materialization") + self._controller.defer_until_transfer = True + + def materialize_deferred_stage(self, stage: Any, num_envs: int) -> bool: + """Complete a deferred Stage preparation on the claimed render owner. + + Args: + stage: The environment USD stage. + num_envs: Number of environments; the lease supports exactly one. + + Returns: + Whether a deferred preparation was completed. ``False`` means nothing was deferred. + + Raises: + RuntimeError: If called off the render-owner thread, or deferral is selected and + the owner transfer has not been claimed. + """ + return self._controller.materialize_deferred_stage(self, stage, num_envs) + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application-thread scene publisher before render ownership transfers. + + Call this on the owner thread while the lease is unbound and before + :meth:`offer_owner_transfer`. The publisher stays bound to the calling thread; see + :class:`~isaaclab_ov.renderers.OvLibrariesScenePublisher` for the publication + protocol. + + Returns: + The lease's only scene publisher. + + Raises: + RuntimeError: If called off the owner thread or while a transfer is offered, after the + Stage is bound, or when the lease already has a publisher. + """ + return self._controller.capture_scene_publisher(self) + + @contextlib.contextmanager + def ovstage_owner_lane(self) -> Iterator[None]: + """Guard one non-reentrant operation on the render-owner thread. + + The application's publication guard must separately exclude concurrent + producer writes while this operation uses the borrowed stage. A failed + producer write poisons publication only: the lane stays available so the + borrower can destroy its consumers and retire the pair. + """ + self._validate_live(allow_poisoned=True) + owner = self._controller + with owner.owner_lane("external renderer operation", lease=self): + yield + + def configure_external_camera_scene_partitions( + self, environment_root_path: str, camera_paths: Sequence[str] + ) -> str: + """Assign the live environment partition token to external cameras. + + This method must be called inside :meth:`ovstage_owner_lane` after the + external camera prims exist. The environment root's resolved token is + authoritative; empty and token-zero values fail closed. + """ + self.validate_live() + owner = self._controller + return owner.configure_camera_partitions(self, environment_root_path, camera_paths) + + def validate_external_scene_partitions(self, ordinal: int) -> dict[str, str]: + """Read back and prove environment/camera partition equality.""" + self.validate_live() + owner = self._controller + return owner.validate_partitions(self, ordinal) + + def release(self) -> None: + """Release the borrowing intent without destroying either pair member.""" + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + return + owner = self._controller + owner.release(self) + self._renderer = None + self._stage = None + self._state = _OvLibrariesRenderLeaseState.RELEASED + + def validate_live(self) -> None: + """Verify that this lease still refers to its live, unpoisoned renderer and stage.""" + self._validate_live(allow_poisoned=False) + + def _validate_live(self, *, allow_poisoned: bool) -> None: + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OvLibrariesRenderLease is not bound to a prepared Stage") + owner = self._controller + owner.validate_live( + self, + cast("Renderer", self._renderer), + cast("OvStage", self._stage), + self._generation, + allow_poisoned=allow_poisoned, + ) + + @classmethod + def _create_unbound(cls, owner: RenderLeaseController) -> OvLibrariesRenderLease: + lease = object.__new__(cls) + lease._controller = owner + lease._state = _OvLibrariesRenderLeaseState.UNBOUND + lease._renderer = None + lease._stage = None + lease._generation = 0 + lease._required_environment_scene_partitions = () + lease._external_camera_scene_partitions = () + lease._external_camera_scene_partitions_pending = False + lease._auxiliary_cameras = [] + return lease + + def _bind(self, renderer: Renderer, stage: OvStage, generation: int) -> None: + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("OvLibrariesRenderLease can only bind once") + self._renderer = renderer + self._stage = stage + self._generation = generation + self._state = _OvLibrariesRenderLeaseState.BOUND + + def _validate_active(self) -> None: + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + raise RuntimeError("OvLibrariesRenderLease is released") + owner = self._controller + owner.validate_active(self) + + +class OvLibrariesScenePublisher: + """Application-thread writer for complete provider-derived OVStage batches. + + Obtain it with :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.capture_scene_publisher`. + While the lease is active, the renderer skips its own eager transform, geometry and material + writes, and the borrower publishes each complete physics state through this object instead: + + 1. On the application thread, call :meth:`publication_required` with the identity of the + current physics state. + 2. If it returns ``True``, reserve a nonzero Stage ordinal and call + :meth:`write_complete_scene` with it. The write does not advance the Stage write floor: + the borrower seals that ordinal afterwards, or abandons it if the write fails. + 3. Hold one application publication guard around both calls and around every operation the + render owner performs in :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.ovstage_owner_lane`, so that scene + writes and native frame operations never interleave. + + A failed write poisons the pair: later publication and ordinary renderer use are rejected, + but the borrower can still retire its consumers and release the lease. The renderer must then + be recreated. :meth:`close` stops publication; releasing the lease also closes the publisher. + The OVXR teleoperation adapter, ``isaaclab_teleop.OvLibrariesXrSession``, follows this protocol. + """ + + def __init__(self, owner: RenderLeaseController, lease: OvLibrariesRenderLease) -> None: + self._controller = owner + self._lease = lease + self._application_thread = threading.current_thread() + self._lock = threading.Lock() + self._closed = False + + def publication_required(self, physics_step_id: Hashable) -> bool: + """Return whether a new physics step or dirty material requires a scene write. + + Args: + physics_step_id: Hashable identity of the complete physics state to publish. Only the + last written identity is remembered, so the identity must change whenever the + state does, including after a reset that does not advance the physics step count. + + Returns: + Whether ``physics_step_id`` differs from the last written batch or a visual material + has pending updates. + + Raises: + RuntimeError: If called off the captured application thread, the publisher is closed + or inactive, the lease is not bound, or a failed write poisoned the pair. + TypeError: If ``physics_step_id`` is not hashable. + """ + self._require_application_thread() + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + return self._controller.producer_publication_required(self._lease, physics_step_id) + + def write_complete_scene(self, ordinal: int, physics_step_id: Hashable) -> None: + """Write, but do not publish, one complete batch at ``ordinal``. + + Writes the current transforms, geometries and pending visual materials at ``ordinal`` and + records ``physics_step_id`` as written. The caller seals the ordinal afterwards. + + Args: + ordinal: Nonzero Stage ordinal reserved by the caller for this batch. + physics_step_id: The identity passed to the preceding :meth:`publication_required` call. + + Raises: + ValueError: If ``ordinal`` is not positive. + RuntimeError: If called off the captured application thread, the publisher is closed or + inactive, the state needs no publication, or the write failed. A failed write + poisons the pair. + """ + self._require_application_thread() + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + self._controller.write_producer_scene(self._lease, ordinal, physics_step_id) + + def close(self) -> None: + """Stop scene publication after any in-flight batch finishes. Safe to call repeatedly.""" + with self._lock: + self._closed = True + + def _require_application_thread(self) -> None: + if threading.current_thread() is not self._application_thread: + raise RuntimeError("OVStage scene publisher must run on its captured application thread") + + +class RenderLeaseController: + """Coordinate external borrowing without owning the renderer's native resources. + + Lease state, thread transfer and publication ordering live here. The renderer + retains scene/camera storage and supplies the component-write operations. + """ + + def __init__(self, renderer: OVRTXRenderer) -> None: + self._renderer = renderer + self._owner_thread = threading.current_thread() + self._transfer_lock = threading.Lock() + self._transfer_capability: object | None = None + self._transfer_claimed = False + self.active: OvLibrariesRenderLease | None = None + self._publisher: OvLibrariesScenePublisher | None = None + self._generation = 0 + self._lane_operation: str | None = None + self._lane_lease: OvLibrariesRenderLease | None = None + self._last_step_id: Hashable = _NO_PHYSICS_STEP_ID + self._poisoned = False + self.defer_until_transfer = False + self.stage_deferred = False + + def acquire(self) -> OvLibrariesRenderLease: + """Reserve the exact OVRTX/OVStage pair before stage preparation.""" + self.require_owner_thread("acquire a render lease") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise RuntimeError("OvLibrariesRenderLease requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") + if os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1": + raise RuntimeError("The render lease requires a private rendering OVStage") + if self._renderer._closed: + raise RuntimeError("Cannot acquire OvLibrariesRenderLease from a closed renderer") + if self.active is not None: + raise RuntimeError("An OvLibrariesRenderLease is already active") + if self._renderer.backend is not None or self._renderer._initialized_scene or self._generation != 0: + raise RuntimeError("OvLibrariesRenderLease must be acquired before clone preparation") + lease = OvLibrariesRenderLease._create_unbound(self) + self.active = lease + return lease + + def require_owner_thread(self, operation: str) -> None: + """Reject lease operations outside the renderer construction thread.""" + if self._transfer_capability is not None: + raise RuntimeError("OVRTXRenderer owner transfer is pending") + if threading.current_thread() is not self._owner_thread: + raise RuntimeError(f"OVRTXRenderer owner thread must {operation}") + + def offer_owner_transfer(self, lease: OvLibrariesRenderLease) -> object: + self.require_owner_thread("offer an owner transfer") + with self._transfer_lock: + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + or self._transfer_capability is not None + or self._transfer_claimed + ): + raise RuntimeError("OVRTXRenderer is not eligible for a pre-use owner transfer") + capability = object() + self._transfer_capability = capability + return capability + + def claim_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + if ( + capability is None + or capability is not self._transfer_capability + or self._transfer_claimed + or self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + ): + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._owner_thread = threading.current_thread() + self._transfer_claimed = True + self._transfer_capability = None + + def cancel_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + # Cancellation is the one original-owner operation allowed while an offer is pending. + if threading.current_thread() is not self._owner_thread: + raise RuntimeError("OVRTXRenderer owner thread must cancel an owner transfer") + if capability is None or capability is not self._transfer_capability or self.active is not lease: + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._transfer_capability = None + + def capture_scene_publisher(self, lease: OvLibrariesRenderLease) -> OvLibrariesScenePublisher: + """Create the scene publisher before render ownership transfers.""" + self.require_owner_thread("capture an OVStage scene publisher") + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._publisher is not None + ): + raise RuntimeError("OVRTXRenderer is not eligible to capture an OVStage scene publisher") + publisher = OvLibrariesScenePublisher(self, lease) + self._publisher = publisher + return publisher + + @contextlib.contextmanager + def owner_lane(self, operation: str, *, lease: OvLibrariesRenderLease | None = None) -> Iterator[None]: + """Reject nested operations and access outside the render-owner thread. + + This lane is not a cross-thread lock. The external consumer must share + its application publication guard with the captured scene publisher. + """ + self.require_owner_thread(f"enter the OVStage owner lane for {operation}") + if self._lane_operation is not None: + raise RuntimeError( + f"OVStage owner lane is already active for {self._lane_operation}; nested writers are forbidden" + ) + self._lane_operation = operation + self._lane_lease = lease + try: + yield + finally: + self._lane_lease = None + self._lane_operation = None + + def validate_active(self, lease: OvLibrariesRenderLease) -> None: + self.require_owner_thread("access a render lease") + if self.active is not lease: + raise RuntimeError("OvLibrariesRenderLease is not active for this renderer") + if self._renderer._closed: + raise RuntimeError("OvLibrariesRenderLease owner is closed or no longer uses OVStage") + + def validate_live( + self, + lease: OvLibrariesRenderLease, + ovrtx_renderer: Renderer, + ovstage_instance: OvStage, + generation: int, + *, + allow_poisoned: bool = False, + ) -> None: + """Validate retained Python-object identity for the live attached pair. + + ``allow_poisoned`` admits retirement work after a failed publication. + """ + self.validate_active(lease) + if self._poisoned and not allow_poisoned: + raise RuntimeError("OvLibrariesRenderLease owner pair is poisoned") + if generation != self._generation: + raise RuntimeError("OvLibrariesRenderLease has a stale Stage generation") + if ovrtx_renderer is not self._renderer.backend.renderer or ovstage_instance is not self._renderer.scene.stage: + raise RuntimeError("OvLibrariesRenderLease no longer matches its exact renderer-owned pair") + if not self._renderer._initialized_scene or not self._renderer.backend.attached: + raise RuntimeError("OvLibrariesRenderLease pair is no longer initialized and attached") + + def release(self, lease: OvLibrariesRenderLease) -> None: + """Retire the lease after borrower destruction and rebase the dormant ordinal.""" + self.validate_active(lease) + if self._lane_operation is not None: + raise RuntimeError("Cannot release OvLibrariesRenderLease while the owner lane is active") + if lease._state is _OvLibrariesRenderLeaseState.BOUND and (not self._poisoned): + with self.owner_lane("lease-release ordinal rebase"): + self._renderer.scene.ordinal = self._next_ordinal() + self._last_step_id = _NO_PHYSICS_STEP_ID + if self._publisher is not None: + self._publisher.close() + self._publisher = None + self._transfer_claimed = False + self.defer_until_transfer = False + self.stage_deferred = False + self.active = None + if self._renderer.backend is not None: + self._renderer.backend._render_lease_owner = None + if self._renderer.scene is not None: + self._renderer.scene._render_lease_owner = None + + def reject_ordinary_operation(self, operation: str, *, retiring: bool = False) -> None: + """Reject ordinary OVRTX operations while an external lease is active. + + A pair poisoned by a failed leased write keeps rejecting ordinary use after + release, because its ordinal is not rebased; only ``retiring`` operations + (closing the renderer) are admitted so the pair can be recreated. + """ + if self.active is not None or ( + self._renderer.backend is not None and self._renderer.backend._render_lease_owner is not None + ): + raise RuntimeError(f"Cannot {operation} while an OvLibrariesRenderLease is active") + if self._poisoned and not retiring: + raise RuntimeError( + f"Cannot {operation}: a failed leased publication poisoned this OVRTX/OVStage pair; recreate it" + ) + + def owns_scene_updates(self) -> bool: + """Whether an active lease's scene publisher owns transform and geometry writes.""" + return self.active is not None + + def producer_publication_required(self, lease: OvLibrariesRenderLease, physics_step_id: Hashable) -> bool: + """Check publication on the application thread without entering the render thread.""" + if self.active is not lease or self._publisher is None: + raise RuntimeError("OVStage scene publisher is not active for this lease") + if self._poisoned or lease._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OVStage scene publisher pair is unavailable") + try: + hash(physics_step_id) + except TypeError as exc: + raise TypeError("physics_step_id must be hashable") from exc + writer_ref = self._renderer._visual_material_writer_ref + writer = writer_ref() if writer_ref is not None else None + return ( + lease._external_camera_scene_partitions_pending + or self._last_step_id != physics_step_id + or (writer is not None and writer.has_pending_updates) + ) + + def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, physics_step_id: Hashable) -> None: + """Write a complete scene batch without advancing its reserved ordinal.""" + if ordinal <= 0: + raise ValueError("OVStage publication ordinal must be nonzero") + if not self.producer_publication_required(lease, physics_step_id): + raise RuntimeError("OVStage scene publication is not required for this complete state") + publish_camera_partitions = lease._external_camera_scene_partitions_pending + try: + self._write_scene_components(ordinal, partition_lease=lease if publish_camera_partitions else None) + except Exception as exc: + self._poisoned = True + raise RuntimeError("Producer scene publication failed; recreate the exact pair") from exc + self._last_step_id = physics_step_id + if publish_camera_partitions: + lease._external_camera_scene_partitions_pending = False + + def camera_for_spec(self, spec: CameraRenderSpec) -> _AuxiliaryCamera | None: + lease = self.active + if lease is None: + return None + for camera in lease._auxiliary_cameras: + expected = camera.cfg + if spec.num_instances != 1 or spec.view_count != 1 or len(spec.camera_prim_paths) != 1: + raise RuntimeError("An external camera requires exactly one camera view") + pattern = expand_env_regex_ns(str(expected.prim_path)) + if ( + int(spec.cfg.width) == int(expected.width) + and int(spec.cfg.height) == int(expected.height) + and tuple(spec.cfg.data_types) == tuple(expected.data_types) + and re.fullmatch(f"(?:{pattern})(?:/camera)?", str(spec.camera_prim_paths[0])) is not None + ): + return camera + return None + + def accept_camera_spec(self, spec: CameraRenderSpec) -> bool: + camera = self.camera_for_spec(spec) + if camera is None: + return False + if camera.spec is None: + camera.spec = spec + elif camera.spec != spec: + raise RuntimeError("Auxiliary camera specification changed after authorization") + return True + + def product_paths(self, lease: OvLibrariesRenderLease) -> tuple[str, ...]: + """Return products in application selection order, independently of sensor initialization order.""" + self.validate_live(lease, lease.ovrtx_renderer, lease.ovstage, lease._generation) + paths = tuple(camera.product_path for camera in lease._auxiliary_cameras) + if any(not isinstance(path, str) or not path.startswith("/") for path in paths) or len(set(paths)) != len( + paths + ): + raise RuntimeError("Auxiliary camera did not materialize exactly one RenderProduct per selection") + return paths + + def configure_camera_partitions( + self, + lease: OvLibrariesRenderLease, + environment_root_path: str, + camera_paths: Sequence[str], + ) -> str: + """Read the authoritative environment token and arm matching camera writes.""" + self.validate_active(lease) + lease.validate_live() + if self._lane_lease is not lease: + raise RuntimeError("configure_external_camera_scene_partitions() needs the active lease owner lane") + if not environment_root_path or not camera_paths or any(not path for path in camera_paths): + raise ValueError("One environment root and at least one non-empty external camera path are required") + + floor = self._next_ordinal() - 1 + attribute = "primvars:omni:scenePartition" + actual = self._read_partition_tokens([environment_root_path], (attribute,), floor) + token = actual.get((environment_root_path, attribute), "") + if token in {"", "0"}: + raise RuntimeError( + f"Environment root {environment_root_path!r} resolved to invalid scene-partition token {token!r}" + ) + + lease._required_environment_scene_partitions = ((environment_root_path, token),) + lease._external_camera_scene_partitions = tuple((str(path), token) for path in camera_paths) + lease._external_camera_scene_partitions_pending = True + return token + + def validate_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> dict[str, str]: + """Prove the configured root and external cameras resolve to one valid token.""" + self.validate_active(lease) + lease.validate_live() + if self._lane_lease is not lease: + raise RuntimeError("validate_external_scene_partitions() needs the active lease owner lane") + expected_rows = [ + *( + (path, "primvars:omni:scenePartition", token) + for path, token in lease._required_environment_scene_partitions + ), + *((path, "omni:scenePartition", token) for path, token in lease._external_camera_scene_partitions), + ] + if len(lease._required_environment_scene_partitions) != 1: + raise RuntimeError("Exactly one authoritative environment scene partition is required") + if not lease._external_camera_scene_partitions: + raise RuntimeError("At least one external camera scene partition is required") + paths = list(dict.fromkeys(path for path, _attribute, _token in expected_rows)) + attributes = tuple(dict.fromkeys(attribute for _path, attribute, _token in expected_rows)) + actual = self._read_partition_tokens(paths, attributes, ordinal) + partitions: dict[str, str] = {} + mismatches = [] + for path, attribute, expected_token in expected_rows: + actual_token = actual.get((path, attribute), "") + partitions[path] = actual_token + if actual_token in {"", "0"} or actual_token != expected_token: + mismatches.append(f"{path}.{attribute}: expected {expected_token!r}, got {actual_token!r}") + if mismatches: + raise RuntimeError("External scene-partition proof failed: " + "; ".join(mismatches)) + return partitions + + def prepare_stage(self, num_envs: int) -> bool: + """Validate or defer leased preparation; ordinary rendering is immediately allowed. + + Preparation is deferred until the owner transfer is claimed when the borrower opted in + or selected auxiliary cameras. + + Returns: + Whether the renderer should materialize the Stage now. + """ + lease = self.active + if lease is not None: + if (self.defer_until_transfer or lease._auxiliary_cameras) and not self._transfer_claimed: + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + self.stage_deferred = True + return False + self.require_owner_thread("prepare the leased Stage") + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + if num_envs != 1: + raise RuntimeError("The render lease requires exactly one environment") + if not self._renderer._use_ovstage or self._renderer.backend._render_lease_owner is not self._renderer: + raise RuntimeError("The render lease requires its reserved private OVStage renderer") + return True + + def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: + """Materialize a deferred Stage, with any selected cameras, after render-thread ownership transfer.""" + self.require_owner_thread("materialize a deferred Stage") + # Unclaimed, a fallback prepare_stage would only defer again; reject even if nothing is deferred yet. + if (self.defer_until_transfer or self.stage_deferred) and not self._transfer_claimed: + raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") + if not self.stage_deferred: + return False + if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Deferred Stage is not eligible for materialization") + if any(camera.spec is None or camera.render_data is None for camera in lease._auxiliary_cameras): + raise RuntimeError("Deferred auxiliary camera Stage is not eligible for materialization") + self._renderer.prepare_stage(stage, num_envs) + return True + + def bind_stage(self) -> None: + """Bind the prepared native pair after renderer initialization succeeds.""" + lease = self.active + if lease is not None: + self._materialize_camera_products(lease) + self.stage_deferred = False + self._generation += 1 + lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) + + def populate_stage(self, usd: str) -> None: + """Populate the renderer's scene, with the camera schedule an active lease authorizes.""" + lease = self.active + if lease is not None and lease._auxiliary_cameras: + usd = self._author_camera_schedules(usd, lease) + scheduled_paths = ( + [camera.spec.camera_prim_paths[0] for camera in lease._auxiliary_cameras if camera.max_fps is not None] + if lease is not None + else [] + ) + if scheduled_paths: + population = ovstage.population + description = population.Desc( + domains=self._renderer.scene.cfg.population_domains, + selectors=[ + population.Selector( + population.PrimPredicate.has_path(*scheduled_paths), + population.PropertyPredicate.has_name( + "omni:sensor:frameRate", "omni:sensor:Core:partialOutputs" + ), + ) + ], + ) + self._renderer.scene.populate(usd, population_desc=description) + else: + self._renderer.scene.populate(usd) + + def queue_camera_update(self, render_data, positions, orientations, intrinsics) -> bool: + """Retain the latest authorized external-camera sample until publication.""" + if self.active is not None: + for camera in self.active._auxiliary_cameras: + if camera.render_data is render_data: + camera.state = (render_data, positions, orientations, intrinsics) + return True + return False + + def reserve_pair(self) -> None: + """Reserve a newly acquired private native pair during clone preparation.""" + renderer = self._renderer + if renderer.backend._render_lease_owner is not None or renderer.scene._render_lease_owner is not None: + raise RuntimeError("The native pair already has a render lease") + with renderer.scene.stage.get_attribute_write_floor() as floor: + floor.wait() + if int(floor.fetch()) != 0: + raise RuntimeError("The render lease requires an unused private OVStage") + renderer.backend._render_lease_owner = renderer + renderer.scene._render_lease_owner = renderer + + def _next_ordinal(self) -> int: + """Return ``floor + 1`` from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None: + raise RuntimeError("Cannot query an OVStage floor before scene materialization") + with self._renderer.scene.stage.get_attribute_write_floor() as floor_query: + floor_query.wait() + floor = int(floor_query.fetch()) + if floor >= (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + return floor + 1 + + def _write_scene_components(self, ordinal: int, *, partition_lease: OvLibrariesRenderLease | None) -> None: + """Write selected scene components; the caller owns validation and commit.""" + self._renderer.scene.ordinal = ordinal + with self._renderer._publish_visual_materials(): + self._renderer._update_transforms() + self._renderer._update_geometries() + self._update_cameras() + if partition_lease is not None: + self._write_camera_partitions(partition_lease, ordinal) + + def _write_camera_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> None: + """Write the external camera tokens inside the complete scene batch.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot write external camera partitions before Stage materialization") + camera_scene_partitions = lease._external_camera_scene_partitions + if not camera_scene_partitions: + raise RuntimeError("No external camera scene partitions are configured") + camera_paths = [path for path, _token in camera_scene_partitions] + token_ids = np.array( + [self._renderer.scene.paths.intern_token(token) for _path, token in camera_scene_partitions], + dtype=np.uint64, + ) + camera_paths_list = self._renderer.scene.paths.create_path_list_from_strings(camera_paths) + camera_query = None + try: + camera_query = self._renderer.scene.stage.query_from_path_list(camera_paths_list) + if camera_query is None: + raise RuntimeError("External camera scene-partition query is None") + self._renderer.scene.stage.write_attribute( + camera_query, + "omni:scenePartition", + ordinal=ordinal, + tensors=token_ids, + is_array=False, + semantic=ovstage.AttributeSemantic.TOKEN_ID, + ).wait() + finally: + if camera_query is not None: + self._renderer.scene.stage.release_query(camera_query).wait() + self._renderer.scene.paths.destroy_path_list(camera_paths_list) + + def _read_partition_tokens( + self, paths: list[str], attributes: tuple[str, ...], ordinal: int + ) -> dict[tuple[str, str], str]: + """Read resolved partition tokens from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot read scene partitions before Stage materialization") + path_list = self._renderer.scene.paths.create_path_list_from_strings(paths) + attribute_tokens = [self._renderer.scene.paths.intern_token(attribute) for attribute in attributes] + resolved: dict[tuple[str, str], str] = {} + try: + with self._renderer.scene.stage.query_from_path_list(path_list) as query: + query.wait() + with self._renderer.scene.stage.read_attributes( + query, attribute_tokens, ovstage.OrdinalRange.latest(ordinal) + ) as read: + read.wait() + for group in read.groups(): + try: + if group.is_delete or group.is_array or group.tensor_count != 1: + continue + attribute = self._renderer.scene.paths.token_to_string(group.attribute) + if attribute not in attributes: + continue + group_paths = self._renderer.scene.paths.get_path_strings(group.prim_list) + try: + values = np.asarray(group.array(0).copy()).reshape(-1) + except NotImplementedError: + values = torch.from_dlpack(group.dlpack(0)).detach().cpu().numpy().copy().reshape(-1) + logical_count = min(group.prim_count, group.data_count) + for local_index in range(logical_count): + prim_index = group.prim_index(local_index) + data_row_index = group.data_row_index(local_index) + if prim_index >= len(group_paths) or data_row_index >= len(values): + raise RuntimeError("OVStage scene-partition read escaped its group-owned rows") + token_id = int(values[data_row_index]) + token = self._renderer.scene.paths.token_to_string(token_id) if token_id else "" + key = (group_paths[prim_index], attribute) + if key in resolved and resolved[key] != token: + raise RuntimeError(f"OVStage returned conflicting scene-partition rows for {key}") + resolved[key] = token + finally: + self._renderer.scene.stage.release_group(group) + finally: + self._renderer.scene.paths.destroy_path_list(path_list) + return resolved + + def _author_camera_schedules(self, usd: str, lease: OvLibrariesRenderLease) -> str: + """Author selected native camera rates on the private export, preserving the application stage.""" + + layer = Sdf.Layer.CreateAnonymous("borrowed-renderer.usda") + if not layer.ImportFromString(usd): + raise RuntimeError("Cannot parse the exported leased Stage") + stage = Usd.Stage.Open(layer) + for camera in lease._auxiliary_cameras: + if camera.max_fps is None: + continue + if camera.spec is None: + raise RuntimeError("Native-rate auxiliary camera was not prepared") + prim = stage.GetPrimAtPath(camera.spec.camera_prim_paths[0]) + if not prim.IsValid(): + raise RuntimeError("Native-rate auxiliary camera is absent from the exported Stage") + prim.CreateAttribute("omni:sensor:frameRate", Sdf.ValueTypeNames.Double2).Set(Gf.Vec2d(camera.max_fps, 1.0)) + prim.CreateAttribute("omni:sensor:Core:partialOutputs", Sdf.ValueTypeNames.Bool).Set(False) + return stage.ExportToString() + + def _materialize_camera_products(self, lease: OvLibrariesRenderLease) -> None: + for camera in lease._auxiliary_cameras: + if camera.spec is None or camera.render_data is None: + raise RuntimeError("Selected auxiliary camera was not prepared") + self._renderer._register_camera(camera.spec, camera.render_data) + camera.product_path = camera.render_data.render_product_path + if lease._auxiliary_cameras: + self._renderer.scene.commit() + + def _update_cameras(self) -> None: + lease = self.active + if lease is None: + return + for camera in lease._auxiliary_cameras: + if camera.state is None or camera.render_data.camera_xform_binding is None: + raise RuntimeError("Auxiliary camera has no live world-pose sample") + self._renderer._update_camera(*camera.state) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 3138cf964e8a..921b640bc1a0 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -62,8 +62,23 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, start = end self._device = str(device) self._event = wp.Event(device=self._device) + self._pending_groups = groups + self._resources = contextlib.ExitStack() + if renderer._initialized_scene: + self._compile_addresses(renderer) + + @property + def has_pending_updates(self) -> bool: + """Whether material changes require a scene publication even without a physics step.""" + return bool(self._dirty_channels) + + def _compile_addresses(self, renderer: OVRTXRenderer) -> None: + """Bind addresses only after the leased or ordinary scene has been populated.""" + if not renderer._initialized_scene: + raise RuntimeError("Visual materials require a populated renderer scene before publication.") + addresses = [] with contextlib.ExitStack() as resources: - for channel, attribute_name, shader_paths, rows, dtype, shape in groups: + for channel, attribute_name, shader_paths, rows, dtype, shape in self._pending_groups: if renderer._use_ovstage: address = resources.enter_context(renderer.scene.query(shader_paths)) else: @@ -76,8 +91,10 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, flags=BindingFlag.OPTIMIZE, ) resources.callback(address.unbind) - self._addresses.append((channel, address, attribute_name, rows)) + addresses.append((channel, address, attribute_name, rows)) self._resources = resources.pop_all() + self._addresses = addresses + self._pending_groups = [] def __call__(self, material_offsets: dict[str, Any] | None = None, env_ids: Any | None = None) -> None: """Mark channels dirty; OVRTX currently copies each dirty channel's full device buffer.""" @@ -92,6 +109,8 @@ def publish(self) -> None: if not channels: return renderer = self._renderer_ref() + if self._pending_groups: + self._compile_addresses(renderer) operations = [] try: for channel, address, attribute_name, rows in self._addresses: @@ -152,5 +171,6 @@ def close(self) -> None: logger.warning("Error draining in-flight render before material release: %s", error) finally: self._addresses.clear() + self._pending_groups.clear() self._dirty_channels.clear() self._buffers.clear() diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index 77bb368235cf..ba58f153bbae 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -184,11 +184,12 @@ def __init__(self, cfg: OvstageBackendCfg): self.clone_env_paths: list[str] = [] self.population_env_paths: list[str] = [] self.clone_positions: np.ndarray | None = None + self._render_lease_owner = None self.resources = contextlib.ExitStack() """Release borrowers before paths and stage, in reverse registration order.""" self.reset() - def populate(self, usda: str) -> None: + def populate(self, usda: str, *, population_desc: ovstage.population.Desc | None = None) -> None: """Import the exported prototypes and apply the clone context's prepared operations. Population commits ordinal 1. Consumers may then author render products and poses @@ -196,12 +197,18 @@ def populate(self, usda: str) -> None: Args: usda: USD scene containing the routed prototypes and their materials. + population_desc: Optional native property selection for this scene population. """ if self.ordinal: return # Ordinal 0 is the empty state; population and cloning form the first committed write. - ovstage.population.open_usd_from_string(self.stage, usda, ordinal=1, domains=self.cfg.population_domains) + if population_desc is None: + ovstage.population.open_usd_from_string(self.stage, usda, ordinal=1, domains=self.cfg.population_domains) + else: + ovstage.population.open_usd_from_string_with_desc( + self.stage, usda, ordinal=1, time=float("nan"), descs=population_desc + ) ovstage_replicate( self.stage, self.paths, self.clone_copies, self.clone_env_paths, self.clone_positions, ordinal=1 ) @@ -233,12 +240,22 @@ def commit(self) -> int: return ordinal def close(self) -> None: - """Release borrowers, then the path dictionary and stage.""" + """Release borrowers, then the path dictionary and stage. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy or reset a stage with an active render lease") self.resources.close() self.stage = self.paths = None def reset(self) -> None: - """Recreate the native stage after all consumers detach, retaining the prepared clone operations.""" + """Recreate the native stage after all consumers detach, retaining the prepared clone operations. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ self.close() self.ordinal = self.next_camera_id = 0 with contextlib.ExitStack() as resources: @@ -255,6 +272,7 @@ def export_stage_to_string( *, env_paths: Sequence[str] = (), preserve_references: bool = False, + preserve_partition_visibility: bool = False, ) -> str: """Export routed prototypes without mutating the simulation's authored USD stage. @@ -271,6 +289,9 @@ def export_stage_to_string( preserve_references: Retain clean file-backed references and instances instead of baking their geometry into text. Anonymous or unsaved referenced layers require full composition so that the returned string does not depend on their Python lifetime. + preserve_partition_visibility: Keep an authored ``showAllPartitionsByDefault`` value instead + of forcing every scene partition visible to perspective cameras. Leased exports set this + so that an external presenter's scene partition can hide meshes from its cameras. Returns: USDA text containing the selected prototypes. By default, composition arcs are baked. @@ -279,13 +300,16 @@ def export_stage_to_string( export_session.subLayerPaths = [stage.GetSessionLayer().identifier] export_stage = Usd.Stage.Open(stage.GetRootLayer(), export_session) export_stage.SetEditTarget(export_session) - # Perspective cameras see every partition; sensor cameras retain their explicit partition. + # Perspective cameras see every partition, unless a leased export preserves an authored value + # (the XR scene partition hides excluded meshes this way); sensor cameras retain their partition. settings_path = export_stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" settings = UsdRender.Settings.Define(export_stage, settings_path) settings.GetPrim().AddAppliedSchema("OmniRtxSettingsGlobalCommonAPI_1") - settings.GetPrim().CreateAttribute( + show_all = settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ).Set(True) + ) + if not (preserve_partition_visibility and show_all.HasAuthoredValue()): + show_all.Set(True) export_stage.SetMetadata("renderSettingsPrimPath", settings_path) prim_paths = [] if len(clone_env_paths) > 1: diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py index dcc61931efad..b382280a0870 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py @@ -555,6 +555,31 @@ def test_scene_settings(monkeypatch, manager, device, rate, override, expected): assert prim.GetAttribute("physxScene:enableSceneQuerySupport").Get() is sim_cfg.enable_scene_query_support +def test_scene_contact_settings_are_authored_only_when_selected(monkeypatch, manager): + """Optional contact tuning reaches the scene prim; unset fields keep the stage value.""" + from isaaclab_ov.physics.ovphysx_manager_cfg import OvPhysxCfg + + from pxr import Usd + + monkeypatch.setattr( + manager, "_sim", SimpleNamespace(cfg=SimpleNamespace(dt=1.0 / 60.0, enable_scene_query_support=False)) + ) + stage = Usd.Stage.CreateInMemory() + default_prim = stage.DefinePrim("/World/DefaultScene", "PhysicsScene") + # Contact settings are authored the same way on every device, so one device covers them. + manager._configure_physx_scene_prim(default_prim, OvPhysxCfg(), "cpu") + for attr in ("bounceThreshold", "frictionCorrelationDistance", "solveArticulationContactLast"): + assert not default_prim.GetAttribute(f"physxScene:{attr}").HasAuthoredValue() + prim = stage.DefinePrim("/World/PhysicsScene", "PhysicsScene") + cfg = OvPhysxCfg( + bounce_threshold_velocity=0.01, friction_correlation_distance=0.00625, solve_articulation_contact_last=True + ) + manager._configure_physx_scene_prim(prim, cfg, "cpu") + assert prim.GetAttribute("physxScene:bounceThreshold").Get() == pytest.approx(0.01) + assert prim.GetAttribute("physxScene:frictionCorrelationDistance").Get() == pytest.approx(0.00625) + assert prim.GetAttribute("physxScene:solveArticulationContactLast").Get() is True + + @pytest.mark.parametrize(("device", "gpu_dynamics", "broadphase"), [("cpu", False, "MBP"), ("gpu", True, "GPU")]) def test_scenes_author_device_dynamics_and_broadphase(monkeypatch, manager, device, gpu_dynamics, broadphase): """Every physics scene selects the simulation device, since CPU and GPU scenes share one process.""" diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index 91d0bbc7d634..1d85cb0d570d 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -94,6 +94,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: clone_positions=None, render_products=set(), cfg=SimpleNamespace(read_gpu_transforms=False), + _render_lease_owner=None, ) renderer.scene = renderer.backend renderer.backend.renderer = SimpleNamespace( @@ -104,7 +105,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: write_array_attribute=lambda *args, **kwargs: None, write_attribute=lambda *args, **kwargs: None, ) - renderer._warp_device = SimpleNamespace(ordinal=0) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True) renderer.scene.next_camera_id = 0 return renderer @@ -284,6 +285,7 @@ def test_create_render_data_writes_combined_stage_on_the_simulation_device(tmp_p class Device: ordinal = 1 + is_cuda = True def __str__(self): return "cuda:1" diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 529ab0a469b5..edc27f8e13be 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -42,11 +42,11 @@ def _simulation(monkeypatch): def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) - renderer.backend = SimpleNamespace(renderer=MagicMock()) + renderer.backend = SimpleNamespace(renderer=MagicMock(), _render_lease_owner=None) renderer.scene = renderer.backend renderer.scene.query = OvstageBackend.query.__get__(renderer.scene) renderer._device = "cpu" - renderer._warp_device = SimpleNamespace(ordinal=0, stream=SimpleNamespace(cuda_stream=99)) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True, stream=SimpleNamespace(cuda_stream=99)) return renderer, renderer.backend.renderer @@ -57,7 +57,7 @@ def test_geometry_bindings_follow_mixed_sdp_points_and_pointer_swaps(mode): renderer.cfg.async_rendering = mode == "async" use_ovstage = mode == "ovstage" renderer._use_ovstage = use_ovstage - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=42)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=42)) points = { path: wp.array(np.arange(count * 3, dtype=np.float32).reshape(count, 3), dtype=wp.vec3f, device="cpu") for path, count in ( @@ -189,7 +189,7 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc stage = Usd.Stage.CreateInMemory() UsdGeom.Xform.Define(stage, paths[0]).AddScaleOp().Set(Gf.Vec3d(2, 3, 4) if scaled else Gf.Vec3d(1)) renderer.scene.clone_copies = [(paths[0], [])] - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=0)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=0)) renderer._use_ovstage = use_ovstage renderer.scene.ordinal = 5 writes = [] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index df07df7d221a..95358da1ed14 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -37,7 +37,7 @@ ] if not _MISSING_MODULES: - from isaaclab_ov.renderers import OVRTXBackendCfg, OVRTXRendererCfg # noqa: E402 + from isaaclab_ov.renderers import OvLibrariesScenePublisher, OVRTXBackendCfg, OVRTXRendererCfg # noqa: E402 from isaaclab_ov.renderers import ovrtx_renderer as ovrtx_renderer_module # noqa: E402 from isaaclab_ov.renderers.ovrtx_renderer import ( # noqa: E402 _DISABLE_LINUX_CUDA_CPU_SYNC_ENV, @@ -81,6 +81,7 @@ def _make_ovrtx_camera_render_data() -> OVRTXCameraRenderData: def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) renderer.backend = OVRTXBackend.__new__(OVRTXBackend) + renderer.backend._render_lease_owner = None renderer.scene = renderer.backend from isaaclab_ov.stage import OvstageBackend @@ -1041,7 +1042,7 @@ def test_ovrtx_render_var_sync_rejects_non_boolean_values(monkeypatch, value): @pytest.mark.parametrize( ("gpu_side", "cuda_stream", "expected_stream", "consumer_fails"), - [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False)], + [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False), (False, None, None, True)], ) def test_ovrtx_map_render_var_orders_the_read_against_render_completion( monkeypatch, gpu_side, cuda_stream, expected_stream, consumer_fails @@ -1049,15 +1050,17 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( """Order reads after rendering and release after reads, including when the consumer raises.""" events = [] mapping = types.SimpleNamespace( - wait=lambda: events.append("wait"), unmap=lambda *, stream: events.append(("release", stream)) + wait=lambda: events.append("wait"), unmap=lambda *, stream=None: events.append(("release", stream)) ) render_var = types.SimpleNamespace(map=MagicMock(return_value=mapping)) sentinel = object() monkeypatch.setattr(ovrtx_renderer_module, "_gpu_side_render_var_sync_enabled", lambda: gpu_side) - monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: sentinel) + monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: events.append("wrap") or sentinel) renderer = _make_ovrtx_renderer_without_backend() - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=cuda_stream)) + renderer._warp_device = types.SimpleNamespace( + is_cuda=cuda_stream is not None, stream=types.SimpleNamespace(cuda_stream=cuda_stream) + ) with pytest.raises(ValueError, match="consumer failed") if consumer_fails else contextlib.nullcontext(): with renderer._map_render_var_to_dlpack(render_var) as array: assert array is sentinel @@ -1065,8 +1068,11 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( if consumer_fails: raise ValueError("consumer failed") - assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) - assert events == ([] if gpu_side else ["wait"]) + ["consume", ("release", expected_stream)] + if cuda_stream is None: + assert render_var.map.call_args.kwargs == {"device": ovrtx_renderer_module.Device.CPU} + else: + assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) + assert events == ([] if gpu_side else ["wait"]) + ["wrap", "consume", ("release", expected_stream)] @pytest.mark.parametrize( @@ -1225,7 +1231,7 @@ def handle(raw_id): assert not vars(renderer).keys() & {"_camera_xform_binding", "_camera_xform_query", "_camera_paths_list"} monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(wp, "synchronize_stream", lambda stream: None) - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=99)) + renderer._warp_device = types.SimpleNamespace(is_cuda=True, stream=types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(ovrtx_renderer_module, "xform_tensor_from_warp", lambda value: value) positions = wp.array([[1, 2, 3], [4, 5, 6]], dtype=wp.vec3f, device="cpu") rotations = np.array([[1, 2, 3, 4], [0, 0, 0, 1]], dtype=np.float32) @@ -1249,6 +1255,7 @@ def handle(raw_id): renderer.update_camera_intrinsics(cameras[1], wp.zeros(2, dtype=wp.mat33f, device="cpu"), parameters) if use_ovstage: assert renderer.scene.stage.write_attributes.call_args.args[0] is cameras[1].camera_xform_binding + assert all(desc.cuda_stream is None for desc in renderer.scene.stage.write_attributes.call_args.args[1]) bound_paths = [call.args[0] for call in renderer.scene.paths.create_path_list_from_strings.call_args_list] assert bound_paths == [ entry @@ -1307,6 +1314,7 @@ def query(paths): events.append(f"release_query:{paths}") renderer.scene = OvstageBackend.__new__(OvstageBackend) + renderer.scene._render_lease_owner = None SimulationContext.instance()._backend_registry.insert( 0, (OvstageBackendCfg(scene_key=renderer.cfg), renderer.scene) ) @@ -1449,3 +1457,711 @@ def attach(native_stage): assert events.count("disable-camera-reads") == (needs_fallback and stage_mode != "legacy") if stage_mode != "legacy": assert events[-2:] == ["commit", "attach"] + + +@pytest.fixture +def leased_renderer(monkeypatch): + """Exercise real deferred resource acquisition with native SDK endpoints doubled.""" + import isaaclab_ov.stage as stage_module + + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + native, stage, paths = MagicMock(), MagicMock(), MagicMock() + floor = stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 0 + + def advance(*, ordinal, **kwargs): + return types.SimpleNamespace(wait=lambda: setattr(floor.fetch, "return_value", ordinal)) + + stage.advance_write_floor.side_effect = advance + monkeypatch.setattr(ovrtx_renderer_module, "Renderer", lambda _cfg: native) + monkeypatch.setattr(ovrtx_renderer_module, "redirect_shader_cache", lambda _cfg: None) + monkeypatch.setattr(stage_module, "create_ovstage", lambda _name: contextlib.nullcontext(stage)) + monkeypatch.setattr(stage_module.ovstage, "PathDictionary", lambda _: contextlib.nullcontext(paths)) + monkeypatch.setattr(stage_module.ovstage.population, "open_usd_from_string", lambda *args, **kwargs: None) + renderer = SimulationContext.instance().get_or_create_backend(OVRTXRendererCfg()) + assert renderer.backend is renderer.scene is None + return renderer + + +def _prepare_leased_renderer(renderer, *, bind=True): + from isaaclab_ov.cloner import OvrtxReplicateContext + + OvrtxReplicateContext.prepare(renderer._sim, {}) + if bind: + renderer.prepare_stage(renderer._sim.stage, 1) + + +def test_render_lease_borrows_registry_pair_and_blocks_shared_users(leased_renderer, monkeypatch): + renderer = leased_renderer + shared = SimulationContext.instance().get_or_create_backend(renderer.cfg) + assert shared is renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="already active"): + shared.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + with pytest.raises(RuntimeError, match="Lease"): + shared.render_batch((object(),)) + shared.render_batch(()) + with monkeypatch.context() as patch: + patch.setattr(ovrtx_renderer_module, "OVRTXMesh", lambda *_: pytest.fail("constructed a mesh while leased")) + with pytest.raises(RuntimeError, match="log runtime meshes"): + shared.log_mesh("blocked") + assert "blocked" not in shared.meshes + with pytest.raises(RuntimeError, match="active render lease"): + SimulationContext.instance().close_backend(renderer.backend) + with pytest.raises(RuntimeError, match="active render lease"): + renderer.scene.reset() + native = lease.ovrtx_renderer + assert native is shared.backend.renderer + assert lease.ovstage is shared.scene.stage + lease.release() + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene._render_lease_owner is None + native.destroy.assert_not_called() + SimulationContext.instance().close_backend(renderer.backend) + native.destroy.assert_called_once_with() + + +def test_render_lease_rejects_acquisition_after_clone_preparation(leased_renderer): + renderer = leased_renderer + _prepare_leased_renderer(renderer, bind=False) + with pytest.raises(RuntimeError, match="before clone preparation"): + renderer.acquire_ov_libraries_render_lease() + assert renderer.backend._render_lease_owner is None + + +def test_render_lease_rejects_calibration_before_scene_access(leased_renderer, monkeypatch): + """An ordinary calibration write cannot bypass the external owner's publication lane.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + camera = _make_ovrtx_camera_render_data() + intrinsics = wp.zeros(2, dtype=wp.mat33f, device="cpu") + parameters = wp.zeros((5, 2), dtype=wp.float32, device="cpu") + drain = MagicMock(return_value=[]) + monkeypatch.setattr(renderer, "drain_pending_renders", drain) + monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) + try: + with pytest.raises(RuntimeError, match="Lease"): + renderer.update_camera_intrinsics(camera, intrinsics, parameters) + drain.assert_not_called() + assert renderer.backend is renderer.scene is None + finally: + lease.release() + + +def test_render_lease_owner_transfer_is_single_use(leased_renderer): + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(None) + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(token) + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._render_lease._owner_thread is thread + assert renderer.backend is renderer.scene is None + + +@pytest.fixture +def complete_scene_publication(leased_renderer, monkeypatch): + """Exercise the application publisher with native component endpoints doubled.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + lease._external_camera_scene_partitions = (("/Camera/left", "env"), ("/Camera/right", "env")) + lease._external_camera_scene_partitions_pending = True + stage = renderer.scene.stage + stage.get_attribute_write_floor.reset_mock() + stage.advance_write_floor.reset_mock() + state = types.SimpleNamespace( + renderer=renderer, + lease=lease, + publisher=publisher, + ordinal=23, + events=[], + failure=None, + cause=ValueError("component endpoint failed"), + initial_id=renderer._render_lease._last_step_id, + ) + + def component(name): + state.events.append((name, renderer.scene.ordinal)) + if state.failure == name: + raise state.cause + + writer = types.SimpleNamespace( + has_pending_updates=False, + publish=lambda: component("material"), + drain=lambda: component("material-drain"), + ) + monkeypatch.setattr(renderer, "_visual_material_writer_ref", lambda: writer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: component("transform")) + monkeypatch.setattr(renderer, "_update_geometries", lambda: component("geometry")) + monkeypatch.setattr(renderer._render_lease, "_update_cameras", lambda: component("auxiliary")) + + def partitions(selected, ordinal): + assert selected is lease + assert selected._external_camera_scene_partitions == (("/Camera/left", "env"), ("/Camera/right", "env")) + assert ordinal == renderer.scene.ordinal + component("partition") + + monkeypatch.setattr(renderer._render_lease, "_write_camera_partitions", partitions) + state.publish = lambda step_id="step": publisher.write_complete_scene(state.ordinal, step_id) + try: + yield state + finally: + lease.release() + + +@pytest.mark.parametrize("pending", [False, True]) +def test_render_lease_complete_scene_components_share_the_supplied_ordinal(complete_scene_publication, pending): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + lease._external_camera_scene_partitions_pending = pending + assert state.publisher.publication_required("step") + assert state.publish() is None + names = ["material", "transform", "geometry", "auxiliary"] + (["partition"] if pending else []) + ["material-drain"] + assert state.events == [(name, state.ordinal) for name in names] + assert renderer._render_lease._last_step_id == "step" + assert not lease._external_camera_scene_partitions_pending + assert renderer.scene.ordinal == state.ordinal + assert not state.publisher.publication_required("step") + with pytest.raises(RuntimeError, match="publication is not required"): + state.publish() + assert state.events == [(name, state.ordinal) for name in names] + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + renderer.scene.stage.advance_write_floor.assert_not_called() + assert not renderer._render_lease._poisoned + + +@pytest.mark.parametrize("failure", ["material", "transform", "geometry", "auxiliary", "partition", "material-drain"]) +def test_render_lease_complete_scene_failure_preserves_commit_boundary(complete_scene_publication, failure): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + state.failure = failure + with pytest.raises(RuntimeError, match="Producer scene publication failed") as caught: + state.publish() + assert caught.value.__cause__ is state.cause + order = ["material", "transform", "geometry", "auxiliary", "partition", "material-drain"] + expected = order[: order.index(failure) + 1] + if failure != "material-drain": + expected.append("material-drain") + assert state.events == [(name, state.ordinal) for name in expected] + assert renderer._render_lease._last_step_id == state.initial_id + assert lease._external_camera_scene_partitions_pending + assert renderer.scene.ordinal == state.ordinal + assert renderer._render_lease._poisoned + with pytest.raises(RuntimeError, match="poisoned"): + _ = lease.ovstage + previous_events = list(state.events) + with pytest.raises(RuntimeError, match="unavailable|poisoned"): + state.publish("next-step") + assert state.events == previous_events + renderer.scene.stage.advance_write_floor.assert_not_called() + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + + +def test_render_lease_complete_scene_public_authority_precedes_components(complete_scene_publication): + import threading + + state = complete_scene_publication + errors = [] + + def foreign_thread(): + try: + state.publish() + except BaseException as exc: + errors.append(exc) + + thread = threading.Thread(target=foreign_thread) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert "thread" in str(errors[0]) + with pytest.raises(TypeError, match="physics_step_id must be hashable"): + state.publish([]) + with pytest.raises(ValueError, match="ordinal must be nonzero"): + state.publisher.write_complete_scene(0, "step") + state.publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + state.publish() + assert not state.events + state.renderer.scene.stage.get_attribute_write_floor.assert_not_called() + state.renderer.scene.stage.advance_write_floor.assert_not_called() + assert not state.renderer._render_lease._poisoned + + +def test_render_lease_closed_publisher_reports_no_pending_publication(leased_renderer, monkeypatch): + """A closed publisher must not report a write that :meth:`write_complete_scene` would then reject.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + assert isinstance(publisher, OvLibrariesScenePublisher) + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: None) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + assert publisher.publication_required("step-1") + publisher.close() + publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.publication_required("step-1") + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.write_complete_scene(18, "step-1") + lease.release() + + +def test_render_lease_poison_blocks_publication_but_not_retirement(leased_renderer, monkeypatch): + """A failed producer write must not prevent the borrower from retiring the pair (Capture's close order).""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", MagicMock(side_effect=ValueError("write failed"))) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + with pytest.raises(RuntimeError, match="scene publication failed"): + publisher.write_complete_scene(19, "step-1") + with pytest.raises(RuntimeError, match="unavailable"): + publisher.publication_required("step-2") + with pytest.raises(RuntimeError, match="poisoned"): + lease.validate_live() + # OvxrFramePump.close: close publication, destroy the OVXR context inside the owner lane, release the lease. + publisher.close() + with lease.ovstage_owner_lane(): + pass + ordinal = renderer.scene.ordinal + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene.ordinal == ordinal # The abandoned ordinal is not rebased onto a sealed floor. + with pytest.raises(RuntimeError, match="poisoned this OVRTX/OVStage pair; recreate it"): + renderer.render_batch((object(),)) + with pytest.raises(RuntimeError, match="recreate it"): + renderer.update_transforms() + renderer.close() + + +def test_render_lease_release_rebases_ordinal_and_restores_ordinary_use(leased_renderer): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + floor = renderer.scene.stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 41 + lease.release() + assert renderer.scene.ordinal == 42 + renderer.update_transforms() + renderer.update_geometries() + + +@pytest.mark.parametrize( + ("variable", "value"), [("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0"), ("ISAAC_LAB_SHARE_OVSTAGE", "1")] +) +def test_render_lease_rejects_explicit_stage_opt_out(leased_renderer, monkeypatch, variable, value): + monkeypatch.setenv(variable, value) + with pytest.raises(RuntimeError, match="OVSTAGE=1|private rendering OVStage"): + leased_renderer.acquire_ov_libraries_render_lease() + + +def test_render_lease_defers_application_thread_stage_until_owner_transfer(leased_renderer): + """Camera initialization prepares every renderer on the application thread before XR transfers ownership.""" + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + lease.defer_stage_until_owner_transfer() + _prepare_leased_renderer(renderer, bind=False) + context = renderer._sim.render_context + context.ensure_prepare_stage(renderer._sim.stage, 1) + assert not renderer._initialized_scene + assert renderer._render_lease.stage_deferred + # The deferred call must leave the pair eligible for the XR worker handoff. + publisher = lease.capture_scene_publisher() + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + assert not lease.materialize_deferred_stage(renderer._sim.stage, 1) + lease.validate_live() + publisher.close() + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._initialized_scene + + +def test_render_lease_eager_scene_updates_defer_to_the_publisher(leased_renderer, monkeypatch): + """``lazy_sensor_update=False`` refreshes every renderer; the leased renderer's publisher owns those writes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + transforms, geometries = MagicMock(), MagicMock() + monkeypatch.setattr(renderer, "_update_transforms", transforms) + monkeypatch.setattr(renderer, "_update_geometries", geometries) + renderer._sim.render_context.update_scene_state(1) + transforms.assert_not_called() + geometries.assert_not_called() + lease.release() + renderer._sim.render_context.update_scene_state(2) + transforms.assert_called_once_with() + geometries.assert_called_once_with() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA material buffers") +@pytest.mark.parametrize("deferred", [False, True]) +def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch, deferred): + """Real material composition and the lease publisher must reach native writes, including before physics changes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + context = renderer._sim.render_context + if deferred: + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = "/World/Camera" + lease.configure_auxiliary_camera(cfg) + # Real lifecycle: clone preparation, then environment reset finalizes material consumers, + # then the XR borrower binds the Stage. + _prepare_leased_renderer(renderer, bind=False) + if deferred: + # Camera initialization requests the Stage on the application thread; the lease defers it. + renderer.prepare_stage(renderer._sim.stage, 1) + material = types.SimpleNamespace( + channels=("color",), + is_per_env=False, + num_instances=1, + _material_paths=("/Looks/Feedback",), + _shader_paths=("/Looks/Feedback/Shader",), + _input_names={"color": "diffuseColor"}, + _offsets={}, + _values={"color": torch.tensor([[1.0, 0.0, 0.0]], device="cuda")}, + ) + context.register_visual_material(material) + context.finalize_consumers([]) + assert renderer._visual_material_writer_ref is not None + if deferred: + spec = CameraRenderSpec(cfg, "cpu", 1, (cfg.prim_path,), 1) + data = renderer.create_render_data(spec) + monkeypatch.setattr(ovrtx_renderer_module.ovstage.population, "add_usd_reference_from_string", lambda *a: 1) + monkeypatch.setattr(ovrtx_renderer_module.ovstage.population, "apply_usd_changes", lambda *a, **kw: None) + token = lease.offer_owner_transfer() + lease.claim_owner_transfer(token) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + renderer.update_camera( + data, + wp.zeros(1, dtype=wp.vec3f, device="cpu"), + wp.array([[0.0, 0.0, 0.0, 1.0]], dtype=wp.quatf, device="cpu"), + None, + ) + else: + renderer.prepare_stage(renderer._sim.stage, 1) + stage = renderer.scene.stage + try: + for ordinal, color in ((23, [0.0, 1.0, 0.0]), (24, [0.0, 0.0, 1.0])): + context.write_visual_materials([material], {"color": torch.tensor([color], device="cuda")}) + assert publisher.publication_required("unchanged-physics") + stage.write_attribute.reset_mock() + completion = MagicMock() + stage.write_attribute.return_value = completion + publisher.write_complete_scene(ordinal, "unchanged-physics") + writes = [c for c in stage.write_attribute.call_args_list if c.args[1] == "inputs:diffuseColor"] + assert len(writes) == 1 + assert writes[0].kwargs["ordinal"] == ordinal + torch.testing.assert_close(writes[0].kwargs["tensors"].cpu(), torch.tensor([color])) + assert completion.wait.called + assert not publisher.publication_required("unchanged-physics") + finally: + lease.release() + for writer in context._visual_material_writers: + writer.close() + + +@pytest.mark.parametrize("use_ovstage, asynchronous", [(True, False), (False, False), (False, True)]) +def test_cpu_scene_publication_uses_host_tensors_without_a_cuda_stream(use_ovstage, asynchronous): + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = use_ovstage + renderer.cfg = OVRTXRendererCfg(async_rendering=asynchronous) + renderer._warp_device = wp.get_device("cpu") + renderer._device = "cpu" + renderer._object_xform_binding = MagicMock() + renderer._geometry_points_binding = MagicMock() + renderer._transforms_timestamp = renderer._geometry_timestamp = None + renderer._object_scales = None + renderer._geometry_paths = ["/World/Object"] + matrices = wp.array(np.eye(4, dtype=np.float64)[None], dtype=wp.mat44d, device="cpu") + points = wp.array([[1.0, 2.0, 3.0]], dtype=wp.vec3f, device="cpu") + renderer._transform_writes = _AsyncWriteBuffers([types.SimpleNamespace(matrices=matrices)]) + renderer._geometry_writes = _AsyncWriteBuffers([wp.empty_like(points)]) + renderer._geometry_offsets = {"/World/Object": 0} + + def get_points(*, output=None, offsets=None): + if output is None: + return {"/World/Object": points} + wp.copy(output, points) + return None + + renderer._sdp = types.SimpleNamespace( + backend=types.SimpleNamespace(transforms_timestamp=1, geometry_timestamp=1), + get_transforms=lambda *_args, **_kwargs: True, + get_geometry_points=get_points, + ) + stage = MagicMock() + renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) + renderer.update_transforms() + renderer.update_geometries() + bindings = (renderer._object_xform_binding, renderer._geometry_points_binding) + writes = [getattr(binding, "write_async" if asynchronous else "write") for binding in bindings] + calls = stage.write_attribute.call_args_list if use_ovstage else [write.call_args for write in writes] + assert len(calls) == 2 and all(c.kwargs["cuda_stream"] is None for c in calls) + if use_ovstage: + assert [c.args[1] for c in calls] == ["omni:xform", "points"] + assert all(c.kwargs["ordinal"] == 11 for c in calls) + else: + np.testing.assert_array_equal(calls[0].args[0].numpy(), matrices.numpy()) + np.testing.assert_array_equal(calls[1].args[0][0].numpy(), points.numpy()) + # A repeated publication of unchanged state must not duplicate either write. + renderer.update_transforms() + renderer.update_geometries() + assert stage.write_attribute.call_count == 2 if use_ovstage else all(write.call_count == 1 for write in writes) + # Draining the retained buffers waits for previous writes without a CUDA stream. + renderer._transform_writes.close() + renderer._geometry_writes.close() + + +@pytest.mark.parametrize("use_ovstage", [True, False]) +def test_cpu_camera_pose_updates_use_host_tensors_without_a_cuda_stream(use_ovstage): + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = use_ovstage + renderer.cfg = OVRTXRendererCfg() + renderer._warp_device = wp.get_device("cpu") + renderer._device = "cpu" + stage = MagicMock() + renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) + camera = _make_ovrtx_camera_render_data() + camera.camera_xform_binding = MagicMock() + camera.camera_writes = _AsyncWriteBuffers([wp.empty(2, dtype=wp.mat44d, device="cpu")]) + positions = wp.array([[1.0, 2.0, 3.0]] * 2, dtype=wp.vec3f, device="cpu") + orientations = wp.array([[0.0, 0.0, 0.0, 1.0]] * 2, dtype=wp.quatf, device="cpu") + update = renderer.update_camera + update(camera, positions, orientations, None) + call = stage.write_attribute.call_args if use_ovstage else camera.camera_xform_binding.write_async.call_args + assert call.kwargs["cuda_stream"] is None + if use_ovstage: + assert call.args == (camera.camera_xform_binding, "omni:xform") + assert call.kwargs["ordinal"] == 11 + else: + # Reusing the camera ring waits for previous writes without a CUDA stream. + update(camera, positions, orientations, None) + update(camera, positions, orientations, None) + camera.camera_writes.close() + + +@pytest.mark.parametrize("cuda_ordinal", [None, 2]) +def test_camera_registration_authors_valid_render_device_selection(monkeypatch, cuda_ordinal): + """CPU camera consumers leave GPU selection automatic; CUDA consumers retain their device.""" + from pxr import Usd + + class AuthoredProduct(Exception): + pass + + renderer = _make_ovrtx_renderer_without_backend() + renderer.cfg = OVRTXRendererCfg() + + class Device: + is_cuda = cuda_ordinal is not None + ordinal = -1 if cuda_ordinal is None else cuda_ordinal # Warp reports ordinal -1 for the CPU. + + def __str__(self): + return f"cuda:{cuda_ordinal}" if self.is_cuda else "cpu" + + renderer._warp_device = Device() + renderer._device = str(renderer._warp_device) + monkeypatch.setattr(ovrtx_renderer_module.wp, "get_device", lambda _device: renderer._warp_device) + renderer.scene.clone_env_paths = [] + renderer.drain_pending_renders = lambda: [] + spec = types.SimpleNamespace( + cfg=_make_camera_cfg(["rgb"]), + device=renderer._device, + num_instances=1, + camera_prim_paths=["/World/envs/env_0/Camera"], + render_settings={}, + ) + build_product = ovrtx_renderer_module.build_render_product_as_string + + def consume_product(spec, render_data, **kwargs): + stage = Usd.Stage.CreateInMemory() + assert stage.GetRootLayer().ImportFromString(build_product(spec, render_data, **kwargs)) + product = stage.GetPrimAtPath(render_data.render_product_path) + devices = product.GetAttribute("deviceIds") + if cuda_ordinal is None: + assert not devices.HasAuthoredValueOpinion() + else: + assert list(devices.Get()) == [cuda_ordinal] + # Native population is outside this parser/selection regression. + raise AuthoredProduct + + monkeypatch.setattr(ovrtx_renderer_module, "build_render_product_as_string", consume_product) + with pytest.raises(AuthoredProduct): + renderer.create_render_data(spec) + + +def test_leased_camera_cleanup_waits_for_lease_release(leased_renderer, monkeypatch): + """Releasing a leased camera would remove its product from the borrowed Stage; it waits for release.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = "/World/Camera" + lease.configure_auxiliary_camera(cfg) + _prepare_leased_renderer(renderer, bind=False) + renderer.prepare_stage(renderer._sim.stage, 1) + data = renderer.create_render_data(CameraRenderSpec(cfg, "cpu", 1, (cfg.prim_path,), 1)) + population = ovrtx_renderer_module.ovstage.population + removed = [] + monkeypatch.setattr(population, "add_usd_reference_from_string", lambda *args: 7) + monkeypatch.setattr(population, "apply_usd_changes", lambda *args, **kwargs: None) + monkeypatch.setattr(population, "remove_usd", lambda _stage, reference: removed.append(reference)) + lease.claim_owner_transfer(lease.offer_owner_transfer()) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + + with pytest.raises(RuntimeError, match="release camera render data while an OvLibrariesRenderLease is active"): + renderer.cleanup(data) + assert removed == [] + assert data in renderer._camera_render_data + assert data.camera_xform_binding is not None + + lease.release() + renderer.close() + assert removed == [7] + assert data not in renderer._camera_render_data + + +def test_selected_external_camera_order_is_independent_of_sensor_preparation(leased_renderer): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + specs = [] + for name, hz in zip(("Head", "Left", "Right", "Overview"), (20, 10, 10, 5), strict=True): + cfg = types.SimpleNamespace(prim_path=f"/World/{name}", width=640, height=480, data_types=("rgb",)) + lease.configure_auxiliary_camera(cfg, max_fps=hz) + specs.append(types.SimpleNamespace(cfg=cfg, num_instances=1, view_count=1, camera_prim_paths=(cfg.prim_path,))) + for spec in reversed(specs): + assert renderer._render_lease.accept_camera_spec(spec) + assert [camera.spec for camera in lease._auxiliary_cameras] == specs + assert [camera.max_fps for camera in lease._auxiliary_cameras] == [20, 10, 10, 5] + lease.release() + + +def test_external_camera_partition_validation_is_not_stereo_specific(leased_renderer, monkeypatch): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + cameras = ("/Camera/front", "/Camera/side", "/Camera/rear") + tokens = { + ("/World/envs/env_0", "primvars:omni:scenePartition"): "env_0", + **{(path, "omni:scenePartition"): "env_0" for path in cameras}, + } + monkeypatch.setattr(renderer._render_lease, "_read_partition_tokens", lambda *_: tokens) + try: + with lease.ovstage_owner_lane(): + lease.configure_external_camera_scene_partitions("/World/envs/env_0", cameras) + assert lease.validate_external_scene_partitions(1) == { + "/World/envs/env_0": "env_0", + **dict.fromkeys(cameras, "env_0"), + } + tokens[(cameras[-1], "omni:scenePartition")] = "other" + with pytest.raises(RuntimeError, match="scene-partition proof failed"): + lease.validate_external_scene_partitions(1) + finally: + lease.release() + + +def test_external_camera_schedules_reach_native_stage(): + """Leased population authors the selected cameras' native rates and retains only those properties.""" + import ovrtx + import ovstage + from isaaclab_ov.stage import OvstageBackend, OvstageBackendCfg + + ovrtx.register_schema_paths() + camera_paths = [f"/World/envs/env_0/{name}" for name in ("Head", "Wrist", "Unselected")] + # The application export has no camera schedules; the unselected camera's own rate must not populate. + source = """#usda 1.0 +def Xform "World" { + def Xform "envs" { + def Xform "env_0" { + def Camera "Head" { + } + def Camera "Wrist" { + } + def Camera "Unselected" { + double2 omni:sensor:frameRate = (999, 1) + } + } + } +} +""" + renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) + lease = renderer.acquire_ov_libraries_render_lease() + with contextlib.ExitStack() as resources: + resources.callback(lease.release) + for path, max_fps in zip(camera_paths[:2], (20.0, 10.0), strict=True): + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = path + lease.configure_auxiliary_camera(cfg, max_fps=max_fps) + for camera in lease._auxiliary_cameras: + camera.spec = types.SimpleNamespace(camera_prim_paths=[camera.cfg.prim_path]) + renderer.scene = OvstageBackend(OvstageBackendCfg()) + resources.callback(renderer.scene.close) + renderer._render_lease.populate_stage(source) + + stage, paths = renderer.scene.stage, renderer.scene.paths + names = ("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs") + resolved = {} + plist = paths.create_path_list_from_strings(camera_paths) + try: + with stage.query_from_path_list(plist) as query: + query.wait() + with stage.read_attributes( + query, + [paths.intern_token(name) for name in names], + ovstage.OrdinalRange.latest(renderer.scene.ordinal - 1), + ) as read: + read.wait() + for group in read.groups(): + try: + values = np.asarray(group.array(0).copy()).reshape(group.data_count, -1) + group_paths = paths.get_path_strings(group.prim_list) + attribute = paths.token_to_string(group.attribute) + for index in range(min(group.prim_count, group.data_count)): + path = group_paths[group.prim_index(index)] + resolved[path, attribute] = values[group.data_row_index(index)].tolist() + finally: + stage.release_group(group) + finally: + paths.destroy_path_list(plist) + assert resolved.get((camera_paths[0], names[0])) == [20.0, 1.0] + assert resolved.get((camera_paths[1], names[0])) == [10.0, 1.0] + assert resolved.get((camera_paths[0], names[1])) == [False] + assert resolved.get((camera_paths[1], names[1])) == [False] + assert (camera_paths[2], names[0]) not in resolved diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 522a4d5d4790..7b30437e37a7 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -528,27 +528,53 @@ def test_export_stage_heterogeneous_keeps_multiple_sources(): assert 'def Camera "Camera"' not in exported -@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) -def test_export_stage_preserves_authored_scene(settings_path): - """Renderer settings and clone trimming are confined to the exported scene.""" +@pytest.mark.parametrize("preserve_partition_visibility", [False, True], ids=["forced", "preserved"]) +@pytest.mark.parametrize( + "settings_path, authored_show_all, preserved_show_all", + [ + (None, None, True), + ("/Render/AuthoredSettings", False, False), + ("/Render/AuthoredSettings", True, True), + # A value block authors no value, so the export keeps every partition visible. + ("/Render/AuthoredSettings", "blocked", True), + ], +) +def test_export_stage_preserves_authored_scene( + settings_path, authored_show_all, preserved_show_all, preserve_partition_visibility +): + """Renderer settings and clone trimming are confined to the exported scene. + + Only a partition-preserving export keeps an authored ``showAllPartitionsByDefault``; any other + export forces every partition visible. + """ num_envs = 4 stage = _make_multi_env_stage(num_envs) if settings_path is not None: settings = UsdRender.Settings.Define(stage, settings_path) settings.CreateResolutionAttr(Gf.Vec2i(1280, 720)) + show_all = settings.GetPrim().CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ) + if authored_show_all == "blocked": + show_all.Block() + else: + show_all.Set(authored_show_all) stage.SetMetadata("renderSettingsPrimPath", settings_path) before = stage.ExportToString() layer = Sdf.Layer.CreateAnonymous() layer.ImportFromString( export_stage_to_string( - stage, tuple(f"/World/envs/env_{i}" for i in range(num_envs)), source_paths=("/World/envs/env_0",) + stage, + tuple(f"/World/envs/env_{i}" for i in range(num_envs)), + source_paths=("/World/envs/env_0",), + preserve_partition_visibility=preserve_partition_visibility, ) ) exported = Usd.Stage.Open(layer) assert exported.HasAuthoredMetadata("renderSettingsPrimPath") settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) show_all = settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault") - assert show_all and show_all.Get() is True + assert show_all and show_all.Get() is (preserved_show_all if preserve_partition_visibility else True) if settings_path is not None: assert settings.GetPath() == settings_path assert settings.GetAttribute("resolution").Get() == (1280, 720) diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 6c7b0c9efc29..ca892860c012 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -23,6 +23,7 @@ if not _MISSING_MODULES: from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXRendererCfg + from isaaclab_ov.renderers.render_lease import RenderLeaseController from ovrtx import DataAccess from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -121,7 +122,8 @@ def destroy_path_list(self, path_list): def _renderer(*, use_ovstage: bool = False): events: list[str] = [] renderer = OVRTXRenderer.__new__(OVRTXRenderer) - renderer.backend = SimpleNamespace(render_products=set()) + renderer._render_lease = RenderLeaseController(renderer) + renderer.backend = SimpleNamespace(render_products=set(), _render_lease_owner=None) renderer.scene = renderer.backend renderer._initialized_scene = True renderer._needs_transform_initialization = False diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst new file mode 100644 index 000000000000..0b8bc9cb4153 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst @@ -0,0 +1,10 @@ +* Added G1 and GR1T2 kitless teleoperation setup, including portable GR1T2 kinematics, + backend-specific actuators, collision preparation, and bounded action targets. Kitless preparation + replaces the GR1T2 Pink action term with + :class:`~isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`; + Isaac Sim PhysX keeps the Pink term. +* Added public OvPhysX and Newton selection while retaining Kit defaults and supporting + standalone backend preparation without optional teleoperation configuration. + Registered construction prepares the selected backend and preserves prepared configuration copies + and serialized spawn callbacks. +* Prepared packing-table mesh collisions for standalone physics. diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst new file mode 100644 index 000000000000..6b14bcf7ecff --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -0,0 +1,10 @@ +* Changed the gym entry point of ``IsaacContrib-PickPlace-GR1T2-Abs``, + ``IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs``, ``IsaacContrib-PickPlace-Locomanipulation-G1-Abs`` + and ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` to + :class:`~isaaclab_tasks.contrib.teleop_env.HumanoidTeleopEnv`, which prepares the selected physics + backend before construction. Code that constructs the environment without that entry point should + first call :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment`, + which also applies the preset defaults to a configuration built without task resolution. +* Changed the G1 hand contact-sensor paths to name the ``left_hand``/``right_hand`` parent of the hand + links in the Isaac 6.2 ``g1.usd``. Newton matches whole body paths, so it needs that parent; Isaac Sim + PhysX and OvPhysX resolve the same links by name as before. diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.minor b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst new file mode 100644 index 000000000000..bdc60b1dcec7 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst @@ -0,0 +1,4 @@ +* Changed the SO101 IK-Abs stack task to reset the arm to its nominal joint pose without joint + noise and to use a full-pose clutch home. Policies or datasets that relied on randomized + initial arm joints should restore ``events.randomize_joint_state.params["std"]`` to its + previous value of ``0.02``. diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst new file mode 100644 index 000000000000..4ad6c82a7c1a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst @@ -0,0 +1,2 @@ +* Fixed standalone OvPhysX selection for SO101 teleoperation while retaining the Kit default and + Newton settings. The OvPhysX selection applies the stack family's grasp contact tuning. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst new file mode 100644 index 000000000000..d197eef489f5 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst @@ -0,0 +1,4 @@ +* Added fixed-base G1 wrist cameras and isolated robot head visuals for OVXR views. +* Added an ``rgb`` preset to ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` that adds the + head and wrist RGB images to its policy observations (``presets=rgb``). The default policy + observations stay state-only. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst new file mode 100644 index 000000000000..07d4286ff83a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst @@ -0,0 +1,5 @@ +* Changed Kit ``--xr`` camera panels for the fixed-base G1 task: it shows head and wrist panels, + and head-panel uploads are capped at 20 Hz (wrist panels at 10 Hz). +* Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), so + its head and wrist images are not stale after a reset. This adds render work per reset when cameras + are rendered. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.minor b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 2ccb195fb5d7..85751c594646 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -58,3 +58,51 @@ gym.register( > **Note:** As a practice, we specify all the environments in a single file to avoid name conflicts between different > tasks or environments. However, this practice is debatable and we are open to suggestions to deal with a large > scaling in the number of tasks or environments. + +## Standalone physics for GR1T2 and G1 teleoperation tasks + +For standalone OVPhysX, the fixed-base G1 task's backend preparation hook moves the +tabletop collision anchor onto its existing meshes, using the same preparation as GR1T2. +OVPhysX cooks the steering wheel's authored collision meshes. The bin's existing +primitive colliders and the object's drop termination remain enabled. +Registered GR1T2 and G1 tasks prepare the selected backend before scene construction, +including when created through `gym.make()` or `instantiate(cfg)` without the +teleoperation launcher. Newton uses its separate backend preparation. Resolve a +fresh task configuration when changing backends after preparation; copying a prepared +configuration preserves its selected backend and caller overrides. + +## Fixed-base G1 manipulation cameras + +`IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` provides three 640×480 RGB +cameras: `robot_pov_cam` at 20 Hz, and `left_wrist_cam` and `right_wrist_cam` at +10 Hz each. The head uses the calibrated G1 head-camera pose; wrist cameras +are mounted on their respective wrist-yaw links and look along wrist-local +X +with +Z up. Their offsets are configurable in the task's scene configuration. + +The cameras feed the task's XR camera panels. The policy observation space stays +state-only by default. Select `presets=rgb` to add the three images to the policy +group, for example to record demonstrations with them: + +```bash +uv run isaaclab teleop record --task IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs \ + --xr presets=rgb +``` + +`image_obs_list` names the three cameras; image consumers such as robomimic use +only the names present in the policy observations. + +The same names identify the policy image observations and the task's XR camera +feeds, with the head panel centered and wrist panels to its left and right. +In Kit, sensor `update_period` controls image sampling in simulation time; feed +`max_update_hz` caps uploads. In OVXR, each panel renders natively at the lower of +`--ovxr_pip_fps` and its feed's nonzero `max_update_hz`; `update_period` does not +apply. Actual delivery also depends on simulation and renderer throughput. Camera +sampling rates are configuration targets, not measured headset frame rates. + +With `--xr`, the teleoperation launcher's `--disable_external_cameras` option removes +these cameras and their image observations. The kitless XR launcher accepts up to +three camera names with `--ovxr_pip_camera`. Camera feeds are opt-in: without +this option, live OVXR and MCAP replay omit task cameras and PiP layers. Selected +OVXR cameras feed the panels only: their policy image observations are removed even +with `presets=rgb`, so the application does not request a second camera render. +The RGB recording example above uses Kit `--xr`. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py index eea4a34bf045..46922bb00ca4 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py @@ -12,7 +12,7 @@ gym.register( id="IsaacContrib-PickPlace-Locomanipulation-G1-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.locomanipulation_g1_env_cfg:LocomanipulationG1EnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", @@ -22,7 +22,7 @@ gym.register( id="IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.fixed_base_upper_body_ik_g1_env_cfg:FixedBaseUpperBodyIKG1EnvCfg", }, diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index e4e9a1a1e59d..08a05a0130eb 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -6,6 +6,7 @@ import isaaclab.envs.mdp as base_mdp import isaaclab.sim as sim_utils from isaaclab.assets import ArticulationCfg, AssetBaseCfg, RigidObjectCfg +from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.managers import ObservationGroupCfg as ObsGroup from isaaclab.managers import ObservationTermCfg as ObsTerm @@ -13,6 +14,7 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR @@ -20,7 +22,13 @@ from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE from isaaclab_tasks.contrib.locomanip_pick_place import mdp as locomanip_mdp from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp -from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg +from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg, robot_pov_camera_cfg +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + check_kitless_backend_preparation, + prepare_humanoid_teleop_environment, +) +from isaaclab_tasks.utils import preset from isaaclab_assets.robots.unitree import G1_29DOF_CFG @@ -261,18 +269,35 @@ class FixedBaseUpperBodyIKG1SceneCfg(InteractiveSceneCfg): # Unitree G1 Humanoid robot - fixed base configuration robot: ArticulationCfg = replace(G1_29DOF_CFG, prim_path="{ENV_REGEX_NS}/Robot") - robot_pov_cam = g1_robot_pov_camera_cfg() + # Share the calibrated head view with the locomanipulation G1 task. + robot_pov_cam = g1_robot_pov_camera_cfg().replace(update_period=1.0 / 20.0) + + # Mount above each hand, looking along wrist-local +X with +Z up. + left_wrist_cam = robot_pov_camera_cfg( + parent_prim_path="{ENV_REGEX_NS}/Robot/left_wrist_yaw_link", + offset_pos=(0.06, 0.0, 0.06), + offset_rot=(-0.5, 0.5, -0.5, 0.5), + ).replace( + prim_path="{ENV_REGEX_NS}/Robot/left_wrist_yaw_link/LeftWristCam", + update_period=1.0 / 10.0, + height=480, + width=640, + spawn=sim_utils.PinholeCameraCfg(focal_length=15.0, horizontal_aperture=20.955, clipping_range=(0.01, 5.0)), + ) + right_wrist_cam = left_wrist_cam.replace( + prim_path="{ENV_REGEX_NS}/Robot/right_wrist_yaw_link/RightWristCam", + ) # Per-hand contact sensors over all finger links, used to drive controller # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. left_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/left_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/left_hand/left_hand_[^/]*_link", update_period=0.0, history_length=3, ) right_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/right_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/right_hand/right_hand_[^/]*_link", update_period=0.0, history_length=3, ) @@ -340,8 +365,34 @@ def __post_init__(self): self.enable_corruption = False self.concatenate_terms = False + @configclass + class RgbPolicyCfg(PolicyCfg): + """Policy observations with state values and head/wrist RGB images [uint8, H x W x 3].""" + + robot_pov_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "normalize": False, + }, + ) + left_wrist_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("left_wrist_cam"), + "normalize": False, + }, + ) + right_wrist_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("right_wrist_cam"), + "normalize": False, + }, + ) + # observation groups - policy: PolicyCfg = PolicyCfg() + policy: PolicyCfg = preset(default=PolicyCfg(), rgb=RgbPolicyCfg()) @configclass @@ -374,11 +425,28 @@ class FixedBaseUpperBodyIKG1EnvCfg(ManagerBasedRLEnvCfg): controlled using upper body IK. """ + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + kitless_preparation: dict[str, str] = { + "ovphysx": ( + "isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg" + ":prepare_fixed_base_g1_for_kitless_ovphysx" + ), + "newton_mjwarp": ( + "isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg" + ":prepare_fixed_base_g1_for_kitless_newton_mjwarp" + ), + } + """Preparation function, as a ``"module:function"`` reference, for each supported kitless backend.""" + haptic_feedback: object | None = None + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: FixedBaseUpperBodyIKG1SceneCfg = FixedBaseUpperBodyIKG1SceneCfg(num_envs=1, env_spacing=2.5) # MDP settings terminations: TerminationsCfg = TerminationsCfg() observations: ObservationsCfg = ObservationsCfg() + image_obs_list: list[str] = ["robot_pov_cam", "left_wrist_cam", "right_wrist_cam"] actions: ActionsCfg = ActionsCfg() # Unused managers @@ -388,12 +456,15 @@ class FixedBaseUpperBodyIKG1EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 4 self.episode_length_s = 20.0 # simulation settings self.sim.dt = 1 / 200 # 200Hz self.sim.render_interval = 2 + self.num_rerenders_on_reset = 3 # Set the URDF and mesh paths for the IK controller urdf_omniverse_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 @@ -418,9 +489,11 @@ def __post_init__(self): self.xr = XrCfg( anchor_pos=(0.0, 0.0, -0.30), anchor_rot=(0.0, 0.0, 0.0, 1.0), + excluded_visual_meshes=[f"{DEFAULT_ENV_TEMPLATE.format(0)}/Robot/torso_link/visuals/head_link/mesh"], ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: _build_g1_upper_body_pipeline()[0], # retargeters_to_tune=lambda: _build_g1_upper_body_pipeline()[1], sim_device=self.sim.device, @@ -429,11 +502,26 @@ def __post_init__(self): xr_camera_feeds=[ XrCameraFeedCfg( camera_name="robot_pov_cam", + label="Head", enable_dlss_ray_reconstruction=True, dlss_exec_mode="quality", offset_m=(0.0, -0.15), - max_update_hz=0.0, - ) + max_update_hz=20.0, + ), + XrCameraFeedCfg( + camera_name="left_wrist_cam", + label="Left wrist", + panel_width_m=0.30, + offset_m=(-0.43, -0.15), + max_update_hz=10.0, + ), + XrCameraFeedCfg( + camera_name="right_wrist_cam", + label="Right wrist", + panel_width_m=0.30, + offset_m=(0.43, -0.15), + max_update_hz=10.0, + ), ], ) @@ -442,3 +530,37 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + +def prepare_fixed_base_g1_for_kitless_ovphysx(env_cfg: FixedBaseUpperBodyIKG1EnvCfg) -> None: + """Prepare the table collision authoring for kitless OVPhysX; OVPhysX cooks the wheel meshes. + + Args: + env_cfg: Fixed-base G1 task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "ovphysx"): + return + from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + + configure_packing_table_collisions(env_cfg.scene.packing_table, physx_offsets=True) + env_cfg._prepared_kitless_backend = "ovphysx" + + +def prepare_fixed_base_g1_for_kitless_newton_mjwarp(env_cfg: FixedBaseUpperBodyIKG1EnvCfg) -> None: + """Configure G1 actions, assets and contacts for kitless Newton. + + Args: + env_cfg: Fixed-base G1 task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "newton_mjwarp"): + return + from .g1_kitless_newton import configure_g1_for_kitless_newton + + configure_g1_for_kitless_newton(env_cfg) + env_cfg._prepared_kitless_backend = "newton_mjwarp" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py new file mode 100644 index 000000000000..7d9925c0d14a --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -0,0 +1,325 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""G1 action and actuator boundaries for kitless Newton teleoperation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import numpy as np +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + +if TYPE_CHECKING: + from isaaclab.assets import ArticulationCfg + from isaaclab.envs import ManagerBasedRLEnv + + from .fixed_base_upper_body_ik_g1_env_cfg import FixedBaseUpperBodyIKG1EnvCfg + +POSITION_MAX_STEP_M = 0.10 +"""Maximum Cartesian wrist-target change accepted in one active frame.""" + +ROTATION_MAX_STEP_RAD = 0.60 +"""Maximum wrist-target rotation accepted in one active frame.""" + +JOINT_TARGET_MAX_STEP_RAD = 0.10 +"""Maximum Pink joint-target change accepted in one active frame.""" + +JOINT_LIMIT_MARGIN_RAD = 1.0e-3 +"""Margin retained inside the common Pink/articulation joint limits.""" + +EXPECTED_ACTION_DIM = 28 +"""Two 7-D wrist poses followed by fourteen G1 hand targets.""" + +WRIST_QUATERNION_NORM_BOUNDS = (0.25, 2.0) +"""Reject malformed wrist rotations before the conditioner normalizes them.""" + +HAND_TARGET_MAX_MAGNITUDE_RAD = 6.5 +"""Reject hand joint targets outside the task's accepted angular input range [rad].""" + +ACTUATOR_PROFILE = { + "waist": {"stiffness": 500.0, "damping": 50.0, "armature": 0.10, "joint_effort_limit": 250.0}, + "arms": {"stiffness": 1200.0, "damping": 60.0, "armature": 0.30, "joint_effort_limit": 800.0}, + "hands": {"stiffness": 5.0, "damping": 0.5, "armature": 0.02, "joint_effort_limit": 20.0}, +} +"""Newton implicit-drive settings for G1 joint groups.""" + + +def teleop_action_target_issue(action: torch.Tensor) -> str | None: + """Return why an incoming G1 controller action must not reach Pink/Newton. + + Args: + action: One G1 action of :data:`EXPECTED_ACTION_DIM` elements, in any shape that flattens to + it: left wrist position [m] and quaternion (x, y, z, w), right wrist position [m] and + quaternion, then fourteen hand joint targets [rad]. + + Returns: + A description of the first problem found, or ``None`` when the action is acceptable. + """ + flat = action.detach().flatten() + if flat.numel() != EXPECTED_ACTION_DIM: + return f"unexpected G1 controller action size {flat.numel()} (expected {EXPECTED_ACTION_DIM})" + wrists = flat[:14].reshape(2, 7) + quaternion_norms = torch.linalg.norm(wrists[:, 3:7], dim=1) + min_norm, max_norm = WRIST_QUATERNION_NORM_BOUNDS + invalid_quaternion = ((quaternion_norms < min_norm) | (quaternion_norms > max_norm)).any() + invalid_hand = (torch.abs(flat[14:]) > HAND_TARGET_MAX_MAGNITUDE_RAD).any() + # CPU flags are already local; accelerator flags share one host transfer. + flags = (torch.isfinite(flat).all(), invalid_quaternion, invalid_hand) + finite, invalid_quaternion, invalid_hand = ( + tuple(bool(flag) for flag in flags) if flat.device.type == "cpu" else torch.stack(flags).cpu().tolist() + ) + if not finite: + return "non-finite action target" + if invalid_quaternion: + return f"invalid wrist quaternion norms {quaternion_norms.detach().cpu().tolist()}" + if invalid_hand: + return "unreasonable hand joint target magnitude" + return None + + +def bound_pink_joint_targets( + current: torch.Tensor, + proposed: torch.Tensor, + lower: torch.Tensor, + upper: torch.Tensor, +) -> torch.Tensor: + """Keep one Pink update finite, inside limits, and close to current state. + + Targets are first limited to :data:`JOINT_TARGET_MAX_STEP_RAD` from ``current``, then + clamped :data:`JOINT_LIMIT_MARGIN_RAD` inside ``[lower, upper]``. The joint limits take + precedence: when ``current`` lies outside those limits, the final change can exceed + the step bound. + + Args: + current: Current joint positions [rad]. Shape is (num_envs, num_joints), in the Pink action's + Isaac Lab controlled-joint order. + proposed: Pink joint targets [rad], with the same shape and joint order. + lower: Lower joint limits [rad], with the same shape and joint order. + upper: Upper joint limits [rad], with the same shape and joint order. + + Returns: + The bounded targets [rad], with the same shape and joint order. + + Raises: + RuntimeError: If any input element is non-finite, or the margin-reduced limits of any joint do + not overlap. The check covers the whole batch, so no environment's targets are returned. + """ + safe_lower = lower + JOINT_LIMIT_MARGIN_RAD + safe_upper = upper - JOINT_LIMIT_MARGIN_RAD + flags = ( + torch.isfinite(current).all(), + torch.isfinite(proposed).all(), + torch.isfinite(lower).all(), + torch.isfinite(upper).all(), + (safe_lower > safe_upper).any(), + ) + *finite, empty_intersection = ( + tuple(bool(flag) for flag in flags) if current.device.type == "cpu" else torch.stack(flags).cpu().tolist() + ) + if not all(finite): + raise RuntimeError("non-finite Pink joint target boundary") + if empty_intersection: + raise RuntimeError("Pink/articulation joint-limit intersection is empty") + + stepped = torch.clamp(proposed, min=current - JOINT_TARGET_MAX_STEP_RAD, max=current + JOINT_TARGET_MAX_STEP_RAD) + return torch.maximum(torch.minimum(stepped, safe_upper), safe_lower) + + +class G1NewtonBoundedPinkInverseKinematicsAction(PinkInverseKinematicsAction): + """Bound Pink outputs without changing G1's public 28-D action layout.""" + + def __init__(self, cfg, env): + super().__init__(cfg, env) + self._pink_joint_limits = None + + def _compute_ik_solutions(self) -> torch.Tensor: + proposed = super()._compute_ik_solutions() + current = self._asset.data.joint_pos.torch[:, self._isaaclab_controlled_joint_ids] + articulation_limits = self._asset.data.joint_pos_limits.torch[:, self._isaaclab_controlled_joint_ids, :] + + # Pink's controlled model/order are immutable for this action's lifetime. + # Keep articulation limits live: users may update them during a run. + if self._pink_joint_limits is None: + self._pink_joint_limits = tuple( + torch.as_tensor( + np.stack( + [ + np.asarray(getattr(controller.pink_configuration.controlled_model, limit_name))[ + controller.pink_to_isaac_lab_controlled_ordering + ] + for controller in self._ik_controllers + ] + ), + device=self.device, + dtype=proposed.dtype, + ) + for limit_name in ("lowerPositionLimit", "upperPositionLimit") + ) + pink_lower, pink_upper = self._pink_joint_limits + lower = torch.maximum(pink_lower, articulation_limits[..., 0]) + upper = torch.minimum(pink_upper, articulation_limits[..., 1]) + return bound_pink_joint_targets(current, proposed, lower, upper) + + +def _wrist_poses(env, state: dict[str, Any]): + """Return current task-selected wrist poses with body indices cached in state.""" + try: + robot = env.scene["robot"] + if "_wrist_indices" in state: + left_index, right_index = state["_wrist_indices"] + else: + target_links = env.cfg.actions.upper_body_ik.target_eef_link_names + left_index = robot.body_names.index(target_links["left_wrist"]) + right_index = robot.body_names.index(target_links["right_wrist"]) + state["_wrist_indices"] = (left_index, right_index) + pose_w = robot.data.body_link_pose_w.torch[0] + except (KeyError, ValueError): + return None + return ( + pose_w[left_index, :3].clone(), + pose_w[left_index, 3:7].clone(), + pose_w[right_index, :3].clone(), + pose_w[right_index, 3:7].clone(), + ) + + +def _step_vector_towards(current: torch.Tensor, target: torch.Tensor, max_step: float) -> torch.Tensor: + delta = target - current + distance = torch.linalg.norm(delta) + alpha = torch.clamp(max_step / torch.clamp(distance, min=1.0e-6), max=1.0) + return current + delta * alpha + + +def _step_quaternion_towards(current: torch.Tensor, target: torch.Tensor, max_step: float) -> torch.Tensor: + current = torch.nn.functional.normalize(current, dim=0) + target = torch.nn.functional.normalize(target, dim=0) + target = torch.where(torch.sum(current * target) < 0.0, -target, target) + dot = torch.clamp(torch.abs(torch.sum(current * target)), 0.0, 1.0) + angle = 2.0 * torch.acos(dot) + alpha = torch.clamp(max_step / torch.clamp(angle, min=1.0e-6), max=1.0) + half_angle = angle * 0.5 + denominator = torch.clamp(torch.sin(half_angle), min=1.0e-6) + interpolated = ( + torch.sin((1.0 - alpha) * half_angle) * current + torch.sin(alpha * half_angle) * target + ) / denominator + # Coincident quaternions need no interpolation and avoid a zero denominator. + return torch.nn.functional.normalize(torch.where(angle < 1.0e-6, target, interpolated), dim=0) + + +_WRIST_RAMP_KEYS = ("left_pos", "left_quat", "right_pos", "right_quat") + + +def condition_kitless_newton_g1_action( + env: ManagerBasedRLEnv, action: torch.Tensor, state: dict[str, Any] +) -> torch.Tensor: + """Ramp both wrist targets from the current G1 poses before Pink solves IK. + + Each call moves the left and right wrist targets at most :data:`POSITION_MAX_STEP_M` and + :data:`ROTATION_MAX_STEP_RAD` toward the requested poses; hand targets pass through unchanged. + The ramp re-seeds from the measured wrist poses whenever environment 0 starts a new episode, + including termination auto-resets inside ``env.step`` that the teleoperation loop does not observe. + + Args: + env: The single-environment G1 teleoperation environment. + action: One G1 action in the layout described by :func:`teleop_action_target_issue`. The Pink + action reads its wrist poses relative to the environment origin, while the ramp seeds from + the measured world-frame wrist poses, so environment 0's origin must be the world origin. + state: Mutable dictionary owned by the caller; pass the same dictionary to every call of one + session, starting empty. The ramp stores its targets and cached wrist body indices there. + + Returns: + A conditioned copy of ``action`` with the same shape. An action whose size is not + :data:`EXPECTED_ACTION_DIM` is returned unchanged and clears ``state``. + + Raises: + RuntimeError: If the task-selected wrist bodies cannot be resolved. ``state`` is cleared. + """ + flat = action.detach().flatten() + if flat.numel() != EXPECTED_ACTION_DIM: + state.clear() + return action + + # The global step at which environment 0's episode began changes with every reset, including a + # termination on the episode's first step, which leaves the episode counter at 0 between frames. + episode_start = env.common_step_counter - int(env.episode_length_buf[0]) + if state.get("_episode_start") != episode_start: + for key in _WRIST_RAMP_KEYS: + state.pop(key, None) + state["_episode_start"] = episode_start + + conditioned = action.clone() + conditioned_flat = conditioned.flatten() + if "left_pos" not in state: + current_wrist_poses = _wrist_poses(env, state) + if current_wrist_poses is None: + state.clear() + raise RuntimeError("could not resolve the task-selected G1 wrist poses") + left_pos, left_quat, right_pos, right_quat = current_wrist_poses + state["left_pos"] = left_pos.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["left_quat"] = left_quat.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["right_pos"] = right_pos.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["right_quat"] = right_quat.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + + state["left_pos"] = _step_vector_towards(state["left_pos"], flat[0:3], POSITION_MAX_STEP_M) + state["left_quat"] = _step_quaternion_towards(state["left_quat"], flat[3:7], ROTATION_MAX_STEP_RAD) + state["right_pos"] = _step_vector_towards(state["right_pos"], flat[7:10], POSITION_MAX_STEP_M) + state["right_quat"] = _step_quaternion_towards(state["right_quat"], flat[10:14], ROTATION_MAX_STEP_RAD) + conditioned_flat[0:3] = state["left_pos"] + conditioned_flat[3:7] = state["left_quat"] + conditioned_flat[7:10] = state["right_pos"] + conditioned_flat[10:14] = state["right_quat"] + return conditioned + + +def apply_g1_newton_actuator_profile(robot_cfg: ArticulationCfg) -> None: + """Apply :data:`ACTUATOR_PROFILE` to the G1 actuator groups used by teleoperation. + + Args: + robot_cfg: The G1 articulation configuration, modified in place. + + Raises: + RuntimeError: If the configuration lacks the ``waist``, ``arms`` or ``hands`` actuator group. + """ + groups = robot_cfg.actuators + if not set(ACTUATOR_PROFILE).issubset(groups): + raise RuntimeError(f"Expected G1 waist/arms/hands actuator groups; found {sorted(groups)}") + for name, selected in ACTUATOR_PROFILE.items(): + actuator = groups[name] + for key, value in selected.items(): + setattr(actuator, key, value) + + +def configure_g1_for_kitless_newton(env_cfg: FixedBaseUpperBodyIKG1EnvCfg) -> None: + """Configure the controller, asset and solver for kitless Newton. + + Args: + env_cfg: The G1 environment configuration with a Newton MJWarp physics selection, modified + in place. + + Raises: + RuntimeError: If the physics configuration has no MJWarp solver configuration, or the robot + lacks an expected actuator group. + """ + from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions + + solver_cfg = getattr(getattr(env_cfg.sim, "physics", None), "solver_cfg", None) + if solver_cfg is None: + raise RuntimeError("G1 kitless Newton stabilization requires an MJWarp solver config") + solver_cfg.nconmax = 4096 + solver_cfg.njmax = 1536 + + apply_g1_newton_actuator_profile(env_cfg.scene.robot) + env_cfg.actions.upper_body_ik.controller.fail_on_joint_limit_violation = False + env_cfg.actions.upper_body_ik.class_type = G1NewtonBoundedPinkInverseKinematicsAction + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.action_conditioner = condition_kitless_newton_g1_action + env_cfg.isaac_teleop.action_validator = teleop_action_target_issue + + configure_packing_table_collisions(env_cfg.scene.packing_table) + condition_newton_steering_wheel_detail_collisions(env_cfg.scene.object) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index 3b8952eadb8d..43b04469373b 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -13,6 +13,7 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR @@ -23,7 +24,13 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.agile_locomotion_observation_cfg import ( AgileTeacherPolicyObservationsCfg, ) +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + check_kitless_backend_preparation, + prepare_humanoid_teleop_environment, +) from isaaclab_assets.robots.unitree import G1_29DOF_CFG @@ -296,12 +303,12 @@ class LocomanipulationG1SceneCfg(InteractiveSceneCfg): # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. left_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/left_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/left_hand/left_hand_[^/]*_link", update_period=0.0, history_length=3, ) right_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/right_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/right_hand/right_hand_[^/]*_link", update_period=0.0, history_length=3, ) @@ -430,6 +437,23 @@ class LocomanipulationG1EnvCfg(ManagerBasedRLEnvCfg): behaviors. """ + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + kitless_preparation: dict[str, str] = { + "ovphysx": ( + "isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg" + ":prepare_locomanipulation_g1_for_kitless_ovphysx" + ), + "newton_mjwarp": ( + "isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg" + ":prepare_locomanipulation_g1_for_kitless_newton_mjwarp" + ), + } + """Preparation function, as a ``"module:function"`` reference, for each supported kitless backend.""" + haptic_feedback: object | None = None + image_obs_list: list[str] = ["robot_pov_cam"] + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: LocomanipulationG1SceneCfg = LocomanipulationG1SceneCfg(num_envs=1, env_spacing=2.5) # MDP settings @@ -444,6 +468,10 @@ class LocomanipulationG1EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return + # The pretrained Agile policy consumes and produces PhysX-ordered joints. + self.scene.robot.joint_ordering = "physx" # general settings self.decimation = 4 self.episode_length_s = 20.0 @@ -455,8 +483,6 @@ def __post_init__(self): # Set the URDF path for the IK controller. Path resolution (Nucleus → local) happens at runtime. self.actions.upper_body_ik.controller.urdf_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 - self.image_obs_list = ["robot_pov_cam"] - # The scene always contains hand contact sensors, so their target bodies must have the # contact reporter API even when the optional teleop package is unavailable. self.scene.robot.spawn.activate_contact_sensors = True @@ -480,6 +506,7 @@ def __post_init__(self): self.xr.anchor_rotation_mode = XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=_build_g1_locomanipulation_pipeline, sim_device=self.sim.device, xr_cfg=self.xr, @@ -500,3 +527,44 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + +def prepare_locomanipulation_g1_for_kitless_ovphysx(env_cfg: LocomanipulationG1EnvCfg) -> None: + """Provide the packing-table support shape required by standalone OVPhysX. + + Args: + env_cfg: G1 locomanipulation task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "ovphysx"): + return + configure_packing_table_collisions(env_cfg.scene.packing_table, physx_offsets=True) + env_cfg._prepared_kitless_backend = "ovphysx" + + +def prepare_locomanipulation_g1_for_kitless_newton_mjwarp(env_cfg: LocomanipulationG1EnvCfg) -> None: + """Prepare shared task collision geometry for kitless Newton without replacing the locomotion policy. + + Args: + env_cfg: G1 locomanipulation task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "newton_mjwarp"): + return + from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions + + solver_cfg = env_cfg.sim.physics.solver_cfg + # Match the common locomotion integrator for the floating-base robot. + solver_cfg.integrator = "implicitfast" + env_cfg.sim.physics.num_substeps = 2 + # The upper body, hands, packing table and wheel match fixed-base G1, plus foot-ground + # contacts; keep at least its contact/constraint budget (MJWarp defaults: njmax=300). + solver_cfg.nconmax = max(solver_cfg.nconmax or 0, 4096) + solver_cfg.njmax = max(solver_cfg.njmax or 0, 1536) + configure_packing_table_collisions(env_cfg.scene.packing_table) + condition_newton_steering_wheel_detail_collisions(env_cfg.scene.object) + env_cfg._prepared_kitless_backend = "newton_mjwarp" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py new file mode 100644 index 000000000000..07e37ff21787 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared collision preparation for the packing-table USD asset.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import MISSING +from typing import TYPE_CHECKING + +from isaaclab.assets import AssetBaseCfg +from isaaclab.sim import UsdFileCfg +from isaaclab.utils import configclass + +from .spawn_wrapping import wrap_spawn + +if TYPE_CHECKING: + from pxr import Usd + + +def _spawn_with_mesh_collisions(prim_path: str, cfg: PackingTableUsdFileCfg, *args, **kwargs) -> Usd.Prim: + from pxr import Usd, UsdGeom, UsdPhysics + + import isaaclab.sim as sim_utils + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + stage = prim.GetStage() + sim_utils.make_uninstanceable(str(prim.GetPath()), stage=stage) + for child in list(Usd.PrimRange(prim)): + if not child.HasAPI(UsdPhysics.MeshCollisionAPI) or child.IsA(UsdGeom.Gprim): + continue + approximation = UsdPhysics.MeshCollisionAPI(child).GetApproximationAttr().Get() + child.RemoveAPI(UsdPhysics.MeshCollisionAPI) + child.RemoveAPI(UsdPhysics.CollisionAPI) + for mesh in Usd.PrimRange(child): + if mesh.IsA(UsdGeom.Mesh): + path = str(mesh.GetPath()) + sim_utils.apply_collision_properties( + path, + [sim_utils.UsdPhysicsCollisionCfg(collision_enabled=True)], + stage=stage, + create_if_missing=True, + ) + sim_utils.apply_mesh_collision_properties( + path, + [sim_utils.UsdPhysicsMeshCollisionCfg(mesh_approximation_name=approximation or "boundingCube")], + stage=stage, + ) + if cfg.physx_offsets: + from isaaclab_physx.sim.schemas import PhysxCollisionCfg + + sim_utils.apply_collision_properties( + str(prim.GetPath()) + "/.*", + [PhysxCollisionCfg(contact_offset=0.001, rest_offset=0.0)], + stage=stage, + ) + return prim + + +@configclass +class PackingTableUsdFileCfg(UsdFileCfg): + """Declared spawn-time collision preparation for the packing-table asset.""" + + original_spawn: Callable | str = MISSING + physx_offsets: bool = False + func: Callable | str = _spawn_with_mesh_collisions + + +def configure_packing_table_collisions(table_cfg: AssetBaseCfg, *, physx_offsets: bool = False) -> None: + """Move the asset's non-geometric collision anchor onto its existing meshes. + + The source USD attaches a bounding-cube collision approximation to an Xform. + Both standalone backends need concrete geometry. Preserve the authored mesh + transforms and approximation, then use the common schema writers for each + mesh. The five bin colliders are left intact; no replacement geometry or + backend-specific cooking implementation is needed. + """ + if table_cfg.spawn is None: + return + table_cfg.spawn = wrap_spawn(table_cfg.spawn, PackingTableUsdFileCfg, _spawn_with_mesh_collisions) + table_cfg.spawn.physx_offsets = physx_offsets diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py index 2e004118995e..1d82b7c783c0 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py @@ -9,7 +9,7 @@ gym.register( id="IsaacContrib-PickPlace-GR1T2-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.pickplace_gr1t2_env_cfg:PickPlaceGR1T2EnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", @@ -39,7 +39,7 @@ gym.register( id="IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.pickplace_gr1t2_waist_enabled_env_cfg:PickPlaceGR1T2WaistEnabledEnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf new file mode 100644 index 000000000000..4b911c3c036d --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf @@ -0,0 +1,848 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py new file mode 100644 index 000000000000..a8d5f8e030f5 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py @@ -0,0 +1,196 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GR1T2 Pink action conditioning for standalone OVPhysX and Newton.""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedEnv + + from .gr1t2_kitless_action_cfg import GR1T2KitlessPinkInverseKinematicsActionCfg + + +def bound_ik_joint_targets( + targets: torch.Tensor, + current: torch.Tensor, + max_delta_rad: float, + max_magnitude_rad: float, +) -> torch.Tensor: + """Bound finite IK targets and hold rows containing an invalid target. + + Args: + targets: Proposed joint positions [rad], shape (num_envs, num_joints), in the + controlled-joint order of the Pink action. + current: Measured joint positions [rad], with the same shape, joint order, dtype and device. + max_delta_rad: Maximum requested change from ``current`` [rad], before the magnitude clamp. + max_magnitude_rad: Maximum absolute target position [rad]. This clamp takes precedence + over the step bound if ``current`` lies outside the magnitude limit. + + Returns: + New targets [rad] with the same shape, joint order, dtype and device. A row containing + any non-finite target holds its entire ``current`` row without clamping it. Inputs are + not mutated; callers must supply valid measured positions and nonnegative bounds. + """ + finite_rows = torch.isfinite(targets).all(dim=1, keepdim=True) + safe_targets = torch.where(finite_rows, targets, current) + delta = torch.clamp(safe_targets - current, -max_delta_rad, max_delta_rad) + bounded = torch.clamp(current + delta, -max_magnitude_rad, max_magnitude_rad) + return torch.where(finite_rows, bounded, current) + + +def constrain_hand_joint_targets( + targets: torch.Tensor, + mimic_pairs: tuple[tuple[int, int, float, float], ...], + lower: torch.Tensor, + upper: torch.Tensor, +) -> torch.Tensor: + """Apply mimic relations in tuple order, then clamp the hand targets. + + Args: + targets: Hand joint positions [rad], shape (num_envs, num_hand_joints), in the action's + controlled hand-joint order. + mimic_pairs: Tuples of (mimic_index, reference_index, factor, bias). Indices address + columns of ``targets``; the factor is dimensionless and the bias is in radians. + Each relation reads the targets updated by preceding relations. + lower: Lower positions [rad], broadcastable to ``targets`` in the same joint order, dtype and device. + upper: Upper positions [rad], broadcastable to ``targets`` in the same joint order, dtype and device. + + Returns: + Clamped targets [rad] with the same shape, joint order, dtype and device. The input + targets are cloned before applying mimic relations; none of the inputs are mutated. + """ + constrained = targets.clone() + for mimic_index, reference_index, factor, bias in mimic_pairs: + constrained[:, mimic_index] = factor * constrained[:, reference_index] + bias + return torch.maximum(torch.minimum(constrained, upper), lower) + + +class _HandTargetSlewState: + """Per-environment hand target history, independent of measured joint state.""" + + def __init__(self, prototype: torch.Tensor, max_step_rad: float) -> None: + self._previous = torch.zeros_like(prototype) + self._initialized = torch.zeros(prototype.shape[0], dtype=torch.bool, device=prototype.device) + self._max_step_rad = max_step_rad + + def condition(self, targets: torch.Tensor) -> torch.Tensor: + slewed = self._previous + torch.clamp(targets - self._previous, -self._max_step_rad, self._max_step_rad) + result = torch.where(self._initialized[:, None], slewed, targets) + self._previous.copy_(result) + self._initialized.fill_(True) + return result + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + if env_ids is None: + env_ids = slice(None) + self._previous[env_ids] = 0.0 + self._initialized[env_ids] = False + + +class GR1T2KitlessPinkInverseKinematicsAction(PinkInverseKinematicsAction): + """Pink action that owns the GR1T2 kitless target-stability boundary. + + Kit realizes the hand's PhysX mimic constraints and sanitizes the raw asset. + The kitless task removes those hard constraints before physics parsing, so + this term reproduces their target relation in software and bounds hand + targets. OVPhysX widens physical hand limits; Newton retains them. It also + bounds cold-start IK demands against measured state to avoid a first-step + drive impulse. + """ + + cfg: GR1T2KitlessPinkInverseKinematicsActionCfg + + def __init__(self, cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, env: ManagerBasedEnv) -> None: + super().__init__(cfg, env) + + from .pickplace_gr1t2_env_cfg import GR1T2KitlessUsdFileCfg + + spawn_cfg = self._asset.cfg.spawn + if isinstance(spawn_cfg, GR1T2KitlessUsdFileCfg) and spawn_cfg.action_cfg is not None: + # Asset construction copies spawn settings before conditioning the USD. Read + # its result here, rather than coupling independently copied environment configs. + cfg.hand_joint_limits = dict(spawn_cfg.action_cfg.hand_joint_limits) + cfg.hand_mimic_relations = dict(spawn_cfg.action_cfg.hand_mimic_relations) + + for name, value in ( + ("max_ik_joint_delta_rad", cfg.max_ik_joint_delta_rad), + ("max_ik_joint_magnitude_rad", cfg.max_ik_joint_magnitude_rad), + ("max_hand_target_step_rad", cfg.max_hand_target_step_rad), + ): + if not math.isfinite(value) or value <= 0.0: + raise ValueError(f"{name} must be finite and positive.") + if not math.isfinite(cfg.hand_target_margin_rad) or cfg.hand_target_margin_rad < 0.0: + raise ValueError("hand_target_margin_rad must be finite and non-negative.") + + hand_indices = {name: index for index, name in enumerate(self._hand_joint_names)} + missing_limits = [name for name in self._hand_joint_names if name not in cfg.hand_joint_limits] + if missing_limits: + raise ValueError(f"Missing pre-conditioned hand limits for: {missing_limits}") + + pairs = [] + for mimic_name, (reference_name, gearing, offset) in cfg.hand_mimic_relations.items(): + if mimic_name not in hand_indices or reference_name not in hand_indices: + raise ValueError( + f"Mimic relation {mimic_name!r} -> {reference_name!r} is outside the Pink hand joint set." + ) + if not all(math.isfinite(value) for value in (gearing, offset)): + raise ValueError(f"Mimic relation for {mimic_name!r} must be finite.") + # PhysX authors q_mimic + gearing * q_reference + offset = 0. + pairs.append((hand_indices[mimic_name], hand_indices[reference_name], -gearing, -offset)) + self._mimic_pairs = tuple(pairs) + + lower = [] + upper = [] + for name in self._hand_joint_names: + joint_lower, joint_upper = cfg.hand_joint_limits[name] + if not all(math.isfinite(value) for value in (joint_lower, joint_upper)) or joint_lower > joint_upper: + raise ValueError(f"Invalid hand limits for {name!r}: {(joint_lower, joint_upper)}") + lower.append(joint_lower - cfg.hand_target_margin_rad) + upper.append(joint_upper + cfg.hand_target_margin_rad) + self._hand_lower = torch.tensor(lower, dtype=self._raw_actions.dtype, device=self.device) + self._hand_upper = torch.tensor(upper, dtype=self._raw_actions.dtype, device=self.device) + self._hand_target_slew = _HandTargetSlewState( + torch.zeros( + (self.num_envs, len(self._hand_joint_names)), + dtype=self._raw_actions.dtype, + device=self.device, + ), + cfg.max_hand_target_step_rad, + ) + + def process_actions(self, actions: torch.Tensor) -> None: + """Process input without mutating the action-manager input during coupling.""" + super().process_actions(actions) + constrained = constrain_hand_joint_targets( + self._target_hand_joint_positions, + self._mimic_pairs, + self._hand_lower, + self._hand_upper, + ) + self._target_hand_joint_positions = self._hand_target_slew.condition(constrained) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Clear commanded-target history when the owning session/environment resets.""" + super().reset(env_ids) + self._hand_target_slew.reset(env_ids) + + def _compute_ik_solutions(self) -> torch.Tensor: + targets = super()._compute_ik_solutions() + current = self._asset.data.joint_pos.torch[:, self._isaaclab_controlled_joint_ids] + return bound_ik_joint_targets( + targets, + current, + self.cfg.max_ik_joint_delta_rad, + self.cfg.max_ik_joint_magnitude_rad, + ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py new file mode 100644 index 000000000000..e453f80dda6e --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py @@ -0,0 +1,55 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure-data configuration for the GR1T2 standalone action boundary.""" + +from __future__ import annotations + +from dataclasses import field +from typing import TYPE_CHECKING + +from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg +from isaaclab.utils.configclass import configclass + +if TYPE_CHECKING: + from .gr1t2_kitless_action import GR1T2KitlessPinkInverseKinematicsAction + + +@configclass +class GR1T2KitlessPinkInverseKinematicsActionCfg(PinkInverseKinematicsActionCfg): + """Pink action with backend-specific target conditioning for GR1T2. + + Kitless OVPhysX and Newton preparation replace the task's Pink action term + with this one; the regular Kit/PhysX path keeps the base term. Preparation + fills the hand metadata while conditioning the spawned USD before physics + parses it. + """ + + class_type: type[GR1T2KitlessPinkInverseKinematicsAction] | str = ( + "{DIR}.gr1t2_kitless_action:GR1T2KitlessPinkInverseKinematicsAction" + ) + + max_ik_joint_delta_rad: float = 0.2 + """Maximum IK target offset from measured joint position on each IK application [rad]. + + The bound is re-evaluated when the action is applied. OVPhysX applies actions + on each environment-managed physics substep; Newton can manage decimation + within a physics step. This is not a per-policy-step travel limit. + """ + + max_ik_joint_magnitude_rad: float = 3.5 + """Symmetric absolute bound for commanded IK-joint positions [rad].""" + + max_hand_target_step_rad: float = 0.079 + """Maximum hand-target change per policy step [rad].""" + + hand_target_margin_rad: float = 0.5 + """Extra target travel beyond anatomical limits, used to sustain grip effort [rad].""" + + hand_mimic_relations: dict[str, tuple[str, float, float]] = field(default_factory=dict) + """Mimic joint to ``(reference joint, gearing, offset)`` from the spawned USD.""" + + hand_joint_limits: dict[str, tuple[float, float]] = field(default_factory=dict) + """Original anatomical hand-joint limits recorded before physical limits are widened [rad].""" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index 2a049e692bc5..d67934f9f556 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -3,12 +3,21 @@ # # SPDX-License-Identifier: BSD-3-Clause +from __future__ import annotations + import os import tempfile +from collections.abc import Callable +from dataclasses import MISSING +from typing import TYPE_CHECKING + +from isaaclab_physx.sim.schemas import PhysxArticulationCfg, PhysxRigidBodyCfg import isaaclab.envs.mdp as base_mdp import isaaclab.sim as sim_utils +from isaaclab.actuators import ImplicitActuatorCfg from isaaclab.assets import ArticulationCfg, AssetBaseCfg, RigidObjectCfg +from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg @@ -19,18 +28,33 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path from isaaclab.visualizers import VisualizerCfg from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions +from isaaclab_tasks.contrib.spawn_wrapping import wrap_spawn +from isaaclab_tasks.contrib.steering_wheel import ( + condition_newton_steering_wheel_detail_collisions, +) +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + check_kitless_backend_preparation, + prepare_humanoid_teleop_environment, +) from . import mdp +from .gr1t2_kitless_action_cfg import GR1T2KitlessPinkInverseKinematicsActionCfg from isaaclab_assets.robots.fourier import GR1T2_HIGH_PD_CFG # isort: skip from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip +if TYPE_CHECKING: + from pxr import Usd + def build_gr1t2_pickplace_pipeline(): """Build an Isaac Capture retargeting pipeline for GR1T2 pick-place teleoperation. @@ -582,6 +606,19 @@ class EventCfg: class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + kitless_preparation: dict[str, str] = { + "ovphysx": "isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg:prepare_gr1t2_for_kitless_ovphysx", + "newton_mjwarp": ( + "isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg:prepare_gr1t2_for_kitless_newton_mjwarp" + ), + } + """Preparation function, as a ``"module:function"`` reference, for each supported kitless backend.""" + haptic_feedback: object | None = None + image_obs_list: list[str] = ["robot_pov_cam"] + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: PickPlaceGR1T2SceneCfg = PickPlaceGR1T2SceneCfg(num_envs=1, env_spacing=2.5) # Basic settings @@ -643,6 +680,9 @@ class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + # Prepared config copies already contain their backend and teleop settings. + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 6 self.episode_length_s = 20.0 @@ -657,8 +697,6 @@ def __post_init__(self): self.actions.upper_body_ik.controller.usd_path = self.scene.robot.spawn.usd_path self.actions.upper_body_ik.controller.urdf_output_dir = self.temp_urdf_dir - self.image_obs_list = ["robot_pov_cam"] - if ISAACLAB_TELEOP_AVAILABLE: from isaaclab_teleop import GloveHapticFeedbackCfg, IsaacTeleopCfg, XrCameraFeedCfg, XrCfg @@ -666,8 +704,12 @@ def __post_init__(self): self.xr = XrCfg( anchor_pos=(0.0, 0.0, 0.0), anchor_rot=(0.0, 0.0, 0.0, 1.0), + # The shared anchor places the default eye inside the robot head. + # Exclude visual meshes from the eye partition, not source visibility. + excluded_visual_meshes=[f"{DEFAULT_ENV_TEMPLATE.format(0)}/Robot/head_.*_link/visuals/.*"], ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, @@ -690,3 +732,282 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + +def prepare_gr1t2_for_kitless_ovphysx(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Configure a GR1T2 task's asset and action boundaries for kitless OVPhysX. + + Replaces the Pink action term with the kitless GR1T2 term and selects the vendored kinematics + URDF for it, requires initial bimanual hand tracking, restores actuator coverage, conditions the + robot spawn, and authors PhysX contact offsets on the packing table. The configured controlled + joints (with or without the waist) are kept. A prepared configuration only re-attaches its spawn + conditioning. + + Args: + env_cfg: GR1T2 pick-place task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "ovphysx"): + condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik) + return + action_cfg = _use_kitless_action_term(env_cfg) + # Do not apply retargeter defaults before hand tracking starts. + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True + apply_kitless_ovphysx_drive_gains(env_cfg.scene.robot) + condition_kitless_ovphysx_robot_spawn_properties(env_cfg.scene.robot) + condition_kitless_gr1t2_stage(env_cfg.scene.robot, action_cfg) + configure_packing_table_collisions(env_cfg.scene.packing_table, physx_offsets=True) + env_cfg._prepared_kitless_backend = "ovphysx" + + +def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Configure a GR1T2 task's actuators, contacts and action boundaries for kitless Newton. + + Shares the kitless action term and its URDF with :func:`prepare_gr1t2_for_kitless_ovphysx`, + applies Newton actuator gains and MJWarp contact budgets, and keeps the configured + controlled joints. Hand joint limits are not widened on Newton. + + Args: + env_cfg: GR1T2 pick-place task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "newton_mjwarp"): + condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik, widen_hand_limits=False) + return + action_cfg = _use_kitless_action_term(env_cfg) + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True + _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg) + condition_kitless_gr1t2_stage(env_cfg.scene.robot, action_cfg, widen_hand_limits=False) + configure_packing_table_collisions(env_cfg.scene.packing_table) + env_cfg._prepared_kitless_backend = "newton_mjwarp" + + +def _use_kitless_action_term(env_cfg: ManagerBasedRLEnvCfg) -> GR1T2KitlessPinkInverseKinematicsActionCfg: + """Replace the Pink action term with the kitless GR1T2 term and select the vendored URDF. + + The kitless term keeps the task's Pink settings, including caller changes made before preparation. + """ + action_cfg = env_cfg.actions.upper_body_ik + if not isinstance(action_cfg, GR1T2KitlessPinkInverseKinematicsActionCfg): + settings = {name: value for name, value in vars(action_cfg).items() if name != "class_type"} + action_cfg = GR1T2KitlessPinkInverseKinematicsActionCfg(**settings) + env_cfg.actions.upper_body_ik = action_cfg + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + return action_cfg + + +def _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Apply Newton actuator, allocation-budget, and wheel-collision preparation.""" + from isaaclab_newton.sim.schemas import NewtonArticulationCfg + + # Match the kitless OVPhysX task: internal finger contacts must not force + # compliant thumb drives away from their commands. Object contacts remain. + fragments = env_cfg.scene.robot.spawn.articulation_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, NewtonArticulationCfg): + fragment.self_collision_enabled = False + + groups = env_cfg.scene.robot.actuators + contracts = { + "head": (["head_.*"], 80.0, 8.0, 0.03, 80.0), + "trunk": (["waist_.*"], 500.0, 50.0, 0.10, 250.0), + "legs": ([".*_hip_.*", ".*_knee_.*", ".*_ankle_.*"], 300.0, 30.0, 0.125, 250.0), + "right-arm": (["right_shoulder_.*", "right_elbow_.*", "right_wrist_.*"], 1200.0, 60.0, 0.30, 800.0), + "left-arm": (["left_shoulder_.*", "left_elbow_.*", "left_wrist_.*"], 1200.0, 60.0, 0.30, 800.0), + } + for name, (joint_names, stiffness, damping, armature, effort_limit) in contracts.items(): + actuator = groups.get(name) + if actuator is None: + actuator = ImplicitActuatorCfg(joint_names_expr=joint_names) + groups[name] = actuator + actuator.joint_names_expr = joint_names + actuator.stiffness = stiffness + actuator.damping = damping + actuator.armature = armature + actuator.joint_effort_limit = effort_limit + + for name in ("left-hand", "right-hand"): + # The action conditions all eleven targets to the asset's mimic relation. Give Newton the + # same compliant hand drives as the OVPhysX path. + hand = groups[name] + hand.stiffness = 5.0 + hand.damping = 0.5 + hand.armature = 0.02 + hand.joint_effort_limit = 20.0 + + solver_cfg = env_cfg.sim.physics.solver_cfg + solver_cfg.nconmax = 4096 + solver_cfg.njmax = 1536 + condition_newton_steering_wheel_detail_collisions(env_cfg.scene.object) + + +def apply_kitless_ovphysx_drive_gains(robot_cfg: ArticulationCfg) -> None: + """Restore stable actuator coverage around the Kit-tuned arm drives.""" + groups = { + "head": (["head_.*"], 80.0, 8.0, 0.03, 80.0), + "legs": ([".*_hip_.*", ".*_knee_.*", ".*_ankle_.*"], 300.0, 30.0, 0.10, 250.0), + "right-hand": (None, 5.0, 0.5, 0.001, 20.0), + "left-hand": (None, 5.0, 0.5, 0.001, 20.0), + } + for group_name, (joint_expr, stiffness, damping, armature, effort_limit) in groups.items(): + actuator = robot_cfg.actuators.get(group_name) + if actuator is None: + assert joint_expr is not None + actuator = ImplicitActuatorCfg(joint_names_expr=list(joint_expr)) + robot_cfg.actuators[group_name] = actuator + actuator.stiffness = stiffness + actuator.damping = damping + actuator.armature = armature + actuator.joint_effort_limit = effort_limit + + if robot_cfg.spawn is not None and robot_cfg.spawn.articulation_props is not None: + fragments = robot_cfg.spawn.articulation_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, PhysxArticulationCfg): + fragment.enabled_self_collisions = False + + +def condition_kitless_ovphysx_robot_spawn_properties(robot_cfg: ArticulationCfg) -> None: + """Limit robot contact depenetration for kitless OVPhysX.""" + if robot_cfg.spawn is not None and robot_cfg.spawn.rigid_props is not None: + fragments = robot_cfg.spawn.rigid_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, PhysxRigidBodyCfg): + fragment.max_depenetration_velocity = 0.1 + + +def _author_kitless_contact_offsets(prim) -> bool: + """Author Kit-parity PhysX contact offsets on a collision prim before kitless parsing. + + Both kitless backends receive the same spawned stage: OVPhysX consumes these + ``physxCollision`` offsets, and Newton parses the same collision prims. + """ + from pxr import Sdf + + schemas = prim.GetMetadata("apiSchemas") + applied = [str(schema) for schema in (schemas.GetAddedOrExplicitItems() if schemas else [])] + if not any(schema.startswith("PhysicsCollisionAPI") for schema in applied): + return False + if not any(schema.startswith("PhysxCollisionAPI") for schema in applied): + list_op = Sdf.TokenListOp() + list_op.explicitItems = [*applied, "PhysxCollisionAPI"] + prim.SetMetadata("apiSchemas", list_op) + prim.CreateAttribute("physxCollision:contactOffset", Sdf.ValueTypeNames.Float).Set(0.001) + prim.CreateAttribute("physxCollision:restOffset", Sdf.ValueTypeNames.Float).Set(0.0) + return True + + +def _spawn_and_condition(prim_path: str, cfg: GR1T2KitlessUsdFileCfg, *args, **kwargs) -> Usd.Prim: + import math + + from pxr import Gf, Sdf, Usd + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + action_cfg = cfg.action_cfg + action_cfg.hand_joint_limits.clear() + action_cfg.hand_mimic_relations.clear() + + # Collision meshes below instanceable ``collisions`` scopes are + # immutable proxies. De-instance those scopes so the parser-visible + # collision prims can receive the OVPhysX contact offsets. + for child in Usd.PrimRange(prim): + if child.GetName() == "collisions" and child.IsInstanceable(): + child.SetInstanceable(False) + + for child in Usd.PrimRange(prim): + _author_kitless_contact_offsets(child) + inertia = child.GetAttribute("physics:diagonalInertia") + if inertia and inertia.HasValue(): + value = inertia.Get() + if min(value) < 1.0e-4: + inertia.Set(Gf.Vec3f(*(max(component, 1.0e-4) for component in value))) + + is_hand_joint = child.GetName()[:2] in ("L_", "R_") and child.GetTypeName() == "PhysicsRevoluteJoint" + schemas = child.GetMetadata("apiSchemas") + applied = list(schemas.GetAddedOrExplicitItems()) if schemas else [] + kept = [schema for schema in applied if not str(schema).startswith("PhysxMimicJointAPI")] + if len(kept) != len(applied): + gearing = child.GetAttribute("physxMimicJoint:rotZ:gearing") + offset = child.GetAttribute("physxMimicJoint:rotZ:offset") + reference = child.GetRelationship("physxMimicJoint:rotZ:referenceJoint") + targets = reference.GetTargets() if reference else [] + if gearing and gearing.HasValue() and targets: + action_cfg.hand_mimic_relations[child.GetName()] = ( + targets[0].name, + float(gearing.Get()), + float(offset.Get()) if offset and offset.HasValue() else 0.0, + ) + list_op = Sdf.TokenListOp() + list_op.explicitItems = kept + child.SetMetadata("apiSchemas", list_op) + for property_name in [name for name in child.GetPropertyNames() if name.startswith("physxMimicJoint:")]: + if not child.RemoveProperty(property_name): + mimic_attr = child.GetAttribute(property_name) + if mimic_attr and mimic_attr.GetTypeName() == Sdf.ValueTypeNames.Float: + mimic_attr.Set(0.0) + + if is_hand_joint: + lower = child.GetAttribute("physics:lowerLimit") + upper = child.GetAttribute("physics:upperLimit") + if lower and upper and lower.HasValue() and upper.HasValue(): + action_cfg.hand_joint_limits[child.GetName()] = ( + math.radians(float(lower.Get())), + math.radians(float(upper.Get())), + ) + if cfg.widen_hand_limits: + lower.Set(-720.0) + upper.Set(720.0) + return prim + + +@configclass +class GR1T2KitlessUsdFileCfg(UsdFileCfg): + """Serializable GR1T2 spawn preparation and asset-owned hand metadata.""" + + original_spawn: Callable | str = MISSING + action_cfg: GR1T2KitlessPinkInverseKinematicsActionCfg | None = None + widen_hand_limits: bool = True + func: Callable | str = _spawn_and_condition + + +def condition_kitless_gr1t2_stage( + robot_cfg: ArticulationCfg, + action_cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, + *, + widen_hand_limits: bool = True, +) -> None: + """Condition the spawned GR1T2 hand description before kitless physics parses it. + + The raw asset has near-singular finger inertia, hard PhysX-only mimic + constraints, and hand rest poses exactly on physical limits. Those inputs + are stable after Kit's asset preparation but are ill-conditioned when read + directly by a kitless backend. This spawn boundary runs for both OVPhysX and + Newton. It: + + - de-instances ``collisions`` scopes and authors Kit-parity PhysX contact + offsets (contact 1 mm, rest 0) on every collision prim; + - floors every diagonal inertia component below 1e-4 [kg·m²] on every body, + which in this asset affects the finger links; + - removes the hard mimic constraints that conflict with direct hand drives; + - optionally widens the hand joint limits (``widen_hand_limits``, used for + OVPhysX). Newton retains the anatomical limits because bounding drive + targets alone does not bound simulated positions. + + The original mimic relations and anatomical limits are recorded on the typed + action config for target conditioning. The source USD and normal Kit path + remain unchanged. + """ + if robot_cfg.spawn is None: + return + robot_cfg.spawn = wrap_spawn(robot_cfg.spawn, GR1T2KitlessUsdFileCfg, _spawn_and_condition) + # Seed metadata from this environment; Asset construction owns its independent spawn copy. + robot_cfg.spawn.action_cfg = action_cfg + robot_cfg.spawn.widen_hand_limits = widen_hand_limits diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py index 9e04c5afe254..e6be228b44ff 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py @@ -5,10 +5,16 @@ import tempfile +from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE from isaaclab.envs import ManagerBasedRLEnvCfg +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + prepare_humanoid_teleop_environment, +) from .pickplace_gr1t2_env_cfg import ( ActionsCfg, @@ -24,6 +30,21 @@ class PickPlaceGR1T2WaistEnabledEnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + kitless_preparation: dict[str, str] = { + "ovphysx": "isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg:prepare_gr1t2_for_kitless_ovphysx", + "newton_mjwarp": ( + "isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg:prepare_gr1t2_for_kitless_newton_mjwarp" + ), + } + """Preparation function, as a ``"module:function"`` reference, for each supported kitless backend. + + The shared GR1T2 preparation keeps the configured controlled joints, including the waist. + """ + haptic_feedback: object | None = None + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: ObjectTableSceneCfg = ObjectTableSceneCfg(num_envs=1, env_spacing=2.5) # Basic settings @@ -43,6 +64,8 @@ class PickPlaceGR1T2WaistEnabledEnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 6 self.episode_length_s = 20.0 @@ -66,8 +89,11 @@ def __post_init__(self): self.xr = XrCfg( anchor_pos=(0.0, 0.0, 0.0), anchor_rot=(0.0, 0.0, 0.0, 1.0), + # Exclude head visuals from the operator camera, not the source scene. + excluded_visual_meshes=[f"{DEFAULT_ENV_TEMPLATE.format(0)}/Robot/head_.*_link/visuals/.*"], ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/spawn_wrapping.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/spawn_wrapping.py new file mode 100644 index 000000000000..8dae2b8d3e63 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/spawn_wrapping.py @@ -0,0 +1,48 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared spawn-function wrapping for contributed asset preparation.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, TypeVar + +from isaaclab.sim import UsdFileCfg + +if TYPE_CHECKING: + from pxr import Usd + +_WrapperCfgT = TypeVar("_WrapperCfgT", bound=UsdFileCfg) + + +def wrap_spawn( + spawn_cfg: UsdFileCfg, wrapper_cfg_type: type[_WrapperCfgT], wrapper: Callable[..., Usd.Prim] +) -> _WrapperCfgT: + """Route a USD spawn configuration through ``wrapper``, keeping its current spawn function. + + The wrapper configuration declares ``original_spawn``, which ``wrapper`` calls before + preparing the spawned prim. Wrapping is idempotent: an already wrapped configuration keeps + its original spawn function, while a function that replaced the wrapper after an earlier + preparation becomes the new original. ``func`` may be the wrapper itself or its + ``"module:function"`` string form after configuration serialization. + + Args: + spawn_cfg: Spawn configuration to wrap. + wrapper_cfg_type: Configuration type that declares ``original_spawn`` and the wrapper's settings. + wrapper: Spawn function that calls ``cfg.original_spawn`` and prepares the result. + + Returns: + The wrapped spawn configuration. It is ``spawn_cfg`` itself when it already has + ``wrapper_cfg_type``, otherwise a new configuration copied from it. + """ + if not isinstance(spawn_cfg, wrapper_cfg_type): + original_spawn = spawn_cfg.func + spawn_cfg = wrapper_cfg_type(**vars(spawn_cfg)) + spawn_cfg.original_spawn = original_spawn + elif spawn_cfg.func is not wrapper and spawn_cfg.func != f"{wrapper.__module__}:{wrapper.__name__}": + spawn_cfg.original_spawn = spawn_cfg.func + spawn_cfg.func = wrapper + return spawn_cfg diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py index d491fbc2a012..efd52af8ce0a 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py @@ -6,8 +6,10 @@ from dataclasses import MISSING import numpy as np +from isaaclab_ov.physics import OvPhysxCfg from isaaclab.actuators import ImplicitActuatorCfg +from isaaclab.physics import PhysxAutoCfg from isaaclab.utils import configclass, replace from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE @@ -37,8 +39,15 @@ # the home orientation target. Capture 1.7 seeds both channels from this transform on reset and # applies controller motion relative to each engage, retaining the commanded pose on re-clutch. # The base rebase is handled upstream by ``target_frame_prim_path``; no live EE pose is read here. -_BASE_T_GRIPPER_HOME = np.eye(4, dtype=np.float32) -_BASE_T_GRIPPER_HOME[:3, 3] = (0.01918, -0.18852, 0.18887) +_BASE_T_GRIPPER_HOME = np.array( + [ + [0.00000711, 1.0, -0.00000682, 0.02061444], + [-0.71735179, 0.00000985, 0.69671114, -0.18883384], + [0.69671114, -0.00000006, 0.71735179, 0.19226513], + [0.0, 0.0, 0.0, 1.0], + ], + dtype=np.float32, +) def _build_so101_stack_pipeline(): @@ -181,6 +190,24 @@ class SO101CubeStackEnvCfg(stack_joint_pos_env_cfg.SO101CubeStackEnvCfg): def __post_init__(self): # post init of parent super().__post_init__() + # The clutch resets to a configured full pose, so the arm must return to the same + # nominal joints on every reset. Keep the reset event active, without joint noise. + self.events.randomize_joint_state.params["std"] = 0.0 + isaacsim_physx = self.sim.physics.isaacsim_physx + # Match the stack family's Isaac Sim PhysX grasp tuning on the kitless OvPhysX backend. + ovphysx = OvPhysxCfg( + bounce_threshold_velocity=isaacsim_physx.bounce_threshold_velocity, + friction_correlation_distance=isaacsim_physx.friction_correlation_distance, + solve_articulation_contact_last=isaacsim_physx.solve_articulation_contact_last, + ) + self.sim.physics = preset( + default=self.sim.physics.default, + isaacsim_physx=isaacsim_physx, + # The automatic selector picks the same tuned OvPhysX config when Isaac Sim is absent. + physx=PhysxAutoCfg(isaacsim_physx=isaacsim_physx, ovphysx=ovphysx), + newton_mjwarp=self.sim.physics.newton_mjwarp, + ovphysx=ovphysx, + ) # Switch to a stiffer PD controller so the arm tracks IK targets well. Replace the robot # configured by the parent so its task-specific seated root and stack joint pose are retained. @@ -201,7 +228,9 @@ def __post_init__(self): variants={ "Robot": "robot", "Sensor": "sensors", - "Physics": preset(default="physx", isaacsim_physx="physx", physx="physx", newton_mjwarp="physics"), + "Physics": preset( + default="physx", isaacsim_physx="physx", physx="physx", ovphysx="physx", newton_mjwarp="physics" + ), }, ), actuators={ diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py new file mode 100644 index 000000000000..f96d2d20926e --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py @@ -0,0 +1,68 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared collision preparation for the contributed steering-wheel asset.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import MISSING +from typing import TYPE_CHECKING + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg +from isaaclab.utils import configclass + +from .spawn_wrapping import wrap_spawn + +if TYPE_CHECKING: + from pxr import Usd + +# These are the graspable wheel surfaces. Their decomposition preserves the +# rim opening and spoke gaps that a single hull would fill. +_STEERING_WHEEL_DECOMPOSED_MESHES = frozenset( + { + "sm_steeringwheel_a01_wheel_rim_01", + "sm_steeringwheel_a01_wheel_spokes_01", + } +) + + +def _spawn_with_selective_collision(prim_path: str, cfg: SteeringWheelUsdFileCfg, *args, **kwargs) -> Usd.Prim: + from pxr import Usd, UsdGeom + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + for root_path in sim_utils.find_matching_prim_paths(prim_path): + root = prim.GetStage().GetPrimAtPath(root_path) + if not root.IsValid(): + continue + for mesh_prim in Usd.PrimRange(root): + if not mesh_prim.IsA(UsdGeom.Mesh) or mesh_prim.GetName() in _STEERING_WHEEL_DECOMPOSED_MESHES: + continue + approximation = mesh_prim.GetAttribute("physics:approximation") + if approximation and approximation.Get() is not None: + approximation.Set("convexHull") + return prim + + +@configclass +class SteeringWheelUsdFileCfg(sim_utils.UsdFileCfg): + """Declared selective collision preparation for the steering-wheel asset.""" + + original_spawn: Callable | str = MISSING + func: Callable | str = _spawn_with_selective_collision + + +def condition_newton_steering_wheel_detail_collisions(object_cfg: RigidObjectCfg) -> None: + """Hull only non-graspable wheel details that MuJoCo cannot compile as solids. + + The authored wheel uses convex decomposition throughout. Tiny hub and + quick-release decomposition pieces can have near-zero volume, which MuJoCo + rejects for solid inertia. The rim and spokes remain decomposed so the + wheel remains graspable; only detail meshes are switched to convex hulls. + """ + if object_cfg.spawn is None: + return + object_cfg.spawn = wrap_spawn(object_cfg.spawn, SteeringWheelUsdFileCfg, _spawn_with_selective_collision) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py new file mode 100644 index 000000000000..d182b85279cd --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Construction boundary for the contributed GR1T2 and G1 teleoperation tasks.""" + +from isaaclab.envs import ManagerBasedRLEnv, ManagerBasedRLEnvCfg + +from .teleop_physics_cfg import prepare_humanoid_teleop_environment + + +class HumanoidTeleopEnv(ManagerBasedRLEnv): + """Prepare the selected task backend before physics and scene construction.""" + + def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, **kwargs): + prepare_humanoid_teleop_environment(cfg) + super().__init__(cfg, render_mode, **kwargs) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py new file mode 100644 index 000000000000..2c013b596e48 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py @@ -0,0 +1,85 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Backend selection for humanoid teleoperation tasks with task-owned preparation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg +from isaaclab_ov.physics import OvPhysxCfg +from isaaclab_physx.physics import PhysxCfg + +from isaaclab.physics import PhysxAutoCfg +from isaaclab.utils import configclass, string_to_callable + +from isaaclab_tasks.utils import PresetCfg, resolve_presets + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedRLEnvCfg + + +def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: + """Prepare humanoid task settings before simulation or environment construction. + + Dispatches to the preparation function that the configuration's ``kitless_preparation`` + mapping names for the selected standalone backend (``"ovphysx"`` or ``"newton_mjwarp"``). + Kit PhysX needs no preparation. Repeated calls on a prepared configuration are no-ops. A + configuration constructed directly, without task resolution, uses the default of every + remaining preset. + + Args: + cfg: Resolved humanoid teleoperation task configuration. + + Raises: + ValueError: If ``cfg`` was already prepared for a different physics backend. + """ + resolve_presets(cfg) + physics = cfg.sim.physics + if isinstance(physics, OvPhysxCfg): + backend = "ovphysx" + elif isinstance(physics, NewtonCfg) and isinstance(physics.solver_cfg, MJWarpSolverCfg): + backend = "newton_mjwarp" + else: + check_kitless_backend_preparation(cfg, None) + return + string_to_callable(cfg.kitless_preparation[backend])(cfg) + + +@configclass +class TeleopPhysicsCfg(PresetCfg): + """Keep Kit as the default and expose the supported standalone backends.""" + + isaacsim_physx = PhysxCfg() + ovphysx = OvPhysxCfg() + newton_mjwarp = NewtonCfg(solver_cfg=MJWarpSolverCfg()) + physx = PhysxAutoCfg(isaacsim_physx=isaacsim_physx, ovphysx=ovphysx) + default = isaacsim_physx + + +def check_kitless_backend_preparation(env_cfg: ManagerBasedRLEnvCfg, backend: str | None) -> bool: + """Check a task's kitless preparation against the selected backend. + + Preparation is recorded on the configuration, so it stays stable across repeated callers and + configuration copies. + + Args: + env_cfg: Task configuration that declares ``_prepared_kitless_backend``. + backend: Selected backend name (``"ovphysx"``, ``"newton_mjwarp"``) or ``None`` for Kit PhysX. + + Returns: + Whether ``env_cfg`` is already prepared for ``backend``. + + Raises: + ValueError: If ``env_cfg`` was prepared for a different backend. + """ + prepared = env_cfg._prepared_kitless_backend + if prepared is not None and prepared != backend: + raise ValueError( + f"Task configuration was already prepared for {prepared}; resolve a fresh task configuration " + "before selecting a different physics backend." + ) + return prepared is not None diff --git a/source/isaaclab_tasks/pyproject.toml b/source/isaaclab_tasks/pyproject.toml index b47f281770b4..29a6393215f6 100644 --- a/source/isaaclab_tasks/pyproject.toml +++ b/source/isaaclab_tasks/pyproject.toml @@ -39,3 +39,4 @@ include = ["isaaclab_tasks", "isaaclab_tasks.*"] [tool.setuptools.package-data] "*" = ["*.pyi"] +"isaaclab_tasks.contrib.pick_place" = ["assets/*.urdf"] diff --git a/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py b/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py index 01fbd819f859..2ad44d7e5865 100644 --- a/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py +++ b/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py @@ -132,3 +132,51 @@ def test_env_cfg_arm_action_is_pose_and_ordering_matches_pipeline(): "arm_action", "gripper_action", ] + + +def test_ik_reset_restores_nominal_joints_without_changing_joint_control_randomization(): + """The IK task's configured reset event restores its nominal joints without noise.""" + from types import SimpleNamespace + + from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import SO101CubeStackEnvCfg + from isaaclab_tasks.contrib.stack.config.so101.stack_joint_pos_env_cfg import ( + _SO101_STACK_INIT_JOINT_POS, + ) + from isaaclab_tasks.contrib.stack.config.so101.stack_joint_pos_env_cfg import ( + SO101CubeStackEnvCfg as JointPositionCfg, + ) + + cfg = SO101CubeStackEnvCfg() + assert JointPositionCfg().events.randomize_joint_state.params["std"] == 0.02 + defaults = torch.tensor([list(_SO101_STACK_INIT_JOINT_POS.values())]) + written = {} + + class Robot: + data = SimpleNamespace( + default_joint_pos=SimpleNamespace(torch=defaults), + default_joint_vel=SimpleNamespace(torch=torch.zeros_like(defaults)), + soft_joint_pos_limits=SimpleNamespace(torch=torch.tensor([[[-10.0, 10.0]] * defaults.shape[1]])), + ) + + def find_joints(self, names): + return [list(_SO101_STACK_INIT_JOINT_POS).index(name) for name in names], names + + def set_joint_position_target_index(self, target, env_ids): + written["target"] = target.clone() + + def set_joint_velocity_target_index(self, target, env_ids): + written["velocity_target"] = target.clone() + + def write_joint_position_to_sim_index(self, position, env_ids): + written["position"] = position.clone() + + def write_joint_velocity_to_sim_index(self, velocity, env_ids): + written["velocity"] = velocity.clone() + + env = SimpleNamespace(cfg=cfg, scene={"robot": Robot()}) + event = cfg.events.randomize_joint_state + event.func(env, torch.tensor([0]), **event.params) + torch.testing.assert_close(written["position"], defaults) + torch.testing.assert_close(written["target"], defaults) + torch.testing.assert_close(written["velocity"], torch.zeros_like(defaults)) + torch.testing.assert_close(written["velocity_target"], torch.zeros_like(defaults)) diff --git a/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py b/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py new file mode 100644 index 000000000000..5410ef913a19 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Geometry-preserving common collision preparation, independent of a physics engine.""" + +from types import SimpleNamespace + +import pytest + +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + + +@pytest.mark.parametrize("physx_offsets", [False, True]) +def test_existing_mesh_transforms_and_bin_colliders_are_preserved(physx_offsets): + stage = Usd.Stage.CreateInMemory() + expected = {} + + def spawn(path, cfg): + root = stage.DefinePrim(path, "Xform") + root.SetInstanceable(True) + parent = stage.DefinePrim(path + "/Table", "Xform") + parent.SetInstanceable(True) + UsdGeom.Xformable(parent).AddTranslateOp().Set((0.3, -0.2, 0.4)) + UsdPhysics.CollisionAPI.Apply(parent) + UsdPhysics.MeshCollisionAPI.Apply(parent).CreateApproximationAttr().Set("boundingCube") + parent.CreateAttribute("horde:filterOwner", Sdf.ValueTypeNames.String).Set("table") + for index in range(4): + mesh = UsdGeom.Mesh.Define(stage, path + f"/Table/Mesh_{index}") + mesh.CreatePointsAttr().Set([(0, 0, 0), (1, 0, 0), (0, 1, 0)]) + mesh.CreateFaceVertexCountsAttr().Set([3]) + mesh.CreateFaceVertexIndicesAttr().Set([0, 1, 2]) + UsdGeom.Xformable(mesh).AddTranslateOp().Set((index, index / 2, index / 4)) + stage.DefinePrim(path + "/Bin", "Xform").SetInstanceable(True) + for index in range(5): + cube = UsdGeom.Cube.Define(stage, path + f"/Bin/Cube_{index}") + cube.CreateSizeAttr().Set(0.1 + index) + UsdPhysics.CollisionAPI.Apply(cube.GetPrim()) + cache = UsdGeom.XformCache() + for prim in Usd.PrimRange(root): + if prim.IsA(UsdGeom.Gprim): + expected[str(prim.GetPath())] = cache.GetLocalToWorldTransform(prim) + return root + + cfg = SimpleNamespace(spawn=SimpleNamespace(func=spawn)) + configure_packing_table_collisions(cfg, physx_offsets=physx_offsets) + prepared = cfg.spawn.func + configure_packing_table_collisions(cfg, physx_offsets=physx_offsets) + assert cfg.spawn.func is prepared + root = prepared("/World/PackingTable", cfg.spawn) + shapes = [p for p in Usd.PrimRange(root) if p.HasAPI(UsdPhysics.CollisionAPI)] + assert len(shapes) == 9 + assert {str(p.GetPath()) for p in shapes} == set(expected) + assert sum(p.IsA(UsdGeom.Mesh) for p in shapes) == 4 + assert sum(p.IsA(UsdGeom.Cube) for p in shapes) == 5 + assert not stage.GetPrimAtPath("/World/PackingTable/Table").HasAPI(UsdPhysics.CollisionAPI) + assert stage.GetPrimAtPath("/World/PackingTable/Table").GetAttribute("horde:filterOwner").Get() == "table" + assert not any(prim.IsInstance() or prim.IsInstanceProxy() for prim in Usd.PrimRange(root)) + cache = UsdGeom.XformCache() + for shape in shapes: + assert cache.GetLocalToWorldTransform(shape) == expected[str(shape.GetPath())] + assert UsdPhysics.CollisionAPI(shape).GetCollisionEnabledAttr().Get() + assert UsdGeom.Imageable(shape).GetPurposeAttr().Get() == UsdGeom.Tokens.default_ + if shape.IsA(UsdGeom.Mesh): + assert UsdPhysics.MeshCollisionAPI(shape).GetApproximationAttr().Get() == "boundingCube" + assert list(UsdGeom.Mesh(shape).GetFaceVertexIndicesAttr().Get()) == [0, 1, 2] + if physx_offsets: + assert shape.GetAttribute("physxCollision:contactOffset").Get() == pytest.approx(0.001) + assert shape.GetAttribute("physxCollision:restOffset").Get() == 0.0 diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py new file mode 100644 index 000000000000..cfc29f15fe29 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py @@ -0,0 +1,306 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sim-free tests for the G1 kitless Newton task boundary.""" + +import math +from types import SimpleNamespace + +import pytest +import torch +from isaaclab_newton.physics import NewtonCfg + +from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + prepare_fixed_base_g1_for_kitless_newton_mjwarp, +) +from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( + ACTUATOR_PROFILE, + JOINT_LIMIT_MARGIN_RAD, + JOINT_TARGET_MAX_STEP_RAD, + POSITION_MAX_STEP_M, + ROTATION_MAX_STEP_RAD, + G1NewtonBoundedPinkInverseKinematicsAction, + bound_pink_joint_targets, + condition_kitless_newton_g1_action, + teleop_action_target_issue, +) +from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg +from isaaclab_tasks.contrib.steering_wheel import SteeringWheelUsdFileCfg + +RECORDED_FIRST_ACTION = torch.tensor( + [ + -0.11201994, + 0.43588385, + 0.89166349, + -0.12487421, + -0.11215594, + 0.69946498, + 0.69467705, + 0.14433567, + 0.41181007, + 0.91755670, + 0.00914924, + -0.05320583, + 0.65801126, + 0.75107032, + *([0.0] * 14), + ], + dtype=torch.float32, +) + + +def _quat_angle(a: torch.Tensor, b: torch.Tensor) -> float: + a = torch.nn.functional.normalize(a, dim=0) + b = torch.nn.functional.normalize(b, dim=0) + return float(2.0 * torch.acos(torch.clamp(torch.abs(torch.sum(a * b)), 0.0, 1.0))) + + +def _fake_env(): + left = torch.tensor([0.23491086, 0.21602505, 0.63518274, -0.13474128, 0.79248637, -0.02669795, -0.59422028]) + right = torch.tensor([-0.30430639, 0.52459145, 0.78784978, 0.19782960, 0.52823287, 0.41972736, 0.71109974]) + poses = torch.stack((left, right)).unsqueeze(0) + robot = SimpleNamespace( + body_names=["left_wrist_yaw_link", "right_wrist_yaw_link"], + data=SimpleNamespace(body_link_pose_w=SimpleNamespace(torch=poses)), + ) + cfg = SimpleNamespace( + actions=SimpleNamespace( + upper_body_ik=SimpleNamespace( + target_eef_link_names={ + "left_wrist": "left_wrist_yaw_link", + "right_wrist": "right_wrist_yaw_link", + } + ) + ) + ) + env = SimpleNamespace( + scene={"robot": robot}, cfg=cfg, common_step_counter=0, episode_length_buf=torch.zeros(1, dtype=torch.long) + ) + return env, left, right + + +def test_g1_newton_contract_is_backend_scoped_and_complete() -> None: + default_cfg = FixedBaseUpperBodyIKG1EnvCfg() + newton_cfg = FixedBaseUpperBodyIKG1EnvCfg() + newton_cfg.sim.physics = NewtonCfg() + prepare_fixed_base_g1_for_kitless_newton_mjwarp(newton_cfg) + + assert newton_cfg.sim.physics.solver_cfg.nconmax == 4096 + assert newton_cfg.sim.physics.solver_cfg.njmax == 1536 + for name, selected in ACTUATOR_PROFILE.items(): + actuator = newton_cfg.scene.robot.actuators[name] + assert {key: getattr(actuator, key) for key in selected} == selected + assert newton_cfg.actions.upper_body_ik.class_type is G1NewtonBoundedPinkInverseKinematicsAction + assert newton_cfg.isaac_teleop.action_conditioner is condition_kitless_newton_g1_action + assert newton_cfg.isaac_teleop.action_validator is teleop_action_target_issue + assert isinstance(newton_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert isinstance(newton_cfg.scene.object.spawn, SteeringWheelUsdFileCfg) + + assert default_cfg.isaac_teleop.action_conditioner is None + assert default_cfg.isaac_teleop.action_validator is None + assert not isinstance(default_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert not isinstance(default_cfg.scene.object.spawn, SteeringWheelUsdFileCfg) + + +def test_locomanipulation_preserves_agile_policy_order_and_action_contract() -> None: + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import ( + LocomanipulationG1EnvCfg, + prepare_locomanipulation_g1_for_kitless_newton_mjwarp, + ) + + cfg = LocomanipulationG1EnvCfg() + controller = cfg.actions.lower_body_joint_pos.copy() + upper_body_class = cfg.actions.upper_body_ik.class_type + cfg.sim.physics = NewtonCfg() + prepare_locomanipulation_g1_for_kitless_newton_mjwarp(cfg) + + assert cfg.scene.robot.joint_ordering == "physx" + assert cfg.actions.lower_body_joint_pos == controller + assert cfg.actions.upper_body_ik.class_type == upper_body_class + assert cfg.isaac_teleop.action_conditioner is None + assert cfg.sim.physics.solver_cfg.integrator == "implicitfast" + assert cfg.sim.physics.num_substeps == 2 + # Same robot and props as fixed-base G1, plus foot contacts: keep at least its MJWarp budget. + assert cfg.sim.physics.solver_cfg.nconmax >= 4096 + assert cfg.sim.physics.solver_cfg.njmax >= 1536 + + +def test_recorded_first_action_is_ramped_and_finite() -> None: + env, left, right = _fake_env() + conditioned = condition_kitless_newton_g1_action(env, RECORDED_FIRST_ACTION, {}) + + assert teleop_action_target_issue(conditioned) is None + assert torch.isfinite(conditioned).all() + assert float(torch.linalg.norm(conditioned[:3] - left[:3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert float(torch.linalg.norm(conditioned[7:10] - right[:3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert _quat_angle(conditioned[3:7], left[3:7]) <= ROTATION_MAX_STEP_RAD + 1.0e-5 + assert _quat_angle(conditioned[10:14], right[3:7]) <= ROTATION_MAX_STEP_RAD + 1.0e-5 + assert torch.equal(conditioned[14:], RECORDED_FIRST_ACTION[14:]) + + +def test_wrist_ramp_reseeds_after_termination_auto_reset() -> None: + """A success/drop auto-reset inside env.step moves the wrists; the ramp must restart from there.""" + env, left, _ = _fake_env() + poses = env.scene["robot"].data.body_link_pose_w.torch + + def step(terminated: bool = False) -> None: + # Advance the counters like ManagerBasedRLEnv.step; a terminated episode resets to a lower pose. + env.common_step_counter += 1 + env.episode_length_buf += 1 + if terminated: + env.episode_length_buf[0] = 0 + poses[0, :, :3] += torch.tensor([0.0, 0.0, -0.2]) + + state = {} + target = RECORDED_FIRST_ACTION.clone() + conditioned = condition_kitless_newton_g1_action(env, target, state) + for _ in range(58): + step() + conditioned = condition_kitless_newton_g1_action(env, target, state) + # The ramp has converged near the operator's hands, far from the reset pose. + assert float(torch.linalg.norm(conditioned[:3] - left[:3])) > 2 * POSITION_MAX_STEP_M + # The episode terminates, then the next one terminates on its first step: its counter goes + # 0 -> 1 -> 0 between two frames without decreasing. + for _ in range(2): + step(terminated=True) + conditioned = condition_kitless_newton_g1_action(env, target, state) + assert float(torch.linalg.norm(conditioned[:3] - poses[0, 0, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert float(torch.linalg.norm(conditioned[7:10] - poses[0, 1, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + + +@pytest.mark.parametrize( + "angle, axis, sign", [(0.9, (1, 0, 0), 1), (1.2, (0, 1, 1), -1), (0.1, (0, 0, 1), 1), (0, (1, 0, 0), -1)] +) +def test_conditioned_wrist_rotation_obeys_angular_cap(angle, axis, sign): + """The public action conditioner follows the short rotation without exceeding its cap.""" + env, _, _ = _fake_env() + pose = env.scene["robot"].data.body_link_pose_w.torch + # Any fixed unit quaternion works as the start: slerp is convention-independent. This is the + # (x, y, z, w) = (1, 0, 0, 0) half-turn about x, not identity. + pose[0, :, 3:7] = torch.tensor([1.0, 0, 0, 0]) + direction = torch.nn.functional.normalize(torch.tensor(axis, dtype=torch.float32), dim=0) + target = sign * torch.cat((torch.tensor([math.cos(angle / 2)]), direction * math.sin(angle / 2))) + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = action[10:14] = target + conditioned = condition_kitless_newton_g1_action(env, action, {}) + start = torch.tensor([1.0, 0, 0, 0]) + for result in (conditioned[3:7], conditioned[10:14]): + assert torch.isfinite(result).all() + assert torch.linalg.norm(result).item() == pytest.approx(1, abs=1.0e-6) + assert _quat_angle(start, result) <= min(angle, ROTATION_MAX_STEP_RAD) + 1.0e-5 + assert _quat_angle(result, target) == pytest.approx(max(0, angle - ROTATION_MAX_STEP_RAD), abs=1.0e-5) + + +def test_invalid_actions_fail_closed() -> None: + nonfinite = RECORDED_FIRST_ACTION.clone() + nonfinite[0] = float("nan") + assert teleop_action_target_issue(nonfinite) == "non-finite action target" + + oversized = RECORDED_FIRST_ACTION.clone() + oversized[14] = 7.0 + assert teleop_action_target_issue(oversized) == "unreasonable hand joint target magnitude" + assert "unexpected G1 controller action size" in teleop_action_target_issue(torch.zeros(27)) + + missing_wrist_env, _, _ = _fake_env() + missing_wrist_env.scene = {} + with pytest.raises(RuntimeError, match="could not resolve the task-selected G1 wrist poses"): + condition_kitless_newton_g1_action(missing_wrist_env, RECORDED_FIRST_ACTION, {}) + + +@pytest.mark.parametrize("nonfinite", [False, True]) +def test_invalid_action_errors_preserve_priority(nonfinite) -> None: + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = 0.0 + action[10:14] = torch.tensor([3.0, 0.0, 0.0, 0.0]) + action[14] = 7.0 + if nonfinite: + action[0] = float("nan") + expected = "non-finite action target" if nonfinite else "invalid wrist quaternion norms [0.0, 3.0]" + assert teleop_action_target_issue(action) == expected + + +def test_action_validation_accepts_inclusive_thresholds() -> None: + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = torch.tensor([0.25, 0.0, 0.0, 0.0]) + action[10:14] = torch.tensor([2.0, 0.0, 0.0, 0.0]) + action[14:] = 6.5 + assert teleop_action_target_issue(action) is None + + +def test_pink_targets_are_slew_limited_and_clamped_to_common_limits() -> None: + current = torch.tensor([0.25, 0.0, -0.5]) + proposed = torch.tensor([0.5215, -4.0, 4.0]) + lower = torch.tensor([-0.2617994, -2.0, -1.0]) + upper = torch.tensor([0.2617994, 2.0, 1.0]) + + bounded = bound_pink_joint_targets(current, proposed, lower, upper) + + assert torch.all(bounded >= lower + JOINT_LIMIT_MARGIN_RAD) + assert torch.all(bounded <= upper - JOINT_LIMIT_MARGIN_RAD) + assert float(bounded[0]) == pytest.approx(0.2617994 - JOINT_LIMIT_MARGIN_RAD) + assert float(bounded[1]) == pytest.approx(-JOINT_TARGET_MAX_STEP_RAD) + assert float(bounded[2]) == pytest.approx(-0.5 + JOINT_TARGET_MAX_STEP_RAD) + + +@pytest.mark.parametrize("invalid_input", ["current", "proposed", "lower", "upper"]) +def test_nonfinite_pink_target_fails_closed(invalid_input) -> None: + inputs = { + "current": torch.zeros((2, 1)), + "proposed": torch.zeros((2, 1)), + "lower": -torch.ones((2, 1)), + "upper": torch.ones((2, 1)), + } + inputs[invalid_input][1, 0] = float("nan") + with pytest.raises(RuntimeError, match="non-finite Pink joint target boundary"): + bound_pink_joint_targets(**inputs) + + +@pytest.mark.parametrize("nonfinite", [False, True]) +def test_empty_pink_limit_intersection_fails_closed(nonfinite) -> None: + current = torch.zeros((2, 1)) + if nonfinite: + current[0, 0] = float("nan") + lower = torch.tensor([[-1.0], [0.0]]) + upper = torch.tensor([[1.0], [0.0]]) + message = "non-finite Pink joint target boundary" if nonfinite else "joint-limit intersection is empty" + with pytest.raises(RuntimeError, match=message): + bound_pink_joint_targets(current, torch.zeros_like(current), lower, upper) + + +def test_pink_limit_cache_keeps_live_articulation_limits(monkeypatch): + from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + + proposed = torch.tensor([[0.6, -0.4], [0.8, 0.8]]) + monkeypatch.setattr(PinkInverseKinematicsAction, "_compute_ik_solutions", lambda self: proposed) + action = G1NewtonBoundedPinkInverseKinematicsAction.__new__(G1NewtonBoundedPinkInverseKinematicsAction) + action._env = SimpleNamespace(device="cpu") + limits = torch.tensor([[[-1.0, 1.0]] * 3, [[-0.8, 0.8]] * 3]) + action._asset = SimpleNamespace( + data=SimpleNamespace( + joint_pos=SimpleNamespace(torch=torch.tensor([[0.2, 0.0, 0.0], [0.4, 0.0, -0.2]])), + joint_pos_limits=SimpleNamespace(torch=limits), + ) + ) + action._isaaclab_controlled_joint_ids = [2, 0] + action._ik_controllers = [ + SimpleNamespace( + pink_configuration=SimpleNamespace( + controlled_model=SimpleNamespace( + lowerPositionLimit=lower, + upperPositionLimit=upper, + ) + ), + pink_to_isaac_lab_controlled_ordering=[1, 0], + ) + for lower, upper in [([-0.25, -0.05], [0.5, 0.06]), ([-0.3, -0.4], [0.45, 0.7])] + ] + action._pink_joint_limits = None + torch.testing.assert_close(action._compute_ik_solutions(), torch.tensor([[0.059, 0.1], [-0.1, 0.449]])) + cached = action._pink_joint_limits + limits[0, 2, 1] = 0.03 + limits[1, 0, 1] = 0.42 + torch.testing.assert_close(action._compute_ik_solutions(), torch.tensor([[0.029, 0.1], [-0.1, 0.419]])) + assert action._pink_joint_limits is cached diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py new file mode 100644 index 000000000000..7c80715cea8b --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py @@ -0,0 +1,138 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused regressions for the G1 kitless OVPhysX packing-table contract.""" + +import gymnasium as gym +import pytest +import torch +from isaaclab_ov.physics import OvPhysxCfg + +from isaaclab.app import launch_simulation + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + prepare_fixed_base_g1_for_kitless_ovphysx, +) +from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import ( + LocomanipulationG1EnvCfg, + prepare_locomanipulation_g1_for_kitless_ovphysx, +) +from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg + +_TASK = "IsaacContrib-PickPlace-Locomanipulation-G1-Abs" + + +def test_fixed_base_g1_ovphysx_conditions_assets_without_mutating_default_configs() -> None: + cfg = FixedBaseUpperBodyIKG1EnvCfg() + default_cfg = FixedBaseUpperBodyIKG1EnvCfg() + action_type = cfg.actions.upper_body_ik.class_type + drop_term = cfg.terminations.object_dropping.func + + prepare_fixed_base_g1_for_kitless_ovphysx(cfg) + table_spawn = cfg.scene.packing_table.spawn.func + wheel_spawn = cfg.scene.object.spawn.func + prepare_fixed_base_g1_for_kitless_ovphysx(cfg) + + assert isinstance(cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert cfg.scene.object.spawn.func == default_cfg.scene.object.spawn.func + assert cfg.scene.object.spawn.usd_path == default_cfg.scene.object.spawn.usd_path + assert cfg.scene.packing_table.spawn.func is table_spawn + assert cfg.scene.object.spawn.func is wheel_spawn + assert cfg.actions.upper_body_ik.class_type is action_type + assert cfg.terminations.object_dropping.func is drop_term + assert not isinstance(default_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + + +@pytest.mark.isaacsim_ci +@pytest.mark.parametrize( + ("cfg_type", "task", "prepare"), + [ + (LocomanipulationG1EnvCfg, _TASK, prepare_locomanipulation_g1_for_kitless_ovphysx), + ( + FixedBaseUpperBodyIKG1EnvCfg, + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + prepare_fixed_base_g1_for_kitless_ovphysx, + ), + ], + ids=["locomanipulation", "fixed_base"], +) +def test_g1_wheel_remains_supported_on_ovphysx_cpu(cfg_type, task, prepare) -> None: + pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") + + env_cfg = cfg_type() + env_cfg.scene.num_envs = 1 + env_cfg.sim.device = "cpu" + env_cfg.sim.physics = OvPhysxCfg() + env_cfg.scene.robot_pov_cam = None + if isinstance(env_cfg, FixedBaseUpperBodyIKG1EnvCfg): + # The default policy observations are state-only, so only the scene cameras are removed. + env_cfg.scene.left_wrist_cam = None + env_cfg.scene.right_wrist_cam = None + else: + env_cfg.observations.policy.robot_pov_cam = None + env_cfg.image_obs_list = [] + prepare(env_cfg) + + env = None + launcher_args = { + "physics": None, + "device": "cpu", + "visualizer": None, + "visualizer_explicit": True, + } + try: + with launch_simulation(env_cfg, launcher_args): + env = gym.make(task, cfg=env_cfg).unwrapped + phases = ("table", "reset", "bin") if cfg_type is FixedBaseUpperBodyIKG1EnvCfg else ("table",) + for phase in phases: + env.reset() + wheel = env.scene["object"] + if phase == "bin": + from pxr import Usd, UsdGeom + + from isaaclab.sim import get_current_stage + + floor = get_current_stage().GetPrimAtPath( + "/World/envs/env_0/PackingTable/container_h20/container_h20_inst/colliders/Cube" + ) + assert floor.IsValid() + # Include invisible collision geometry when measuring the bin floor. + bounds = ( + UsdGeom.BBoxCache(Usd.TimeCode.Default(), [UsdGeom.Tokens.default_], False, True) + .ComputeWorldBound(floor) + .ComputeAlignedRange() + ) + center = bounds.GetMidpoint() + floor_top = bounds.GetMax()[2] + pose = wheel.data.root_pose_w.torch.clone() + pose[0, :3] = torch.tensor([center[0], center[1], floor_top + 0.10]) + wheel.write_root_pose_to_sim_index(root_pose=pose) + wheel.write_root_velocity_to_sim_index(root_velocity=torch.zeros((1, 6))) + + minimum_z = float("inf") + with torch.inference_mode(): + for _ in range(300): + env.sim.step(render=False) + env.scene.update(env_cfg.sim.dt) + minimum_z = min(minimum_z, float(wheel.data.root_pos_w.torch[0, 2].item())) + assert torch.isfinite(wheel.data.root_pose_w.torch).all() + assert torch.isfinite(wheel.data.root_vel_w.torch).all() + object_dropping = env_cfg.terminations.object_dropping.func( + env, **env_cfg.terminations.object_dropping.params + ) + assert not bool(object_dropping[0].item()), phase + + assert 300 * env_cfg.sim.dt == pytest.approx(1.5) + assert minimum_z > env_cfg.terminations.object_dropping.params["minimum_height"] + if phase == "bin": + final_position = wheel.data.root_pos_w.torch[0] + assert float(final_position[2]) >= floor_top - 0.005 + for axis in (0, 1): + assert bounds.GetMin()[axis] < float(final_position[axis]) < bounds.GetMax()[axis] + finally: + if env is not None: + env.close() diff --git a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py new file mode 100644 index 000000000000..f4f356bdbbb3 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py @@ -0,0 +1,375 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sim-free tests for the GR1T2 kitless OVPhysX action boundary.""" + +from copy import deepcopy +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from isaaclab_newton.physics import NewtonCfg +from isaaclab_ov.physics import OvPhysxCfg + +from pxr import Usd, UsdGeom, UsdPhysics + +from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg + +from isaaclab_tasks.contrib.pick_place import pickplace_gr1t2_env_cfg as gr1t2_env_cfg +from isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action import ( + GR1T2KitlessPinkInverseKinematicsAction, + _HandTargetSlewState, + bound_ik_joint_targets, + constrain_hand_joint_targets, +) +from isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg import ( + GR1T2KitlessPinkInverseKinematicsActionCfg, +) +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import ( + PickPlaceGR1T2EnvCfg, + condition_kitless_gr1t2_stage, + condition_kitless_ovphysx_robot_spawn_properties, + prepare_gr1t2_for_kitless_newton_mjwarp, + prepare_gr1t2_for_kitless_ovphysx, +) +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import PickPlaceGR1T2WaistEnabledEnvCfg +from isaaclab_tasks.contrib.steering_wheel import SteeringWheelUsdFileCfg +from isaaclab_tasks.utils import parse_env_cfg + +_PREPARE_GR1T2 = { + "ovphysx": prepare_gr1t2_for_kitless_ovphysx, + "newton_mjwarp": prepare_gr1t2_for_kitless_newton_mjwarp, +} + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +def test_newton_contract_is_backend_scoped_and_complete(config_type): + default_cfg = config_type() + task = ( + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs" + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg + else "IsaacContrib-PickPlace-GR1T2-Abs" + ) + newton_cfg = parse_env_cfg(task, device="cpu", num_envs=1, overrides=["physics=newton_mjwarp"]) + prepare_gr1t2_for_kitless_newton_mjwarp(newton_cfg) + + expected = { + "head": (80.0, 8.0, 0.03, 80.0), + "trunk": (500.0, 50.0, 0.10, 250.0), + "legs": (300.0, 30.0, 0.125, 250.0), + "right-arm": (1200.0, 60.0, 0.30, 800.0), + "left-arm": (1200.0, 60.0, 0.30, 800.0), + "right-hand": (5.0, 0.5, 0.02, 20.0), + "left-hand": (5.0, 0.5, 0.02, 20.0), + } + assert set(newton_cfg.scene.robot.actuators) == set(expected) + for name, values in expected.items(): + actuator = newton_cfg.scene.robot.actuators[name] + assert (actuator.stiffness, actuator.damping, actuator.armature, actuator.joint_effort_limit) == pytest.approx( + values + ) + assert newton_cfg.scene.robot.actuators["left-hand"].joint_names_expr == ["L_.*"] + assert newton_cfg.scene.robot.actuators["right-hand"].joint_names_expr == ["R_.*"] + assert newton_cfg.sim.physics.solver_cfg.nconmax == 4096 + assert newton_cfg.sim.physics.solver_cfg.njmax == 1536 + assert newton_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is True + assert isinstance(newton_cfg.actions.upper_body_ik, GR1T2KitlessPinkInverseKinematicsActionCfg) + assert type(default_cfg.actions.upper_body_ik) is PinkInverseKinematicsActionCfg + assert isinstance(newton_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) + assert not isinstance(default_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) + assert default_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is False + + assert set(default_cfg.scene.robot.actuators) == { + "trunk", + "right-arm", + "left-arm", + "right-hand", + "left-hand", + } + assert "legs" not in default_cfg.scene.robot.actuators + + +def test_newton_teleop_self_collision_matches_ovphysx_without_changing_asset_default(): + from isaaclab_newton.sim.schemas import NewtonArticulationCfg + from isaaclab_physx.sim.schemas import PhysxArticulationCfg + + default_cfg = PickPlaceGR1T2EnvCfg() + for backend, fragment_type, attribute in ( + ("newton_mjwarp", NewtonArticulationCfg, "self_collision_enabled"), + ("ovphysx", PhysxArticulationCfg, "enabled_self_collisions"), + ): + cfg = PickPlaceGR1T2EnvCfg() + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + _PREPARE_GR1T2[backend](cfg) + fragment = next(p for p in cfg.scene.robot.spawn.articulation_props if isinstance(p, fragment_type)) + original = next(p for p in default_cfg.scene.robot.spawn.articulation_props if isinstance(p, fragment_type)) + assert getattr(fragment, attribute) is False + assert getattr(original, attribute) is True + + +@pytest.mark.parametrize("backend", ["ovphysx", "newton_mjwarp"]) +@pytest.mark.parametrize("copy_prepared", [False, True, "deepcopy"]) +@pytest.mark.parametrize("replace_spawn", [False, True]) +def test_hand_physical_limits_are_widened_only_for_ovphysx(monkeypatch, backend, copy_prepared, replace_spawn): + stage = Usd.Stage.CreateInMemory() + + def spawn_robot(path, _cfg, **_kwargs): + root = stage.DefinePrim(path, "Xform") + joint = UsdPhysics.RevoluteJoint.Define(stage, path + "/R_thumb_distal_joint") + joint.CreateLowerLimitAttr().Set(-15.0) + joint.CreateUpperLimitAttr().Set(80.0) + return root + + cfg = PickPlaceGR1T2EnvCfg() + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + cfg.scene.robot.spawn.func = spawn_robot + _PREPARE_GR1T2[backend](cfg) + if replace_spawn: + cfg.scene.robot.spawn.func = spawn_robot + _PREPARE_GR1T2[backend](cfg) + original_cfg = cfg + if copy_prepared: + cfg = deepcopy(cfg) if copy_prepared == "deepcopy" else cfg.copy() + assert cfg.actions.upper_body_ik is not original_cfg.actions.upper_body_ik + _PREPARE_GR1T2[backend](cfg) + from isaaclab.assets import Asset, AssetBaseCfg + from isaaclab.sim import use_stage + + cfg.scene.robot.spawn.semantic_tags = [("class", "gr1t2")] + # AssetBase delegates to this exact constructor, which copies the asset config before spawning. + with use_stage(stage): + asset = Asset(AssetBaseCfg(prim_path="/World/Robot", spawn=cfg.scene.robot.spawn)) + assert asset.cfg.spawn is not cfg.scene.robot.spawn + assert asset.cfg.spawn.action_cfg is not cfg.actions.upper_body_ik + asset.cfg.spawn.semantic_tags.append(("instance", "copy")) + assert cfg.scene.robot.spawn.semantic_tags == [("class", "gr1t2")] + assert cfg.scene.robot.spawn.copy().action_cfg is not cfg.actions.upper_body_ik + from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + + def initialize_solver(self, action_cfg, env): + self.cfg = action_cfg + self._env = env + self._asset = asset + self._hand_joint_names = ["R_thumb_distal_joint"] + self._raw_actions = torch.zeros((1, 8)) + + monkeypatch.setattr(PinkInverseKinematicsAction, "__init__", initialize_solver) + action = GR1T2KitlessPinkInverseKinematicsAction( + cfg.actions.upper_body_ik, SimpleNamespace(device="cpu", num_envs=1) + ) + assert action._hand_lower.shape == action._hand_upper.shape == (1,) + assert cfg.actions.upper_body_ik.hand_joint_limits is not asset.cfg.spawn.action_cfg.hand_joint_limits + joint = UsdPhysics.RevoluteJoint(stage.GetPrimAtPath("/World/Robot/R_thumb_distal_joint")) + expected = (-720.0, 720.0) if backend == "ovphysx" else (-15.0, 80.0) + assert (joint.GetLowerLimitAttr().Get(), joint.GetUpperLimitAttr().Get()) == pytest.approx(expected) + assert cfg.actions.upper_body_ik.hand_joint_limits["R_thumb_distal_joint"] == pytest.approx( + (torch.deg2rad(torch.tensor(-15.0)).item(), torch.deg2rad(torch.tensor(80.0)).item()) + ) + if copy_prepared: + assert not original_cfg.actions.upper_body_ik.hand_joint_limits + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +def test_newton_wheel_wrapper_hulls_only_non_graspable_details(monkeypatch, config_type): + stage = Usd.Stage.CreateInMemory() + + def spawn_wheel(prim_path, _cfg): + root = stage.DefinePrim(prim_path, "Xform") + for name in ( + "sm_steeringwheel_a01_wheel_rim_01", + "sm_steeringwheel_a01_wheel_spokes_01", + "sm_steeringwheel_a01_hub_detail_01", + "sm_steeringwheel_a01_quick_release_detail_01", + ): + mesh = UsdGeom.Mesh.Define(stage, f"{prim_path}/{name}").GetPrim() + UsdPhysics.MeshCollisionAPI.Apply(mesh).CreateApproximationAttr().Set(UsdPhysics.Tokens.convexDecomposition) + return root + + cfg = config_type() + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg: + assert {"waist_yaw_joint", "waist_pitch_joint", "waist_roll_joint"} <= set(controlled_joints) + cfg.sim.physics = NewtonCfg() + spawn = cfg.scene.object.spawn + spawn.func = spawn_wheel + monkeypatch.setattr(gr1t2_env_cfg.sim_utils, "find_matching_prim_paths", lambda _path: ["/World/Wheel"]) + + prepare_gr1t2_for_kitless_newton_mjwarp(cfg) + spawn = cfg.scene.object.spawn + wrapped_spawn = spawn.func + wrapped_spawn("/World/Wheel", spawn) + + for name in ("sm_steeringwheel_a01_wheel_rim_01", "sm_steeringwheel_a01_wheel_spokes_01"): + assert stage.GetPrimAtPath(f"/World/Wheel/{name}").GetAttribute("physics:approximation").Get() == ( + UsdPhysics.Tokens.convexDecomposition + ) + for name in ("sm_steeringwheel_a01_hub_detail_01", "sm_steeringwheel_a01_quick_release_detail_01"): + assert stage.GetPrimAtPath(f"/World/Wheel/{name}").GetAttribute("physics:approximation").Get() == ( + UsdPhysics.Tokens.convexHull + ) + + gr1t2_env_cfg.condition_newton_steering_wheel_detail_collisions(cfg.scene.object) + assert spawn.func is wrapped_spawn + assert cfg.actions.upper_body_ik.pink_controlled_joint_names == controlled_joints + assert not isinstance(config_type().scene.object.spawn, SteeringWheelUsdFileCfg) + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +@pytest.mark.parametrize("backend", ["newton_mjwarp", "ovphysx"]) +def test_kitless_kinematics_model_loads_without_the_kit_exporter(config_type, backend): + from isaaclab.controllers.pink_ik.pink_kinematics_configuration import PinkKinematicsConfiguration + + cfg = config_type() + assert cfg.actions.upper_body_ik.controller.urdf_path is None + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + spawn_usd = cfg.scene.robot.spawn.usd_path + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + _PREPARE_GR1T2[backend](cfg) + action = cfg.actions.upper_body_ik + assert action.pink_controlled_joint_names == controlled_joints + assert cfg.scene.robot.spawn.usd_path == spawn_usd + assert action.controller.urdf_path is not None, "Kitless IK must not invoke Isaac Sim's USD-to-URDF exporter" + kinematics = PinkKinematicsConfiguration( + controlled_joint_names=action.pink_controlled_joint_names, + urdf_path=action.controller.urdf_path, + ) + assert set(kinematics.controlled_joint_names_pinocchio_order) == set(action.pink_controlled_joint_names) + for frame in ("left_hand_pitch_link", "right_hand_pitch_link"): + assert np.isfinite(kinematics.get_transform_frame_to_world(frame).homogeneous).all() + jacobian = kinematics.get_frame_jacobian(frame) + assert jacobian.shape == (6, len(action.pink_controlled_joint_names)) + assert np.isfinite(jacobian).all() + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg: + waist_yaw = kinematics.controlled_joint_names_pinocchio_order.index("waist_yaw_joint") + # A real movable waist must rotate either hand about its vertical axis. + world_axis = kinematics.get_transform_frame_to_world(frame).rotation @ jacobian[3:, waist_yaw] + np.testing.assert_allclose(world_axis, [0.0, 0.0, 1.0], atol=1e-6) + + +def test_kitless_action_metadata_is_not_shared(): + first = GR1T2KitlessPinkInverseKinematicsActionCfg() + second = GR1T2KitlessPinkInverseKinematicsActionCfg() + + first.hand_joint_limits["joint"] = (-1.0, 1.0) + + assert second.hand_joint_limits == {} + + +def test_ik_guard_is_per_environment_and_holds_invalid_rows(): + current = torch.tensor([[0.0, 0.1], [0.25, -0.25], [-0.1, 0.1]]) + targets = torch.tensor([[1.0, -1.0], [float("nan"), 0.0], [-0.15, 0.15]]) + + actual = bound_ik_joint_targets(targets, current, max_delta_rad=0.2, max_magnitude_rad=0.3) + + expected = torch.tensor([[0.2, -0.1], [0.25, -0.25], [-0.15, 0.15]]) + torch.testing.assert_close(actual, expected) + + +def test_hand_constraints_apply_mimic_equation_and_bounds_without_mutating_input(): + targets = torch.tensor([[0.4, 9.0, -0.9], [-0.3, 9.0, 0.8]]) + original = targets.clone() + # q_mimic = 2 * q_reference - 0.1; the first row then reaches its upper bound. + mimic_pairs = ((1, 0, 2.0, -0.1),) + lower = torch.tensor([-0.5, -0.6, -0.7]) + upper = torch.tensor([0.5, 0.6, 0.7]) + + actual = constrain_hand_joint_targets(targets, mimic_pairs, lower, upper) + + torch.testing.assert_close(actual, torch.tensor([[0.4, 0.6, -0.7], [-0.3, -0.6, 0.7]])) + torch.testing.assert_close(targets, original) + + +def test_hand_target_slew_clamps_discontinuity_against_previous_command(): + state = _HandTargetSlewState(torch.zeros((1, 2)), max_step_rad=0.079) + + torch.testing.assert_close(state.condition(torch.tensor([[0.2, -0.2]])), torch.tensor([[0.2, -0.2]])) + torch.testing.assert_close(state.condition(torch.tensor([[0.6, -0.5]])), torch.tensor([[0.279, -0.279]])) + + +def test_hand_target_slew_preserves_smooth_input_without_command_chasing(): + state = _HandTargetSlewState(torch.zeros((1, 2)), max_step_rad=0.08) + state.condition(torch.tensor([[0.2, -0.2]])) + + # The conditioner has no measured-state input: a smooth command passes exactly. + commanded = torch.tensor([[0.25, -0.24]]) + torch.testing.assert_close(state.condition(commanded), commanded) + + +def test_hand_target_slew_reset_clears_previous_command_history(): + state = _HandTargetSlewState(torch.zeros((2, 1)), max_step_rad=0.08) + state.condition(torch.tensor([[0.2], [-0.2]])) + state.condition(torch.tensor([[0.6], [-0.6]])) + + state.reset([0]) + + torch.testing.assert_close(state.condition(torch.tensor([[0.7], [-0.7]])), torch.tensor([[0.7], [-0.36]])) + + +def test_robot_conditioning_authors_collision_offsets_and_deinstances_scopes(): + stage = Usd.Stage.CreateInMemory() + + def spawn_robot(prim_path, _cfg): + robot = stage.DefinePrim(prim_path, "Xform") + collisions = stage.DefinePrim(f"{prim_path}/collisions", "Scope") + collisions.SetInstanceable(True) + collision = UsdGeom.Cube.Define(stage, f"{prim_path}/collision_shape").GetPrim() + UsdPhysics.CollisionAPI.Apply(collision) + return robot + + from isaaclab_physx.sim.schemas import PhysxRigidBodyCfg + + spawn = SimpleNamespace( + func=spawn_robot, + rigid_props=[PhysxRigidBodyCfg(max_depenetration_velocity=1.0)], + ) + robot_cfg = SimpleNamespace(spawn=spawn) + action_cfg = GR1T2KitlessPinkInverseKinematicsActionCfg() + + condition_kitless_ovphysx_robot_spawn_properties(robot_cfg) + condition_kitless_gr1t2_stage(robot_cfg, action_cfg) + spawn = robot_cfg.spawn + spawn.func("/World/Robot", spawn) + + collisions = stage.GetPrimAtPath("/World/Robot/collisions") + collision = stage.GetPrimAtPath("/World/Robot/collision_shape") + schemas = collision.GetMetadata("apiSchemas") + applied = [str(schema) for schema in schemas.GetAddedOrExplicitItems()] + assert not collisions.IsInstanceable() + assert "PhysxCollisionAPI" in applied + assert collision.GetAttribute("physxCollision:contactOffset").Get() == pytest.approx(0.001) + assert collision.GetAttribute("physxCollision:restOffset").Get() == pytest.approx(0.0) + assert spawn.rigid_props[0].max_depenetration_velocity == pytest.approx(0.1) + + +def test_wheel_mesh_collision_geometry_is_not_replaced(): + stage = Usd.Stage.CreateInMemory() + original = {} + + def spawn_wheel(path, cfg): + root = stage.DefinePrim(path, "Xform") + body = stage.DefinePrim(path + "/Body", "Xform") + UsdPhysics.RigidBodyAPI.Apply(body) + for name in ("Rim", "Spokes", "Hub", "QuickRelease"): + mesh = UsdGeom.Mesh.Define(stage, path + "/Body/" + name) + mesh.CreatePointsAttr().Set([(0, 0, 0), (1, 0, 0), (0, 1, 0)]) + mesh.CreateFaceVertexCountsAttr().Set([3]) + mesh.CreateFaceVertexIndicesAttr().Set([0, 1, 2]) + UsdPhysics.CollisionAPI.Apply(mesh.GetPrim()) + UsdPhysics.MeshCollisionAPI.Apply(mesh.GetPrim()).CreateApproximationAttr().Set("convexDecomposition") + original["layer"] = stage.GetRootLayer().ExportToString() + return root + + cfg = PickPlaceGR1T2EnvCfg() + cfg.scene.object.spawn.func = spawn_wheel + prepare_gr1t2_for_kitless_ovphysx(cfg) + root = cfg.scene.object.spawn.func("/World/Wheel", cfg.scene.object.spawn) + assert stage.GetRootLayer().ExportToString() == original["layer"] + colliders = [p for p in Usd.PrimRange(root) if p.HasAPI(UsdPhysics.CollisionAPI)] + assert len(colliders) == 4 + assert {p.GetName() for p in colliders} == {"Rim", "Spokes", "Hub", "QuickRelease"} + assert all(p.IsA(UsdGeom.Mesh) for p in colliders) + assert all(UsdPhysics.CollisionAPI(p).GetCollisionEnabledAttr().Get() for p in colliders) + assert all(UsdPhysics.MeshCollisionAPI(p).GetApproximationAttr().Get() == "convexDecomposition" for p in colliders) diff --git a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py index e90483653d8e..a2bffd6544fb 100644 --- a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py +++ b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py @@ -7,13 +7,17 @@ import numpy as np import pytest +from scipy.spatial.transform import Rotation pytest.importorskip("isaacteleop") from isaacteleop.retargeting_engine.interface import ComputeContext, ExecutionEvents, ExecutionState, TensorGroup from isaacteleop.schema import ControllerInputState, ControllerPose, ControllerSnapshot, Point, Pose, Quaternion -from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import _build_so101_stack_pipeline +from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import ( + _BASE_T_GRIPPER_HOME, + _build_so101_stack_pipeline, +) def test_so101_pipeline_tracks_engage_relative_pose_and_gripper(): @@ -37,11 +41,14 @@ def step(position, orientation, squeeze=1.0, trigger=1.0, reset=False): assert action.shape == (8,) and action.dtype == np.float32 return action.copy() - home = [0.01918, -0.18852, 0.18887, 0.0, 0.0, 0.0, 1.0, 1.0] + home_rotation = Rotation.from_matrix(_BASE_T_GRIPPER_HOME[:3, :3]) + home = [*_BASE_T_GRIPPER_HOME[:3, 3], *home_rotation.as_quat(), 1.0] # The first squeeze emits the configured home even with a rotated controller. np.testing.assert_allclose(step((0.2, 0.3, 0.4), (0.0, 0.0, 2**-0.5, 2**-0.5)), home, atol=1e-6) # Relative to the initial Z quarter-turn, this controller quaternion gives [1, 2, 3, 4] / sqrt(30). - moved = [0.11918, 0.01148, 0.48887, *np.array([1, 2, 3, 4]) / np.sqrt(30), 1.0] + # The clutch applies that world-frame rotation and the controller displacement to the full home pose. + delta = Rotation.from_quat(np.array([1, 2, 3, 4]) / np.sqrt(30)) + moved = [*np.add(home[:3], (0.1, 0.2, 0.3)), *(delta * home_rotation).as_quat(), 1.0] np.testing.assert_allclose(step((0.3, 0.5, 0.7), np.array([3, 1, 7, 1]) / np.sqrt(60)), moved, atol=1e-6) # Release at another pose, then re-engage there: the commanded arm pose stays held. held = [*moved[:7], 0.0] diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py new file mode 100644 index 000000000000..bbe63f3321ee --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -0,0 +1,240 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Check hard-coded teleoperation task paths against the robot USD assets the tasks spawn. + +These tests read the published assets without a simulator. They skip when the asset server is +unreachable. +""" + +from __future__ import annotations + +import math +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from pxr import Usd, UsdPhysics + +from isaaclab.utils.assets import retrieve_file_path + +_ENV_ROBOT = "{ENV_REGEX_NS}/Robot" + + +def _retrieve_asset(usd_path: str) -> str: + try: + return retrieve_file_path(usd_path) + except (FileNotFoundError, RuntimeError) as exc: + # The asset server is optional for local test runs: it may be unreachable or fail the download. + pytest.skip(f"Robot asset is unavailable: {exc}") + + +def _open_asset(usd_path: str) -> Usd.Stage: + local_path = _retrieve_asset(usd_path) + stage = Usd.Stage.Open(local_path) + assert stage is not None, local_path + return stage + + +def _robot_paths(stage: Usd.Stage, pattern: str) -> list[Usd.Prim]: + """Return prims of the asset's default prim (the spawned ``Robot``) that fully match ``pattern``.""" + root = stage.GetDefaultPrim() + assert pattern.startswith(_ENV_ROBOT), pattern + expression = re.compile(re.escape(str(root.GetPath())) + pattern[len(_ENV_ROBOT) :]) + return [ + prim for prim in Usd.PrimRange(root, Usd.TraverseInstanceProxies()) if expression.fullmatch(str(prim.GetPath())) + ] + + +@pytest.mark.parametrize("task", ["fixed_base", "locomanipulation"]) +def test_g1_hand_contact_sensors_resolve_to_rigid_hand_links(task): + """Newton matches whole paths, so the patterns must name the ``left_hand``/``right_hand`` parent prims.""" + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + + cfg = FixedBaseUpperBodyIKG1EnvCfg() if task == "fixed_base" else LocomanipulationG1EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + matched = {} + for side in ("left", "right"): + prims = _robot_paths(stage, getattr(cfg.scene, f"{side}_hand_contact").prim_path) + assert prims, side + assert all(prim.HasAPI(UsdPhysics.RigidBodyAPI) for prim in prims) + matched[side] = {str(prim.GetPath()) for prim in prims} + assert matched["left"].isdisjoint(matched["right"]) + + +def test_gr1t2_kitless_kinematics_urdf_matches_the_spawned_asset(): + """The vendored Pink URDF must keep the spawned USD's movable joints and limits [rad].""" + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + cfg = PickPlaceGR1T2EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + usd_limits = {} + for prim in Usd.PrimRange(stage.GetPseudoRoot(), Usd.TraverseInstanceProxies()): + if prim.IsA(UsdPhysics.RevoluteJoint): + joint = UsdPhysics.RevoluteJoint(prim) + usd_limits[prim.GetName()] = ( + math.radians(joint.GetLowerLimitAttr().Get()), + math.radians(joint.GetUpperLimitAttr().Get()), + ) + elif prim.IsA(UsdPhysics.Joint) and not prim.IsA(UsdPhysics.FixedJoint): + pytest.fail(f"Unexpected non-revolute movable joint {prim.GetPath()}") + + urdf_path = ( + Path(__import__("isaaclab_tasks").__file__).parent + / "contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + urdf_limits = {} + for joint in ET.parse(urdf_path).getroot().findall("joint"): + if joint.get("type") == "fixed": + continue + limit = joint.find("limit") + urdf_limits[joint.get("name")] = (float(limit.get("lower")), float(limit.get("upper"))) + + assert set(urdf_limits) == set(usd_limits) + for name, (lower, upper) in usd_limits.items(): + assert urdf_limits[name] == pytest.approx((lower, upper), abs=1.0e-4), name + + +def test_gr1t2_kitless_kinematics_urdf_reproduces_the_asset_forward_kinematics(): + """Every URDF link must sit where the spawned USD's joint frames place it at an asymmetric pose [m, rad]. + + Matching joint names and limits would not catch a moved joint origin or a reversed joint axis. + """ + import numpy as np + + pin = pytest.importorskip("pinocchio") + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + cfg = PickPlaceGR1T2EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + + def joint_frame(position, orientation) -> np.ndarray: + transform = np.eye(4) + quaternion = pin.Quaternion(orientation.GetReal(), *orientation.GetImaginary()) + transform[:3, :3] = quaternion.normalized().toRotationMatrix() + transform[:3, 3] = position + return transform + + # Child link -> (joint name, parent link, joint frame on parent, joint frame on child, joint axis). + joints, limits = {}, {} + for prim in Usd.PrimRange(stage.GetPseudoRoot(), Usd.TraverseInstanceProxies()): + if prim.IsA(UsdPhysics.RevoluteJoint): + joint = UsdPhysics.RevoluteJoint(prim) + joints[joint.GetBody1Rel().GetTargets()[0].name] = ( + prim.GetName(), + joint.GetBody0Rel().GetTargets()[0].name, + joint_frame(joint.GetLocalPos0Attr().Get(), joint.GetLocalRot0Attr().Get()), + joint_frame(joint.GetLocalPos1Attr().Get(), joint.GetLocalRot1Attr().Get()), + np.eye(3)["XYZ".index(joint.GetAxisAttr().Get())], + ) + limits[prim.GetName()] = ( + math.radians(joint.GetLowerLimitAttr().Get()), + math.radians(joint.GetUpperLimitAttr().Get()), + ) + + def usd_link_poses(angles: dict[str, float]) -> dict[str, np.ndarray]: + """Compose ``parent @ frame0 @ R(axis, angle) @ inv(frame1)`` from ``base_link``.""" + poses = {"base_link": np.eye(4)} + + def pose(link: str) -> np.ndarray: + if link not in poses: + name, parent, frame0, frame1, axis = joints[link] + rotation = np.eye(4) + rotation[:3, :3] = pin.AngleAxis(angles[name], axis).toRotationMatrix() + poses[link] = pose(parent) @ frame0 @ rotation @ np.linalg.inv(frame1) + return poses[link] + + for link in joints: + pose(link) + return poses + + def pose_change(a: np.ndarray, b: np.ndarray) -> float: + return float(np.linalg.norm(a[:3, 3] - b[:3, 3]) + np.linalg.norm(pin.log3(a[:3, :3].T @ b[:3, :3]))) + + # Spread the joints over their ranges at golden-ratio fractions, so neighbouring joints and the + # two sides take different angles. + angles = { + name: lower + (upper - lower) * (0.2 + 0.6 * ((index * 0.618034) % 1.0)) + for index, (name, (lower, upper)) in enumerate(sorted(limits.items())) + } + expected = usd_link_poses(angles) + + urdf_path = ( + Path(__import__("isaaclab_tasks").__file__).parent + / "contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + model = pin.buildModelFromUrdf(str(urdf_path)) + data = model.createData() + configuration = pin.neutral(model) + for name, angle in angles.items(): + assert model.existJointName(name), name + joint = model.joints[model.getJointId(name)] + assert joint.nq == 1, name + configuration[joint.idx_q] = angle + pin.framesForwardKinematics(model, data, configuration) + base_inverse = data.oMf[model.getFrameId("base_link")].inverse() + for link, usd_pose in expected.items(): + assert model.existFrame(link), link + urdf_pose = (base_inverse * data.oMf[model.getFrameId(link)]).homogeneous + assert urdf_pose[:3, 3] == pytest.approx(usd_pose[:3, 3], abs=1.0e-5), link + assert np.linalg.norm(pin.log3(urdf_pose[:3, :3].T @ usd_pose[:3, :3])) < 5.0e-4, link + + # The pose must expose a reversed arm axis: negating any one arm joint moves its Pink wrist frame. + wrists = cfg.actions.upper_body_ik.target_eef_link_names + for side in ("left", "right"): + wrist = wrists[f"{side}_wrist"] + arm_joints = [name for name in angles if re.fullmatch(rf"{side}_(shoulder|elbow|wrist)_\w+_joint", name)] + assert len(arm_joints) == 7, arm_joints + for name in arm_joints: + reversed_pose = usd_link_poses(angles | {name: -angles[name]})[wrist] + assert pose_change(reversed_pose, expected[wrist]) > 1.0e-2, name + + +@pytest.mark.parametrize("task", ["fixed_base_g1", "gr1t2", "gr1t2_waist"]) +def test_xr_head_exclusions_partition_only_visual_head_meshes(task): + """OVXR attach fails closed when an exclusion matches nothing; partition the real spawned asset.""" + from pxr import UsdGeom + + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import ( + PickPlaceGR1T2WaistEnabledEnvCfg, + ) + + cfg_type = { + "fixed_base_g1": FixedBaseUpperBodyIKG1EnvCfg, + "gr1t2": PickPlaceGR1T2EnvCfg, + "gr1t2_waist": PickPlaceGR1T2WaistEnabledEnvCfg, + }[task] + cfg = cfg_type() + if cfg.xr is None or not cfg.xr.excluded_visual_meshes: + pytest.skip("The teleoperation configuration is unavailable") + from isaaclab_teleop.xr_scene_partition import partition_visual_meshes + + # Reference the robot where the single-environment scene spawns it. + stage = Usd.Stage.CreateInMemory() + stage.DefinePrim("/World/envs/env_0/Robot").GetReferences().AddReference( + _retrieve_asset(cfg.scene.robot.spawn.usd_path) + ) + session_sublayers = list(stage.GetSessionLayer().subLayerPaths) + with partition_visual_meshes(stage, cfg.xr.excluded_visual_meshes) as paths: + assert paths + for path in paths: + prim = stage.GetPrimAtPath(path) + assert prim.IsA(UsdGeom.Mesh), path + assert "head" in path and "collisions" not in path.split("/") + assert not prim.HasAPI(UsdPhysics.CollisionAPI) + assert UsdGeom.PrimvarsAPI(prim).GetPrimvar("omni:scenePartition").Get() == "xr_excluded" + # The partition is temporary: exiting removes its session sublayer and the authored primvars. + assert list(stage.GetSessionLayer().subLayerPaths) == session_sublayers + for path in paths: + assert not UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).HasPrimvar("omni:scenePartition") diff --git a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py index 6bf7bff87355..952510f65a20 100644 --- a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py +++ b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py @@ -262,3 +262,70 @@ def hide_teleop(name, *args, **kwargs): assert result.returncode == 0, ( f"Subprocess failed:\n--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" ) + + +def test_humanoid_standalone_preparation_without_teleop(): + """Registered humanoid tasks must prepare standalone physics without the Isaac Teleop package.""" + script = textwrap.dedent("""\ + import builtins + import importlib.util + + original_import = builtins.__import__ + original_find_spec = importlib.util.find_spec + + def block_teleop(name, *args, **kwargs): + if name == "isaaclab_teleop" or name.startswith("isaaclab_teleop."): + raise ModuleNotFoundError("No module named 'isaaclab_teleop'", name="isaaclab_teleop") + return original_import(name, *args, **kwargs) + + def hide_teleop(name, *args, **kwargs): + if name == "isaaclab_teleop" or name.startswith("isaaclab_teleop."): + return None + return original_find_spec(name, *args, **kwargs) + + builtins.__import__ = block_teleop + importlib.util.find_spec = hide_teleop + + import gymnasium + import isaaclab_tasks # noqa: F401 + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import validate + from isaaclab_tasks.utils import resolve_task_config + + class ConstructionReached(Exception): + pass + + def stop_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + raise ConstructionReached + + ManagerBasedRLEnv.__init__ = stop_before_simulation + + standalone_tasks = ( + ("IsaacContrib-PickPlace-GR1T2-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "newton_mjwarp"), + ) + for task_name, physics in standalone_tasks: + cfg, _ = resolve_task_config(task_name, "", overrides=[f"physics={physics}"]) + try: + gymnasium.make(task_name, cfg=cfg) + except ConstructionReached: + pass + else: + raise AssertionError("Registered constructor did not reach the simulation boundary") + validate(cfg) + assert cfg.isaac_teleop is None + assert cfg._prepared_kitless_backend == physics + """) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, ( + f"Subprocess failed:\n--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index e7e9f9e6e78e..1ec006cfbb92 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -12,6 +12,7 @@ import argparse import sys +from copy import deepcopy from pathlib import Path import pytest @@ -269,3 +270,269 @@ def load(task, entry): for phrase in expected_phrases: assert phrase in flat, f"Missing phrase: {phrase!r}" + + +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + "IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", + ], +) +@pytest.mark.parametrize("physics", [None, "isaacsim_physx", "ovphysx", "newton_mjwarp"]) +def test_teleop_tasks_resolve_public_physics_selectors(monkeypatch, task_name, physics): + """Resolve registered tasks through the public flag parser before backend preparation. + + Registered humanoid construction then prepares a standalone backend and leaves Kit physics unprepared. + """ + import gymnasium as gym + from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_physx.physics import PhysxCfg + + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import instantiate + + import isaaclab_tasks # noqa: F401 -- registers the public task IDs + from isaaclab_tasks.utils import parse_env_cfg + from isaaclab_tasks.utils.hydra import setup_preset_cli + + arguments = ["--task", task_name] + if physics is not None: + arguments.extend(["--physics", physics]) + _, selections = setup_preset_cli(_make_parser(), arguments) + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=selections) + expected_type = {None: PhysxCfg, "isaacsim_physx": PhysxCfg, "ovphysx": OvPhysxCfg, "newton_mjwarp": NewtonCfg} + assert isinstance(cfg.sim.physics, expected_type[physics]) + assert cfg.scene.num_envs == 1 and cfg.sim.device == "cpu" + if "SO101" in task_name: + assert cfg.scene.robot.spawn.variants["Physics"] == ("physics" if physics == "newton_mjwarp" else "physx") + if physics == "newton_mjwarp": + assert isinstance(cfg.sim.physics.solver_cfg, MJWarpSolverCfg) + if physics == "ovphysx" and "SO101" in task_name: + # The kitless OvPhysX selection keeps the task's Isaac Sim PhysX grasp tuning. + _, physx_selections = setup_preset_cli(_make_parser(), ["--task", task_name, "--physics", "isaacsim_physx"]) + physx_cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=physx_selections).sim.physics + for name in ("bounce_threshold_velocity", "friction_correlation_distance", "solve_articulation_contact_last"): + assert getattr(cfg.sim.physics, name) == getattr(physx_cfg, name), name + + if "SO101" in task_name: + return + + class ConstructionReached(Exception): + pass + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + assert cfg._prepared_kitless_backend == (physics if physics in ("ovphysx", "newton_mjwarp") else None) + raise ConstructionReached + + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + # Each constructor receives its own unprepared copy, so each must prepare it. + for construct in (lambda env_cfg: gym.make(task_name, cfg=env_cfg), instantiate): + with pytest.raises(ConstructionReached): + construct(deepcopy(cfg)) + + +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + ], +) +@pytest.mark.parametrize("physics", ["ovphysx", "newton_mjwarp"]) +def test_humanoid_preparation_preserves_caller_changes_and_rejects_backend_switch(monkeypatch, task_name, physics): + """Teleop preparation, copies and registered construction share one completed preparation.""" + import gymnasium as gym + from isaaclab_physx.physics import PhysxCfg + + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils.string import string_to_callable + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + string_to_callable(cfg.kitless_preparation[physics])(cfg) + cfg.isaac_teleop.require_initial_bimanual_hand_tracking = ( + not cfg.isaac_teleop.require_initial_bimanual_hand_tracking + ) + policy = { + name: getattr(cfg.isaac_teleop, name) + for name in ("require_initial_bimanual_hand_tracking", "action_conditioner", "action_validator") + } + haptics = cfg.haptic_feedback.to_dict() + images = getattr(cfg, "image_obs_list", None) + if images is not None: + images.append("caller_camera") + images = list(images) + cfg.sim.dt = 0.003 + actuator_name = next(iter(cfg.scene.robot.actuators)) + cfg.scene.robot.actuators[actuator_name].stiffness = 123.0 + if physics == "newton_mjwarp": + cfg.sim.physics.solver_cfg.nconmax = 5123 + if "GR1T2" in task_name: + cfg.actions.upper_body_ik.controller.urdf_path = "caller-selected.urdf" + + class ConstructionReached(Exception): + pass + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + assert cfg.actions.upper_body_ik.pink_controlled_joint_names == controlled_joints + assert all(getattr(cfg.isaac_teleop, name) == value for name, value in policy.items()) + assert cfg.haptic_feedback.to_dict() == haptics + assert getattr(cfg, "image_obs_list", None) == images + assert cfg.sim.dt == 0.003 + assert cfg.scene.robot.actuators[actuator_name].stiffness == 123.0 + if physics == "newton_mjwarp": + assert cfg.sim.physics.solver_cfg.nconmax == 5123 + if "GR1T2" in task_name: + assert cfg.actions.upper_body_ik.controller.urdf_path == "caller-selected.urdf" + raise ConstructionReached + + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + for prepared in (cfg, cfg.copy()): + string_to_callable(prepared.kitless_preparation[physics])(prepared) + serialized = prepared.to_dict() + assert serialized["haptic_feedback"] == haptics + if "GR1T2" in task_name: + assert serialized["scene"]["robot"]["spawn"]["func"].endswith(":_spawn_and_condition") + with pytest.raises(ConstructionReached): + gym.make(task_name, cfg=prepared) + prepared.sim.physics = PhysxCfg() + with pytest.raises(ValueError, match="resolve a fresh task configuration"): + gym.make(task_name, cfg=prepared) + + +def test_humanoid_preparation_resolves_directly_built_configs(): + """A configuration built without task resolution is prepared with the default of every preset.""" + from isaaclab_physx.physics import PhysxCfg + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.teleop_physics_cfg import prepare_humanoid_teleop_environment + + cfg = FixedBaseUpperBodyIKG1EnvCfg() + prepare_humanoid_teleop_environment(cfg) + assert type(cfg.sim.physics) is PhysxCfg + assert type(cfg.scene.robot_pov_cam.renderer_cfg) is IsaacRtxRendererCfg + + +def _spawn_task_roundtrip_fixture(path, _cfg, **_kwargs): + """Module-level source spawner, preserving the public callable serialization contract.""" + from pxr import UsdGeom, UsdPhysics + + from isaaclab.sim import get_current_stage + + stage = get_current_stage() + root = stage.DefinePrim(path, "Xform") + joint = UsdPhysics.RevoluteJoint.Define(stage, path + "/R_thumb_distal_joint") + joint.CreateLowerLimitAttr().Set(-15.0) + joint.CreateUpperLimitAttr().Set(80.0) + table = stage.DefinePrim(path + "/Table", "Xform") + UsdPhysics.CollisionAPI.Apply(table) + UsdPhysics.MeshCollisionAPI.Apply(table).CreateApproximationAttr().Set("boundingCube") + UsdGeom.Mesh.Define(stage, path + "/Table/Mesh") + for name in ("sm_steeringwheel_a01_wheel_rim_01", "hub_detail"): + mesh = UsdGeom.Mesh.Define(stage, path + "/" + name).GetPrim() + UsdPhysics.MeshCollisionAPI.Apply(mesh).CreateApproximationAttr().Set("convexDecomposition") + return root + + +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + ], +) +@pytest.mark.parametrize("physics", ["ovphysx", "newton_mjwarp"]) +@pytest.mark.parametrize("restore_target", ["copy", "fresh_prepared"]) +def test_prepared_task_spawn_callbacks_survive_public_config_roundtrip(monkeypatch, task_name, physics, restore_target): + """Restored prepared configurations reach registered construction with resolvable spawners.""" + import gymnasium as gym + + from pxr import Usd, UsdPhysics + + import isaaclab.sim as sim_utils + from isaaclab.assets import Asset, AssetBaseCfg + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import to_dict, update_from_dict + from isaaclab.utils.string import string_to_callable + + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + for asset in (cfg.scene.robot, cfg.scene.packing_table, cfg.scene.object): + asset.spawn.func = _spawn_task_roundtrip_fixture + string_to_callable(cfg.kitless_preparation[physics])(cfg) + cfg.sim.dt = 0.003 + serialized = to_dict(cfg) + if "GR1T2" in task_name: + cfg.actions.upper_body_ik.hand_target_margin_rad = 0.123 + if restore_target == "copy": + restored = cfg.copy() + else: + restored = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + string_to_callable(restored.kitless_preparation[physics])(restored) + update_from_dict(restored, serialized) + if "GR1T2" in task_name: + assert cfg.actions.upper_body_ik.hand_target_margin_rad == 0.123 + + class ConstructionReached(Exception): + pass + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + assert cfg.sim.dt == 0.003 + assets = [cfg.scene.packing_table] + if physics == "newton_mjwarp": + assets.append(cfg.scene.object) + if "GR1T2" in task_name: + assets.append(cfg.scene.robot) + stage = Usd.Stage.CreateInMemory() + for index, asset in enumerate(assets): + callback = to_dict(asset.spawn)["func"] + assert callable(string_to_callable(callback)) + path = f"/World/Asset_{index}" + with sim_utils.use_stage(stage): + authored = Asset(AssetBaseCfg(prim_path=path, spawn=asset.spawn)) + root = authored.prim + assert root == stage.GetPrimAtPath(path) + if asset is cfg.scene.packing_table: + assert not stage.GetPrimAtPath(path + "/Table").HasAPI(UsdPhysics.CollisionAPI) + assert stage.GetPrimAtPath(path + "/Table/Mesh").HasAPI(UsdPhysics.CollisionAPI) + elif asset is cfg.scene.object: + assert ( + stage.GetPrimAtPath(path + "/hub_detail").GetAttribute("physics:approximation").Get() + == "convexHull" + ) + assert ( + stage.GetPrimAtPath(path + "/sm_steeringwheel_a01_wheel_rim_01") + .GetAttribute("physics:approximation") + .Get() + == "convexDecomposition" + ) + else: + limits = authored.cfg.spawn.action_cfg.hand_joint_limits["R_thumb_distal_joint"] + assert limits == pytest.approx((-0.2617993878, 1.3962634016)) + if "GR1T2" in task_name: + assert cfg.scene.robot.spawn.action_cfg is cfg.actions.upper_body_ik + raise ConstructionReached + + monkeypatch.setattr(sim_utils, "find_matching_prim_paths", lambda path: [path]) + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + with pytest.raises(ConstructionReached): + gym.make(task_name, cfg=restored) diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py index 67630cca26c5..745b9d76f7cb 100644 --- a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -48,18 +48,33 @@ def test_g1_xr_camera_uses_calibration_and_head_locked_panel(env_cfg_type): assert feed_cfg.distance_m == 0.8 assert feed_cfg.enable_dlss_ray_reconstruction is True assert feed_cfg.dlss_exec_mode == "quality" - assert feed_cfg.max_update_hz == 0.0 + assert feed_cfg.max_update_hz == (20.0 if env_cfg_type is FixedBaseUpperBodyIKG1EnvCfg else 0.0) assert camera_cfg.renderer_cfg.default == IsaacRtxRendererCfg() assert camera_cfg.renderer_cfg.isaacsim_rtx == IsaacRtxRendererCfg() -def test_fixed_base_g1_pip_preserves_policy_observations_and_fixed_root(): - """PiP adds a sensor without changing the fixed-base policy observation schema.""" - cfg = FixedBaseUpperBodyIKG1EnvCfg() +@pytest.mark.parametrize("selection", [[], ["presets=rgb"]]) +def test_fixed_base_g1_pip_keeps_state_policy_unless_rgb_selected_and_fixed_root(selection): + """Three cameras feed XR panels; their policy image terms are opt-in through ``presets=rgb``.""" + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg - assert hasattr(cfg.scene, "robot_pov_cam") - assert not hasattr(cfg.observations.policy, "robot_pov_cam") + cfg = parse_env_cfg( + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", device="cpu", num_envs=1, overrides=selection + ) + + assert cfg.image_obs_list == ["robot_pov_cam", "left_wrist_cam", "right_wrist_cam"] + for name in ("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"): + assert hasattr(cfg.scene, name) + assert hasattr(cfg.observations.policy, name) is bool(selection) assert cfg.scene.robot.spawn.fix_root_link is True + assert [feed.camera_name for feed in cfg.isaac_teleop.xr_camera_feeds] == [ + "robot_pov_cam", + "left_wrist_cam", + "right_wrist_cam", + ] + assert [feed.max_update_hz for feed in cfg.isaac_teleop.xr_camera_feeds] == [20.0, 10.0, 10.0] + assert cfg.xr.excluded_visual_meshes def test_locomanipulation_g1_retains_recorded_camera(): @@ -85,3 +100,16 @@ def test_xr_camera_reference_renderer_resolves_for_supported_backends(): assert default == expected_rtx assert isaacsim_rtx == expected_rtx assert isinstance(ovrtx, OVRTXRendererCfg) + + +def test_fixed_base_g1_direct_construction_prepares_the_state_policy_default(): + """Registered construction without task resolution uses the state-only policy preset default.""" + from isaaclab.managers import ObservationGroupCfg + + from isaaclab_tasks.contrib.teleop_physics_cfg import prepare_humanoid_teleop_environment + + cfg = FixedBaseUpperBodyIKG1EnvCfg() + prepare_humanoid_teleop_environment(cfg) + assert isinstance(cfg.observations.policy, ObservationGroupCfg) + assert not hasattr(cfg.observations.policy, "robot_pov_cam") + assert hasattr(cfg.observations.policy, "robot_joint_pos") diff --git a/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.added.rst b/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.added.rst new file mode 100644 index 000000000000..750fe8c5a544 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.added.rst @@ -0,0 +1,5 @@ +* Added task hooks to :class:`~isaaclab_teleop.IsaacTeleopCfg`: ``prepare_environment``, + ``action_conditioner``, ``action_validator`` and ``require_initial_bimanual_hand_tracking``. + The live and replay agents run them before creating simulation resources and before each step. +* Added the :attr:`~isaaclab_teleop.IsaacTeleopDevice.hand_tracking_presence` property, which + reports left and right hand presence in the latest input frame. diff --git a/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.minor b/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst new file mode 100644 index 000000000000..0565d24bd9f3 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst @@ -0,0 +1,6 @@ +* Added :class:`~isaaclab_teleop.OvLibrariesXrSession`, an OVXR application adapter and session + bootstrap that borrow the simulation-owned OVRTX renderer and OVStage, publish scene updates, + and coordinate render-thread ownership and shutdown with Isaac Capture. + Publication failures close admission, and failed lease retirement retains borrowed resources. +* Added :class:`~isaaclab_teleop.OvLibrariesXrCfg` in :mod:`isaaclab_teleop.ovxr_cfg`, which + imports without the native OVXR packages. diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor b/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst new file mode 100644 index 000000000000..04fd6d18ab78 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -0,0 +1,13 @@ +* Added optional OVXR camera panels and a desktop mirror as application-configured + Isaac Capture frame consumers using completed render products, with OVUI's default + Vulkan backend for the mirror. :class:`~isaaclab_teleop.OvLibrariesXrCameraCfg` selects a panel. +* Enabled the mirror by default in OVXR live and MCAP replay entrypoints; ``--headless`` + disables the local window while preserving XR and camera-panel rendering. +* Supported task-configured automatic layouts for head-locked panels + in live teleoperation and replay. ``--ovxr_pip_head_locked`` places a task layout that is + not head-locked in the OpenXR VIEW space. Replay accepts the live ``--ovxr_pip_fps``, + ``--ovxr_pip_view_position`` and ``--ovxr_desktop_mirror_fps`` options. +* Added :attr:`~isaaclab_teleop.XrCfg.excluded_visual_meshes`, which hides matching visual meshes + from the OVXR eye cameras and camera panels without changing the source scene's visibility. +* Added the ``keep`` argument to :func:`~isaaclab_teleop.remove_camera_configs` to retain selected + cameras while removing their observation terms. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.minor b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst new file mode 100644 index 000000000000..81f21c36ff2f --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -0,0 +1,7 @@ +* Added kitless OVXR teleoperation and MCAP replay with independently paced rendering, + backend-derived anchors, and ordered shutdown. +* Added ``--start_on_replay`` to the MCAP replay agent. It starts playback at startup and after + each recorded RESET, for captures made without recorded START messages. +* Added external OVXR inputs to :class:`~isaaclab_teleop.IsaacTeleopDevice`, together with the + ``target_frame_prim_path``, ``xr_render_anchor_path``, ``external_anchor_state``, + ``required_openxr_extensions`` and ``session_cleanup_failed`` properties. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst new file mode 100644 index 000000000000..0963882a4989 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst @@ -0,0 +1,6 @@ +* Fixed replay ending at a recorded RESET; it now resets and waits for the next recorded START. +* Deferred anchor USD imports until use so Kit can initialize its own USD provider. +* Fixed teleoperation cleanup stopping at the first failure: a retargeting UI or XR anchor + cleanup failure no longer skips the Isaac Capture session and CloudXR shutdown, and a failed + device entry stops a runtime it already launched. The first failure is still reported. +* Reported OVXR replay session-exit failures and retained their resources instead of suppressing them as Kit XR teardown. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.minor b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md new file mode 100644 index 000000000000..9fe4bae4b412 --- /dev/null +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -0,0 +1,128 @@ +# OVXR application adapter + +`isaaclab_teleop.OvLibrariesXrSession` connects an Isaac Lab environment to +Isaac Capture's OVXR session owner. It uses the simulation's existing OVRTX +renderer and OVStage through `isaaclab_ov.renderers.OvLibrariesRenderLease`. + +## Ownership + +- `isaaclab_ov` owns the renderer, stage, scene publication and their lifetime. + Its lease validates access and enforces render-thread ownership. The adapter's + publication guard serializes application writes with Capture's frame operations. + It does not create an XR session or select teleoperation behavior. +- `isaaclab_teleop.ovxr_session.OvxrSessionBootstrap` creates the simulation + context and reserves the lease before environment construction. It binds the + completed environment and reads current target-frame transforms. +- `isaaclab_teleop.ov_libraries_xr.OvLibrariesXrSession` supplies the application + adapter to Isaac Capture. It declares XR render products, publishes scene and + anchor updates, transfers render-thread ownership and coordinates shutdown. +- Isaac Capture owns the OVXR session and render worker. Its borrowed OpenXR + handles are available to the teleoperation input session. + +## Lifecycle + +Prepare the adapter before constructing the environment so that both use the +same simulation context and native renderer/scene pair. Start the render worker +after the environment exists. Publish simulation updates through the session; +the worker owns rendering. Stop input consumers before closing the XR session, +and retire the lease before destroying the simulation's renderer or stage. +If startup fails before the worker takes ownership, the caller closes the +environment after releasing the lease. If input retirement fails, request +shutdown and retain the borrowed resources until their retirement is proved. + +The lease currently requires one environment and a private rendering OVStage. +Ownership transfer is single-use. Failed retirement must preserve resources +for the owning thread to finish cleanup; a failed drain is not successful +shutdown. Importing `OvLibrariesXrSession` requires a Capture release with OVXR +session support; see [Installing kitless XR](kitless_xr_source.md). The +`OvLibrariesXrCfg` dataclass lives in `isaaclab_teleop.ovxr_cfg` and imports +without those packages. + +A failed scene write or seal stops further publication. Recreate the session +after retiring the failed pair; retrying the same generation cannot return an +older scene as if the failed update succeeded. The failure poisons producer +publication only: the render-owner lane stays available so Capture can destroy +its OVXR context and the adapter can release the lease and close the +environment. After a poisoned lease is released, the renderer keeps rejecting +ordinary use (except closing it) because its write ordinal was not rebased. +During shutdown, publication storage drains before lease guards are removed. +The adapter retains the environment if lease retirement cannot establish completion. + +## Environment requirements + +- `OvLibrariesXrSession.prepare()` creates the `SimulationContext` from the + `sim_cfg` it receives, and the environment reuses that context. Pass the + environment's own `env_cfg.sim`; before the worker claims the lease, + `start_render_worker()` rejects an environment whose simulation device or + time step differs, or whose `num_envs` is not 1. +- Only environments that accept a pre-created simulation context are + supported. Manager-based environments do; `DirectRLEnv` always creates its own + context and fails with "Simulation context already exists". +- Camera sensors on other renderers are supported: their initialization + prepares every registered renderer on the application thread, and the leased + renderer defers its Stage until the XR worker claims ownership. A camera that + resolves to the leased renderer itself (for example an OVRTX camera with the + default `OVRTXRendererCfg()`) is rejected while the lease is active. +- Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its + scene publisher writes the same physics state at the reserved ordinal. +- CPU simulation devices publish scene transforms, but visual-material + publication requires CUDA material buffers, as it does for ordinary OVRTX + cameras. With `--device cpu`, an environment that has `VisualMaterial` assets + fails at `sim.reset()`. +- Dynamic anchoring takes any object implementing + `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects + other objects before the worker claims the lease. + +## Backend implementation + +The ordinary renderer remains in `isaaclab_ov.renderers.ovrtx_renderer`. +`render_lease.RenderLeaseController` owns the lease state, thread-transfer +capability and per-owner operation guards; it borrows the renderer's native resources +and delegates component writes back to that renderer. Public renderer and lease +imports remain available from `isaaclab_ov.renderers`. + +Acquiring a lease selects a private rendering OVStage without changing process +environment variables. An explicit OVStage disable or shared-stage selection is +rejected. Without an active lease, the controller permits +ordinary renderer operations and does not start an XR session. While a consumer +holds the lease, ordinary camera rendering, scene writes and renderer destruction +are rejected to prevent concurrent access to the borrowed resources. The same +generic borrowing interface is available to applications outside teleoperation. + +## Optional presentation + +After resolving the selected camera render products, the Lab adapter supplies +Capture with configured `CameraPanelConsumer` and `OwnedDesktopMirror` instances +in `OvxrSessionBinding.frame_consumers`. Each panel configuration pairs its product +path with its placement and image bound. The mirror receives its camera choices +separately, so it can run without headset panels. + +Capture runs both consumers through the same frame lifecycle. It shares CPU +snapshots within a frame and keeps native layers alive through submission; Lab +continues to own sensor production and scene publication. These interfaces are +application-neutral and do not require Lab-specific scene types. + +The mirror uses OVUI's default Vulkan backend. Neither Lab nor Capture modifies +`OMNIUI_BACKEND`; the mirror rejects an active non-Vulkan backend. Neither optional +consumer is created when disabled. + +## Native validation + +The adapter and lease unit tests double the native OVRTX and OVStage endpoints. +An opt-in harness runs the render-owner transfer and complete scene publication +against the real libraries, on the manager-based `Isaac-Cartpole` task with its +`ovphysx` physics preset and one environment on `cuda:0`. The transfer case +also runs with CPU physics, which publishes host scene tensors to the GPU +renderer. Both cases also run `IsaacContrib-PickPlace-GR1T2-Abs` with CPU +OvPhysX, which its registered entry point prepares for kitless physics; its +cameras are removed as in an `--ovxr` launch. The harness needs a CUDA GPU, +OVRTX, OVStage, OvPhysX and an Isaac Capture release with OVXR session support; +it starts no XR session or CloudXR runtime. +Each case runs in a child process: + +```bash +ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1 uv run python -m pytest \ + source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py +``` + +Without `ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1`, every case is skipped. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 266f5470dda9..aff2eb8cbee1 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -21,9 +21,15 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. parameters when `retargeters_to_tune` is provided. - **XR camera feedback** -- selected scene cameras can be shown through Kit Scene UI using a feed-owned CUDA view of their existing RTX render product, with camera-buffer fallback. + The optional [OVXR path](kitless_xr.md) shows them as OpenXR quad layers; its desktop mirror + shows eye images without those layers. ## Architecture +The [OVXR application adapter](OVXR_ADAPTER.md) connects a simulation-owned +renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; +`isaaclab_ov` supplies the generic renderer/scene lease and lifetime checks. + `IsaacTeleopDevice` composes three focused collaborators: | Component | Responsibility | @@ -32,6 +38,15 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. | `TeleopSessionLifecycle` | Pipeline building, OpenXR handle acquisition, session create/destroy, action-tensor extraction | | `CommandHandler` | Callback registration and XR message-bus command dispatch | +For kitless XR, `isaaclab_teleop` owns session preparation and tracking anchors. +It borrows the application renderer through `isaaclab_ov`'s `OvLibrariesRenderLease` +interface. The renderer retains ownership of native resources, thread synchronization +and GPU scene publication. Robot-specific action conditioning and physics preparation +remain in the task package. +`isaaclab_teleop` also owns camera-panel selection and presentation policy; live +teleoperation and replay share the camera-feed configuration helpers in +`isaaclab_teleop.camera_feed`. + ## Usage ### 1. Configure Your Environment @@ -98,6 +113,9 @@ uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ --task My-Isaac-Capture-Env-v0 --xr ``` +For kitless teleoperation, installation, MCAP replay, camera panels and the +optional desktop mirror, see [Kitless XR teleoperation](kitless_xr.md). + See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for reference tasks, camera selection, layout, placement, renderer, disable, and kitless behavior. @@ -136,6 +154,9 @@ rendering without blocking. | `sim_device` | `str` | `"cuda:0"` | Torch device for output action tensors | | `retargeting_execution` | `RetargetingExecutionConfig \| None` | `None` (resolved at session start to `mode="pipelined", pacing=DeadlinePacingConfig(safety_margin_s=0.025)`) | Isaac Capture retargeting execution settings; deferred so the config imports without `isaacteleop` | | `teleoperation_active_default` | `bool` | `False` | Whether teleoperation is active on session start | +| `require_initial_bimanual_hand_tracking` | `bool` | `False` | Wait for initial hand presence before stepping | +| `action_conditioner` | `Callable \| None` | `None` | Task-owned stateful pre-step action conditioning | +| `action_validator` | `Callable \| None` | `None` | Task-owned fail-closed action validation | | `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the Isaac Capture session | The 25 ms `DeadlinePacingConfig` safety margin staggers Isaac Capture's Python work behind Isaac Lab's @@ -148,17 +169,36 @@ contend for the GIL at the start of the step. |---|---|---|---| | `anchor_pos` | `tuple[float, float, float]` | `(0, 0, 0)` | XR anchor position in world frame | | `anchor_rot` | `tuple[float, float, float, float]` | `(0, 0, 0, 1)` | XR anchor rotation (quaternion xyzw) | -| `anchor_prim_path` | `str \| None` | `None` | Prim to attach anchor to for dynamic positioning | +| `excluded_visual_meshes` | `list[str]` | `[]` | Visual-mesh regexes assigned to a separate render-export partition, without changing source USD visibility | +| `anchor_prim_path` | `str \| None` | `None` | Prim to attach anchor to for dynamic positioning; OVXR reads committed physics poses and shares the same anchor policy | | `anchor_rotation_mode` | `XrAnchorRotationMode` | `FIXED` | How anchor rotation tracks the reference prim | | `anchor_rotation_smoothing_time` | `float` | `1.0` | Slerp time constant (seconds) for `FOLLOW_PRIM_SMOOTHED` mode | | `anchor_rotation_custom_func` | `Callable` | identity | Custom rotation function for `CUSTOM` mode | | `near_plane` | `float` | `0.15` | Near clipping plane distance for the XR device | | `fixed_anchor_height` | `bool` | `True` | Fix anchor height to initial value of the reference prim | +For single-environment OVXR, prefer `excluded_visual_meshes` for a robot +head surrounding the eyes. Match only visual mesh paths, not head links or +collision geometry. Environment cameras select `env_0`; excluded meshes use +`xr_excluded`, with `showAllPartitionsByDefault=false`. This also excludes the +head from PiP cameras selecting `env_0`; other camera views require an explicit +partition policy. The source Stage's visibility and instancing are restored after +render export. GR1T2 and fixed-base G1 use this policy. + +## Capture and replay boundaries + +MCAP records DeviceIO tracker/control input, not historical simulation scenes. +Environment/control reset does not rewind MCAP input. HDF5 +demonstration scripts use the Kit/Isaac Sim application path; these OVXR +commands do not qualify HDF5 recording/replay. +PiP and mirror consume latest completed images rather than a history of simulation +scenes; they are not lossless recorders or rolling-shutter playback. + ## Utilities -- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy +- **`remove_camera_configs(env_cfg, keep=())`** -- strips camera sensors and their associated policy observation terms. + Cameras named in `keep` stay in the scene, but their observation terms are still removed. ## Run with Docker @@ -177,8 +217,13 @@ In the Isaac Sim UI, set the AR panel to **System OpenXR Runtime** and click **S For a fully headless experience, omit `--visualizer kit` when running Docker, and the XR teleop session will run automatically. +For OVXR, use the separate [kitless XR installation and runtime guide](kitless_xr.md). + ## Dependencies - **`isaaclab`** -- core Isaac Lab framework +- **`isaaclab_ov`** -- required by the optional OVXR session adapter; the standalone `isaaclab_teleop` package does not install this backend automatically - **`isaacteleop`** -- Isaac Capture retargeting engine, device I/O, and session management - **`isaacsim`** -- Isaac Sim runtime (provides the Kit XR bridge for OpenXR handle acquisition) +- **OVXR / OVRTX / OVStage** -- matched native runtime and stock peers for OVXR +- **OVUI** -- optional desktop-mirror extra; headset-only operation does not import it diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md new file mode 100644 index 000000000000..8a19bc54f24b --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -0,0 +1,182 @@ +# Kitless XR teleoperation + +OVXR presents Isaac Lab's OVRTX renderer and OVStage scene while Isaac Capture +provides tracking and retargeting. This workflow supports Linux x86-64 with +Newton or OVPhysX physics and one environment. + +## Prerequisites + +Before installing, complete the workstation and streaming setup in the +[CloudXR prerequisites](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#prerequisites). +Use the OVXR package selection below for this workflow; the Kit installation +commands in that guide belong to Kit XR. + +- Follow [CloudXR network setup](https://docs.nvidia.com/cloudxr-sdk/latest/requirement/network_setup.html) + for connectivity and firewall requirements between the workstation and client. +- Follow [XR client setup](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#connect-an-xr-device) + for a supported headset or the CloudXR.js browser client, including certificate setup. + The [Isaac Capture Quick Start](https://nvidia.github.io/IsaacCapture/main/teleop/quick_start.html#connect-an-xr-headset) + also covers the desktop IWER client. +- Review the [CloudXR license](https://github.com/NVIDIA/IsaacCapture/blob/main/deps/cloudxr/CLOUDXR_LICENSE) + and complete the first-run EULA acceptance described in + [CloudXR startup](https://nvidia.github.io/IsaacCapture/main/references/cloudxr.html#start-the-service) + before launching without an interactive terminal. + +## Installation + +Use Python 3.12 and the Isaac Capture wheel selected by the installation +guide. +Configure the `ovxr` dependency selection using +[Installing kitless XR](kitless_xr_source.md), then install and +check that selection: + +```bash +uv sync --python 3.12 --locked --extra ovxr +uv pip check +``` + +The runtime commands below use this prepared `ovxr` selection. + +The live and MCAP workflows are implemented in `isaaclab_teleop.entrypoints`. +The command-line entrypoints load registered downstream tasks before parsing +and restore the caller's arguments on return. The scripts under +`scripts/environments/teleoperation/` delegate to these package entrypoints. + +## Live teleoperation + +Start with CPU OvPhysX using the command below. The same registered-task +selectors apply to live input and MCAP replay: + +| Task | Physics selectors | +| --- | --- | +| `IsaacContrib-PickPlace-GR1T2-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-Locomanipulation-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-Stack-Cube-SO101-IK-Abs-v0` | `ovphysx`, `isaacsim_physx` | + +Omitting `--physics` retains the Kit/Isaac Sim default. Use `--physics ovphysx +--device cpu` for standalone CPU physics, or `--physics newton_mjwarp --device +cuda:0` for Newton; these standalone selections support `--ovxr`. Kit's +`isaacsim_physx` selection uses its separate `--xr` workflow and Isaac Sim +installation, not `--ovxr`. The task presets retain backend-specific solver +settings and, for SO101, choose the matching USD Physics variant. SO101 also +defines a `newton_mjwarp` preset, but the environment catalog +(`tools/environ_docs.py`) disables that combination after the environment smoke +matrix found runtime constraints, so it is not listed here. + +For example, replace the task in this command with any row above: + +```bash +uv run --extra ovxr isaaclab teleop run \ + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ + --device cpu --physics ovphysx --ovxr +``` + +`--ovxr` selects OVStage and starts CloudXR with the `cloudxrjs` profile by +default. Connect a supported CloudXR client, then use its Play and Reset +controls. Pause/resume is available only on clients that expose it; Reset is a +separate action. + +The SO101 IK task resets its joints to the configured nominal pose without +joint-position noise, giving the clutch a repeatable reset pose. Its configured +home transform supplies the clutch calibration. This applies to its Kit and +kitless backends. The separate joint-control task retains its randomized reset. + +For SO101, check right-controller grip/squeeze engagement, release and +re-engagement, proportional trigger control and Reset separately from cube +grasping. Object retention and cube stacking also depend on contact properties +and grasp placement; validate them separately from the input controls. + +For desktop IWER, select **VR** and set **Reference Space** to **local-floor** +under **Teleoperator adjustment** before connecting. This keeps the emulated +head and controllers relative to the floor. Between trials, end the previous XR +session and reload the client page before reconnecting; recheck both settings. +A blank or below-scene main view with a working camera panel warrants checking +this client origin before changing the renderer. + +`--ovxr` and Kit's `--xr` select separate XR runtimes and are mutually exclusive. +An explicit `ISAAC_LAB_OVRTX_USE_OVSTAGE=0` is incompatible with OVXR. + +OVXR live teleoperation and MCAP replay open a desktop mirror by default. Add +`--headless` to disable the local window, including on hosts without a desktop +display. Headset rendering still uses the GPU and works with `DISPLAY` unset +when `--headless` is selected. PiP remains available in headless mode. +Renderer startup has a bounded attachment timeout; it does not bound waiting for a +client to establish its OpenXR session. Installing Capture and resolving +Python packages do not warm application-specific RTX shaders or Newton/Warp +physics kernels. First runtime preparation and the first physics step can take +longer than subsequent runs. Retain startup logs and distinguish stage/render +preparation from client connection and active simulation. + +Use `--step_hz 20` to cap live control at 20 updates per second while XR rendering +runs independently. Time spent reading input, stepping physics and publishing +the scene counts toward that interval. Slow steps do not cause catch-up steps. +Omit the option or use zero for unthrottled control. Paused polling uses the +environment action interval when no explicit rate is set. This option requires +`--ovxr` and does not change physics dt or decimation. + +## MCAP replay + +```bash +uv run --extra ovxr python scripts/environments/teleoperation/teleop_replay_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ + --device cpu --physics ovphysx --ovxr \ + --replay_file /absolute/path/to/session.mcap +``` + +MCAP supplies tracker and control input; the connected headset supplies the +view. A recorded STOP, task success/failure, or `--max_replay_duration_s` ends +the run. Playback waits for recorded START messages, including after a recorded +RESET. Reset does not rewind recorded input. For legacy captures started locally +without recorded START messages, add `--start_on_replay` to start automatically +at startup and after RESET. `--max_stage_load_wait_s` bounds +renderer startup separately; the replay duration starts after warmup and +includes any lazy first-step physics initialization. Deadlines are checked +between steps. Application render-call timing statistics are unavailable when +OVXR owns rendering independently. + +HDF5 demonstration recording and replay continue to use the Kit/Isaac Sim path. + +## Camera panels and desktop mirror + +Select up to three task cameras and a head-locked layout: + +```bash +uv run --extra ovxr isaaclab teleop run --ovxr \ + --task IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs \ + --num_envs 1 --device cpu --physics ovphysx \ + --ovxr_pip_camera robot_pov_cam left_wrist_cam right_wrist_cam \ + --ovxr_pip_head_locked +``` + +`--ovxr_pip_fps` limits panel updates. `--ovxr_pip_view_position X Y Z` overrides +one panel's position; multiple panels use the task layout. Panel cameras must be +enabled in the task's `xr_camera_feeds`. World and viewer-start layouts are not +supported by the OVXR presenter: `--ovxr_pip_head_locked` places such a layout's +panels in the OpenXR VIEW space, and without it the command fails before launch. +The session snapshots its camera selection at +preparation; create a new session to change it. Panels use the same renderer as +the headset and do not require a desktop display. OVXR panels show no label; a +feed's `label` applies to Kit panels only, and the desktop mirror lists each camera +by its sensor name. + +GR1T2 and G1 locomanipulation provide `robot_pov_cam`; fixed-base G1 also provides +`left_wrist_cam` and `right_wrist_cam`. SO101 does not declare a PiP camera in +its scene. Use the desktop mirror without `--ovxr_pip_camera` for that task. + +Selected OVXR cameras feed the panels only. Their policy image observations are +removed even with `presets=rgb`, so the application does not request a second +camera render while OVXR owns rendering. The `presets=rgb` policy-image selection +applies to Kit workflows. + +The default desktop mirror shows completed eye or camera images locally. +`--headless` disables it. `--ovxr_desktop_mirror_fps` limits its update rate. The mirror requires the +`desktop-mirror` extra, Python 3.12 or newer, and a usable display. On Linux, +use a GPU-backed desktop session for the mirror. OVUI's default Vulkan backend +requires no `OMNIUI_BACKEND` setting. Lab and Capture preserve the application's +backend choice; an already initialized OVUI backend cannot be reconfigured. +The mirror requires Vulkan; remove conflicting backend settings before OVUI +initialization. The mirror opens after the first submitted +XR image; closing it requests application shutdown. Eye images exclude the +separate OpenXR camera panels; select a camera product to inspect that feed. diff --git a/source/isaaclab_teleop/docs/kitless_xr_source.md b/source/isaaclab_teleop/docs/kitless_xr_source.md new file mode 100644 index 000000000000..b1fc336fbf9e --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr_source.md @@ -0,0 +1,66 @@ +# Installing kitless XR + +Use Python 3.12 on Linux x86-64. The Isaac Capture wheel includes the +OVXR session integration, native camera-panel helper and desktop mirror modules. + +Complete the [CloudXR, client, network and license prerequisites](kitless_xr.md#prerequisites) +before starting a session. + +From the Isaac Lab root, select the Capture version in an `ovxr` project +extra. Lab already configures NVIDIA's public Python index and allows prereleases: + +```bash +uv add --optional ovxr --frozen \ + "isaaccapture[ovxr,retargeters,ui,cloudxr,desktop-mirror]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +uv add --optional ovxr --frozen \ + "isaaclab-dev[ov] ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-teleop ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-mimic ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "ipywidgets>=8.1.5 ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +``` + +The `ov` extra supplies the compatible OVRTX, OVStage, OVPhysX and PhysX USD +schemas. Keep its provider requirements and the workspace package metadata unchanged. + +Add these mutually exclusive selections to `tool.uv.conflicts`, preserving existing entries: + +```toml +[tool.uv] +conflicts = [ + [{ extra = "ovxr" }, { extra = "teleop" }], + [{ extra = "ovxr" }, { extra = "isaacsim" }], + [{ extra = "ovxr" }, { extra = "viser" }], + [{ extra = "ovxr" }, { extra = "all" }], +] +``` + +The `teleop` extra selects Capture packages for Kit and non-Kit input +workflows. Use the separate `ovxr` selection for the XR commands here. +Isaac Sim and Viser require older WebSocket versions than this Capture +release, so these selections cannot be combined. The `all` extra +includes Viser. The wheel requirement includes the optional `desktop-mirror` extra; +omit that extra only when running with `--headless`. OVXR live and MCAP replay +open the mirror by default. The mirror requires Python 3.12 or newer and a +usable GPU-backed desktop display. + +Resolve the selected environment and check its dependencies: + +```bash +uv lock --refresh-package isaaccapture --upgrade-package isaaccapture \ + --upgrade-package numpy==2.3.5 --upgrade-package packaging==23.2 +uv sync --python 3.12 --locked --extra ovxr +uv run --extra ovxr python -c 'import importlib.metadata as m; print(m.version("isaaccapture"), m.version("ovxr"))' +uv pip check +``` + +The lock selects compatible NumPy 2.3.5 and packaging 23.2 versions without +changing project-wide overrides. OVPhysX declares `packaging<24`; the explicit +selection above satisfies its metadata in this non-Kit selection. Stop applications before changing selections. +The Kit and OVXR provider selections are different; preserve the source metadata +and lockfile when changing between them, and check the resulting dependency graph. +Repeat this selection if a later lock upgrade replaces it. Isaac Capture's +`ovxr` extra selects OVXR and its compatible OVRTX/OVStage dependencies; see its +OVXR guide for application version overrides. + +Continue with [live teleoperation](kitless_xr.md#live-teleoperation). diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index d810345978e6..4e94d8cbdaaa 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,6 +15,9 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", + "OvLibrariesXrCameraCfg", + "OvLibrariesXrCfg", + "OvLibrariesXrSession", "SupportsControlEvents", "SystemCheckItem", "SystemCheckResult", @@ -42,8 +45,12 @@ __all__ = [ from .camera_feed import XrCameraFeedSession from .control_events import TELEOP_CONTROL_CHANNEL_UUID, ControlEvents, SupportsControlEvents, poll_control_events from .device_pipelines import ( - se2_gamepad_teleop_cfg, se2_keyboard_teleop_cfg, se2_spacemouse_teleop_cfg, - se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, + se2_gamepad_teleop_cfg, + se2_keyboard_teleop_cfg, + se2_spacemouse_teleop_cfg, + se3_gamepad_teleop_cfg, + se3_keyboard_teleop_cfg, + se3_spacemouse_teleop_cfg, ) from .haptic_feedback import ( ControllerHapticFeedbackCfg, @@ -62,6 +69,8 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device +from .ov_libraries_xr import OvLibrariesXrSession +from .ovxr_cfg import OvLibrariesXrCameraCfg, OvLibrariesXrCfg from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/action_policy.py b/source/isaaclab_teleop/isaaclab_teleop/action_policy.py new file mode 100644 index 000000000000..01f0af91953e --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/action_policy.py @@ -0,0 +1,53 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Task-owned action validation shared by live and replay.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .isaac_teleop_device import IsaacTeleopDevice + + +def prepare_teleop_action( + env: Any, + action: Any, + conditioner: Callable[[Any, Any, dict[str, Any]], Any] | None, + validator: Callable[[Any], str | None] | None, + state: dict[str, Any], +) -> tuple[Any, str | None]: + """Validate and condition one task-owned action without stepping physics. + + The conditioner runs on a copy of ``state``, which is committed only when + the conditioned action also passes validation. + """ + issue = validator(action) if validator is not None else None + if issue is None and conditioner is not None: + candidate = dict(state) + action = conditioner(env, action, candidate) + issue = validator(action) if validator is not None else None + if issue is None: + state.clear() + state.update(candidate) + return action, issue + + +def initial_hand_tracking_issue( + teleop_interface: IsaacTeleopDevice, initialized_sides: set[str], *, required: bool = True +) -> str | None: + """Wait for initial hand presence without changing later tracking-loss behavior.""" + required_sides = {"left", "right"} + if not required or required_sides.issubset(initialized_sides): + return None + presence = teleop_interface.hand_tracking_presence + if presence is None: + return "hand-tracking presence is unavailable" + initialized_sides.update(side for side, present in zip(("left", "right"), presence, strict=True) if present) + missing = sorted(required_sides - initialized_sides) + return f"waiting for initial {'/'.join(missing)} hand tracking" if missing else None diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index 624ce3f42b82..2e362089fdf0 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -12,6 +12,7 @@ import math import time from collections import Counter +from collections.abc import Sequence from contextlib import suppress from copy import deepcopy from dataclasses import dataclass @@ -25,13 +26,85 @@ from .isaac_teleop_cfg import XrCameraFeedCfg, XrCameraFeedLayoutCfg if TYPE_CHECKING: - from isaaclab.sensors import Camera + from isaaclab_ov.renderers import OVRTXRendererCfg + + from isaaclab.sensors import Camera, CameraCfg + + from .ovxr_cfg import OvLibrariesXrCameraCfg logger = logging.getLogger(__name__) _DLSS_EXEC_MODES = frozenset({"performance", "balanced", "quality", "auto", "rtxaa", "manual"}) +def select_ovxr_camera_feeds( + env_cfg: Any, names: Sequence[str], renderer_cfg: OVRTXRendererCfg +) -> list[tuple[str, CameraCfg, XrCameraFeedCfg]]: + """Validate selected PiP cameras and assign their shared renderer before launch. + + Selection order is retained. Each name must identify a scene camera and exactly + one enabled feed. Validation completes before any camera configuration changes. + """ + from isaaclab.sensors import CameraCfg + + if len(names) > 3 or len(names) != len(set(names)): + raise ValueError("Select up to three unique OVXR PiP cameras") + selected = [] + for name in names: + camera = getattr(env_cfg.scene, name, None) + if not isinstance(camera, CameraCfg): + raise ValueError(f"OVXR PiP camera {name!r} is not a configured CameraCfg") + feeds = [feed for feed in env_cfg.isaac_teleop.xr_camera_feeds if feed.enabled and feed.camera_name == name] + if len(feeds) != 1: + raise ValueError(f"OVXR PiP camera {name!r} must have exactly one enabled XrCameraFeedCfg") + selected.append((name, camera, feeds[0])) + for _name, camera, _feed in selected: + camera.renderer_cfg = renderer_cfg + return selected + + +def resolve_ovxr_camera_configs( + selected: Sequence[tuple[str, CameraCfg, XrCameraFeedCfg]], + layout: XrCameraFeedLayoutCfg, + *, + max_fps: float, + position_override: tuple[float, float, float] | None = None, +) -> tuple[OvLibrariesXrCameraCfg, ...]: + """Resolve selected feeds into OVXR panels without creating native resources. + + A feed's positive update cap limits the caller's rate; zero leaves that rate + unchanged. A position override applies only to a single selected panel. + """ + from .ovxr_cfg import OvLibrariesXrCameraCfg + + _validate_layout_cfg(layout) + if selected and layout.placement != "head_locked": + raise ValueError("Kitless XR camera quads require head_locked placement") + if position_override is not None and len(selected) > 1: + raise ValueError("A PiP position override requires one camera; multiple panels use task layout") + for _name, _camera, feed in selected: + _XrCameraFeedManager._validate_cfg(feed, layout.placement) + feeds = _layout_feed_cfgs( + [feed for _name, _camera, feed in selected], + [(camera.width, camera.height) for _name, camera, _feed in selected], + layout, + ) + cameras = [] + for (name, camera, _feed), feed in zip(selected, feeds, strict=True): + cameras.append( + OvLibrariesXrCameraCfg( + name=name, + camera_cfg=camera, + max_fps=min(max_fps, feed.max_update_hz) if feed.max_update_hz > 0.0 else max_fps, + panel_width_m=feed.panel_width_m, + position=(float(feed.offset_m[0]), float(feed.offset_m[1]), -float(feed.distance_m)) + if position_override is None + else tuple(position_override), + ) + ) + return tuple(cameras) + + @lru_cache(maxsize=1) def _camera_type() -> type[Camera]: """Load the concrete camera type only when feeds are presented.""" diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py new file mode 100644 index 000000000000..4540b0c0c563 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -0,0 +1,190 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch checks shared by the live and replay OVXR entrypoints.""" + +from __future__ import annotations + +import argparse +import math +import os +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +from ..xr_cfg import remove_camera_configs + +if TYPE_CHECKING: + from ..ovxr_cfg import OvLibrariesXrCameraCfg + + +def prepare_ovxr_launch(env_cfg: Any, task: str, num_envs: int, *, keep: tuple[str, ...] = ()) -> Any: + """Check an OVXR launch before runtime startup and remove the task's camera sensors. + + Args: + env_cfg: Resolved environment configuration. + task: Task name, for the error message. + num_envs: Requested number of environments. + keep: Camera sensor names to retain for the OVXR camera panels. + + Returns: + The configuration without camera sensors other than ``keep``. + + Raises: + ValueError: If ``num_envs`` is not 1, ``ISAAC_LAB_OVRTX_USE_OVSTAGE`` is set to a value + other than ``1``, or the task needs Kit physics (see :func:`require_kitless_physics`). + """ + if num_envs != 1: + raise ValueError("--ovxr currently requires --num_envs 1") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") + require_kitless_physics(env_cfg, task) + return remove_camera_configs(env_cfg, keep=keep) + + +def require_kitless_physics(env_cfg: Any, task: str) -> None: + """Reject a task whose physics selection needs the Kit runtime before an OVXR launch. + + A task that leaves ``sim.physics`` unset selects Isaac Sim PhysX, which needs Kit. + The kitless OVXR session cannot run it, so fail before launch. Such a task normally + declares no physics preset, so the error points at ``--xr`` or the task configuration + rather than at a ``physics=`` selector. + + Args: + env_cfg: Resolved environment configuration. + task: Task name, for the error message. + + Raises: + ValueError: If ``env_cfg.sim.physics`` is ``None``. + """ + if env_cfg.sim.physics is None: + raise ValueError( + f"--ovxr needs a task that selects kitless physics (OvPhysxCfg or NewtonCfg), but task '{task}' leaves" + " sim.physics unset, which runs Isaac Sim PhysX through Kit. Run this task with --xr instead, or give" + " its configuration a kitless physics preset." + ) + + +def add_ovxr_presentation_args(parser: argparse.ArgumentParser) -> None: + """Add the OVXR camera-panel and desktop-mirror options shared by live teleoperation and replay.""" + parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") + parser.add_argument( + "--ovxr_desktop_mirror_fps", + type=float, + default=30.0, + help="Maximum desktop-mirror update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_camera", + type=str, + nargs="+", + default=None, + help="Show up to three task-configured cameras as application-owned XR panels.", + ) + parser.add_argument( + "--ovxr_pip_fps", + type=float, + default=30.0, + help="Maximum OVXR PiP update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_head_locked", + action="store_true", + help="Place the application camera quads in the OpenXR VIEW space, overriding the task layout.", + ) + parser.add_argument( + "--ovxr_pip_view_position", + type=float, + nargs=3, + metavar=("X", "Y", "Z"), + help="Optional VIEW-space camera panel center in meters; requires --ovxr_pip_head_locked.", + ) + + +def validate_ovxr_presentation_args( + parser: argparse.ArgumentParser, args: argparse.Namespace, *, using_ovxr: bool +) -> None: + """Validate the shared OVXR presentation options. + + Args: + parser: Parser used to report errors. + args: Parsed namespace. + using_ovxr: Whether the command selected ``--ovxr``. + """ + if args.ovxr_pip_camera is not None and not using_ovxr: + parser.error("--ovxr_pip_camera requires --ovxr") + if (args.ovxr_pip_head_locked or args.ovxr_pip_view_position is not None) and not using_ovxr: + parser.error("OVXR presentation options require --ovxr") + if args.headless and not using_ovxr: + parser.error("--headless requires --ovxr; Kit runs headless unless --visualizer is given") + if args.ovxr_pip_camera is not None and args.disable_external_cameras: + parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") + if not math.isfinite(args.ovxr_pip_fps) or args.ovxr_pip_fps <= 0.0: + parser.error("--ovxr_pip_fps must be finite and positive") + if args.ovxr_pip_view_position is not None: + if not args.ovxr_pip_head_locked or args.ovxr_pip_camera is None: + parser.error("--ovxr_pip_view_position requires a head-locked PiP camera") + if len(args.ovxr_pip_camera) > 1: + parser.error("A PiP position override requires one camera; multiple panels use task layout") + if not all(math.isfinite(value) for value in args.ovxr_pip_view_position): + parser.error("--ovxr_pip_view_position must be finite") + if not math.isfinite(args.ovxr_desktop_mirror_fps) or args.ovxr_desktop_mirror_fps <= 0: + parser.error("--ovxr_desktop_mirror_fps must be finite and positive") + + +def resolve_ovxr_pip_cameras( + env_cfg: Any, args: argparse.Namespace, renderer_cfg: Any +) -> tuple[OvLibrariesXrCameraCfg, ...]: + """Select the requested PiP cameras and resolve their OVXR panels before launch. + + Selection assigns ``renderer_cfg`` to each selected camera, so the OVXR session and its panels + share one OVRTX configuration. Resolution creates no native resources, so an invalid selection or + task layout fails before the simulation launches. + + Args: + env_cfg: Resolved environment configuration. + args: Parsed namespace with the shared OVXR presentation options. + renderer_cfg: OVRTX configuration shared by the OVXR session and its PiP cameras. + + Returns: + The resolved panels in selection order, or an empty tuple without ``--ovxr_pip_camera``. + + Raises: + ValueError: If a selected camera or its feed is invalid, or if the task layout is not + head-locked and ``--ovxr_pip_head_locked`` is not given. + """ + if args.ovxr_pip_camera is None: + return () + + from ..camera_feed import resolve_ovxr_camera_configs, select_ovxr_camera_feeds + from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg + + selected = select_ovxr_camera_feeds(env_cfg, args.ovxr_pip_camera, renderer_cfg) + layout = env_cfg.isaac_teleop.xr_camera_feed_layout or XrCameraFeedLayoutCfg() + if args.ovxr_pip_head_locked: + layout = layout.replace(placement="head_locked") + elif layout.placement != "head_locked": + raise ValueError( + f"OVXR camera panels are head-locked, but the task places them with {layout.placement!r} placement." + " Pass --ovxr_pip_head_locked to place them in the OpenXR VIEW space." + ) + return resolve_ovxr_camera_configs( + selected, layout, max_fps=args.ovxr_pip_fps, position_override=args.ovxr_pip_view_position + ) + + +def reject_ovxr_runtime_conflicts(configs: Sequence[Any], kit_sources: Sequence[str]) -> None: + """Reject validated launch components that conflict with an OVXR-owned frame loop. + + Args: + configs: Runtime configurations returned by :func:`~isaaclab.app.validate_launch`. + kit_sources: Components that :func:`~isaaclab.app.validate_launch` found to require Kit. + + Raises: + ValueError: If a component requires Kit or a local visualizer is configured. + """ + from isaaclab.visualizers import VisualizerBackendCfg + + if kit_sources or any(isinstance(cfg, VisualizerBackendCfg) for cfg in configs): + raise ValueError("--ovxr requires no Kit runtime or local visualizer") diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_presentation.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_presentation.py new file mode 100644 index 000000000000..807f1e8ec908 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_presentation.py @@ -0,0 +1,170 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""OVXR session lifecycle shared by the live and replay OVXR entrypoints.""" + +from __future__ import annotations + +import argparse +from collections.abc import Iterable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from isaaclab_ov.renderers import OVRTXRendererCfg + + from ..ov_libraries_xr import OvLibrariesXrSession + from ..ovxr_cfg import OvLibrariesXrCameraCfg + + +class OvxrPresentation: + """Own one OVXR session from preparation through render-worker startup, publication and close. + + The input device that borrows the session's OpenXR handles is assigned to :attr:`input_device` + once it exists. :meth:`start` reads its anchors, and :meth:`close` releases the session only + after that device drained. + """ + + def __init__(self, session: OvLibrariesXrSession, *, excluded_visual_meshes: Iterable[str] = ()) -> None: + self.session = session + """Prepared OVXR session.""" + self._excluded_visual_meshes = excluded_visual_meshes + self.input_device: Any = None + """Input device that borrows the session, or ``None`` before it is created.""" + self.started = False + """Whether the render worker started.""" + self._reset_generation = 0 + self._pending: tuple[int, int] | None = None + self._force = False + + @classmethod + def prepare( + cls, + env_cfg: Any, + args: argparse.Namespace, + *, + application_name: str, + auxiliary_cameras: tuple[OvLibrariesXrCameraCfg, ...] = (), + renderer_cfg: OVRTXRendererCfg | None = None, + ) -> OvxrPresentation: + """Prepare the OVXR session before the environment is constructed. + + Args: + env_cfg: Resolved environment configuration. Its ``isaac_teleop.xr_cfg`` selects the visual + meshes excluded from the XR view. + args: Parsed namespace with ``ovxr_session_mode``, ``cloudxr_env``, ``auto_launch_cloudxr`` + and the shared OVXR presentation options (see + :func:`~isaaclab_teleop.entrypoints._ovxr_launch.add_ovxr_presentation_args`). + application_name: OpenXR application name. + auxiliary_cameras: Resolved PiP camera panels. The session renders with their shared OVRTX + configuration. + renderer_cfg: OVRTX configuration without camera panels. Defaults to + :class:`~isaaclab_ov.renderers.OVRTXRendererCfg`. + + Returns: + The prepared presentation. + """ + from isaaclab_ov.renderers import OVRTXRendererCfg + + from .. import OvLibrariesXrCfg, OvLibrariesXrSession + + if auxiliary_cameras: + # Camera selection assigned one OVRTX configuration to every panel; the session shares it. + renderer_cfg = auxiliary_cameras[0].camera_cfg.renderer_cfg + elif renderer_cfg is None: + renderer_cfg = OVRTXRendererCfg() + session = OvLibrariesXrSession.prepare( + env_cfg.sim, + OvLibrariesXrCfg( + application_name=application_name, + session_mode=args.ovxr_session_mode, + cloudxr_env_file=args.cloudxr_env, + auto_launch_cloudxr=args.auto_launch_cloudxr, + desktop_mirror_enabled=not args.headless, + desktop_mirror_max_fps=args.ovxr_desktop_mirror_fps, + auxiliary_cameras=auxiliary_cameras, + ), + renderer_cfg=renderer_cfg, + ) + return cls(session, excluded_visual_meshes=env_cfg.isaac_teleop.xr_cfg.excluded_visual_meshes) + + def start(self, env: Any, *, timeout_s: float, required_openxr_extensions: Iterable[str] = ()) -> bool: + """Start the render worker for :attr:`input_device`, or adopt its anchor once started. + + Args: + env: Constructed environment sharing the prepared simulation context. + timeout_s: Maximum worker startup wait [s]. + required_openxr_extensions: Extensions required by the input pipeline. + + Returns: + Whether the render worker runs. ``False`` means the session shut down during startup. + """ + device = self.input_device + if self.started: + self.session.set_external_anchor_state(device.external_anchor_state) + return True + self.started = self.session.start_render_worker( + env, + tracking_anchor_path=device.xr_render_anchor_path, + required_openxr_extensions=required_openxr_extensions, + excluded_visual_meshes=self._excluded_visual_meshes, + external_anchor_state=device.external_anchor_state, + timeout_s=timeout_s, + ) + return self.started + + @property + def reset_pending(self) -> bool: + """Whether a reset publication still waits for admission.""" + return self._force + + def mark_changed(self, env: Any, *, reset: bool = False) -> None: + """Record that the scene changed, without requesting a render. + + Args: + env: Environment whose physics step identifies the publication. + reset: Whether the change is a reset. Its publication is forced and retained until admitted. + """ + if reset: + self._reset_generation += 1 + self._force = True + self._pending = (self._reset_generation, env.sim.get_physics_step_count()) + + def publish(self, env: Any, *, changed: bool = False, reset: bool = False, report_timing: bool = True) -> bool: + """Offer the latest unadmitted scene to the render worker. + + Args: + env: Environment whose physics step identifies the publication. + changed: Whether the scene changed since the last call. + reset: Whether the change is a reset (see :meth:`mark_changed`). + report_timing: Whether this call completes a host frame, whose physics rate the + desktop mirror reports. + + Returns: + Whether no publication remains pending. + """ + if changed or reset: + self.mark_changed(env, reset=reset) + if self._pending is not None and self.session.request_render(self._pending, force=self._force) is not None: + self._pending = None + self._force = False + if report_timing: + self.session.report_host_timing(env.sim.get_physics_step_count(), env.sim.get_physics_dt()) + return self._pending is None + + def close(self) -> None: + """Close the session once :attr:`input_device` drained; otherwise only request shutdown. + + Raises: + RuntimeError: If the input device failed to drain. The session and the environment stay + borrowed, so the caller must not release them. + """ + if getattr(self.input_device, "session_cleanup_failed", False) is True: + error = RuntimeError("OVXR input did not drain; retaining the borrowed XR session and environment") + try: + self.session.request_shutdown() + except Exception as shutdown_error: + error.add_note(f"Requesting XR shutdown also failed: {shutdown_error!r}") + raise error + self.session.close() diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index c7c046856894..298bc4ebd246 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -12,6 +12,17 @@ ``--max_replay_duration_s`` elapses, or the simulator is closed). The user-journey teleop script remains ``teleop_se3_agent.py``. +OVXR viewer: + ``--ovxr`` presents replay through an independently owned OVXR session while + tracker and control input remain sourced from MCAP. See + ``source/isaaclab_teleop/docs/kitless_xr.md`` for installation and launch + instructions. + + Optional camera quads use ``--ovxr_pip_camera NAME`` with the same rate, placement + and position options as live teleoperation (``--ovxr_pip_head_locked`` overrides a + task layout that is not head-locked). The desktop mirror is enabled by default; + ``--headless`` disables it. + Inputs: ``--replay_file`` is an MCAP capture produced by Isaac Capture's ``McapRecordingConfig`` path (typically written by ``record_demos.py @@ -31,6 +42,9 @@ ``ctrl.is_active`` (so pre-START operator setup frames are rendered without stepping), and handles mid-demo ``ctrl.should_reset`` events by running the full sim/env/teleop reset cycle. + Recorded controls govern playback by default. For legacy captures started + locally without recorded START messages, pass ``--start_on_replay`` to start + at replay startup and resume after each recorded RESET. End-of-replay termination: Four distinct signals end the current run by breaking out of the @@ -87,6 +101,8 @@ remaining replays never start and no stdout summary or JSON file is written. Partial reports are never emitted: CI relies on "no JSON output" as an unambiguous failure signal. + OVXR owns rendering independently, so application render-call timing + statistics remain empty for OVXR replays. Each active iteration also emits one ``GpuStatsProvider.sample()`` call. The default :class:`NvmlGpuStatsProvider` snapshots GPU @@ -117,6 +133,8 @@ "task": "IsaacContrib-PickPlace-GR1T2-Abs", "replay_file": "/tmp/pickplace_gr1t2.mcap", "num_replays": 5, + "requested_num_replays": 5, + "batch_complete": true, "outcomes": {"success": 4, "failure": 1, "incomplete": 0, "timeout": 0}, "success_rate": 0.8, "env_cfg": { @@ -141,6 +159,8 @@ "run_index": 0, "outcome": "success", "active_iterations": 322, + "rejected_action_frames": 0, + "last_action_rejection": null, "active_duration_s": 21.503, "success_step_count": 1, "cpu_frame_time_ms": { # per-render ms (wall time between sim.render calls) @@ -252,6 +272,7 @@ import json import logging import os +import signal import statistics import sys import time @@ -265,23 +286,34 @@ import torch from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab.app import add_launcher_args, launch_simulation +from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks from isaaclab.devices.openxr import remove_camera_configs from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.envs.utils._manual_success import _prepare_success_term +from isaaclab.utils.version import has_kit import isaaclab_tasks # noqa: F401 -from isaaclab_tasks.utils import parse_env_cfg +from isaaclab_tasks.utils import parse_env_cfg, setup_preset_cli from .. import ( CLOUDXR_AVP_ENV, CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopDevice, + OvLibrariesXrCameraCfg, create_isaac_teleop_device, poll_control_events, ) +from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action +from ._ovxr_launch import ( + add_ovxr_presentation_args, + prepare_ovxr_launch, + reject_ovxr_runtime_conflicts, + resolve_ovxr_pip_cameras, + validate_ovxr_presentation_args, +) +from ._ovxr_presentation import OvxrPresentation logger = logging.getLogger(__name__) @@ -301,12 +333,22 @@ def _build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--task", type=str, required=True, help="Name of the task.") parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.") + parser.add_argument( + "--ovxr", action="store_true", help="Present recorded motion through kitless OVXR; input stays MCAP." + ) + add_ovxr_presentation_args(parser) + parser.add_argument("--ovxr_session_mode", choices=("ar", "vr"), default="ar") parser.add_argument( "--replay_file", type=str, required=True, help="Absolute path to the Isaac Capture MCAP capture to replay.", ) + parser.add_argument( + "--start_on_replay", + action="store_true", + help="Start automatically at startup and after RESET for legacy captures without recorded START messages.", + ) parser.add_argument( "--num_success_steps", type=int, @@ -371,6 +413,7 @@ def _build_parser() -> argparse.ArgumentParser: "Safety cap on how long to wait for omni.usd to finish loading the stage before" " proceeding anyway. Hit only when something is misconfigured (missing asset, slow" " Nucleus, etc.); a warning is logged and replay continues. Default is 300s." + " With --ovxr it also bounds render-worker startup, which fails and drains on expiry." ), ) parser.add_argument( @@ -421,7 +464,9 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: sys.argv = [sys.argv[0], *argv] parser = _build_parser() # forward unrecognized args as Hydra-style task config overrides - args_cli, hydra_overrides = parser.parse_known_args() + args_cli, hydra_overrides = setup_preset_cli(parser) + validate_ovxr_presentation_args(parser, args_cli, using_ovxr=args_cli.ovxr) + sys.argv = [sys.argv[0]] + hydra_overrides # Enable external camera rendering by default so the replay mimics the production teleop env (perf # parity with live camera rendering); ``--disable_external_cameras`` turns it off. The @@ -429,10 +474,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # the launcher; this selects a camera-rendering experience that provides RTX/DLSS. # Everywhere else we read ``args_cli.disable_external_cameras`` directly. args_cli.enable_cameras = not args_cli.disable_external_cameras - # the stage-load wait and the RTX settings use Kit APIs directly - args_cli.require_kit = True + # the stage-load wait and the RTX settings use Kit APIs directly; the OVXR viewer replaces Kit + args_cli.require_kit = not args_cli.ovxr # Resolve CLI profile names once; runtime construction receives a path or None. + if args_cli.ovxr and args_cli.cloudxr_env is None: + args_cli.cloudxr_env = "cloudxrjs" if args_cli.cloudxr_env is not None: if not args_cli.cloudxr_env.strip() or args_cli.cloudxr_env.lower() == "none": args_cli.cloudxr_env = None @@ -482,6 +529,8 @@ class _RunStats: outcome: str = "incomplete" # "success" | "failure" | "incomplete" | "timeout" active_frame_times_ms: list[float] = field(default_factory=list) active_iterations: int = 0 + rejected_action_frames: int = 0 + last_action_rejection: str | None = None active_render_calls: int = 0 active_duration_s: float = 0.0 success_step_count: int = 0 # final consecutive-success counter at terminator @@ -495,6 +544,8 @@ def to_dict(self, run_index: int) -> dict: "run_index": run_index, "outcome": self.outcome, "active_iterations": self.active_iterations, + "rejected_action_frames": self.rejected_action_frames, + "last_action_rejection": self.last_action_rejection, "active_duration_s": round(self.active_duration_s, 6), "success_step_count": self.success_step_count, "cpu_frame_time_ms": _compute_frame_stats(self.active_frame_times_ms), @@ -811,6 +862,8 @@ def _build_report(args, env_cfg, all_runs: list[_RunStats]) -> dict: "task": args.task, "replay_file": args.replay_file, "num_replays": len(all_runs), + "requested_num_replays": args.num_replays, + "batch_complete": len(all_runs) == args.num_replays, "outcomes": outcomes_count, "success_rate": success_rate, "env_cfg": env_perf_cfg, @@ -1030,7 +1083,7 @@ def _prepare_env_cfg( num_envs: int, device: str, overrides: Sequence[str] = (), -) -> tuple[ManagerBasedRLEnvCfg, object | None]: +) -> tuple[ManagerBasedRLEnvCfg, object | None, tuple[OvLibrariesXrCameraCfg, ...]]: """Build and tweak an env config suitable for non-interactive replay. Mirrors the env-config mutations performed by ``record_demos.py``'s @@ -1059,6 +1112,13 @@ def _prepare_env_cfg( cycle (sim reinit + teleop device reset) so Pink IK starts the next attempt with fresh articulation views. + It also checks the launch before any runtime starts and runs the task's + :attr:`~isaaclab_teleop.IsaacTeleopCfg.prepare_environment` hook, which may + modify the returned configuration. With ``--ovxr`` it also removes camera + sensors other than the selected PiP cameras and rejects a launch that needs + more than one environment, a stage path other than OVStage, Kit or a local + visualizer. + Args: task: Registered task name to load the env config for. num_envs: Number of parallel environments. @@ -1066,8 +1126,9 @@ def _prepare_env_cfg( overrides: Hydra-style ``key=value`` overrides forwarded to :func:`parse_env_cfg`. Returns: - Tuple ``(env_cfg, success_term)``. ``success_term`` is ``None`` when + Tuple ``(env_cfg, success_term, selected_cameras)``. ``success_term`` is ``None`` when the env doesn't define a ``success`` termination term. + ``selected_cameras`` holds the resolved OVXR PiP panels, empty without ``--ovxr_pip_camera``. """ env_cfg = parse_env_cfg(task, device=device, num_envs=num_envs, overrides=overrides) env_cfg.env_name = task.split(":")[-1] @@ -1088,10 +1149,43 @@ def _prepare_env_cfg( # renders them for production parity; otherwise strip them for a lighter headless replay. if args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) - return env_cfg, success_term + selected_cameras = () + if args_cli.ovxr: + if args_cli.xr: + raise ValueError("--xr and --ovxr are mutually exclusive OpenXR frame-loop owners") + if args_cli.ovxr_pip_camera is not None: + from isaaclab_ov.renderers import OVRTXRendererCfg + + # Resolve the panels before launch preparation removes the other cameras, so that an + # invalid selection or task layout fails before launch. + selected_cameras = resolve_ovxr_pip_cameras(env_cfg, args_cli, OVRTXRendererCfg()) + env_cfg = prepare_ovxr_launch(env_cfg, task, num_envs, keep=tuple(args_cli.ovxr_pip_camera or ())) + # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. + configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) + if args_cli.ovxr: + reject_ovxr_runtime_conflicts(configs, kit_sources) + if env_cfg.isaac_teleop is not None and env_cfg.isaac_teleop.prepare_environment is not None: + env_cfg.isaac_teleop.prepare_environment(env_cfg) + return env_cfg, success_term, selected_cameras + + +def _replay_requires_hard_sim_reset(physics_backend: str | None) -> bool: + """Return whether replay must rebuild the simulator before an environment reset.""" + # Scene identity comes from the constructed manager, so preset aliases all + # resolve to NewtonManager.backend_name == "newton" before this policy runs. + return physics_backend is None or not physics_backend.startswith("newton") + + +def _reset_replay_environment(env: gym.Env, *, hard_sim_reset: bool, reset_recorder: bool) -> None: + """Reset replay-owned environment state in backend-safe order.""" + if hard_sim_reset: + env.sim.reset() + if reset_recorder: + env.recorder_manager.reset() + env.reset() -def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice) -> None: +def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice, *, hard_sim_reset: bool) -> None: """Run the full env+teleop reset cycle used by ``record_demos.py``. Mirrors :func:`isaaclab_teleop.entrypoints.record_demos.handle_reset` (sans the @@ -1102,11 +1196,11 @@ def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice) -> None: recorders are configured (the default for this script), but kept for parity with record_demos.py so future recorder additions don't have to re-derive the call sequence. + With ``hard_sim_reset=False`` (Newton), the hard reinit is skipped so the + solver constructed before replay stays alive. """ print("Resetting environment...") - env.sim.reset() - env.recorder_manager.reset() - env.reset() + _reset_replay_environment(env, hard_sim_reset=hard_sim_reset, reset_recorder=True) teleop_interface.reset() @@ -1146,15 +1240,77 @@ def _process_success_condition( binding, material warm-up) typically needs a few more event-loop ticks to converge. Thirty frames at the default render cadence is ~0.5 s on most machines and is deterministic per-machine -- unlike a wall-clock delay it does not have to be tuned for hardware. +Used only when Kit is active. """ +def _warmup_replay(env: gym.Env, run_index: int, presentation: OvxrPresentation | None) -> None: + """Wait for Kit assets or pump the independently owned viewer before input.""" + # Deterministic warmup is only required on the first run -- once the + # stage has been streamed in and the renderer settled, subsequent + # runs share the same stage and only need the per-run reset in + # ``_run_single_replay``. Skipping it on later runs saves the + # renderer-settle frames at the cost of doing nothing measurable (the + # wait returns immediately on a fully-loaded stage anyway). + if run_index == 0: + # Only an active Kit application owns the asynchronous USD loader. + # Kitless environments finish their stage construction before this call. + if has_kit(): + from isaaclab_physx.renderers.isaac_rtx_renderer_utils import wait_for_stage_load + + print("Waiting for USD stage to finish loading...") + wait_for_stage_load(args_cli.max_stage_load_wait_s, settle_frames=_RENDERER_SETTLE_FRAMES) + + # Optional extra wall-clock buffer on top of the deterministic + # wait. Useful as an escape hatch when the deterministic check + # is not enough (e.g. very slow shader compilation paths). + if args_cli.replay_start_delay_s > 0: + print( + f"Additional warmup buffer: rendering for {args_cli.replay_start_delay_s:.1f}s" + " before consuming MCAP frames." + ) + buffer_start_s = time.monotonic() + while env.sim.is_running() and time.monotonic() - buffer_start_s < args_cli.replay_start_delay_s: + if presentation is None: + env.sim.render() + else: + if presentation.session.shutdown_requested: + raise RuntimeError("OVXR viewer closed before replay started") + presentation.publish(env) + time.sleep(0.01) + + +def _start_replay_presentation(env: gym.Env, presentation: OvxrPresentation) -> None: + """Start the viewer for the first run, or adopt the current device's anchor, then publish the reset.""" + if not presentation.start(env, timeout_s=args_cli.max_stage_load_wait_s): + raise RuntimeError("OVXR viewer shut down before replay startup") + presentation.publish(env, reset=True) + + +def _advance_replay_input(device: IsaacTeleopDevice, presentation: OvxrPresentation | None) -> torch.Tensor | None: + """Read recorded input using the viewer owner's target transform when needed.""" + target_path = device.target_frame_prim_path if presentation is not None else None + if target_path is None: + return device.advance() + return device.advance(target_T_world=presentation.session.get_target_frame_T_world(target_path)) + + +def _render_replay_idle(env: gym.Env, presentation: OvxrPresentation | None) -> None: + """Keep the selected presentation responsive without advancing physics.""" + if presentation is None: + env.sim.render() + else: + presentation.publish(env) + time.sleep(env.step_dt) + + def _run_single_replay( env: gym.Env, isaac_teleop_cfg, success_term: object | None, run_index: int, total_runs: int, + presentation: OvxrPresentation | None = None, ) -> _RunStats: """Run a single replay pass against ``env`` and return the per-run stats. @@ -1211,7 +1367,10 @@ def _run_single_replay( auto_launch_cloudxr=False, mcap_replay_path=args_cli.replay_file, enable_debug_visualization=args_cli.enable_debug_visualization, + ovxr_session=presentation is not None, ) + if presentation is not None: + presentation.input_device = teleop_interface if run_index == 0: print(f"Using teleop device: {teleop_interface}") @@ -1250,38 +1409,26 @@ def _patched_sim_render(): # and to compute ``target = curr + delta``; if the articulation view is # stale, every IK call produces zero-delta arm targets while the # hand-finger path (which bypasses IK) keeps tracking. See PR #5507. - env.sim.reset() - env.reset() + # Newton skips the hard reset: its constructed solver must stay alive. + _reset_replay_environment( + env, + hard_sim_reset=_replay_requires_hard_sim_reset(env.scene.physics_backend), + reset_recorder=False, + ) teleop_interface.reset() - - # Deterministic warmup is only required on the first run -- once the - # stage has been streamed in and the renderer settled, subsequent - # runs share the same stage and only need the per-run ``env.sim.reset`` - # above. Skipping it on later runs saves the renderer-settle frames - # at the cost of doing nothing measurable (the wait returns - # immediately on a fully-loaded stage anyway). - if run_index == 0: - from isaaclab_physx.renderers.isaac_rtx_renderer_utils import wait_for_stage_load - - print("Waiting for USD stage to finish loading...") - wait_for_stage_load(args_cli.max_stage_load_wait_s, settle_frames=_RENDERER_SETTLE_FRAMES) - - # Optional extra wall-clock buffer on top of the deterministic - # wait. Useful as an escape hatch when the deterministic check - # is not enough (e.g. very slow shader compilation paths). - if args_cli.replay_start_delay_s > 0: - print( - f"Additional warmup buffer: rendering for {args_cli.replay_start_delay_s:.1f}s" - " before consuming MCAP frames." - ) - buffer_start_s = time.monotonic() - while env.sim.is_running() and time.monotonic() - buffer_start_s < args_cli.replay_start_delay_s: - env.sim.render() + # Locally started legacy captures have no recorded START message. + if args_cli.start_on_replay: + teleop_interface.request_start() + if presentation is not None: + _start_replay_presentation(env, presentation) + _warmup_replay(env, run_index, presentation) print( f"Replay {run_index + 1}/{total_runs} started; replaying MCAP from {args_cli.replay_file}" f" (max_replay_duration_s={args_cli.max_replay_duration_s:.1f})." ) + action_state: dict = {} + initialized_hand_tracking: set[str] = set() teleop_active = False teleop_was_active = False # only terminate on STOP after a real START success_step_count = 0 @@ -1312,7 +1459,11 @@ def _patched_sim_render(): stats.outcome = "timeout" break - action = teleop_interface.advance() + if presentation is not None and presentation.session.shutdown_requested: + print("OVXR viewer requested shutdown before replay completed.") + stats.outcome = "incomplete" + break + action = _advance_replay_input(teleop_interface, presentation) ctrl = poll_control_events(teleop_interface) # Track active state from the replayed _teleop_control @@ -1339,21 +1490,54 @@ def _patched_sim_render(): # without an operator STOP are terminated instead by # the success / failure / wall-clock terminators # below. - if prev_active and not teleop_active and teleop_was_active: + # RESET also pauses the control graph, but must reach + # the reset handler instead of ending the replay. + if prev_active and not teleop_active and teleop_was_active and not ctrl.should_reset: print("Replay end observed (STOP edge); ending run.") break if ctrl.should_reset: - _handle_reset(env, teleop_interface) + action_state.clear() + initialized_hand_tracking.clear() + _handle_reset( + env, + teleop_interface, + hard_sim_reset=_replay_requires_hard_sim_reset(env.scene.physics_backend), + ) + if args_cli.start_on_replay: + teleop_interface.request_start() success_step_count = 0 + if presentation is not None: + presentation.publish(env, changed=True, reset=True) continue # Gate stepping on the active state (mirrors # teleop_se3_agent.py:309-328). Pre-START operator # setup frames render only; the recorded START flips # us into the stepping branch. + if not teleop_active: + action_state.clear() if action is None or not teleop_active: - env.sim.render() + _render_replay_idle(env, presentation) + continue + + issue = initial_hand_tracking_issue( + teleop_interface, + initialized_hand_tracking, + required=isaac_teleop_cfg.require_initial_bimanual_hand_tracking, + ) + if issue is None: + action, issue = prepare_teleop_action( + env, + action, + isaac_teleop_cfg.action_conditioner, + isaac_teleop_cfg.action_validator, + action_state, + ) + if issue is not None: + stats.rejected_action_frames += 1 + stats.last_action_rejection = issue + _render_replay_idle(env, presentation) continue # Gate the render shim to this env.step call so @@ -1370,6 +1554,8 @@ def _patched_sim_render(): sampling_active[0] = False last_active_end_s = time.perf_counter() stats.active_iterations += 1 + if presentation is not None: + presentation.publish(env, changed=True) # Snapshot GPU state right after env.step so the # sample reflects the active workload (post-render @@ -1440,7 +1626,8 @@ def _patched_sim_render(): # Fail loud before returning so the next replay never starts and # the batch produces no partial report. See _assert_run_measured. - _assert_run_measured(stats, run_index) + if presentation is None: + _assert_run_measured(stats, run_index) return stats @@ -1457,7 +1644,8 @@ def main() -> int: * Pre-loop warmup: ``wait_for_stage_load`` polls ``omni.usd.UsdContext.get_stage_loading_status`` until the simulator reports zero pending assets, then renders a fixed number of - settle frames (only on ``run_index == 0``). An optional + settle frames (only on ``run_index == 0`` and only when Kit is + active; kitless runs do not import Kit's asset loader). An optional ``--replay_start_delay_s`` buffer can be appended for hardware that needs more grace. ``advance()`` is not called during warmup so ``ReplaySession.update`` does not consume MCAP @@ -1508,6 +1696,11 @@ def main() -> int: Resource cleanup is wrapped in a ``try/finally`` so that ``env.close()`` always runs, even when device construction or any subsequent setup raises -- otherwise the USD stage would leak across CI runs. + With ``--ovxr``, cleanup first drains replay input and the OVXR viewer, + which may own the environment. A failed input drain retains the borrowed + environment and raises rather than releasing a stage that a worker may + still use. Repeated shutdown signals request owner shutdown without + interrupting its drain. Returns: The host process exit code, mapped from the worst-of-N outcome @@ -1515,11 +1708,12 @@ def main() -> int: ``success_term`` fired, ``2`` if any run hit ``--max_replay_duration_s``, otherwise ``1`` (any failure or incomplete run). + A batch that stops before ``--num_replays`` runs complete returns ``1``. """ if args_cli.num_replays < 1: raise ValueError(f"--num_replays must be >= 1; got {args_cli.num_replays}") - env_cfg, success_term = _prepare_env_cfg( + env_cfg, success_term, selected_cameras = _prepare_env_cfg( args_cli.task, args_cli.num_envs, args_cli.device, @@ -1532,19 +1726,34 @@ def main() -> int: "MCAP replay requires env_cfg.isaac_teleop to be set." ) - with launch_simulation(env_cfg, args_cli): - exit_code = _run_replay_batch(env_cfg, success_term) + with launch_simulation(env_cfg, args_cli, cleanup_context=not args_cli.ovxr): + exit_code = _run_replay_batch(env_cfg, success_term, selected_cameras) return exit_code -def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None) -> int: +def _run_replay_batch( + env_cfg: ManagerBasedRLEnvCfg, + success_term: object | None, + selected_cameras: tuple[OvLibrariesXrCameraCfg, ...], +) -> int: """Build the env, run the ``--num_replays`` replays, and report the stats; see :func:`main`. + Args: + env_cfg: Environment configuration prepared by :func:`_prepare_env_cfg`. + success_term: Extracted ``success`` termination term, or ``None``. + selected_cameras: OVXR PiP panels resolved by :func:`_prepare_env_cfg`. + Returns: The host process exit code, see :func:`main`. """ + from isaaclab.sim import SimulationContext + + previous_context = SimulationContext.instance() env: gym.Env | None = None cloudxr_launcher = None + presentation = None + previous_handlers = {} + shutting_down = False all_runs: list[_RunStats] = [] try: @@ -1555,9 +1764,29 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None # ``_run_single_replay``). This is what lets ``--num_replays > 1`` # work in XR-active mode without losing the OpenXR runtime # IPC between runs. - cloudxr_launcher = _maybe_launch_cloudxr(args_cli.cloudxr_env, args_cli.auto_launch_cloudxr) + if args_cli.ovxr: + presentation = OvxrPresentation.prepare( + env_cfg, + args_cli, + application_name=f"isaaclab-ovxr-replay-{args_cli.task}", + auxiliary_cameras=selected_cameras, + ) + + def request_shutdown(_signum, _frame): + nonlocal shutting_down + if not presentation.started and not shutting_down: + # Begin normal unwinding during construction or peer-gated + # startup; subsequent signals must not interrupt owner drain. + shutting_down = True + raise KeyboardInterrupt + shutting_down = True + presentation.session.request_shutdown() + + previous_handlers = {sig: signal.signal(sig, request_shutdown) for sig in (signal.SIGINT, signal.SIGTERM)} + else: + cloudxr_launcher = _maybe_launch_cloudxr(args_cli.cloudxr_env, args_cli.auto_launch_cloudxr) - if not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr: + if args_cli.require_kit and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings apply_isaac_rtx_global_settings(IsaacRtxRendererGlobalSettingsCfg(antialiasing_mode="DLSS")) @@ -1571,18 +1800,33 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None success_term=success_term, run_index=run_idx, total_runs=args_cli.num_replays, + presentation=presentation, ) all_runs.append(run_stats) print(f"Replay {run_idx + 1}/{args_cli.num_replays} outcome: {run_stats.outcome}") - if not env.sim.is_running(): + if not env.sim.is_running() or (presentation is not None and presentation.session.shutdown_requested): # The simulator was closed externally mid-batch; stop the # outer loop rather than spawning a fresh device against # a dead app. break finally: - if env is not None: - env.close() - print("Environment closed") + shutting_down = True + try: + if presentation is not None: + presentation.close() + if env is not None: + if presentation is None or not presentation.session.async_worker_owns_environment: + env.close() + print("Environment closed") + elif presentation is None or not presentation.session.async_worker_owns_environment: + # Construction may have created a context before returning env. + # Only run after proven input/viewer drain; preserve older contexts. + context = SimulationContext.instance() + if context is not None and context is not previous_context: + SimulationContext.clear_instance() + finally: + for sig, handler in previous_handlers.items(): + signal.signal(sig, handler) if cloudxr_launcher is not None: try: cloudxr_launcher.stop() @@ -1594,6 +1838,8 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None _print_stdout_summary(report) if args_cli.stats_output_file is not None: _write_json_report(args_cli.stats_output_file, report) + if len(all_runs) != args_cli.num_replays: + return 1 return _exit_code_for_outcomes(all_runs) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index e0a26de2024c..f4288074aeb3 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -18,15 +18,17 @@ import argparse import logging +import math +import signal import sys +import time from collections.abc import Callable -from contextlib import ExitStack import gymnasium as gym import torch from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab.app import add_launcher_args, launch_simulation +from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks from isaaclab.devices import DeviceCfg from isaaclab.devices.openxr import remove_camera_configs @@ -44,14 +46,27 @@ CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg, + OvLibrariesXrCameraCfg, se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, ) +from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action from ..device_selection import create_teleop_device, select_teleop_cfg +from ._ovxr_launch import ( + add_ovxr_presentation_args, + prepare_ovxr_launch, + reject_ovxr_runtime_conflicts, + resolve_ovxr_pip_cameras, + validate_ovxr_presentation_args, +) +from ._ovxr_presentation import OvxrPresentation logger = logging.getLogger(__name__) +_RENDERER_ATTACH_TIMEOUT_S = 180.0 +"""Allow cold RTX preparation before failing attachment; client connection waits separately.""" + args_cli: argparse.Namespace """Parsed command line, set by :func:`_parse_args`.""" @@ -72,6 +87,12 @@ def _build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--task", type=str, default=None, help="Name of the task.") parser.add_argument("--sensitivity", type=float, default=1.0, help="Sensitivity factor.") + parser.add_argument( + "--step_hz", + type=float, + default=0.0, + help="Maximum host-control rate with --ovxr; 0 leaves active control unthrottled. XR rendering is independent.", + ) parser.add_argument( "--cloudxr_env", type=str, @@ -79,7 +100,7 @@ def _build_parser() -> argparse.ArgumentParser: help=( "Path to a CloudXR .env file, or a shorthand: 'cloudxrjs' (Quest/Pico), 'avp' (Apple Vision Pro)," " or 'standalone' (headless, no XR client). Set to 'none' to disable CloudXR auto-launch entirely." - " When unset, defaults to 'cloudxrjs' with --xr and 'standalone' without --xr." + " When unset, defaults to 'cloudxrjs' with --xr or --ovxr and 'standalone' otherwise." ), ) parser.add_argument( @@ -109,6 +130,20 @@ def _build_parser() -> argparse.ArgumentParser: " and improve XR performance)." ), ) + parser.add_argument( + "--ovxr", + dest="using_ovxr", + action="store_true", + default=False, + help="Use the OVXR OVRTX/OVStage/OVXR path instead of Kit XR.", + ) + parser.add_argument( + "--ovxr_session_mode", + choices=("ar", "vr"), + default="ar", + help="OVXR session mode. Defaults to 'ar'; pass 'vr' for an opaque background.", + ) + add_ovxr_presentation_args(parser) add_launcher_args(parser) return parser @@ -127,6 +162,13 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # the launcher; this selects a camera-rendering experience that provides RTX/DLSS support. # Everywhere else we read ``args_cli.disable_external_cameras`` directly. args_cli.enable_cameras = not args_cli.disable_external_cameras + if args_cli.xr and args_cli.using_ovxr: + parser.error("--xr and --ovxr are mutually exclusive OpenXR frame-loop owners") + if not math.isfinite(args_cli.step_hz) or args_cli.step_hz < 0.0: + parser.error("--step_hz must be finite and non-negative; 0 disables active-control pacing") + if args_cli.step_hz > 0.0 and not args_cli.using_ovxr: + parser.error("--step_hz requires --ovxr") + validate_ovxr_presentation_args(parser, args_cli, using_ovxr=args_cli.using_ovxr) # Call an external callback if requested. if args_cli.external_callback: @@ -137,7 +179,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # Resolve CLI profile names once; runtime construction receives a path or None. if args_cli.cloudxr_env is None: - args_cli.cloudxr_env = "cloudxrjs" if args_cli.xr else "standalone" + args_cli.cloudxr_env = "cloudxrjs" if args_cli.xr or args_cli.using_ovxr else "standalone" if not args_cli.cloudxr_env.strip() or args_cli.cloudxr_env.lower() == "none": args_cli.cloudxr_env = None else: @@ -159,6 +201,15 @@ def main() -> None: # noqa: C901 Returns: None """ + env_cfg, teleop_cfg, use_isaac_teleop, selected_cameras = _resolve_env_cfg() + with launch_simulation(env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): + run_teleoperation(env_cfg, teleop_cfg, use_isaac_teleop, selected_cameras) + + +def _resolve_env_cfg() -> tuple[ + ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, tuple[OvLibrariesXrCameraCfg, ...] +]: + """Resolve task configuration and teleop ownership before runtime launch.""" # Resolve the task configuration through Hydra so CLI presets are applied. env_cfg, _ = resolve_task_config(args_cli.task, "") env_cfg.sim.device = args_cli.device @@ -202,6 +253,9 @@ def main() -> None: # noqa: C901 if isinstance(teleop_cfg, IsaacTeleopCfg): env_cfg.isaac_teleop = teleop_cfg + use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) + if args_cli.using_ovxr and not use_isaac_teleop: + raise ValueError("--ovxr requires a task configured with env_cfg.isaac_teleop") # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders # normally, so gate on --xr alone. @@ -210,216 +264,476 @@ def main() -> None: # noqa: C901 # strip them so the XR headset view is the sole render product. if args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) + selected_cameras = () + if args_cli.using_ovxr: + if args_cli.ovxr_pip_camera is not None: + from isaaclab_ov.renderers import OVRTXRendererCfg + + # Resolve the panels before launch preparation removes the other cameras, so that an + # invalid selection or task layout fails before launch. + selected_cameras = resolve_ovxr_pip_cameras(env_cfg, args_cli, OVRTXRendererCfg()) + env_cfg = prepare_ovxr_launch( + env_cfg, args_cli.task, args_cli.num_envs, keep=tuple(args_cli.ovxr_pip_camera or ()) + ) + # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. + configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) + if args_cli.using_ovxr: + reject_ovxr_runtime_conflicts(configs, kit_sources) + if use_isaac_teleop and teleop_cfg.prepare_environment is not None: + teleop_cfg.prepare_environment(env_cfg) + return env_cfg, teleop_cfg, use_isaac_teleop, selected_cameras + + +def _pace_ovxr_control_loop( + using_ovxr: bool, + action_applied: bool, + *, + iteration_started_s: float, + action_period_s: float, + step_hz: float = 0.0, + wait_for_shutdown: Callable[[float], bool] | None = None, +) -> None: + """Limit host control independently of the XR frame clock. + + Credit elapsed input, physics and publication work; an overrun adds no wait + or catch-up steps. With no explicit limit, active control remains unpaced + and paused polling retains the environment's action interval. + """ + if not using_ovxr or (action_applied and step_hz == 0.0): + return + period_s = 1.0 / step_hz if step_hz > 0.0 else action_period_s + remaining_s = period_s - (time.perf_counter() - iteration_started_s) + if remaining_s > 0.0: + if wait_for_shutdown is None: + time.sleep(remaining_s) + else: + wait_for_shutdown(remaining_s) + - with launch_simulation(env_cfg, args_cli), ExitStack() as cleanup: - run_teleoperation(env_cfg, teleop_cfg, cleanup) +def _apply_teleop_frame(env, action, issue: str | None, haptic_driver, *, render: bool = True) -> bool: + """Step one valid action, or hold the scene while an unsafe tracker frame is rejected. + ``render`` is false under OVXR, which owns rendering and republishes the unchanged scene. + """ + if issue is not None: + if render: + env.sim.render() + if haptic_driver is not None: + haptic_driver.stop() + return False + # process actions + actions = action.repeat(env.num_envs, 1) + # apply actions + env.step(actions) + # render controller haptics from post-step contact forces + if haptic_driver is not None: + haptic_driver.update() + return True + + +def _install_ovxr_shutdown_handlers(presentation: OvxrPresentation | None) -> dict: + """Make owner-script signals request the same orderly shutdown as OVUI.""" + if presentation is None: + return {} + + def request_shutdown(_signum, _frame) -> None: + presentation.session.request_shutdown() + + return {signum: signal.signal(signum, request_shutdown) for signum in (signal.SIGINT, signal.SIGTERM)} + + +def _restore_ovxr_shutdown_handlers(previous_handlers: dict) -> None: + for signum, handler in previous_handlers.items(): + signal.signal(signum, handler) + + +def _close_teleop_resources( + env, + camera_feed_session, + presentation: OvxrPresentation | None, + previous_shutdown_handlers, + teleop_interface=None, + *, + previous_context=None, +) -> None: + """Preserve the first failure and drain XR before freeing its borrowed environment.""" + failure = sys.exception() + cleanup_failure = None + xr_drained = True + camera_drained = True + + def report_cleanup_error(error: Exception) -> None: + nonlocal cleanup_failure + logger.exception("Teleoperation cleanup failed") + if failure is not None: + failure.add_note(f"Cleanup also failed: {error!r}") + elif cleanup_failure is None: + cleanup_failure = error + else: + cleanup_failure.add_note(f"Cleanup also failed: {error!r}") -def run_teleoperation( - env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, cleanup: ExitStack -) -> None: # noqa: C901 + try: + input_drain_failed = getattr(teleop_interface, "session_cleanup_failed", False) is True + xr_drained = not input_drain_failed + if presentation is not None: + # Releases the session only after its input device drained. + try: + presentation.close() + except Exception as error: + xr_drained = False + report_cleanup_error(error) + elif input_drain_failed: + report_cleanup_error( + RuntimeError("Isaac Capture cleanup failed; borrowed XR/environment drain is unproved") + ) + if not input_drain_failed: + # A successfully bound feed exits before Capture. This is also the + # cleanup path for preparation that never reached bind(). + try: + camera_feed_session.close() + except Exception as error: + camera_drained = False + report_cleanup_error(error) + if env is not None and xr_drained and camera_drained: + if presentation is None or not presentation.session.async_worker_owns_environment: + try: + env.close() + except Exception as error: + report_cleanup_error(error) + elif env is None and xr_drained and camera_drained: + from isaaclab.sim import SimulationContext + + if presentation is None or not presentation.session.async_worker_owns_environment: + context = SimulationContext.instance() + if context is not None and context is not previous_context: + try: + SimulationContext.clear_instance() + except Exception as error: + report_cleanup_error(error) + finally: + # Keep repeated SIGINT/SIGTERM on the owner throughout this cleanup attempt. + try: + _restore_ovxr_shutdown_handlers(previous_shutdown_handlers) + except Exception as error: + report_cleanup_error(error) + if cleanup_failure is not None: + raise cleanup_failure + if env is not None and xr_drained and camera_drained: + print("Environment closed") + + +def run_teleoperation( # noqa: C901 + env_cfg: ManagerBasedRLEnvCfg, + teleop_cfg: IsaacTeleopCfg | DeviceCfg, + use_isaac_teleop: bool, + selected_cameras: tuple[OvLibrariesXrCameraCfg, ...], +) -> None: """Create the environment and teleop device, then run the teleoperation loop until the app is closed.""" - from .. import XrCameraFeedSession, poll_control_events - use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) + from isaaclab.sim import SimulationContext + from .. import XrCameraFeedSession, poll_control_events + + previous_context = SimulationContext.instance() camera_feed_session = XrCameraFeedSession.prepare( env_cfg, enabled=args_cli.xr and use_isaac_teleop, - camera_rendering_enabled=not args_cli.disable_external_cameras, + camera_rendering_enabled=not args_cli.disable_external_cameras and not args_cli.using_ovxr, ) - cleanup.callback(camera_feed_session.close) - - # Apply the RTX/DLSS global settings when Kit runs an RTX render pipeline (Kit visualizer, - # external cameras, or XR). - if has_kit() and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): - from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings - - apply_isaac_rtx_global_settings( - IsaacRtxRendererGlobalSettingsCfg( - antialiasing_mode="DLSS", - carb_settings=( - {"/rtx/dldenoiser/responsiveDenoising": True} - if camera_feed_session.requires_responsive_denoising - else None - ), - ), - ) + env = None + teleop_interface = None + presentation = None + previous_shutdown_handlers = {} try: - # create environment - env = gym.make(args_cli.task, cfg=env_cfg).unwrapped - # check environment name (for reach , we don't allow the gripper) - if "Reach" in args_cli.task: - logger.warning( - f"The environment '{args_cli.task}' does not support gripper control. The device command will be" - " ignored." + # Apply the RTX/DLSS global settings when Kit runs an RTX render pipeline (Kit visualizer, + # external cameras, or XR). + if has_kit() and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): + from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings + + apply_isaac_rtx_global_settings( + IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + carb_settings=( + {"/rtx/dldenoiser/responsiveDenoising": True} + if camera_feed_session.requires_responsive_denoising + else None + ), + ), ) - except Exception as e: - logger.error(f"Failed to create environment: {e}") - return - - # Flags for controlling teleoperation flow - should_reset_recording_instance = False - teleoperation_active = True - - # Callback handlers - def reset_recording_instance() -> None: - """ - Reset the environment to its initial state. - - Sets a flag to reset the environment on the next simulation step. - - Returns: - None - """ - nonlocal should_reset_recording_instance - should_reset_recording_instance = True - print("Reset triggered - Environment will reset on next step") - def start_teleoperation() -> None: - """ - Activate teleoperation control of the robot. - - Enables the application of teleoperation commands to the environment. + if args_cli.using_ovxr: + presentation = OvxrPresentation.prepare( + env_cfg, + args_cli, + application_name=f"isaaclab-ovxr-teleop-{args_cli.task}", + auxiliary_cameras=selected_cameras, + ) - Returns: - None - """ - nonlocal teleoperation_active - teleoperation_active = True - print("Teleoperation activated") - - def stop_teleoperation() -> None: - """ - Deactivate teleoperation control of the robot. - - Disables the application of teleoperation commands to the environment. - - Returns: - None - """ - nonlocal teleoperation_active - teleoperation_active = False - print("Teleoperation deactivated") - - # Create device config if not already in env_cfg - teleoperation_callbacks: dict[str, Callable[[], None]] = { - "R": reset_recording_instance, - "START": start_teleoperation, - "STOP": stop_teleoperation, - "RESET": reset_recording_instance, - } - - # For XR devices (hand tracking or Isaac Capture), default to inactive. Without - # --xr, teleop is started locally (see ``request_start`` below) rather than by - # a headset, so it still begins running -- but it flows through the same state - # machine, so keyboard/host pause/resume keeps working. - if use_isaac_teleop or args_cli.xr: - teleoperation_active = env_cfg.isaac_teleop.teleoperation_active_default if use_isaac_teleop else False - else: - # Always active for other devices + try: + # create environment + env = gym.make(args_cli.task, cfg=env_cfg).unwrapped + # check environment name (for reach , we don't allow the gripper) + if "Reach" in args_cli.task: + logger.warning( + f"The environment '{args_cli.task}' does not support gripper control. The device command will be" + " ignored." + ) + except Exception as e: + logger.error(f"Failed to create environment: {e}") + return + + # Flags for controlling teleoperation flow + should_reset_recording_instance = False teleoperation_active = True - try: - teleop_interface = create_teleop_device( - teleop_cfg, - env_prim_path=env.scene.env_prim_paths[0], - sim_device=args_cli.device, - callbacks=teleoperation_callbacks, - visualizers=env.sim.visualizers, - cloudxr_env_file=args_cli.cloudxr_env, - auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - enable_debug_visualization=args_cli.enable_debug_visualization, - use_kit_xr_bridge=args_cli.xr, - haptic_cfg=getattr(env_cfg, "haptic_feedback", None), - ) - except Exception as e: - logger.error(f"Failed to create teleop device: {e}") - env.close() - return - - print(f"Using teleop device: {teleop_interface}") - - haptic_driver = None - if use_isaac_teleop: - from .. import create_haptic_feedback_driver - - haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - - if use_isaac_teleop: - # the device reads these from the focused visualizer window - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - - def run_loop(): - """Inner function to run the teleop loop with access to nonlocal variables.""" - nonlocal should_reset_recording_instance, teleoperation_active - - # reset environment - env.reset() - teleop_interface.reset() + # Callback handlers + def reset_recording_instance() -> None: + """ + Reset the environment to its initial state. + + Sets a flag to reset the environment on the next simulation step. + + Returns: + None + """ + nonlocal should_reset_recording_instance + should_reset_recording_instance = True + print("Reset triggered - Environment will reset on next step") + + def start_teleoperation() -> None: + """ + Activate teleoperation control of the robot. + + Enables the application of teleoperation commands to the environment. + + Returns: + None + """ + nonlocal teleoperation_active + teleoperation_active = True + print("Teleoperation activated") + + def stop_teleoperation() -> None: + """ + Deactivate teleoperation control of the robot. + + Disables the application of teleoperation commands to the environment. + + Returns: + None + """ + nonlocal teleoperation_active + teleoperation_active = False + print("Teleoperation deactivated") + + # Create device config if not already in env_cfg + teleoperation_callbacks: dict[str, Callable[[], None]] = { + "R": reset_recording_instance, + "START": start_teleoperation, + "STOP": stop_teleoperation, + "RESET": reset_recording_instance, + } + + # For XR devices (hand tracking or Isaac Capture), default to inactive. Without + # --xr, teleop is started locally (see ``request_start`` below) rather than by + # a headset, so it still begins running -- but it flows through the same state + # machine, so keyboard/host pause/resume keeps working. + if use_isaac_teleop or args_cli.xr: + teleoperation_active = env_cfg.isaac_teleop.teleoperation_active_default if use_isaac_teleop else False + else: + # Always active for other devices + teleoperation_active = True + + try: + teleop_interface = create_teleop_device( + teleop_cfg, + env_prim_path=env.scene.env_prim_paths[0], + sim_device=args_cli.device, + callbacks=teleoperation_callbacks, + visualizers=env.sim.visualizers, + cloudxr_env_file=(None if args_cli.using_ovxr else args_cli.cloudxr_env), + auto_launch_cloudxr=args_cli.auto_launch_cloudxr and not args_cli.using_ovxr, + enable_debug_visualization=args_cli.enable_debug_visualization, + use_kit_xr_bridge=args_cli.xr, + external_oxr_handles_provider=( + presentation.session.external_oxr_handles if presentation is not None else None + ), + haptic_cfg=getattr(env_cfg, "haptic_feedback", None), + ) + except Exception as e: + logger.error(f"Failed to create teleop device: {e}") + return - # Without --xr there is no headset to send START, so start locally ([B]/[P] can - # still pause/resume). The reset() above is a host reset (a pure pulse), so it does - # not cancel this start. - if use_isaac_teleop and not args_cli.xr: - teleop_interface.request_start() + if presentation is not None: + presentation.input_device = teleop_interface + print(f"Using teleop device: {teleop_interface}") - stack_name = "Isaac Capture" if use_isaac_teleop else "native" - print(f"{stack_name} teleoperation started. Press 'R' to reset the environment.") + haptic_driver = None + if use_isaac_teleop: + from .. import create_haptic_feedback_driver - # simulate environment - while env.sim.is_running(): - try: - # run everything in inference mode - with torch.inference_mode(): - # get device command - action = teleop_interface.advance() - - if use_isaac_teleop: - ctrl = poll_control_events(teleop_interface) - if ctrl.is_active is not None: - teleoperation_active = ctrl.is_active - if ctrl.should_reset: - should_reset_recording_instance = True - - # action is None when Isaac Capture session hasn't started yet - # (e.g. waiting for user to click "Start AR") - if action is None or not teleoperation_active: - env.sim.render() - if haptic_driver is not None: - haptic_driver.stop() - else: - # process actions - actions = action.repeat(env.num_envs, 1) - # apply actions - env.step(actions) - # render controller haptics from post-step contact forces - if haptic_driver is not None: - haptic_driver.update() - - if should_reset_recording_instance: - env.reset() - teleop_interface.reset() - camera_feed_session.refresh() - should_reset_recording_instance = False - print("Environment reset complete") - except Exception as e: - logger.error(f"Error during simulation step: {e}") - break - - # Run the teleoperation loop - # Isaac Capture requires a context manager, native devices don't - if use_isaac_teleop: - with teleop_interface, camera_feed_session.bind(env): - run_loop() - else: - with camera_feed_session.bind(env): - run_loop() + haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - # close the simulator - env.close() - print("Environment closed") + require_initial_bimanual_tracking = ( + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking if use_isaac_teleop else False + ) + initialized_hand_tracking: set[str] = set() + skipped_tracking_frames = 0 + action_conditioner = env_cfg.isaac_teleop.action_conditioner if use_isaac_teleop else None + action_validator = env_cfg.isaac_teleop.action_validator if use_isaac_teleop else None + action_state: dict = {} + + if use_isaac_teleop: + # the device reads these from the focused visualizer window + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") + + def run_loop(): + """Inner function to run the teleop loop with access to nonlocal variables.""" + nonlocal should_reset_recording_instance, skipped_tracking_frames, teleoperation_active + + # reset environment + env.reset() + teleop_interface.reset() + initialized_hand_tracking.clear() + action_state.clear() + if presentation is not None: + presentation.mark_changed(env, reset=True) + + # Without --xr there is no headset to send START, so start locally ([B]/[P] can + # still pause/resume). The reset() above is a host reset (a pure pulse), so it does + # not cancel this start. + if use_isaac_teleop and not (args_cli.xr or args_cli.using_ovxr): + teleop_interface.request_start() + + stack_name = "Isaac Capture" if use_isaac_teleop else "native" + print(f"{stack_name} teleoperation started. Press 'R' to reset the environment.") + + # simulate environment + while env.sim.is_running() and not (presentation is not None and presentation.session.shutdown_requested): + iteration_started_s = time.perf_counter() if presentation is not None else 0.0 + try: + # run everything in inference mode + with torch.inference_mode(): + # Reset invalidates the tracking anchor. Publish the new epoch + # before reading input, retaining force across deferred admissions. + if ( + presentation is not None + and presentation.reset_pending + and not presentation.publish(env, report_timing=False) + ): + presentation.session.wait_for_shutdown(env.step_dt) + continue + # get device command + target_frame_prim_path = ( + teleop_interface.target_frame_prim_path + if presentation is not None and use_isaac_teleop + else None + ) + if target_frame_prim_path is None: + action = teleop_interface.advance() + else: + target_T_world = presentation.session.get_target_frame_T_world(target_frame_prim_path) + action = teleop_interface.advance(target_T_world=target_T_world) + + if use_isaac_teleop: + ctrl = poll_control_events(teleop_interface) + if ctrl.is_active is not None: + teleoperation_active = ctrl.is_active + if ctrl.should_reset: + should_reset_recording_instance = True + + if not teleoperation_active: + action_state.clear() + + # action is None when Isaac Capture session hasn't started yet + # (e.g. waiting for user to click "Start AR") + if action is None or not teleoperation_active: + if presentation is None: + env.sim.render() + if haptic_driver is not None: + haptic_driver.stop() + action_applied = False + else: + issue = initial_hand_tracking_issue( + teleop_interface, + initialized_hand_tracking, + required=require_initial_bimanual_tracking, + ) + if issue is None: + action, issue = prepare_teleop_action( + env, action, action_conditioner, action_validator, action_state + ) + action_applied = _apply_teleop_frame( + env, action, issue, haptic_driver, render=presentation is None + ) + if issue is not None: + skipped_tracking_frames += 1 + if skipped_tracking_frames <= 5 or skipped_tracking_frames % 200 == 0: + logger.warning( + "Rendering without a physics step for unsafe teleop frame (%s); skipped=%d", + issue, + skipped_tracking_frames, + ) + + reset_applied = should_reset_recording_instance + if should_reset_recording_instance: + env.reset() + teleop_interface.reset() + initialized_hand_tracking.clear() + action_state.clear() + camera_feed_session.refresh() + should_reset_recording_instance = False + print("Environment reset complete") + + if presentation is not None: + presentation.publish(env, changed=action_applied, reset=reset_applied) + + _pace_ovxr_control_loop( + presentation is not None, + action_applied, + iteration_started_s=iteration_started_s, + action_period_s=env.step_dt, + step_hz=args_cli.step_hz, + wait_for_shutdown=( + presentation.session.wait_for_shutdown if presentation is not None else None + ), + ) + except Exception as e: + logger.error(f"Error during simulation step: {e}") + break + + # Install after environment/device initialization because embedded native + # libraries may replace inherited signal dispositions during startup. + previous_shutdown_handlers = _install_ovxr_shutdown_handlers(presentation) + # Run the teleoperation loop + # Isaac Capture requires a context manager, native devices don't + if use_isaac_teleop: + if presentation is not None: + # Capture releases borrowed handles before the outer cleanup + # closes OVXR and then the environment. + with teleop_interface, camera_feed_session.bind(env): + if presentation.start( + env, + timeout_s=_RENDERER_ATTACH_TIMEOUT_S, + required_openxr_extensions=teleop_interface.required_openxr_extensions, + ): + run_loop() + else: + with teleop_interface, camera_feed_session.bind(env): + run_loop() + else: + with camera_feed_session.bind(env): + run_loop() + finally: + _close_teleop_resources( + env, + camera_feed_session, + presentation, + previous_shutdown_handlers, + teleop_interface, + previous_context=previous_context, + ) def run_teleop_se3_agent_cli(argv: list[str] | None = None) -> int: diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py index 675a32e85761..b58a283291cb 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py @@ -10,7 +10,7 @@ from collections.abc import Callable from dataclasses import MISSING, field from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from isaaclab.utils import configclass @@ -38,6 +38,8 @@ from isaacteleop.retargeting_engine.interface import BaseRetargeter, OutputCombiner from isaacteleop.teleop_session_manager import PluginConfig, RetargetingExecutionConfig + from isaaclab.envs import ManagerBasedRLEnvCfg + @configclass class XrCameraFeedCfg: @@ -246,6 +248,38 @@ def build_pipeline(): ``"START"`` command is received from xr_core via the message bus. """ + require_initial_bimanual_hand_tracking: bool = False + """Wait for both hands to be present before the first environment step. + + An environment or device reset clears the initialization latch. Later + tracking loss follows the configured retargeting pipeline's behavior. + """ + + prepare_environment: Callable[[ManagerBasedRLEnvCfg], None] | None = None + """Optional task-owned preparation of the resolved environment configuration. + + Live and replay launchers call this before creating any simulation resources, + including the OVXR bootstrap. Preparation must tolerate repeated calls from + the task constructor. Keep this callable's imports safe before simulator startup. + It runs after launcher validation, so it must not change the runtime selections, + such as the physics or renderer backend. + """ + + action_conditioner: Callable[[Any, Any, dict[str, Any]], Any] | None = None + """Optional task-owned pre-step action conditioner. + + The callable receives ``(env, action, state)``. The launcher retains the + state dictionary across active frames and clears it on pause or reset. + State updates from a frame that fails validation after conditioning are discarded. + """ + + action_validator: Callable[[Any], str | None] | None = None + """Optional task-owned fail-closed action validator. + + Return ``None`` for a valid action or a concise reason to render without a + physics step. Validation runs both before and after conditioning. + """ + retargeters_to_tune: Callable[[], list[BaseRetargeter]] | None = None """Optional callable returning retargeters to expose in the tuning UI. diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index 25749b3fcaaa..ccf8b165d41d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -26,6 +26,8 @@ from .xr_anchor_manager import XrAnchorManager if TYPE_CHECKING: + from isaacteleop.oxr import OpenXRSessionHandles + from .haptic_feedback import HapticFeedbackCfg from .session_lifecycle import SupportsDLPack from .visualizers.controller_aim_visualizer import ControllerAimVisualizer @@ -127,6 +129,9 @@ def __init__( enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, visualizers: Sequence[BaseVisualizer] = (), + *, + external_oxr_handles_provider: Callable[[], OpenXRSessionHandles | None] | None = None, + ovxr_session: bool = False, ): """Initialize the Isaac Capture device. @@ -144,6 +149,17 @@ def __init__( standalone (``False``) with ``isaacteleop`` owning its own OpenXR session through the CloudXR runtime -- teleop I/O with no Kit XR rendering. Typically wired to the ``--xr`` CLI flag. + external_oxr_handles_provider: Optional callback returning handles + borrowed from an application-owned OVXR context. + Supplying this capability delegates XR rendering/anchoring to + that owner. Static anchors use the configured pose; dynamic + anchors use the owner's committed physics pose and the configured + height/rotation policy. The owner must commit a scene pose before + input is read initially and after reset. + ovxr_session: Use application-owned presentation and anchoring + without borrowing live input handles. Static and dynamic anchors + follow the same policy as live OVXR, allowing recorded input to + drive a separately presented scene. mcap_record_path: Optional MCAP file path to record the live teleop session into. Mutually exclusive with *mcap_replay_path*. Debug-grade only -- the produced file @@ -168,14 +184,20 @@ def __init__( """ self._cfg = cfg self._target_world_matrix = None + self._ovxr_session = ovxr_session or external_oxr_handles_provider is not None - self._anchor_manager = XrAnchorManager(cfg.xr_cfg) + # Borrowed OVXR handles imply external anchor ownership. + self._anchor_manager = XrAnchorManager( + cfg.xr_cfg, ovxr_session=ovxr_session or external_oxr_handles_provider is not None + ) self._command_handler = CommandHandler() self._session_lifecycle = TeleopSessionLifecycle( cfg, cloudxr_env_file=cloudxr_env_file, auto_launch_cloudxr=auto_launch_cloudxr, use_kit_xr_bridge=use_kit_xr_bridge, + external_oxr_handles_provider=external_oxr_handles_provider, + ovxr_session=self._ovxr_session, mcap_record_path=mcap_record_path, mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, @@ -231,36 +253,89 @@ def __str__(self) -> str: def __enter__(self) -> IsaacTeleopDevice: """Enter the context manager and prepare the Isaac Capture session. - Binds the configured target frame, builds the retargeting pipeline, and acquires OpenXR handles - from Kit's XR bridge extension. If the handles are not yet available - (e.g. the user has not clicked "Start AR"), session creation is deferred - and will be retried automatically on each :meth:`advance` call. + Builds the retargeting pipeline and starts the selected live or replay + input session. Kit sessions bind the configured target frame through + Fabric. External OVXR callers supply target transforms to :meth:`advance` + when rebasing is needed. If live OpenXR handles are not yet available, + session creation is deferred and retried on each :meth:`advance` call. Returns: Self for context manager protocol. Raises: - ValueError: If the configured target frame does not exist. + ValueError: If the configured Kit target frame does not exist. """ - if self._cfg.target_frame_prim_path is not None: - import usdrt - from usdrt import Rt - - stage = usdrt.Usd.Stage.Attach(get_current_stage_id()) - prim = stage.GetPrimAtPath(self._cfg.target_frame_prim_path) - if not prim.IsValid(): - raise ValueError(f"No target frame at {self._cfg.target_frame_prim_path!r}.") - self._target_world_matrix = Rt.Xformable(prim).GetFabricHierarchyWorldMatrixAttr() - self._session_lifecycle.start() + try: + if not self._ovxr_session and self._cfg.target_frame_prim_path is not None: + import usdrt + from usdrt import Rt + + stage = usdrt.Usd.Stage.Attach(get_current_stage_id()) + prim = stage.GetPrimAtPath(self._cfg.target_frame_prim_path) + if not prim.IsValid(): + raise ValueError(f"No target frame at {self._cfg.target_frame_prim_path!r}.") + self._target_world_matrix = Rt.Xformable(prim).GetFabricHierarchyWorldMatrixAttr() + self._session_lifecycle.start() + except BaseException as error: + # A failed __enter__ has no automatic __exit__; unwind any runtime, + # pipeline or UI acquired before the failure without replacing it. + self.__exit__(type(error), error, error.__traceback__) + raise return self def __exit__(self, exc_type, exc_val, exc_tb): - """Exit the context manager and clean up the Isaac Capture session.""" + """Exit the context manager and clean up the Isaac Capture session. + + Cleanup failures propagate when no body exception is active; otherwise + they are attached as notes to that original exception. A failed native + input drain retains its resources, as indicated by + :attr:`session_cleanup_failed`. + """ self._target_world_matrix = None - self._anchor_manager.cleanup() - self._session_lifecycle.stop(exc_type, exc_val, exc_tb) + error = exc_val + try: + self._session_lifecycle.stop(exc_type, exc_val, exc_tb) + except Exception as cleanup_error: + if error is None: + error = cleanup_error + else: + error.add_note(f"Isaac Capture cleanup failed: {cleanup_error!r}") + try: + self._anchor_manager.cleanup() + except Exception as cleanup_error: + if error is None: + error = cleanup_error + else: + error.add_note(f"XR anchor cleanup failed: {cleanup_error!r}") + if exc_val is None and error is not None: + raise error return False + @property + def target_frame_prim_path(self) -> str | None: + """Concrete target frame for external scene readers, after environment-path expansion.""" + return self._cfg.target_frame_prim_path + + @property + def xr_render_anchor_path(self) -> str: + """Anchor prim that frames the rendered XR view.""" + return self._anchor_manager.anchor_headset_path + + @property + def external_anchor_state(self) -> XrAnchorManager | None: + """The dynamic anchor capability consumed by Lab's external scene publisher.""" + return self._anchor_manager if self._anchor_manager.external_reference_path is not None else None + + @property + def required_openxr_extensions(self) -> tuple[str, ...]: + """Return extensions required by the built teleop graphs.""" + return self._session_lifecycle.required_openxr_extensions() + + @property + def session_cleanup_failed(self) -> bool: + """Whether borrowed XR handles could not be safely drained.""" + return self._session_lifecycle.session_cleanup_failed + def reset(self, pause: bool = False) -> None: """Reset the device state. @@ -306,10 +381,25 @@ def last_control_events(self) -> ControlEvents: Returns a :class:`ControlEvents` derived from the teleop control pipeline. When no control channel is configured, returns a - default (no-op) :class:`ControlEvents`. + default (no-op) :class:`ControlEvents`. Repeated property reads return + the same events. RESET is emitted only once per returned input frame, + even if subsequent :meth:`advance` calls reuse that frame. """ return self._session_lifecycle.last_control_events + @property + def hand_tracking_presence(self) -> tuple[bool, bool] | None: + """Left/right hand presence in the latest input frame, or None before hand data is available.""" + lifecycle = self._session_lifecycle + result = lifecycle.last_step_result + if result is None: + return None + left = result.get(lifecycle.HAND_LEFT_KEY) + right = result.get(lifecycle.HAND_RIGHT_KEY) + if left is None or right is None: + return None + return not left.is_none, not right.is_none + def add_callback(self, key: str, func: Callable) -> None: """Add a callback function for teleop commands. @@ -324,8 +414,8 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N If the Isaac Capture session has not been started yet (because the OpenXR handles were not available at ``__enter__`` time), this method will - attempt to start it on each call. Once the user clicks "Start AR" and - the handles become available, the session is created transparently. + attempt to start it on each call. Once the selected Kit or external XR + provider exposes handles, the session is created transparently. Args: target_T_world: Optional 4x4 transform matrix that rebases all @@ -343,10 +433,14 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N (``__dlpack__``), including :class:`numpy.ndarray`, :class:`torch.Tensor`, and ``wp.array``. - When ``None`` and + For Kit sessions, when ``None`` and :attr:`~IsaacTeleopCfg.target_frame_prim_path` is set, the transform is computed automatically by reading the prim's - world matrix from Fabric and inverting it. + world matrix from Fabric and inverting it. External OVXR + sessions do not read Fabric: callers must supply this matrix + to rebase into a target frame. With ``None``, external output + remains in the simulation world frame, using the configured + static anchor or the owner's latest committed dynamic anchor. Returns: A flattened action :class:`torch.Tensor` ready for the Isaac Lab @@ -354,7 +448,8 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N (e.g. still waiting for the user to start AR). Raises: - RuntimeError: If called outside of a context manager. + RuntimeError: If called outside of a context manager or an external + dynamic anchor has no committed pose for the current reset epoch. """ if target_T_world is None and self._target_world_matrix is not None: matrix = self._target_world_matrix.Get() @@ -557,6 +652,8 @@ def create_isaac_teleop_device( *, env_prim_path: str | None = None, visualizers: Sequence[BaseVisualizer] = (), + external_oxr_handles_provider: Callable[[], OpenXRSessionHandles | None] | None = None, + ovxr_session: bool = False, ) -> IsaacTeleopDevice: """Create an :class:`IsaacTeleopDevice` with required Omniverse extension setup. @@ -596,6 +693,12 @@ def create_isaac_teleop_device( ``isaacteleop`` creates its own OpenXR session through the CloudXR runtime, so teleop I/O works headless without Kit XR rendering. Typically wired to the ``--xr`` CLI flag. + external_oxr_handles_provider: Optional callback returning handles + borrowed from an application-owned OVXR context. That owner supplies + static anchoring or commits dynamic anchor poses before input reads. + ovxr_session: Use application-owned OVXR presentation and the configured + static or dynamic anchor policy even when input comes from an MCAP + reader rather than live XR handles. mcap_record_path: Optional MCAP file path to record the live teleop session into. Debug-grade only. Mutually exclusive with *mcap_replay_path*. @@ -651,6 +754,8 @@ def create_isaac_teleop_device( cloudxr_env_file=cloudxr_env_file, auto_launch_cloudxr=auto_launch_cloudxr, use_kit_xr_bridge=use_kit_xr_bridge, + external_oxr_handles_provider=external_oxr_handles_provider, + ovxr_session=ovxr_session, mcap_record_path=mcap_record_path, mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py new file mode 100644 index 000000000000..658db8ffe7e6 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -0,0 +1,764 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Lab host adapter for Isaac Capture-owned OVXR presentation.""" + +from __future__ import annotations + +import contextlib +import logging +import math +import threading +import time +from collections import deque +from collections.abc import Callable, Hashable, Iterable +from dataclasses import replace +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +import numpy as np +from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger +from isaaclab_ov.renderers import OVRTXRendererCfg + +from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE + +from .ovxr_cfg import OvLibrariesXrCameraCfg, OvLibrariesXrCfg +from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation + +if TYPE_CHECKING: + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + from isaacteleop.oxr import OpenXRSessionHandles + + from isaaclab.sim import SimulationCfg + +__all__ = ["OvLibrariesXrCameraCfg", "OvLibrariesXrCfg", "OvLibrariesXrSession"] + +logger = logging.getLogger(__name__) + +_DEFAULT_OVXR_RENDER_PRODUCT_PATHS = ( + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", +) +_DEFAULT_OVXR_RENDER_VAR_PATHS = ( + "/Render/Vars/LdrColor", + "/Render/Vars/XRDepth", +) +_DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" +_ENVIRONMENT_ROOT_PATH = DEFAULT_ENV_TEMPLATE.format(0) + + +class _CounterRate: + """Measure a cumulative application counter over a recent wall-clock window.""" + + def __init__(self, window_s: float = 1.0) -> None: + self._window_ns = int(window_s * 1_000_000_000) + self._samples: deque[tuple[int, int]] = deque() + + def sample(self, count: int, now_ns: int | None = None) -> float: + now_ns = time.perf_counter_ns() if now_ns is None else now_ns + if self._samples and count < self._samples[-1][1]: + self._samples.clear() + self._samples.append((now_ns, count)) + while len(self._samples) > 2 and self._samples[1][0] <= now_ns - self._window_ns: + self._samples.popleft() + if len(self._samples) < 2: + return 0.0 + elapsed_s = (self._samples[-1][0] - self._samples[0][0]) / 1_000_000_000 + count_delta = self._samples[-1][1] - self._samples[0][1] + return 0.0 if elapsed_s <= 0.0 else max(0, count_delta) / elapsed_s + + +def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrCfg) -> None: + """Declare type-only OVXR prims for the application Stage's lifetime.""" + if not cfg.create_render_products: + return + + unwrapped = getattr(env, "unwrapped", env) + stage = unwrapped.scene.stage + if stage is None: + raise RuntimeError("OVXR runtime prim declaration requires an environment Stage") + + render_products = cfg.render_products or _DEFAULT_OVXR_RENDER_PRODUCT_PATHS + declarations = ( + *((path, "Camera") for path in cfg.camera_paths), + *((path, "RenderProduct") for path in render_products), + *((path, "RenderVar") for path in _DEFAULT_OVXR_RENDER_VAR_PATHS), + (_DEFAULT_OVXR_RENDER_SETTINGS_PATH, "RenderSettings"), + ) + expected_types: dict[str, str] = {} + for path, type_name in declarations: + if not path.startswith("/") or path == "/": + raise ValueError(f"OVXR runtime prim path must be absolute and non-root: {path!r}") + previous_type = expected_types.setdefault(path, type_name) + if previous_type != type_name: + raise ValueError(f"OVXR runtime prim path {path!r} has conflicting requested types") + + for path, type_name in expected_types.items(): + existing = stage.GetPrimAtPath(path) + existing_type = str(existing.GetTypeName()) if existing and existing.IsValid() else "" + if existing_type and existing_type != type_name: + raise RuntimeError( + f"OVXR runtime prim {path!r} already has incompatible type {existing_type!r}; expected {type_name!r}" + ) + stage.DefinePrim(path, type_name) + + +class _StageOrdinalLedger(StageOrdinalLedger): + """Reserve explicit OVStage ordinals for the application publication lane.""" + + def __init__(self, stage: Any) -> None: + self._stage = stage + self._state_lock = threading.Lock() + self._active_ordinal: int | None = None + self._closed = False + self._floor = self._read_floor() + + def _read_floor(self) -> int: + with self._stage.get_attribute_write_floor() as query: + query.wait() + floor = int(query.fetch()) + if floor < 0 or floor > (1 << 64) - 1: + raise RuntimeError(f"OVStage returned an invalid write floor: {floor}") + return floor + + def _require_active(self, ordinal: int) -> None: + if self._active_ordinal != ordinal: + raise RuntimeError( + f"OVStage ordinal {ordinal} is not the active application reservation ({self._active_ordinal=})" + ) + + def reserve(self) -> int: + """Reserve exactly ``floor + 1`` while the application publication guard is held.""" + with self._state_lock: + if self._closed: + raise RuntimeError("OVStage ordinal ledger is closed") + if self._active_ordinal is not None: + raise RuntimeError(f"OVStage ordinal {self._active_ordinal} is still unresolved") + if self._floor == (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + self._active_ordinal = self._floor + 1 + return self._active_ordinal + + def publish(self, ordinal: int) -> None: + """Seal one application-written reservation and mark it resolved.""" + with self._state_lock: + self._require_active(ordinal) + self._stage.advance_write_floor(ordinal=ordinal).wait() + self._floor = ordinal + self._active_ordinal = None + + def resolve_external(self, ordinal: int, *, expected_published: bool | None) -> bool: + """Resolve an external write and verify its documented publication result.""" + with self._state_lock: + self._require_active(ordinal) + if expected_published is True: + published = True + else: + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + published = floor == ordinal + # Record native side effects even when the response contradicts the + # expected result. A reported error cannot make a sealed ordinal reusable. + if published: + self._floor = ordinal + if expected_published is not None and published != expected_published: + self._active_ordinal = None + state = "sealed" if published else "unsealed" + expected = "sealed" if expected_published else "unsealed" + raise RuntimeError(f"OVStage ordinal {ordinal} is {state} after an external write; expected {expected}") + self._active_ordinal = None + return published + + def abandon(self, ordinal: int) -> None: + """Reconcile a failed writer before releasing its reservation.""" + with self._state_lock: + self._require_active(ordinal) + # A publication wait can fail after the native Stage has sealed it. + # Query only on the failure/abandon path; an unreadable or unrelated + # floor leaves the reservation unresolved so it cannot be reused. + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + self._floor = floor + self._active_ordinal = None + + def destroy(self) -> None: + """Close the ledger after every application and external write is resolved.""" + with self._state_lock: + if self._active_ordinal is not None: + raise RuntimeError(f"Cannot close with unresolved OVStage ordinal {self._active_ordinal}") + self._closed = True + + +class _IsaacLabOvxrApplicationInterface(OvxrApplicationInterface): + """Supply only the renderer/Stage/coordinator and simulation publication.""" + + def __init__( + self, + bootstrap: OvxrSessionBootstrap, + env: Any, + cfg: OvLibrariesXrCfg, + excluded_visual_meshes: Iterable[str] = (), + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, + ) -> None: + self._bootstrap = bootstrap + self._env = env + self._cfg = cfg + self._excluded_visual_meshes = tuple(excluded_visual_meshes) + self._publication_lock = threading.Lock() + self._scene_publisher = None + self._lease = None + self._ordinal_ledger = None + self._last_published_ordinal: int | None = None + self._publication_closed = False + self._auxiliary_cameras = cfg.auxiliary_cameras + self.desktop_mirror = None + self._external_anchor = external_anchor_state + self._anchor_snapshot: tuple[int, np.ndarray] | None = None + self._last_anchor_frame_time: float | None = None + self._anchor_resources = contextlib.ExitStack() + self._anchor_query = None + self._anchor_stage = None + self._anchor_retirement_failed = False + + def offer_render_owner_transfer(self) -> object: + self._scene_publisher = self._bootstrap.capture_scene_publisher() + return self._bootstrap.offer_render_owner_transfer() + + def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + + self._bootstrap.claim_render_owner_transfer(capability) + unwrapped = self._env.unwrapped + if self._auxiliary_cameras: + # Fail before native Stage binding and partition export. + sensors = unwrapped.scene.sensors + for camera in self._auxiliary_cameras: + if sensors.get(camera.name) is None: + raise RuntimeError(f"Configured auxiliary camera {camera.name!r} is not present in the environment") + _declare_ovxr_runtime_prims(self._env, self._cfg) + from .xr_scene_partition import partition_visual_meshes + + stage = unwrapped.scene.stage + with partition_visual_meshes(stage, self._excluded_visual_meshes): + self._lease = self._bootstrap.bind_environment(self._env) + auxiliary_paths = () + if self._auxiliary_cameras: + auxiliary_paths = self._lease.auxiliary_render_product_paths + with self._lease.ovstage_owner_lane(): + self._lease.configure_external_camera_scene_partitions(_ENVIRONMENT_ROOT_PATH, self._cfg.camera_paths) + self._ordinal_ledger = _StageOrdinalLedger(self._lease.ovstage) + if self._external_anchor is not None: + import ovstage + + self._anchor_stage = self._lease.ovstage + try: + paths = self._anchor_resources.enter_context(ovstage.PathDictionary(self._anchor_stage)) + path_list = paths.create_path_list_from_strings(self._external_anchor.external_anchor_paths) + self._anchor_resources.callback(paths.destroy_path_list, path_list) + self._anchor_query = self._anchor_resources.enter_context( + self._anchor_stage.query_from_path_list(path_list) + ) + self._anchor_query.wait() + except BaseException as setup_error: + try: + self._anchor_resources.close() + except BaseException as cleanup_error: + self._anchor_retirement_failed = True + setup_error.add_note(f"Anchor query cleanup also failed: {cleanup_error!r}") + raise + from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig, CameraPanelConsumer + + panels = tuple( + CameraPanelConfig( + render_product_path=path, + label=camera.name, + max_image_bytes=int(camera.camera_cfg.width) * int(camera.camera_cfg.height) * 4, + panel_width_m=camera.panel_width_m, + position=camera.position, + orientation_xyzw=camera.orientation_xyzw, + space=camera.space, + ) + for path, camera in zip(auxiliary_paths, self._auxiliary_cameras, strict=True) + ) + consumers = [CameraPanelConsumer(panels)] if panels else [] + if self._cfg.desktop_mirror_enabled: + # Headless sessions never import the optional mirror modules. + from isaaccapture.ovxr_session_manager.desktop_mirror import DesktopMirrorConfig + from isaaccapture.ovxr_session_manager.owned_mirror import MirrorProduct, OwnedDesktopMirror + + self.desktop_mirror = OwnedDesktopMirror( + DesktopMirrorConfig( + max_fps=self._cfg.desktop_mirror_max_fps, + initial_status_text="Waiting for headset", + ), + camera_products=tuple(MirrorProduct(panel.render_product_path, panel.label) for panel in panels), + ) + consumers.append(self.desktop_mirror) + return OvxrSessionBinding( + renderer=self._lease.ovrtx_renderer, + stage=self._lease.ovstage, + ordinal_ledger=self._ordinal_ledger, + frame_consumers=tuple(consumers), + prepare_frame_scene=self.prepare_frame_scene if self._external_anchor is not None else None, + ) + + def publication_guard(self) -> contextlib.AbstractContextManager[None]: + return self._publication_lock + + def owner_lane(self) -> contextlib.AbstractContextManager[None]: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + return self._lease.ovstage_owner_lane() + + def publish_complete_scene(self, publication_id: Hashable) -> int: + if self._publication_closed or self._scene_publisher is None or self._ordinal_ledger is None: + raise RuntimeError("IsaacLab scene publication is not available") + anchor_epoch = None + if self._external_anchor is not None: + # A failed seed can follow the publisher recording its identity. + # Check the terminal policy before dedup may return an older seal. + anchor_epoch = self._external_anchor.external_epoch + if not self._scene_publisher.publication_required(publication_id): + if self._last_published_ordinal is None: + raise RuntimeError("OVStage publication deduplicated before a sealed ordinal exists") + return self._last_published_ordinal + if self._auxiliary_cameras: + sensors = self._env.unwrapped.scene.sensors + for camera in self._auxiliary_cameras: + # OVXR presentation bypasses Camera's image-update path. Refresh only poses + # after simulation/reset kinematics, before publishing the matching scene. + sensors[camera.name].refresh_world_poses() + snapshot = None + seed = False + if self._external_anchor is not None: + epoch, seed = anchor_epoch + reference = self._bootstrap.get_world_transform(self._external_anchor.external_reference_path) + snapshot = (epoch, reference) + ordinal = self._ordinal_ledger.reserve() + try: + self._scene_publisher.write_complete_scene(ordinal, publication_id) + except BaseException as write_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + write_error.add_note(f"Abandon after failed scene write also failed: {abandon_error!r}") + raise + try: + if seed: + epoch, reference = snapshot + + def commit(matrices: np.ndarray) -> None: + self._write_anchor_matrices(matrices, ordinal) + self._ordinal_ledger.publish(ordinal) + + # Reset seeding must be sealed with this scene before input resumes. + # The first pose initializes the filter without an elapsed interval. + if not self._external_anchor.commit_external_pose(reference, 0.0, epoch, commit): + raise RuntimeError("XR anchor reset changed during complete scene publication") + self._last_anchor_frame_time = None + else: + self._ordinal_ledger.publish(ordinal) + except BaseException as publish_error: + # Component writes already consumed this generation. Do not let a + # retry deduplicate it to an older seal after publication failed. + self.close_publication() + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publish_error.add_note(f"Abandon after failed scene publication also failed: {abandon_error!r}") + raise + self._last_published_ordinal = ordinal + self._anchor_snapshot = snapshot + return ordinal + + def _write_anchor_matrices(self, matrices: np.ndarray, ordinal: int) -> None: + import ovstage + from isaaclab_ov.stage import xform_tensor_from_numpy + + # SDP uses column vectors; the Stage's omni:xform stores USD row matrices. + rows = np.ascontiguousarray(matrices.transpose(0, 2, 1), dtype=np.float64) + tensor = xform_tensor_from_numpy(rows) + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:xform", + ordinal=ordinal, + tensors=tensor, + semantic=ovstage.AttributeSemantic.MATRIX, + is_array=False, + ).wait() + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:resetXformStack", + ordinal=ordinal, + tensors=np.ones(len(rows), dtype=np.bool_), + is_array=False, + ).wait() + + def prepare_frame_scene(self) -> None: + """Advance the committed reference with elapsed XR-frame time [s]. + + The first begun frame after reset establishes the monotonic clock with + zero elapsed time. Scene publication never advances this frame clock. + """ + if self._anchor_snapshot is None: + return + epoch, reference = self._anchor_snapshot + now = time.monotonic() + dt = 0.0 if self._last_anchor_frame_time is None else now - self._last_anchor_frame_time + + def commit(matrices: np.ndarray) -> None: + ordinal = self._ordinal_ledger.reserve() + try: + self._write_anchor_matrices(matrices, ordinal) + # OVXR reads sealed anchor state at bind; bind reserves the next ordinal. + self._ordinal_ledger.publish(ordinal) + except BaseException as publication_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publication_error.add_note( + f"Abandon after failed anchor publication also failed: {abandon_error!r}" + ) + raise + + if self._external_anchor.commit_external_pose(reference, dt, epoch, commit): + self._last_anchor_frame_time = now + + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: + """Replace a replay device's policy while preserving native query ownership.""" + if self._external_anchor is not None: + # A new replay manager must not revive a failed borrowed native pair. + _ = self._external_anchor.external_epoch + if (state is None) != (self._external_anchor is None) or ( + state is not None and state.external_anchor_paths != self._external_anchor.external_anchor_paths + ): + raise ValueError("A reused XR environment must preserve its anchor paths") + self._external_anchor = state + self._anchor_snapshot = None + self._last_anchor_frame_time = None + + def validate_first_published_scene(self, ordinal: int) -> None: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + with self._lease.ovstage_owner_lane(): + self._lease.validate_external_scene_partitions(ordinal) + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + return self._bootstrap.get_target_frame_T_world(prim_path) + + def close_publication(self) -> None: + if self._publication_closed: + return + self._publication_closed = True + if self._scene_publisher is not None: + self._scene_publisher.close() + + def close_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + # Unresolved writes may still use the anchor queries and Stage. + # Retain their resources; retirement requires proven completion. + self._ordinal_ledger.destroy() + self._ordinal_ledger = None + try: + self._anchor_resources.close() + except BaseException: + self._anchor_retirement_failed = True + raise + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its borrowed Stage") + self._bootstrap.close() + + def close_environment_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + raise RuntimeError("OVStage write retirement is unproved; retaining its environment") + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its environment") + if not self._bootstrap.closed: + raise RuntimeError("Render lease retirement is unproved; retaining its environment") + self._env.close() + + def abort(self) -> None: + # The caller owns an environment even when worker startup failed. + # Release the lease before it closes that environment and its context. + self._bootstrap.close() + + +class OvLibrariesXrSession: + """Thin Isaac Lab adapter around Isaac Capture's OVXR session.""" + + @runtime_checkable + class ExternalAnchorState(Protocol): + """Dynamic XR-anchor policy sealed together with complete scene publications. + + :class:`~isaaclab_teleop.xr_anchor_manager.XrAnchorManager` implements this protocol. It + drives the anchor when constructed for an OVXR session with ``XrCfg.anchor_prim_path`` set, + which is when :attr:`~isaaclab_teleop.IsaacTeleopDevice.external_anchor_state` returns it. + """ + + @property + def external_anchor_paths(self) -> tuple[str, ...]: + """Anchor Xform paths written together at each committed pose.""" + ... + + @property + def external_reference_path(self) -> str: + """Absolute path of the physics prim whose world transform drives the anchor.""" + ... + + @property + def external_epoch(self) -> tuple[int, bool]: + """Reset identity and whether that epoch still needs its seeding publication. + + Raises: + RuntimeError: If a previous anchor publication failed. + """ + ... + + def commit_external_pose( + self, + reference: np.ndarray, + rendering_dt: float, + epoch: int, + commit: Callable[[np.ndarray], None], + ) -> bool: + """Advance the policy from ``reference`` and seal its matrices through ``commit``. + + Args: + reference: World transform of :attr:`external_reference_path` [4, 4]. + rendering_dt: Elapsed XR-frame time since the last committed pose [s]. + epoch: Reset identity read from :attr:`external_epoch`. + commit: Callable that writes and seals the anchor matrices [N, 4, 4]. + + Returns: + ``False`` when ``epoch`` is stale and nothing was committed. + """ + ... + + def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrCfg) -> None: + self._bootstrap = bootstrap + self._cfg = cfg + self._owner = None + self._desktop_mirror = None + self._host = None + self._closed = False + self._physics_rate = _CounterRate() + + @classmethod + def prepare( + cls, + sim_cfg: SimulationCfg, + cfg: OvLibrariesXrCfg, + renderer_cfg: OVRTXRendererCfg | None = None, + ) -> OvLibrariesXrSession: + """Snapshot session selection, create simulation and reserve its OVRTX lease. + + Renderer settings are process-global on the stock renderer interface; + use one configured session per process. No supported readback exists + here to restore a previous renderer's settings on close. + """ + cfg = replace(cfg, auxiliary_cameras=tuple(cfg.auxiliary_cameras)) + validate_ovxr_session_preparation() + import ovxr + + # Stock settings are applied before Renderer construction. Preserve explicit caller choices. + ovxr.apply_renderer_settings(cfg.renderer_settings.strip()) + bootstrap = OvxrSessionBootstrap(sim_cfg, renderer_cfg) + try: + for camera in cfg.auxiliary_cameras: + bootstrap.configure_auxiliary_camera(camera.camera_cfg, max_fps=camera.max_fps) + return cls(bootstrap, cfg) + except BaseException as preparation_error: + try: + bootstrap.abort() + except BaseException as abort_error: + # Cleanup follows the failure; retain its original cause. + preparation_error.add_note(f"Cleanup after failed XR preparation also failed: {abort_error!r}") + raise + + def start_render_worker( + self, + env: Any, + *, + tracking_anchor_path: str | None = None, + required_openxr_extensions: Iterable[str] = (), + excluded_visual_meshes: Iterable[str] = (), + max_blank_frames: int = 1, + timeout_s: float = 30.0, + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, + ) -> bool: + """Give Isaac Capture the application pair and report whether XR started. + + Args: + env: Constructed environment sharing the prepared simulation context. + tracking_anchor_path: Override the configured USD tracking-anchor path. + required_openxr_extensions: Extensions required by the input pipeline. + excluded_visual_meshes: Mesh patterns excluded from the XR eye partition. + max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. + timeout_s: Maximum worker startup wait [s]. + external_anchor_state: Dynamic anchor policy seeded from committed scene publications. + + Raises: + RuntimeError: If the session is closed or already started, or ``env`` does not reuse the + prepared simulation context, has no completed stage or not exactly one environment, + or selects a different simulation device or physics time step. + TypeError: If ``external_anchor_state`` does not implement :class:`ExternalAnchorState`. + ValueError: If ``max_blank_frames`` is not a nonnegative integer. + """ + self._require_open() + if self._owner is not None: + raise RuntimeError("OVXR async render worker is already started") + if isinstance(max_blank_frames, bool) or not isinstance(max_blank_frames, int) or max_blank_frames < 0: + raise ValueError("max_blank_frames must be a non-negative integer") + _require_external_anchor_state(external_anchor_state) + # Reject an inconsistent environment before the worker claims the lease. + self._bootstrap.validate_environment(env) + from isaaccapture.ovxr_session_manager.ovxr_session import ( + OvxrSession, + OvxrSessionConfig, + ) + + from .session_lifecycle import cloudxr_eula_accepted + + anchor_path = self._cfg.tracking_anchor_path if tracking_anchor_path is None else tracking_anchor_path + host = _IsaacLabOvxrApplicationInterface( + self._bootstrap, env, self._cfg, excluded_visual_meshes, external_anchor_state + ) + self._host = host + self._owner = OvxrSession( + host, + OvxrSessionConfig( + application_name=self._cfg.application_name, + session_mode=self._cfg.session_mode, + cloudxr_env_file=self._cfg.cloudxr_env_file, + auto_launch_cloudxr=self._cfg.auto_launch_cloudxr, + accept_eula=cloudxr_eula_accepted(), + render_products=self._cfg.render_products, + camera_paths=self._cfg.camera_paths, + create_render_products=self._cfg.create_render_products, + meters_per_unit=self._cfg.meters_per_unit, + depth_near_z=self._cfg.depth_near_z, + depth_far_z=self._cfg.depth_far_z, + tracking_anchor_path=anchor_path, + floor_aligned_anchor=False, + host_wait_render_output=self._cfg.host_wait_render_output, + extra_openxr_extensions=self._cfg.extra_openxr_extensions, + ), + required_openxr_extensions=required_openxr_extensions, + max_blank_frames=max_blank_frames, + ) + try: + return self._owner.start(timeout_s=timeout_s) + finally: + self._desktop_mirror = host.desktop_mirror + + def external_oxr_handles(self) -> OpenXRSessionHandles | None: + """Return Isaac Capture's typed external handles once its owner is ready.""" + self._require_open() + return None if self._owner is None else self._owner.external_oxr_handles() + + def request_render(self, physics_step_id: Hashable, *, force: bool = False) -> int | None: + """Offer one complete application generation for Isaac Capture to render. + + Return the admitted ordinal, or ``None`` when render demand is already + satisfied. Retry the latest dirty generation until admitted, including + while paused. ``force`` preserves reset/lifecycle discontinuities. + """ + self._require_started() + return self._owner.request_render(physics_step_id, force=force) + + def report_host_timing(self, physics_step_count: int, physics_dt_s: float | None = None) -> None: + """Publish physics counter and optional simulation timestep [s] to the mirror.""" + self._require_started() + if self._desktop_mirror is None: + return + self._desktop_mirror.record_host_frame() + physics_hz = self._physics_rate.sample(physics_step_count) + if physics_dt_s is not None and math.isfinite(physics_dt_s) and physics_dt_s > 0.0: + target_hz = 1.0 / physics_dt_s + physics_text = f"Physics {physics_hz:5.1f}/{target_hz:.0f} Hz ({physics_hz / target_hz:.2f}× RT)" + else: + physics_text = f"Physics {physics_hz:5.1f} Hz" + mirror_status = self._desktop_mirror.status + if mirror_status is not None: + self._desktop_mirror.set_status_text( + f"Main {mirror_status.main_thread_fps:5.1f} FPS " + f"{physics_text} Render {mirror_status.render_thread_fps:5.1f} FPS" + ) + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + self._require_started() + return self._owner.get_target_frame_T_world(prim_path) + + def close(self) -> None: + """Close Isaac Capture's owner before releasing the application environment.""" + if self._closed: + return + if self._owner is None: + self._bootstrap.close() + else: + self._owner.close() + self._closed = True + + @property + def async_worker_owns_environment(self) -> bool: + """Whether Isaac Capture accepted environment teardown ownership.""" + return self._owner is not None and self._owner.owns_environment + + @property + def shutdown_requested(self) -> bool: + """Whether the OVXR session requested application shutdown.""" + return self._owner is not None and self._owner.shutdown_requested + + def request_shutdown(self) -> None: + """Request orderly OVXR shutdown.""" + if self._owner is not None: + self._owner.request_shutdown() + + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: + """Adopt the current replay device's anchor policy between runs.""" + self._require_started() + _require_external_anchor_state(state) + with self._host.publication_guard(): + self._host.set_external_anchor_state(state) + + def wait_for_shutdown(self, timeout_s: float) -> bool: + """Wait up to timeout_s [s]; return whether application cancellation was requested.""" + self._require_started() + return self._owner.wait_for_shutdown(timeout_s) + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR session is closed") + + def _require_started(self) -> None: + self._require_open() + if self._owner is None: + raise RuntimeError("OVXR session is not started") + + def __enter__(self) -> OvLibrariesXrSession: + self._require_open() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +def _require_external_anchor_state(state: object) -> None: + """Reject anchor policies that lack the external-publication protocol on the caller's thread.""" + if state is None: + return + if not isinstance(state, OvLibrariesXrSession.ExternalAnchorState): + raise TypeError( + f"external_anchor_state must implement OvLibrariesXrSession.ExternalAnchorState; got {type(state).__name__}" + ) + reference_path = state.external_reference_path + if not isinstance(reference_path, str) or not reference_path.startswith("/"): + raise ValueError(f"external_reference_path must be an absolute prim path; got {reference_path!r}") diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py new file mode 100644 index 000000000000..647cab00c302 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -0,0 +1,126 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for an Isaac Capture-owned OVXR session. + +This module has no runtime dependencies, so launchers can build and validate the +configuration without the native OVXR, OVRTX or OVStage packages. +""" + +from __future__ import annotations + +import math +from dataclasses import MISSING +from typing import TYPE_CHECKING, Literal + +from isaaclab.utils import configclass + +if TYPE_CHECKING: + from isaaclab.sensors import CameraCfg + +_DEFAULT_OVXR_CAMERA_PATHS = ( + "/OVXRGeneratedCameras/OVXR_LeftEyeCamera", + "/OVXRGeneratedCameras/OVXR_RightEyeCamera", +) + + +@configclass +class OvLibrariesXrCameraCfg: + """One application-selected sensor and its camera-panel policy.""" + + name: str = MISSING + """Scene sensor name used for camera selection and as the desktop mirror's camera label.""" + camera_cfg: CameraCfg = MISSING + """Selected sensor configuration, assigned to the session's shared renderer.""" + max_fps: float = 30.0 + """Finite positive image update cap [Hz].""" + panel_width_m: float = 0.48 + """Physical panel width [m]; image aspect ratio determines its height.""" + position: tuple[float, float, float] = (0.0, -0.15, -0.8) + """Panel center in OpenXR head space [m], with forward along negative Z.""" + orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + """Panel orientation in head space as an (x, y, z, w) quaternion.""" + space: Literal["head_locked"] = "head_locked" + """Placement space; only head-locked panels are supported.""" + + def __post_init__(self) -> None: + # configclass copies field defaults, so an unset field holds a copy of MISSING. + unset = (type(MISSING), type(None)) + if isinstance(self.name, unset) or not self.name or isinstance(self.camera_cfg, unset): + raise ValueError("camera name and configuration are required") + if not math.isfinite(self.max_fps) or self.max_fps <= 0: + raise ValueError("camera max_fps must be finite and positive") + # Mirror Capture's CameraPanelConfig checks without importing its session stack; the + # session builds the native panel configuration with the resolved product path. + if not math.isfinite(self.panel_width_m) or self.panel_width_m <= 0.0: + raise ValueError("camera panel panel_width_m must be finite and positive") + if len(self.position) != 3 or not all(math.isfinite(value) for value in self.position): + raise ValueError("camera panel position must contain three finite values") + if len(self.orientation_xyzw) != 4 or not all(math.isfinite(value) for value in self.orientation_xyzw): + raise ValueError("camera panel orientation must contain four finite values") + if math.sqrt(sum(value * value for value in self.orientation_xyzw)) <= 0.0: + raise ValueError("camera panel orientation must have non-zero norm") + if self.space != "head_locked": + raise ValueError("camera panel space must be 'head_locked'") + + +@configclass +class OvLibrariesXrCfg: + """Isaac Lab configuration for an Isaac Capture-owned OVXR session.""" + + application_name: str = "isaaclab-kitless-teleop" + """Application identifier passed to the OpenXR runtime.""" + session_mode: Literal["ar", "vr"] = "ar" + """OpenXR presentation mode: augmented or virtual reality.""" + renderer_settings: str = "--/app/hydra/aperture/conform=stretch" + """Process-global OVXR renderer settings applied before renderer construction.""" + cloudxr_env_file: str | None = None + """CloudXR environment-file path; None leaves runtime launch to the caller.""" + auto_launch_cloudxr: bool = True + """Whether the session owner launches the CloudXR runtime automatically.""" + render_products: tuple[str, ...] = () + """Stereo RenderProduct paths; an empty selection uses OVXR's default paths.""" + camera_paths: tuple[str, ...] = _DEFAULT_OVXR_CAMERA_PATHS + """Exactly two non-empty USD camera paths, ordered left eye then right eye.""" + create_render_products: bool = True + """Whether to declare and configure the selected stereo rendering prims.""" + meters_per_unit: float = 1.0 + """Finite positive scene scale [m per USD unit].""" + depth_near_z: float = 0.0 + """Near depth distance [m]; both depth distances zero select provider defaults.""" + depth_far_z: float = 0.0 + """Far depth distance [m]; explicit distances must satisfy 0 < near < far.""" + tracking_anchor_path: str = "" + """USD tracking-anchor path; an empty path uses the stage origin.""" + host_wait_render_output: bool = True + """Whether OVXR waits for render output before returning from frame rendering.""" + desktop_mirror_enabled: bool = False + """Whether to display the owner-managed desktop mirror.""" + desktop_mirror_max_fps: float = 30.0 + """Finite positive desktop mirror refresh cap [Hz].""" + auxiliary_cameras: tuple[OvLibrariesXrCameraCfg, ...] = () + """Up to three uniquely named camera panels; an empty selection disables panels.""" + extra_openxr_extensions: tuple[str, ...] = () + """Additional OpenXR extension names requested when creating the session.""" + + def __post_init__(self) -> None: + cameras = self.auxiliary_cameras + if len(cameras) > 3 or len({camera.name for camera in cameras}) != len(cameras): + raise ValueError("Select at most three uniquely named auxiliary cameras") + if self.session_mode not in ("ar", "vr"): + raise ValueError("session_mode must be 'ar' or 'vr'") + if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: + raise ValueError("meters_per_unit must be finite and positive") + if not math.isfinite(self.desktop_mirror_max_fps) or self.desktop_mirror_max_fps <= 0.0: + raise ValueError("desktop_mirror_max_fps must be finite and positive") + if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( + not math.isfinite(self.depth_near_z) + or not math.isfinite(self.depth_far_z) + or self.depth_near_z <= 0.0 + or self.depth_far_z <= self.depth_near_z + ): + raise ValueError("depth_near_z and depth_far_z must both be zero or finite with 0 < near < far") + if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): + raise ValueError("Kitless stereo requires exactly two non-empty OVXR camera paths") diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py new file mode 100644 index 000000000000..939cd61d5119 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -0,0 +1,300 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Lifecycle bootstrap for an application-owned OVRTX OVXR pair.""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING, Any + +import numpy as np +import warp as wp +from isaaclab_ov.renderers import OvLibrariesRenderLease, OVRTXRenderer, OVRTXRendererCfg + +from isaaclab.app import get_settings_manager +from isaaclab.scene_data import SceneDataFormat +from isaaclab.sim import SimulationContext + +_TERMINAL_MODE_SENTINEL = "ISAAC_LAUNCHED_FROM_TERMINAL" + +if TYPE_CHECKING: + from isaaclab_ov.renderers import OvLibrariesScenePublisher + + from isaaclab.sensors import CameraCfg + from isaaclab.sim import SimulationCfg + + +def validate_ovxr_session_preparation() -> None: + """Reject ineligible preparation before the caller changes renderer globals.""" + if SimulationContext.instance() is not None: + raise RuntimeError("OVXR bootstrap must create the SimulationContext immediately before the environment") + if hasattr(builtins, _TERMINAL_MODE_SENTINEL) and getattr(builtins, _TERMINAL_MODE_SENTINEL) is not True: + raise RuntimeError("OVXR bootstrap cannot reuse a context in extension mode") + + +def resolve_cuda_device(device: str) -> str: + """Resolve a bare ``"cuda"`` device to the GPU that the simulation selects for it. + + :class:`~isaaclab.sim.SimulationContext` and :func:`~isaaclab.app.launch_simulation` read + the ``/physics/cudaDevice`` setting the same way. + + Args: + device: Simulation device, for example ``"cpu"``, ``"cuda"`` or ``"cuda:1"``. + + Returns: + ``"cuda:"`` for a bare ``"cuda"``; any other device unchanged. + """ + if device != "cuda": + return device + cuda_device = get_settings_manager().get("/physics/cudaDevice") + return f"cuda:{max(0, int(cuda_device) if cuda_device is not None else 0)}" + + +class OvxrSessionBootstrap: + """Reserve an OVRTX/OVStage lease before a real environment is constructed. + + The bootstrap creates the :class:`~isaaclab.sim.SimulationContext` from the ``sim_cfg`` + given here, and the environment reuses it. Only environments that accept a pre-created + context are supported (manager-based environments; :class:`~isaaclab.envs.DirectRLEnv` + always creates its own). :meth:`validate_environment` requires ``num_envs == 1`` and + rejects a simulation device or time step that disagrees with the prepared context. + """ + + def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None = None) -> None: + validate_ovxr_session_preparation() + + self._sim: SimulationContext | None = None + self._lease: OvLibrariesRenderLease | None = None + self._owner_transfer_capability: object | None = None + self._bound = False + self._closed = False + self._installed_terminal_mode_sentinel = False + self._target_frame_readers: dict[str, tuple[Any, Any, Any]] = {} + + if not hasattr(builtins, _TERMINAL_MODE_SENTINEL): + setattr(builtins, _TERMINAL_MODE_SENTINEL, True) + self._installed_terminal_mode_sentinel = True + + try: + sim = SimulationContext(sim_cfg) + # The simulation registry owns the renderer, including cloning and + # cleanup; the XR session only borrows its exact native pair. + renderer = sim.get_or_create_backend(renderer_cfg or OVRTXRendererCfg()) + if not isinstance(renderer, OVRTXRenderer): + raise TypeError(f"OVRTXRendererCfg resolved to unexpected renderer {type(renderer).__name__}") + lease = renderer.acquire_ov_libraries_render_lease() + # Camera sensors prepare every registered renderer on the application thread + # during environment construction. Only the claimed render owner materializes + # this pair, so that call is recorded and completed in bind_environment(). + lease.defer_stage_until_owner_transfer() + except BaseException as preparation_error: + sim = SimulationContext.instance() + try: + if sim is not None: + sim.clear_instance() + except BaseException as cleanup_error: + preparation_error.add_note(f"Cleanup after failed XR construction also failed: {cleanup_error!r}") + finally: + self._restore_terminal_mode_sentinel() + raise + + self._sim = sim + self._lease = lease + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application scene publisher before render ownership transfers.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Scene publisher is available only before OVXR Stage binding") + return self._lease.capture_scene_publisher() + + def configure_auxiliary_camera(self, camera_cfg: CameraCfg, *, max_fps: float | None = None) -> None: + """Authorize a camera with an update cap [Hz]; ``None`` renders it with every frame of its product.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Auxiliary camera must be configured before OVXR Stage binding") + self._lease.configure_auxiliary_camera(camera_cfg, max_fps=max_fps) + + @property + def simulation_context(self) -> SimulationContext: + """Return the exact pre-created simulation context.""" + self._require_open() + assert self._sim is not None + return self._sim + + def validate_environment(self, env: Any) -> None: + """Check that a constructed environment uses the prepared simulation context. + + Args: + env: Constructed environment, possibly wrapped. + + Raises: + RuntimeError: If the environment did not reuse the prepared context, has no + completed stage or not exactly one environment, or its simulation + configuration selects a different device or physics time step [s]. + """ + self._require_open() + unwrapped = getattr(env, "unwrapped", env) + if unwrapped.sim is not self._sim: + raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") + if unwrapped.scene.stage is None: + raise RuntimeError("Environment has no completed InteractiveScene Stage") + if unwrapped.num_envs != 1: + raise RuntimeError( + f"The OVXR render lease requires exactly one environment; got num_envs={unwrapped.num_envs!r}" + ) + env_sim_cfg = unwrapped.cfg.sim + prepared_cfg = self._sim.cfg + if env_sim_cfg is not prepared_cfg: + # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. + env_device = resolve_cuda_device(env_sim_cfg.device) + for field, env_value, prepared_value in ( + ("device", env_device, prepared_cfg.device), + ("dt", env_sim_cfg.dt, prepared_cfg.dt), + ): + if env_value != prepared_value: + raise RuntimeError( + f"Environment sim.{field}={getattr(env_sim_cfg, field)!r} differs from the OVXR" + f" prepared simulation context ({prepared_value!r}); prepare the" + " session with the environment's sim configuration" + ) + + def bind_environment(self, env: Any) -> OvLibrariesRenderLease: + """Prepare and bind the completed environment Stage to the reserved pair. + + The bootstrap defers Stage materialization to the render owner, so call this on the + worker after :meth:`claim_render_owner_transfer`. + + Args: + env: Constructed environment, possibly wrapped. + + Returns: + The bound lease. + + Raises: + RuntimeError: If the bootstrap is closed or already bound, the owner transfer has not + been claimed, or :meth:`validate_environment` rejects the environment. + """ + self._require_open() + if self._bound: + raise RuntimeError("OVXR bootstrap is already bound") + try: + self.validate_environment(env) + unwrapped = getattr(env, "unwrapped", env) + scene = unwrapped.scene + num_envs = unwrapped.num_envs + assert self._sim is not None and self._lease is not None + if not self._lease.materialize_deferred_stage(scene.stage, num_envs): + self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) + self._lease.validate_live() + self._bound = True + return self._lease + finally: + self._restore_terminal_mode_sentinel() + + def offer_render_owner_transfer(self) -> object: + """Offer the exact unbound OVXR lease before Stage binding.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Owner transfer is available only before OVXR Stage binding") + capability = self._lease.offer_owner_transfer() + self._owner_transfer_capability = capability + return capability + + def claim_render_owner_transfer(self, capability: object) -> OvLibrariesRenderLease: + """Claim the offered lease on the worker that will bind, use, and close it.""" + self._require_open() + assert self._lease is not None + self._lease.claim_owner_transfer(capability) + self._owner_transfer_capability = None + return self._lease + + def get_world_transform(self, prim_path: str) -> np.ndarray: + """Read target-to-world transform [4, 4], float64, with translation in scene units.""" + self._require_open() + if not self._bound or self._sim is None: + raise RuntimeError("Target-frame transforms require a bound OVXR environment") + if not isinstance(prim_path, str) or not prim_path.startswith("/"): + raise ValueError(f"Target-frame prim path must be absolute: {prim_path!r}") + + provider = self._sim.get_scene_data_provider() + cached = self._target_frame_readers.get(prim_path) + if cached is None or cached[0] is not provider: + transform_paths = list(provider.backend.transform_paths) + match_count = transform_paths.count(prim_path) + if match_count != 1: + raise RuntimeError( + f"Target-frame prim path {prim_path!r} matched {match_count} live backend transforms" + ) + mapping = provider.create_mapping([prim_path]) + output = SceneDataFormat.Matrix44() + cached = (provider, mapping, output) + self._target_frame_readers[prim_path] = cached + + provider, mapping, output = cached + if not provider.get_transforms(output, mapping=mapping, allow_passthrough=False, count=1): + raise RuntimeError(f"No SceneData transform conversion is available for target frame {prim_path!r}") + if output.matrices is None: + raise RuntimeError(f"SceneData returned no matrix buffer for target frame {prim_path!r}") + + # SceneData converts on the device's current stream. Wait only for that stream so a + # publication does not serialize with unrelated work (such as in-flight XR rendering). + if output.matrices.device.is_cuda: + wp.synchronize_stream(wp.get_stream(output.matrices.device)) + world_T_target = np.asarray(output.matrices.numpy()[0], dtype=np.float64) + if world_T_target.shape != (4, 4) or not np.all(np.isfinite(world_T_target)): + raise RuntimeError(f"Target-frame transform {prim_path!r} is missing or non-finite") + if not np.allclose(world_T_target[3], (0.0, 0.0, 0.0, 1.0), atol=1e-5, rtol=0.0): + raise RuntimeError(f"Target-frame transform {prim_path!r} is not homogeneous") + return world_T_target.copy() + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + world_T_target = self.get_world_transform(prim_path) + try: + target_T_world = np.linalg.inv(world_T_target) + except np.linalg.LinAlgError as exc: + raise RuntimeError(f"Target-frame transform {prim_path!r} is singular") from exc + if not np.all(np.isfinite(target_T_world)): + raise RuntimeError(f"Inverse target-frame transform {prim_path!r} is non-finite") + return target_T_world.astype(np.float32) + + @property + def closed(self) -> bool: + """Whether the lease and its retained publication resources retired successfully.""" + return self._closed + + def close(self) -> None: + """Release after every borrower is destroyed; retain ownership for retry on failure.""" + if self._closed: + return + # Keep ownership and the bootstrap retryable until retirement succeeds. + if self._lease is not None: + if self._owner_transfer_capability is not None: + self._lease.cancel_owner_transfer(self._owner_transfer_capability) + self._owner_transfer_capability = None + self._lease.release() + self._target_frame_readers.clear() + self._restore_terminal_mode_sentinel() + self._closed = True + + def abort(self) -> None: + """Release a failed construction; retain its context if lease retirement fails.""" + if self._closed: + return + self.close() + if not self._bound and self._sim is not None and SimulationContext.instance() is self._sim: + self._sim.clear_instance() + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR bootstrap is closed") + + def _restore_terminal_mode_sentinel(self) -> None: + if self._installed_terminal_mode_sentinel: + if hasattr(builtins, _TERMINAL_MODE_SENTINEL): + delattr(builtins, _TERMINAL_MODE_SENTINEL) + self._installed_terminal_mode_sentinel = False diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 5fba3cc33daa..726e77011ca6 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -244,12 +244,10 @@ class TeleopSessionLifecycle: Only chained when debug visualization is enabled.""" HAND_LEFT_KEY = "hand_left" - """Pipeline output key for the left hand ``TensorGroup`` when the debug - outputs are chained (matches ``HandsSource.LEFT``).""" + """Pipeline output key for left-hand presence when ``HandsSource`` exists.""" HAND_RIGHT_KEY = "hand_right" - """Pipeline output key for the right hand ``TensorGroup`` when the debug - outputs are chained (matches ``HandsSource.RIGHT``).""" + """Pipeline output key for right-hand presence when ``HandsSource`` exists.""" _STEP_FAILURE_RESTART_COOLDOWN_S = 1.0 """Minimum delay [s] before recreating the session after a step failure @@ -270,6 +268,9 @@ def __init__( enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, visualizers: Sequence[BaseVisualizer] = (), + *, + external_oxr_handles_provider: Callable[[], OpenXRSessionHandles | None] | None = None, + ovxr_session: bool = False, ): """Initialize the session lifecycle manager. @@ -291,6 +292,14 @@ def __init__( session through the CloudXR runtime, so teleop I/O works headless without Kit XR rendering. Ignored in replay mode (which never touches the XR runtime). + external_oxr_handles_provider: Optional callback returning handles + borrowed from an external OVXR context. Requires + ``use_kit_xr_bridge=False`` and defers session creation while + the callback returns ``None``. + ovxr_session: Whether an application-owned OVXR context presents + the session, including replay without borrowed handles. A + failed session exit is then reported and latched rather than + suppressed as Kit XR teardown. mcap_record_path: Optional path to an MCAP file the live teleop session should be recorded into. Mutually exclusive with *mcap_replay_path*. Debug-grade only -- see the Isaac Lab @@ -322,15 +331,21 @@ def __init__( "mcap_record_path and mcap_replay_path are mutually exclusive; " "set at most one to switch the session between LIVE recording and REPLAY playback." ) + if external_oxr_handles_provider is not None and use_kit_xr_bridge: + raise ValueError("An external OpenXR handle provider requires use_kit_xr_bridge=False") + if external_oxr_handles_provider is not None and mcap_replay_path is not None: + raise ValueError("An external OpenXR handle provider is incompatible with replay mode") self._cfg = cfg self._device = torch.device(cfg.sim_device) self._cloudxr_env_file = cloudxr_env_file self._auto_launch_cloudxr = auto_launch_cloudxr self._use_kit_xr_bridge = use_kit_xr_bridge + self._external_oxr_handles_provider = external_oxr_handles_provider self._mcap_record_path = mcap_record_path self._mcap_replay_path = mcap_replay_path self._is_replay = mcap_replay_path is not None + self._latches_exit_failures = ovxr_session or external_oxr_handles_provider is not None self._enable_debug_visualization = enable_debug_visualization # Haptic feedback (optional): when configured, start() builds a @@ -347,6 +362,7 @@ def __init__( # Session state (populated during start) self._session: TeleopSession | None = None + self._session_cleanup_error: BaseException | None = None self._pipeline = None self._teleop_control_pipeline = None self._message_processor: TeleopMessageProcessor | None = None @@ -359,6 +375,8 @@ def __init__( self._last_right_controller = None self._last_left_controller = None self._last_step_result: dict | None = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id: int | None = None self._session_start_deferred_logged = False # Monotonic deadline gating session re-creation after a step failure self._restart_holdoff_until = 0.0 @@ -445,7 +463,12 @@ def __init__( @property def is_active(self) -> bool: """Whether the teleop session is currently running.""" - return self._session is not None + return self._session is not None and not self.session_cleanup_failed + + @property + def session_cleanup_failed(self) -> bool: + """Whether native/session drain failed and further use is prohibited.""" + return self._session_cleanup_error is not None @property def pipeline(self): @@ -475,9 +498,11 @@ def last_left_controller(self): def last_step_result(self) -> dict | None: """Full pipeline output from the most recent :meth:`step`, or ``None``. - Contains at least ``"action"`` and the right-controller entry. When - debug visualization was enabled at session start, :attr:`HAND_LEFT_KEY` - and :attr:`HAND_RIGHT_KEY` are included when a ``HandsSource`` exists. + Contains at least ``"action"`` and the right-controller entry. + :attr:`HAND_LEFT_KEY` and :attr:`HAND_RIGHT_KEY` are included whenever + a ``HandsSource`` exists; a user pipeline may explicitly provide those + outputs after applying a validity policy. Controller-left and visual + updates remain debug-only. """ return self._last_step_result @@ -490,19 +515,11 @@ def has_control_channel(self) -> bool: def last_control_events(self) -> ControlEvents: """Control events from the most recent :meth:`step`. - When a ``teleop_control_pipeline`` is configured, derives - :class:`ControlEvents` from - ``session.last_context.execution_events``. Otherwise returns a - default (no-op) :class:`ControlEvents`. + Reset is an edge-triggered event. In pipelined execution, multiple app + steps can return the same completed retargeting frame, so the event is + latched by returned frame ID and emitted only on its first return. """ - if self._message_processor is None: - return _NO_OP_EVENTS - if self._session is None: - return _NO_OP_EVENTS - ctx = self._session.last_context - if ctx is None: - return _NO_OP_EVENTS - return _execution_events_to_control(ctx.execution_events) + return self._last_control_events def request_reset(self, pause: bool = False) -> None: """Schedule a reset for the next pipeline step. @@ -569,6 +586,8 @@ def start(self) -> None: "Start AR"), session creation is deferred and will be retried on each :meth:`step` call. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error # Measure the workstation before anything expensive starts. Any notice is # queued now and delivered when the client connects. The queue is not # cleared here: send_client_message promises that messages queued before @@ -598,6 +617,8 @@ def start(self) -> None: self._last_right_controller = None self._last_left_controller = None self._last_step_result = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None self._pipeline = self._build_combined_pipeline(user_pipeline) self._pressed_keys.clear() @@ -648,6 +669,14 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: context manager. If the underlying OpenXR session was already torn down externally (e.g. "Stop AR"), cleanup errors are suppressed. + A retargeting UI failure does not skip session cleanup; it propagates + afterwards, or is attached to the caller's original exception. For an OVXR + session, including OVXR replay, a failed session exit is not suppressed: + it is reported the same way and latched, and the lifecycle prohibits + reuse or restart. The retained session reference does not prove native + drain; the application must not release borrowed resources or stop the + runtime after that failure. + Args: exc_type: Exception type (for context manager protocol). exc_val: Exception value. @@ -656,16 +685,45 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._detach_keyboards() self._pressed_keys.clear() + error = exc_val + + def record_failure(cleanup_error: BaseException) -> None: + nonlocal error + if error is None: + error = cleanup_error + else: + error.add_note(f"Isaac Capture cleanup failed: {cleanup_error!r}") + # Close the retargeting tuning UI first + # A UI failure must not prevent native/session cleanup. if self._retargeting_ui_ctx is not None: - self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) - self._retargeting_ui_ctx = None - self._retargeting_ui = None + try: + self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) + except BaseException as cleanup_error: + record_failure(cleanup_error) + else: + self._retargeting_ui_ctx = None + self._retargeting_ui = None if self._session is not None: + if self._session_cleanup_error is not None: + if self._session_cleanup_error is not exc_val: + record_failure(self._session_cleanup_error) + if exc_val is None: + raise error + return try: self._session.__exit__(exc_type, exc_val, exc_tb) - except Exception as e: + except BaseException as e: + if self._latches_exit_failures: + # OVXR replay failures are terminal; borrowed resources also require a proven drain. + self._session_cleanup_error = e + record_failure(e) + if exc_val is None: + raise error + return + if not isinstance(e, Exception): + raise # The OpenXR session may have already been torn down externally # (e.g. user clicked "Stop AR"), so destroying spaces/action # sets will fail with XR_ERROR_HANDLE_INVALID. This is @@ -680,6 +738,8 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._message_processor = None self._last_step_result = None self._last_left_controller = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None self._haptic_sink = None self._haptic_tracker = None # Undelivered notices belong to the session that queued them; a later @@ -695,6 +755,8 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._cloudxr_launcher = None logger.info("CloudXR runtime stopped") + if exc_val is None and error is not None: + raise error logger.info("Isaac Capture session ended") # ------------------------------------------------------------------ @@ -728,9 +790,9 @@ def _build_combined_pipeline(self, user_pipeline): # the haptic sink, avoiding a second controller action-set attachment. self._button_controllers = ControllersSource("_button_controllers") pipeline_outputs[self._CONTROLLER_RIGHT_KEY] = self._button_controllers.output(ControllersSource.RIGHT) + self._chain_hand_presence_outputs(user_pipeline, pipeline_outputs) if self._enable_debug_visualization: pipeline_outputs[self._CONTROLLER_LEFT_KEY] = self._button_controllers.output(ControllersSource.LEFT) - self._chain_hand_debug_outputs(user_pipeline, pipeline_outputs) return OutputCombiner(pipeline_outputs) # ------------------------------------------------------------------ @@ -794,15 +856,13 @@ def drain_pressed_keys(self) -> list[int]: return pressed @staticmethod - def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: - """Auto-discover a ``HandsSource`` and chain its raw outputs. + def _chain_hand_presence_outputs(user_pipeline, pipeline_outputs: dict) -> None: + """Chain explicit filtered hand outputs or auto-discover raw source outputs. - Walks the leaf nodes of *user_pipeline*; if a ``HandsSource`` is - found, its raw (XR-anchor-frame) ``hand_left`` / ``hand_right`` - outputs are added to *pipeline_outputs* so they appear in the step - result. The anchor-to-world transform is applied on the Isaac Lab - side by the visualizer, so the markers stay world-frame correct even - when a ``target_T_world`` rebase is active in the pipeline. + A user pipeline may expose its own ``hand_left`` / ``hand_right`` + outputs when validity or another application-neutral policy must be + applied before session safety observes presence. Otherwise walk the + leaf nodes and retain the legacy raw ``HandsSource`` behavior. Args: user_pipeline: The pipeline returned by ``pipeline_builder()``. @@ -811,15 +871,23 @@ def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: """ from isaacteleop.retargeting_engine.deviceio_source_nodes import HandsSource + output_types = user_pipeline.output_types() + hand_keys = (TeleopSessionLifecycle.HAND_LEFT_KEY, TeleopSessionLifecycle.HAND_RIGHT_KEY) + if all(key in output_types for key in hand_keys): + for key in hand_keys: + pipeline_outputs[key] = user_pipeline.output(key) + logger.debug("Chained explicit pipeline hand-presence outputs") + return + # HandsSource owns its HandTracker; the tracker API has no reverse # source-node lookup, so use the graph's public leaf discovery API. for leaf in user_pipeline.get_leaf_nodes(): if isinstance(leaf, HandsSource): pipeline_outputs[TeleopSessionLifecycle.HAND_LEFT_KEY] = leaf.output(HandsSource.LEFT) pipeline_outputs[TeleopSessionLifecycle.HAND_RIGHT_KEY] = leaf.output(HandsSource.RIGHT) - logger.debug(f"Auto-discovered HandsSource '{leaf.name}'; chained hand debug outputs") + logger.debug(f"Auto-discovered HandsSource '{leaf.name}'; chained raw hand-presence outputs") return - logger.info("No HandsSource found in the teleop pipeline; hand joint visualization unavailable") + logger.debug("No HandsSource found in the teleop pipeline; raw hand presence unavailable") # ------------------------------------------------------------------ # Control pipeline construction @@ -1066,6 +1134,10 @@ def _on_request_required_extensions(self) -> list[str]: logger.info(f"Required extensions: {required_extensions}") return required_extensions + def required_openxr_extensions(self) -> tuple[str, ...]: + """Return extensions required by the fully built teleop graphs.""" + return tuple(self._on_request_required_extensions()) + def _on_xr_enabled_changed(self, item, event_type): import carb.settings @@ -1137,6 +1209,8 @@ def _try_start_session(self) -> bool: ``True`` if the session was successfully started (or was already running), ``False`` if session creation was deferred. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error if self._session is not None: return True @@ -1166,6 +1240,13 @@ def _try_start_session(self) -> bool: ) self._session_start_deferred_logged = True return False + elif self._external_oxr_handles_provider is not None: + oxr_handles = self._external_oxr_handles_provider() + if oxr_handles is None: + if not self._session_start_deferred_logged: + logger.info("External OpenXR handles are not ready; Isaac Capture session creation deferred") + self._session_start_deferred_logged = True + return False else: # Standalone mode (e.g. no ``--xr``): do not touch Kit's XR bridge. # Passing ``oxr_handles=None`` makes ``TeleopSession`` create and own @@ -1196,8 +1277,9 @@ def _try_start_session(self) -> bool: ) # Create and enter the TeleopSession - self._session = TeleopSession(session_config) - self._session.__enter__() + session = TeleopSession(session_config) + self._enter_session(session) + self._session = session if self._mcap_record_path is not None: logger.info(f"Isaac Capture session started: {self._cfg.app_name} (recording to {self._mcap_record_path})") @@ -1246,12 +1328,25 @@ def _start_replay_session(self) -> bool: sinks=[], ) - self._session = TeleopSession(session_config) - self._session.__enter__() + session = TeleopSession(session_config) + self._enter_session(session) + self._session = session logger.info(f"Isaac Capture replay session started: {self._cfg.app_name} (replaying {self._mcap_replay_path})") return True + def _enter_session(self, session: TeleopSession) -> None: + """Latch Capture's explicit failed-rollback signal before releasing borrowed owners.""" + try: + session.__enter__() + except BaseException as error: + # Capture also rolls back interrupts. Preserve its failed-drain + # signal without converting the original interrupt into an error. + if getattr(session, "cleanup_failed", False) is True: + self._session = session + self._session_cleanup_error = error + raise + # ------------------------------------------------------------------ # Stepping # ------------------------------------------------------------------ @@ -1293,7 +1388,11 @@ def step( Raises: RuntimeError: If called before :meth:`start`. + Exception: If an earlier session cleanup failed, or replay stepping + fails. A failed cleanup prohibits further use and restart. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error if self._pipeline is None: raise RuntimeError("TeleopSessionLifecycle.start() must be called before step()") @@ -1320,7 +1419,6 @@ def step( from isaacteleop.retargeting_engine.interface.execution_events import ExecutionEvents, ExecutionState execution_events = ExecutionEvents(reset=True, execution_state=ExecutionState.RUNNING) - self._pending_reset = False # Execute one step of the teleop session. # If the underlying OpenXR session was destroyed externally (e.g. @@ -1337,7 +1435,9 @@ def step( # The async retarget worker dies permanently on any pipeline # exception, so session re-entry is the only recovery either way; # what differs is the cause and the restart pacing. - if not self._is_replay and self._kit_xr_session_is_active(): + if not self._is_replay and ( + self._external_oxr_handles_provider is not None or self._kit_xr_session_is_active() + ): logger.warning( "Isaac Capture retargeting step failed (pipeline error, XR session still active); " f"restarting the teleop session in {self._STEP_FAILURE_RESTART_COOLDOWN_S:.0f}s: {e}" @@ -1345,10 +1445,24 @@ def step( self._restart_holdoff_until = time.monotonic() + self._STEP_FAILURE_RESTART_COOLDOWN_S else: logger.warning(f"Isaac Capture session step failed (XR session likely torn down): {e}") - self._teardown_dead_session() + try: + self._teardown_dead_session() + except BaseException as cleanup_error: + if not self._latches_exit_failures: + raise + # Failed drain retains the owner; never restart over live resources. + self._session_cleanup_error = e + e.add_note(f"Teleop session drain also failed: {cleanup_error!r}") + raise e from cleanup_error + if self._is_replay: + # Re-entering replay would silently rewind a failed recording. + raise return None + if execution_events is not None: + self._pending_reset = False self._last_step_result = result + self._update_control_events() # Store the controller TensorGroups for button polling / debug viz self._last_right_controller = result.get(self._CONTROLLER_RIGHT_KEY) @@ -1363,6 +1477,21 @@ def step( return action + def _update_control_events(self) -> None: + """Cache the control snapshot associated with the returned retargeting frame.""" + if self._message_processor is None or self._session is None or self._session.last_context is None: + self._last_control_events = _NO_OP_EVENTS + return + + events = _execution_events_to_control(self._session.last_context.execution_events) + step_info = getattr(self._session, "last_step_info", None) + frame_id = getattr(step_info, "returned_frame_id", None) + if frame_id is not None and frame_id == self._last_control_event_frame_id: + events = ControlEvents(is_active=events.is_active, should_reset=False) + else: + self._last_control_event_frame_id = frame_id + self._last_control_events = events + # ------------------------------------------------------------------ # Dead session teardown # ------------------------------------------------------------------ @@ -1376,15 +1505,28 @@ def _teardown_dead_session(self) -> None: resources are already gone), and the internal state is reset so that the deferred-start logic in :meth:`step` can re-acquire handles if the user restarts AR. + + For an OVXR session, including OVXR replay, a failed exit propagates + instead and latches a terminal error. Replay cannot recover by restarting + from the beginning, and borrowed resources require a proven drain. """ if self._session is not None: try: self._session.__exit__(None, None, None) - except Exception as e: + except BaseException as e: + if self._latches_exit_failures: + # Exit may already be terminal. Never infer successful drain from + # a later no-op __exit__, or resume/restart after this failure. + self._session_cleanup_error = e + raise + if not isinstance(e, Exception): + raise logger.debug(f"Suppressed error tearing down dead session: {e}") self._session = None self._session_start_deferred_logged = False self._last_step_result = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None logger.info("Isaac Capture session torn down after external XR shutdown") # ------------------------------------------------------------------ diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py index 39c4f380e356..554ae0fdc062 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py @@ -9,11 +9,12 @@ import contextlib import logging +import threading import numpy as np from scipy.spatial.transform import Rotation -from .xr_anchor_utils import XrAnchorSynchronizer +from .xr_anchor_utils import XrAnchorSynchronizer, _XrAnchorPosePolicy from .xr_cfg import XrCfg # Import Kit components with fallback for sessions without Kit @@ -27,11 +28,31 @@ from omni.kit.xr.core import XRCore, XRCoreEventType from isaaclab.sim.utils.prims import create_prim as _create_prim +from isaaclab.sim.utils.stage import get_current_stage +from isaaclab.sim.utils.transforms import convert_world_pose_to_local, standardize_xform_ops from isaaclab.utils.version import has_kit logger = logging.getLogger(__name__) +def _create_or_update_xr_anchor( + prim_path: str, position: tuple[float, float, float], orientation: tuple[float, float, float, float] +) -> None: + """Apply the configured world pose to a stage-scoped XR anchor.""" + if not _xr_anchor_prim_exists(prim_path): + _create_prim( + prim_path, + prim_type="Xform", + position=np.asarray(position, dtype=np.float64), + orientation=np.asarray(orientation, dtype=np.float64), + ) + return + prim = get_current_stage().GetPrimAtPath(prim_path) + translation, rotation = convert_world_pose_to_local(tuple(position), tuple(orientation), prim.GetParent()) + if not standardize_xform_ops(prim, translation, rotation): + raise ValueError(f"XR anchor {prim_path!r} is not transformable") + + def _xr_anchor_prim_exists(prim_path: str) -> bool: """Return True when ``prim_path`` is already a valid prim on the active stage. @@ -39,16 +60,11 @@ def _xr_anchor_prim_exists(prim_path: str) -> bool: :class:`XrAnchorManager` construction in multi-replay batches (the replay agent rebuilds the device per run, but the prim is stage-scoped). Best-effort: returns ``False`` if the stage cannot - be inspected (e.g. unit tests without omni.usd) so the caller falls + be inspected (e.g. before stage creation) so the caller falls through to the create-and-warn path used historically. """ try: - import omni.usd - - context = omni.usd.get_context() - if context is None: - return False - stage = context.get_stage() + stage = get_current_stage() if stage is None: return False return stage.GetPrimAtPath(prim_path).IsValid() @@ -80,19 +96,35 @@ class XrAnchorManager: dtype=np.float64, ) - def __init__(self, xr_cfg: XrCfg): + def __init__(self, xr_cfg: XrCfg, *, ovxr_session: bool = False): """Initialize the XR anchor manager. - Creates the anchor prim, configures carb XR settings, and sets up - the optional anchor synchronizer for dynamic anchoring. + Creates or refreshes the stage-scoped anchor's configured world pose, + configures carb XR settings, and sets up the optional anchor + synchronizer for dynamic anchoring. Reusing the Stage across sessions + does not retain a previous session's anchor configuration. Args: xr_cfg: XR configuration specifying anchor position, rotation, and optional dynamic anchoring prim path. + ovxr_session: An application-owned renderer/Stage supplies XR. + This path must not configure or synchronize Kit XRCore. + + Raises: + ValueError: An existing anchor prim is not transformable. Other anchor + authoring errors also propagate for OVXR presentation, so + rendering cannot continue with a pose that differs from tracking. """ self._xr_cfg = xr_cfg - self._xr_core = XRCore.get_singleton() if XRCore is not None else None + self._xr_core = XRCore.get_singleton() if XRCore is not None and not ovxr_session else None self._xr_pre_sync_update_subscription = None + self._external_lock = threading.RLock() + self._external_policy = ( + _XrAnchorPosePolicy(xr_cfg) if ovxr_session and xr_cfg.anchor_prim_path is not None else None + ) + self._external_epoch = 0 + self._external_world_matrix = None + self._external_error = None # Resolve the headset anchor path if self._xr_cfg.anchor_prim_path is not None: @@ -103,20 +135,14 @@ def __init__(self, xr_cfg: XrCfg): else: self._xr_anchor_headset_path = "/World/XRAnchor" - # Create the XR anchor prim in USD if it does not already exist. - # The check matters for multi-replay batches (the replay agent - # rebuilds ``IsaacTeleopDevice`` -- and therefore this manager -- - # for each run, but the prim is stage-scoped and survives - # per-run device teardown). XrCfg.anchor_rot is xyzw; create_prim - # orientation expects xyzw. - if not _xr_anchor_prim_exists(self._xr_anchor_headset_path): - x, y, z, w = self._xr_cfg.anchor_rot - try: - pos = np.asarray(self._xr_cfg.anchor_pos, dtype=np.float64) - quat_xyzw = np.asarray([x, y, z, w], dtype=np.float64) - _create_prim(self._xr_anchor_headset_path, prim_type="Xform", position=pos, orientation=quat_xyzw) - except Exception as e: - logger.warning(f"Failed to create XR anchor prim: {e}") + # Anchors survive per-run device teardown. Reuse their paths while + # applying this run's configuration, rather than retaining a stale pose. + try: + _create_or_update_xr_anchor(self._xr_anchor_headset_path, self._xr_cfg.anchor_pos, self._xr_cfg.anchor_rot) + except Exception as e: + if ovxr_session: + raise + logger.warning(f"Failed to create or update XR anchor prim: {e}") # Configure carb settings for XR rendering; carb can import while Kit is not running if carb is not None and has_kit(): @@ -180,19 +206,32 @@ def get_world_matrix(self) -> np.ndarray: core via ``set_world_transform_matrix`` during the most recent ``pre_sync_update``. This works for both dynamic anchoring (``anchor_prim_path`` set) and static anchoring. - * Falls back to raw ``anchor_pos`` / ``anchor_rot`` config values - only when the XR core is unavailable (e.g. unit tests). + * External dynamic sessions read the cached pose sealed by their + frame owner, using application-published physics references. + Unseeded or failed reset epochs raise instead of returning a static pose. + * Static and standalone sessions without Kit retain configured poses. Returns: A (4, 4) float32 numpy array. """ + if self._external_policy is not None: + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + if self._external_world_matrix is None: + raise RuntimeError("Dynamic XR anchor requires a committed post-reset scene pose") + matrix = self._external_world_matrix.copy() + matrix[:3, :3] = matrix[:3, :3] @ self._OXR_TO_USD_ROTATION + return matrix.astype(np.float32) if self._anchor_sync is not None: xform = self._anchor_sync.get_world_transform() if xform is not None: pos, quat_xyzw = xform return self._build_matrix(pos, quat_xyzw) - # Fallback when XR core is unavailable (e.g. unit tests). + # Static and standalone sessions retain their configured-pose fallback. return self._build_matrix(self._xr_cfg.anchor_pos, self._xr_cfg.anchor_rot) def _build_matrix(self, pos, quat_xyzw) -> np.ndarray: @@ -222,14 +261,66 @@ def _build_matrix(self, pos, quat_xyzw) -> np.ndarray: def reset(self) -> None: """Reset the anchor synchronizer state.""" + if self._external_policy is not None: + with self._external_lock: + self._external_epoch += 1 + self._external_policy.reset() + self._external_world_matrix = None if self._anchor_sync is not None: self._anchor_sync.reset() def toggle_anchor_rotation(self) -> None: """Toggle anchor rotation following on the synchronizer.""" + if self._external_policy is not None: + with self._external_lock: + self._external_policy.toggle_anchor_rotation() if self._anchor_sync is not None: self._anchor_sync.toggle_anchor_rotation() + @property + def external_reference_path(self) -> str | None: + """The exact physics reference path, when external dynamic anchoring is active.""" + return self._xr_cfg.anchor_prim_path if self._external_policy is not None else None + + @property + def external_epoch(self) -> tuple[int, bool]: + """Return reset identity and whether the current epoch needs its first publication.""" + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + return self._external_epoch, self._external_world_matrix is None + + @property + def external_anchor_paths(self) -> tuple[str, ...]: + """Native Xforms written together by the OVXR presentation host.""" + return (self._xr_anchor_headset_path,) + + def commit_external_pose(self, reference, rendering_dt: float, epoch: int, commit) -> bool: + """Advance the shared policy once and expose its input pose only after native seal. + + The host serializes this operation with scene publication. Old committed + references cannot reseed a new reset epoch. A failed write invalidates the + cached input pose and propagates; it never falls back to configured static data. + """ + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + if epoch != self._external_epoch: + return False + try: + matrix = np.asarray(self._external_policy.update(reference, rendering_dt)).T.copy() + commit(matrix[np.newaxis]) + except BaseException as exc: + self._external_world_matrix = None + self._external_error = exc + raise + self._external_world_matrix = matrix + return True + def cleanup(self) -> None: """Release event subscriptions.""" self._xr_pre_sync_update_subscription = None diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py index 6e6671a68e8e..a8cfb198b6cc 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py @@ -10,6 +10,7 @@ import contextlib import logging import math +from collections.abc import Callable from typing import Any import numpy as np @@ -23,25 +24,24 @@ with contextlib.suppress(ModuleNotFoundError): import usdrt - from pxr import Gf as pxrGf from usdrt import Rt -class XrAnchorSynchronizer: - """Keeps the XR anchor prim aligned with a reference prim according to XR config.""" +class _XrAnchorPosePolicy: + """Own the shared reference-relative anchor policy, without Kit or Stage I/O.""" - def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): - self._xr_core = xr_core + def __init__(self, xr_cfg): self._xr_cfg = xr_cfg - self._xr_anchor_headset_path = xr_anchor_headset_path + self.reset() + def reset(self): self.__anchor_prim_initial_quat = None self.__anchor_prim_initial_height = None self.__smoothed_anchor_quat = None self.__last_anchor_quat = None self.__anchor_rotation_enabled = True - # Cached anchor world transform (pos, quat_xyzw) set by sync_headset_to_anchor(). + # Cached anchor world transform (pos, quat_xyzw) set by update(). # Reading back from the prim hierarchy is unreliable when the anchor is a child of a # physics-driven prim (e.g. the pelvis) because Fabric computes the hierarchy world # matrix using the physics-updated parent while the local xform was decomposed against @@ -49,6 +49,116 @@ def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): self.__cached_world_pos: np.ndarray | None = None self.__cached_world_quat_xyzw: np.ndarray | None = None + def toggle_anchor_rotation(self): + self.__anchor_rotation_enabled = not self.__anchor_rotation_enabled + logger.info(f"XR: Toggling anchor rotation: {self.__anchor_rotation_enabled}") + + def get_world_transform(self): + if self.__cached_world_pos is None: + return None + return self.__cached_world_pos.copy(), self.__cached_world_quat_xyzw.copy() + + def update(self, world_T_reference, dt: float | Callable[[], float], previous_headpose=None): + # Let the selected runtime initialize USD before loading its native math types. + # An eager standalone USD import can prevent Kit RTX startup. + from pxr import Gf as pxrGf + + if world_T_reference is not None: + rt_matrix = pxrGf.Matrix4d(world_T_reference.T.tolist()) + rt_pos = rt_matrix.ExtractTranslation() + if self.__anchor_prim_initial_quat is None: + self.__anchor_prim_initial_quat = rt_matrix.ExtractRotationQuat() + if self._xr_cfg.fixed_anchor_height: + if self.__anchor_prim_initial_height is None: + self.__anchor_prim_initial_height = rt_pos[2] + rt_pos[2] = self.__anchor_prim_initial_height + pxr_anchor_pos = rt_pos + pxrGf.Vec3d(*self._xr_cfg.anchor_pos) + else: + rt_matrix = None + pxr_anchor_pos = pxrGf.Vec3d(*self._xr_cfg.anchor_pos) + x, y, z, w = self._xr_cfg.anchor_rot + pxr_cfg_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) + + pxr_anchor_quat = pxr_cfg_quat + + if rt_matrix is not None: + if self._xr_cfg.anchor_rotation_mode in ( + XrAnchorRotationMode.FOLLOW_PRIM, + XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, + ): + rt_prim_quat = rt_matrix.ExtractRotationQuat() + rt_delta_quat = rt_prim_quat * self.__anchor_prim_initial_quat.GetInverse() + pxr_delta_quat = pxrGf.Quatd(rt_delta_quat.GetReal(), pxrGf.Vec3d(*rt_delta_quat.GetImaginary())) + + # yaw-only about Z (right-handed, Z-up) + wq = pxr_delta_quat.GetReal() + ix, iy, iz = pxr_delta_quat.GetImaginary() + yaw = math.atan2(2.0 * (wq * iz + ix * iy), 1.0 - 2.0 * (iy * iy + iz * iz)) + cy = math.cos(yaw * 0.5) + sy = math.sin(yaw * 0.5) + pxr_delta_yaw_only_quat = pxrGf.Quatd(cy, pxrGf.Vec3d(0.0, 0.0, sy)) + pxr_anchor_quat = pxr_delta_yaw_only_quat * pxr_cfg_quat + + if self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED: + if self.__smoothed_anchor_quat is None: + self.__smoothed_anchor_quat = pxr_anchor_quat + else: + step_dt = dt() if callable(dt) else dt + alpha = 1.0 - math.exp(-step_dt / max(self._xr_cfg.anchor_rotation_smoothing_time, 1e-6)) + alpha = min(1.0, max(0.05, alpha)) + self.__smoothed_anchor_quat = pxrGf.Slerp(alpha, self.__smoothed_anchor_quat, pxr_anchor_quat) + pxr_anchor_quat = self.__smoothed_anchor_quat + + elif self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.CUSTOM: + if self._xr_cfg.anchor_rotation_custom_func is not None: + rt_prim_quat = rt_matrix.ExtractRotationQuat() + anchor_prim_pose = np.array( + [ + rt_pos[0], + rt_pos[1], + rt_pos[2], + rt_prim_quat.GetImaginary()[0], + rt_prim_quat.GetImaginary()[1], + rt_prim_quat.GetImaginary()[2], + rt_prim_quat.GetReal(), + ], + dtype=np.float64, + ) + prev_head = previous_headpose if previous_headpose is not None else np.zeros(7, dtype=np.float64) + np_array_quat = self._xr_cfg.anchor_rotation_custom_func(prev_head, anchor_prim_pose) + x, y, z, w = np_array_quat + pxr_anchor_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) + + pxr_mat = pxrGf.Matrix4d() + pxr_mat.SetTranslateOnly(pxr_anchor_pos) + + if self.__anchor_rotation_enabled: + pxr_final_quat = pxr_anchor_quat + self.__last_anchor_quat = pxr_anchor_quat + else: + if self.__last_anchor_quat is None: + self.__last_anchor_quat = pxr_anchor_quat + pxr_final_quat = self.__last_anchor_quat + self.__smoothed_anchor_quat = self.__last_anchor_quat + + pxr_mat.SetRotateOnly(pxr_final_quat) + + self.__cached_world_pos = np.array([pxr_anchor_pos[0], pxr_anchor_pos[1], pxr_anchor_pos[2]], dtype=np.float64) + fq_img = pxr_final_quat.GetImaginary() + self.__cached_world_quat_xyzw = np.array( + [fq_img[0], fq_img[1], fq_img[2], pxr_final_quat.GetReal()], dtype=np.float64 + ) + return pxr_mat + + +class XrAnchorSynchronizer: + """Adapt the shared anchor policy to Kit Fabric and XRCore pre-sync events.""" + + def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): + self._xr_core = xr_core + self._xr_cfg = xr_cfg + self._xr_anchor_headset_path = xr_anchor_headset_path + self._policy = _XrAnchorPosePolicy(xr_cfg) stage = get_current_stage() anchor = stage.GetPrimAtPath(xr_anchor_headset_path) self.__anchor_headset_layer_identifier = anchor.GetPrimStack()[0].layer.identifier @@ -61,18 +171,11 @@ def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): self._reference_world_matrix = Rt.Xformable(reference).GetFabricHierarchyWorldMatrixAttr() def reset(self): - self.__anchor_prim_initial_quat = None - self.__anchor_prim_initial_height = None - self.__smoothed_anchor_quat = None - self.__last_anchor_quat = None - self.__anchor_rotation_enabled = True - self.__cached_world_pos = None - self.__cached_world_quat_xyzw = None + self._policy.reset() self.sync_headset_to_anchor() def toggle_anchor_rotation(self): - self.__anchor_rotation_enabled = not self.__anchor_rotation_enabled - logger.info(f"XR: Toggling anchor rotation: {self.__anchor_rotation_enabled}") + self._policy.toggle_anchor_rotation() def get_world_transform(self) -> tuple[np.ndarray, np.ndarray] | None: """Return the anchor world transform. @@ -90,9 +193,7 @@ def get_world_transform(self) -> tuple[np.ndarray, np.ndarray] | None: A ``(position, quat_xyzw)`` tuple of numpy float64 arrays, or ``None`` if :meth:`sync_headset_to_anchor` has not run yet. """ - if self.__cached_world_pos is not None and self.__cached_world_quat_xyzw is not None: - return self.__cached_world_pos, self.__cached_world_quat_xyzw - return None + return self._policy.get_world_transform() def sync_headset_to_anchor(self): """Sync XR anchor pose in USD for both dynamic and static anchoring. @@ -108,102 +209,17 @@ def sync_headset_to_anchor(self): world transform for :meth:`get_world_transform`. """ try: + reference = None if self._reference_world_matrix is not None: - rt_matrix = self._reference_world_matrix.Get() - rt_pos = rt_matrix.ExtractTranslation() - - if self.__anchor_prim_initial_quat is None: - self.__anchor_prim_initial_quat = rt_matrix.ExtractRotationQuat() - - if self._xr_cfg.fixed_anchor_height: - if self.__anchor_prim_initial_height is None: - self.__anchor_prim_initial_height = rt_pos[2] - rt_pos[2] = self.__anchor_prim_initial_height - - pxr_anchor_pos = pxrGf.Vec3d(*rt_pos) + pxrGf.Vec3d(*self._xr_cfg.anchor_pos) - else: - rt_matrix = None - pxr_anchor_pos = pxrGf.Vec3d(*self._xr_cfg.anchor_pos) - - x, y, z, w = self._xr_cfg.anchor_rot - pxr_cfg_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) - - pxr_anchor_quat = pxr_cfg_quat - - if rt_matrix is not None: - if self._xr_cfg.anchor_rotation_mode in ( - XrAnchorRotationMode.FOLLOW_PRIM, - XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, - ): - rt_prim_quat = rt_matrix.ExtractRotationQuat() - rt_delta_quat = rt_prim_quat * self.__anchor_prim_initial_quat.GetInverse() - pxr_delta_quat = pxrGf.Quatd(rt_delta_quat.GetReal(), pxrGf.Vec3d(*rt_delta_quat.GetImaginary())) - - # yaw-only about Z (right-handed, Z-up) - wq = pxr_delta_quat.GetReal() - ix, iy, iz = pxr_delta_quat.GetImaginary() - yaw = math.atan2(2.0 * (wq * iz + ix * iy), 1.0 - 2.0 * (iy * iy + iz * iz)) - cy = math.cos(yaw * 0.5) - sy = math.sin(yaw * 0.5) - pxr_delta_yaw_only_quat = pxrGf.Quatd(cy, pxrGf.Vec3d(0.0, 0.0, sy)) - pxr_anchor_quat = pxr_delta_yaw_only_quat * pxr_cfg_quat - - if self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED: - if self.__smoothed_anchor_quat is None: - self.__smoothed_anchor_quat = pxr_anchor_quat - else: - dt = SimulationContext.instance().get_rendering_dt() - alpha = 1.0 - math.exp(-dt / max(self._xr_cfg.anchor_rotation_smoothing_time, 1e-6)) - alpha = min(1.0, max(0.05, alpha)) - self.__smoothed_anchor_quat = pxrGf.Slerp( - alpha, self.__smoothed_anchor_quat, pxr_anchor_quat - ) - pxr_anchor_quat = self.__smoothed_anchor_quat - - elif self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.CUSTOM: - if self._xr_cfg.anchor_rotation_custom_func is not None: - rt_prim_quat = rt_matrix.ExtractRotationQuat() - anchor_prim_pose = np.array( - [ - rt_pos[0], - rt_pos[1], - rt_pos[2], - rt_prim_quat.GetImaginary()[0], - rt_prim_quat.GetImaginary()[1], - rt_prim_quat.GetImaginary()[2], - rt_prim_quat.GetReal(), - ], - dtype=np.float64, - ) - prev_head = getattr(self, "_previous_headpose", np.zeros(7, dtype=np.float64)) - np_array_quat = self._xr_cfg.anchor_rotation_custom_func(prev_head, anchor_prim_pose) - x, y, z, w = np_array_quat - pxr_anchor_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) - - pxr_mat = pxrGf.Matrix4d() - pxr_mat.SetTranslateOnly(pxr_anchor_pos) - - if self.__anchor_rotation_enabled: - pxr_final_quat = pxr_anchor_quat - self.__last_anchor_quat = pxr_anchor_quat - else: - if self.__last_anchor_quat is None: - self.__last_anchor_quat = pxr_anchor_quat - pxr_final_quat = self.__last_anchor_quat - self.__smoothed_anchor_quat = self.__last_anchor_quat - - pxr_mat.SetRotateOnly(pxr_final_quat) - - self.__cached_world_pos = np.array( - [pxr_anchor_pos[0], pxr_anchor_pos[1], pxr_anchor_pos[2]], dtype=np.float64 + matrix = self._reference_world_matrix.Get() + reference = np.array([[matrix[i][j] for j in range(4)] for i in range(4)]).T + matrix = self._policy.update( + reference, + lambda: SimulationContext.instance().get_rendering_dt(), + getattr(self, "_previous_headpose", None), ) - fq_img = pxr_final_quat.GetImaginary() - self.__cached_world_quat_xyzw = np.array( - [fq_img[0], fq_img[1], fq_img[2], pxr_final_quat.GetReal()], dtype=np.float64 - ) - self._xr_core.set_world_transform_matrix( - self._xr_anchor_headset_path, pxr_mat, self.__anchor_headset_layer_identifier + self._xr_anchor_headset_path, matrix, self.__anchor_headset_layer_identifier ) except Exception as e: logger.warning(f"XR: Anchor sync failed: {e}") diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py index 7ec58e4c1ca0..3366ef05131d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py @@ -51,6 +51,15 @@ class XrCfg: This quantity is only effective if :attr:`xr_anchor_pos` is set. """ + excluded_visual_meshes: list[str] = [] + """Mesh-path regexes excluded from the OVXR eye cameras and camera panels by scene partition. + + This leaves USD visibility and the source scene unchanged. Only matching + visual meshes in the single-environment render export enter the + ``xr_excluded`` partition. Camera selectors remain independent; collisions + and link/camera transforms must not be matched. + """ + anchor_prim_path: str | None = None """Specifies the prim path to attach the XR anchor to for dynamic positioning. @@ -60,6 +69,10 @@ class XrCfg: :func:`~isaaclab_teleop.create_isaac_teleop_device` expands ``{ENV_REGEX_NS}`` when given an ``env_prim_path``; direct device construction requires a concrete path. + Kit sessions use XRCore to follow the reference. Application-owned OVXR presentation + uses committed physics references for the same rotation and height policies. Input cannot + consume an unseeded or failed publication epoch; no static pose is substituted. + If None, the anchor will use the static :attr:`anchor_pos` and :attr:`anchor_rot` values. """ @@ -111,7 +124,7 @@ class XrCfg: from typing import Any -def remove_camera_configs(env_cfg: Any) -> Any: +def remove_camera_configs(env_cfg: Any, *, keep: tuple[str, ...] = ()) -> Any: """Removes cameras from environments when using XR devices. Having additional cameras cause operation performance issues. This function scans the environment @@ -120,6 +133,9 @@ def remove_camera_configs(env_cfg: Any) -> Any: Args: env_cfg: The environment configuration to modify. + keep: Camera attribute names to retain for the OVXR renderer. + Observation terms are removed for retained cameras as well, so the + application loop never requests a second camera render. Returns: The modified environment configuration with cameras removed. @@ -136,8 +152,9 @@ def remove_camera_configs(env_cfg: Any) -> Any: for attr_name in dir(env_cfg.scene): attr = getattr(env_cfg.scene, attr_name) if isinstance(attr, CameraCfg): - delattr(env_cfg.scene, attr_name) - logger.info(f"Removed camera config: {attr_name}") + if attr_name not in keep: + delattr(env_cfg.scene, attr_name) + logger.info(f"Removed camera config: {attr_name}") # Remove any ObsTerms for the camera if hasattr(env_cfg.observations, "policy"): diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py new file mode 100644 index 000000000000..275f2541d209 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py @@ -0,0 +1,95 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Camera-selective visual exclusions for the single-environment XR export.""" + +from __future__ import annotations + +import re +from collections.abc import Iterator, Sequence +from contextlib import contextmanager +from typing import TYPE_CHECKING + +from isaaclab.cloner.cloner_cfg import DEFAULT_ENV_TEMPLATE + +if TYPE_CHECKING: + from pxr import Usd + + +@contextmanager +def partition_visual_meshes(stage: Usd.Stage, patterns: Sequence[str]) -> Iterator[tuple[str, ...]]: + """Exclude matching visual meshes from environment cameras during export. + + A temporary session sublayer is flattened into the borrowed render Stage. + The application's source layers, visibility, physics and cameras are not + changed. Patterns match mesh paths, including instance proxies, not links. + + Args: + stage: Application Stage with exactly one environment, ``/World/envs/env_0``. + patterns: Regular expressions that must each fully match at least one visual mesh path. + + Yields: + The excluded mesh paths, or an empty tuple without patterns. + + Raises: + ValueError: If the Stage has other environments, a pattern matches no visual mesh, + or a match is collision geometry. + RuntimeError: If a stronger source opinion prevents the partition or deinstancing. + """ + if not patterns: + yield () + return + + from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + env_0 = DEFAULT_ENV_TEMPLATE.format(0) + envs = stage.GetPrimAtPath(env_0.rsplit("/", 1)[0]) + if not envs or [str(p.GetPath()) for p in envs.GetChildren()] != [env_0]: + raise ValueError("XR visual partitions require exactly env_0") + expressions = [re.compile(pattern) for pattern in patterns] + matches = [[] for _ in expressions] + for prim in Usd.PrimRange(envs, Usd.TraverseInstanceProxies()): + if not prim.IsA(UsdGeom.Mesh): + continue + for index, expression in enumerate(expressions): + if expression.fullmatch(str(prim.GetPath())): + if prim.HasAPI(UsdPhysics.CollisionAPI) or "collisions" in prim.GetPath().pathString.split("/"): + raise ValueError(f"XR visual exclusion matched collision geometry: {prim.GetPath()}") + matches[index].append(str(prim.GetPath())) + if any(not paths for paths in matches): + raise ValueError("XR visual exclusion pattern matched no visual mesh") + paths = tuple(dict.fromkeys(path for group in matches for path in group)) + layer = Sdf.Layer.CreateAnonymous("xr-visual-partitions.usda") + session = stage.GetSessionLayer() + session.subLayerPaths.insert(0, layer.identifier) + try: + with Usd.EditContext(stage, layer): + for path in paths: + while stage.GetPrimAtPath(path).IsInstanceProxy(): + parent = stage.GetPrimAtPath(path).GetParent() + while parent and not parent.IsInstance(): + parent = parent.GetParent() + if not parent: + raise RuntimeError(f"Cannot author XR visual partition: {path}") + parent.SetInstanceable(False) + if parent.IsInstance(): + raise RuntimeError(f"Stronger source opinion prevents XR deinstancing: {parent.GetPath()}") + partition = UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).CreatePrimvar( + "omni:scenePartition", Sdf.ValueTypeNames.Token, UsdGeom.Tokens.constant + ) + partition.Set("xr_excluded") + if partition.Get() != "xr_excluded": + raise RuntimeError(f"Stronger source opinion prevents XR visual partition: {path}") + settings_path = stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" + settings = stage.DefinePrim(settings_path, "RenderSettings") + show_all = settings.CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ) + show_all.Set(False) + if show_all.Get() is not False: + raise RuntimeError("Stronger source opinion prevents XR camera-selective partitions") + yield paths + finally: + session.subLayerPaths.remove(layer.identifier) diff --git a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py index 15513137ab3c..094234c3fed8 100644 --- a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py +++ b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py @@ -151,6 +151,30 @@ def _make_lifecycle( ) +@pytest.mark.parametrize("entry", ["lifecycle", "device", "factory"]) +def test_positional_recording_options_preserve_public_constructor_contract(entry): + """Existing positional MCAP/debug/haptic arguments must not become XR handles. + + Exercise all three public argument-binding boundaries without opening an XR + session. Native recording/playback correctness has a separate owner. + """ + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device + + cfg = _make_cfg() + if entry == "lifecycle": + lifecycle = TeleopSessionLifecycle(cfg, None, False, False, "capture.mcap", None, True, None) + elif entry == "device": + device = IsaacTeleopDevice(cfg, None, False, False, "capture.mcap", None, True, None) + lifecycle = device._session_lifecycle + else: + device = create_isaac_teleop_device(cfg, "cpu", None, None, False, False, "capture.mcap", None, True, None) + lifecycle = device._session_lifecycle + assert lifecycle._mcap_record_path == "capture.mcap" + assert lifecycle._mcap_replay_path is None + assert lifecycle._enable_debug_visualization is True + assert lifecycle._external_oxr_handles_provider is None + + # ============================================================================ # Shipped .env profile paths # ============================================================================ @@ -209,6 +233,202 @@ def test_session_config_receives_deadline_paced_pipelined_retargeting(self): assert resolved_execution.pacing.safety_margin_s == 0.025 +class TestExternalOpenXRHandles: + """Tests for borrowing OVXR-owned OpenXR handles without the Kit bridge.""" + + def test_provider_handles_are_passed_to_teleop_session(self): + handles = object() + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=lambda: handles, + ) + lifecycle._pipeline = MagicMock() + lifecycle._teleop_control_pipeline = None + session_config_cls = MagicMock(return_value=MagicMock()) + session_cls = MagicMock() + fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"] + + with ( + patch.object(fake_tsm_module, "TeleopSessionConfig", session_config_cls), + patch.object(fake_tsm_module, "TeleopSession", session_cls), + ): + assert lifecycle.try_start_session() is True + + assert session_config_cls.call_args.kwargs["oxr_handles"] is handles + + def test_provider_none_defers_session_creation(self): + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=lambda: None, + ) + lifecycle._pipeline = MagicMock() + lifecycle._teleop_control_pipeline = None + + assert lifecycle.try_start_session() is False + assert lifecycle.is_active is False + + +@pytest.mark.parametrize("replay", [False, True]) +@pytest.mark.parametrize("rollback_fails", [False, True]) +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_failed_session_entry_is_not_published_as_active(tmp_path, monkeypatch, replay, rollback_fails, error_type): + recording = tmp_path / "recording.mcap" + recording.touch() + replay_module = ModuleType("isaacteleop.deviceio_session") + replay_module.McapReplayConfig = MagicMock() + monkeypatch.setitem(sys.modules, replay_module.__name__, replay_module) + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + mcap_replay_path=str(recording) if replay else None, + ) + lifecycle._pipeline = MagicMock() + startup_error = error_type("session entry failed") + session_cls = MagicMock() + session_cls.return_value.cleanup_failed = rollback_fails + session_cls.return_value.__enter__.side_effect = startup_error + fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"] + with patch.object(fake_tsm_module, "TeleopSession", session_cls): + with pytest.raises(error_type) as caught: + lifecycle.try_start_session() + assert caught.value is startup_error + assert not lifecycle.is_active + if rollback_fails: + assert lifecycle._session is session_cls.return_value + assert lifecycle.session_cleanup_failed + for operation in (lifecycle.try_start_session, lifecycle.step, lifecycle.stop): + with pytest.raises(error_type) as repeated: + operation() + assert repeated.value is startup_error + assert session_cls.call_count == 1 + session_cls.return_value.__exit__.assert_not_called() + else: + assert lifecycle._session is None + session_cls.return_value.__enter__.side_effect = None + assert lifecycle.try_start_session() + assert session_cls.call_count == 2 + assert lifecycle._session is session_cls.return_value + + +def test_device_failed_entry_stops_partial_session(): + """Python does not call __exit__ when __enter__ fails after runtime launch.""" + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + + startup_error = RuntimeError("pipeline construction failed") + cfg = _make_cfg() + cfg.pipeline_builder = MagicMock(side_effect=startup_error) + device = IsaacTeleopDevice(cfg, cloudxr_env_file="test.env", use_kit_xr_bridge=False) + launcher_class = MagicMock() + launcher = launcher_class.return_value + with ( + patch.object(device._session_lifecycle, "_run_system_check"), + patch.object(sys.modules["isaacteleop.cloudxr"], "CloudXRLauncher", launcher_class), + patch.dict(os.environ, {"ISAACLAB_CXR_SKIP_AUTOLAUNCH": "0"}), + pytest.raises(RuntimeError) as caught, + ): + with device: + pytest.fail("entry must fail") + assert caught.value is startup_error + launcher.stop.assert_called_once() + assert device._session_lifecycle._cloudxr_launcher is None + + +@pytest.mark.parametrize("body_error", [False, True]) +def test_device_anchor_cleanup_does_not_skip_session_stop(body_error): + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + + device = IsaacTeleopDevice(_make_cfg(), use_kit_xr_bridge=False) + cleanup_error = RuntimeError("anchor subscription cleanup failed") + primary = ValueError("body failed") if body_error else None + device._anchor_manager.cleanup = MagicMock(side_effect=cleanup_error) + device._session_lifecycle.stop = MagicMock() + try: + if primary is None: + with pytest.raises(RuntimeError) as caught: + device.__exit__(None, None, None) + assert caught.value is cleanup_error + else: + assert device.__exit__(type(primary), primary, None) is False + assert any("anchor subscription" in note for note in primary.__notes__) + device._session_lifecycle.stop.assert_called_once() + finally: + device._anchor_manager.cleanup = MagicMock() + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_ui_cleanup_failure_still_drains_session_and_runtime(error_type): + lifecycle = _make_lifecycle() + cleanup_error = error_type("UI cleanup failed") + lifecycle._retargeting_ui_ctx = MagicMock() + lifecycle._retargeting_ui_ctx.__exit__.side_effect = cleanup_error + session, launcher = MagicMock(), MagicMock() + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + with pytest.raises(error_type) as caught: + lifecycle.stop() + assert caught.value is cleanup_error + session.__exit__.assert_called_once() + launcher.stop.assert_called_once() + assert lifecycle._session is None + assert lifecycle._cloudxr_launcher is None + + +@pytest.mark.parametrize("replay", [False, True]) +def test_session_cleanup_error_without_borrowed_handles_is_suppressed(replay): + """Kit "Stop AR" invalidates the handles, so a failed exit still releases the session and runtime.""" + lifecycle = TeleopSessionLifecycle(_make_cfg(), mcap_replay_path="recording.mcap" if replay else None) + session, launcher = MagicMock(), MagicMock() + session.__exit__.side_effect = RuntimeError("XR_ERROR_HANDLE_INVALID") + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + lifecycle.stop() + session.__exit__.assert_called_once() + assert lifecycle._session is None + assert not lifecycle.session_cleanup_failed + launcher.stop.assert_called_once() + + +@pytest.mark.parametrize( + "body_error,error_type,replay", + [ + (False, RuntimeError, False), + (True, RuntimeError, False), + (False, KeyboardInterrupt, False), + (True, KeyboardInterrupt, False), + (False, ValueError, True), + (True, RuntimeError, True), + ], +) +def test_session_exit_failure_is_visible_and_retains_resources(body_error, error_type, replay): + """OVXR replay exit failures stay visible even though input borrows no OpenXR handles.""" + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=None if replay else lambda: object(), + ovxr_session=replay, + mcap_replay_path="recording.mcap" if replay else None, + ) + session, launcher = MagicMock(), MagicMock() + cleanup_error = error_type("native drain failed") + primary = ValueError("body failed") if body_error else None + session.__exit__.side_effect = cleanup_error + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + if primary is None: + with pytest.raises(error_type) as caught: + lifecycle.stop() + assert caught.value is cleanup_error + else: + try: + lifecycle.stop(type(primary), primary, None) + except BaseException as error: + pytest.fail(f"Cleanup replaced the original body failure with {error!r}") + assert any("native drain failed" in note for note in primary.__notes__) + assert lifecycle._session is session + assert lifecycle.session_cleanup_failed + assert lifecycle._cloudxr_launcher is launcher + launcher.stop.assert_not_called() + + # ============================================================================ # _ensure_cloudxr_runtime # ============================================================================ diff --git a/source/isaaclab_teleop/test/test_control_events.py b/source/isaaclab_teleop/test/test_control_events.py index dafe7a2ac18c..556d06185217 100644 --- a/source/isaaclab_teleop/test/test_control_events.py +++ b/source/isaaclab_teleop/test/test_control_events.py @@ -19,7 +19,7 @@ import dataclasses import json import sys -from types import ModuleType +from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock import pytest @@ -95,6 +95,7 @@ def __init__(self, name: str) -> None: _install_stubs() from isaaclab_teleop.control_events import ControlEvents, poll_control_events # noqa: E402 +from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle # noqa: E402 from isaaclab_teleop.teleop_message_processor import ( # noqa: E402 TeleopMessageProcessor, _classify_command, @@ -535,3 +536,29 @@ def last_control_events(self): result = poll_control_events(FakeDevice()) assert result.is_active is False assert result.should_reset is True + + +class TestPipelinedControlEvents: + def test_reset_edge_is_emitted_once_per_returned_frame(self): + from isaacteleop.retargeting_engine.interface.execution_events import ExecutionEvents, ExecutionState + + lifecycle = TeleopSessionLifecycle.__new__(TeleopSessionLifecycle) + lifecycle._message_processor = object() + lifecycle._last_control_event_frame_id = None + lifecycle._last_control_events = ControlEvents() + lifecycle._session = SimpleNamespace( + last_context=SimpleNamespace( + execution_events=ExecutionEvents(reset=True, execution_state=ExecutionState.RUNNING) + ), + last_step_info=SimpleNamespace(returned_frame_id=7), + ) + + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=True) + + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=False) + + lifecycle._session.last_step_info.returned_frame_id = 8 + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=True) diff --git a/source/isaaclab_teleop/test/test_entrypoints.py b/source/isaaclab_teleop/test/test_entrypoints.py index 5f814345e6c8..efffdbcdf577 100644 --- a/source/isaaclab_teleop/test/test_entrypoints.py +++ b/source/isaaclab_teleop/test/test_entrypoints.py @@ -8,6 +8,7 @@ import importlib import importlib.metadata import sys +from contextlib import contextmanager from unittest import mock import pytest @@ -15,6 +16,32 @@ pytestmark = pytest.mark.unit +def test_live_entrypoint_preserves_resolved_camera_selection(monkeypatch): + """The public workflow carries its resolved PiP selection into the selected runtime.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as workflow + + config, teleop_config = object(), object() + selected_cameras = (object(),) + resolved = (config, teleop_config, True, selected_cameras) + monkeypatch.setattr(workflow, "_resolve_env_cfg", lambda: resolved) + calls = [] + + @contextmanager + def launch(env_config, args, *, cleanup_context): + assert env_config is config and args.using_ovxr + assert not cleanup_context + calls.append("launch") + yield + calls.append("close") + + monkeypatch.setattr(workflow, "launch_simulation", launch) + monkeypatch.setattr(workflow, "run_teleoperation", lambda *args: calls.append(args)) + monkeypatch.setattr(sys, "argv", ["caller"]) + assert workflow.run_teleop_se3_agent_cli(["--ovxr", "--ovxr_pip_camera", "robot_pov_cam"]) == 0 + assert calls == ["launch", resolved, "close"] + assert sys.argv == ["caller"] + + @pytest.mark.parametrize( ("module_name", "cli_name"), [ diff --git a/source/isaaclab_teleop/test/test_hand_debug_viz.py b/source/isaaclab_teleop/test/test_hand_debug_viz.py index c67f6d8da4cf..2ed4b29b8a5c 100644 --- a/source/isaaclab_teleop/test/test_hand_debug_viz.py +++ b/source/isaaclab_teleop/test/test_hand_debug_viz.py @@ -7,7 +7,7 @@ """Tests for hand joint debug visualization plumbing. -Covers :meth:`TeleopSessionLifecycle._chain_hand_debug_outputs` (HandsSource +Covers :meth:`TeleopSessionLifecycle._chain_hand_presence_outputs` (HandsSource auto-discovery), :func:`~isaaclab_teleop.visualizers.hand_joint_visualizer._extract_world_joint_positions`, and :func:`~isaaclab_teleop.visualizers.controller_aim_visualizer._aim_world_pose` @@ -192,8 +192,9 @@ def __init__(self, name: str) -> None: class _FakePipeline: """Stand-in for a user pipeline (leaf nodes + action output).""" - def __init__(self, leaves: list) -> None: + def __init__(self, leaves: list, outputs: tuple[str, ...] = ()) -> None: self._leaves = leaves + self._outputs = outputs def get_leaf_nodes(self) -> list: return self._leaves @@ -202,7 +203,7 @@ def output(self, key: str) -> str: return f"selector:user_pipeline:{key}" def output_types(self) -> dict: - return {"action": None} + return dict.fromkeys(("action", *self._outputs), object()) class _FakeTensorGroup: @@ -218,7 +219,7 @@ def __getitem__(self, idx): # =========================================================================== -# _chain_hand_debug_outputs: HandsSource auto-discovery +# _chain_hand_presence_outputs: HandsSource auto-discovery # =========================================================================== @@ -226,9 +227,21 @@ class TestChainHandDebugOutputs: def test_mapping_untouched_without_hands_source(self): pipeline = _FakePipeline([_FakeLeaf("controllers"), _FakeLeaf("head")]) mapping: dict = {"action": "selector:user_pipeline:action"} - TeleopSessionLifecycle._chain_hand_debug_outputs(pipeline, mapping) + TeleopSessionLifecycle._chain_hand_presence_outputs(pipeline, mapping) assert mapping == {"action": "selector:user_pipeline:action"} + def test_explicit_filtered_hand_outputs_take_precedence_over_raw_source(self): + pipeline = _FakePipeline( + [_FakeHandsSource("hands")], + outputs=(TeleopSessionLifecycle.HAND_LEFT_KEY, TeleopSessionLifecycle.HAND_RIGHT_KEY), + ) + mapping: dict = {} + TeleopSessionLifecycle._chain_hand_presence_outputs(pipeline, mapping) + assert mapping == { + TeleopSessionLifecycle.HAND_LEFT_KEY: "selector:user_pipeline:hand_left", + TeleopSessionLifecycle.HAND_RIGHT_KEY: "selector:user_pipeline:hand_right", + } + # =========================================================================== # start(): chaining gated by the enable_debug_visualization lifecycle arg @@ -251,10 +264,10 @@ def test_start_chains_when_enabled(self): assert lifecycle.pipeline.output_mapping[TeleopSessionLifecycle.HAND_RIGHT_KEY] == "selector:hands:hand_right" assert TeleopSessionLifecycle._CONTROLLER_LEFT_KEY in lifecycle.pipeline.output_mapping - def test_start_does_not_chain_when_disabled(self): + def test_start_chains_hand_presence_but_not_left_controller_when_disabled(self): lifecycle = self._start(enable_debug_visualization=False) - assert TeleopSessionLifecycle.HAND_LEFT_KEY not in lifecycle.pipeline.output_mapping - assert TeleopSessionLifecycle.HAND_RIGHT_KEY not in lifecycle.pipeline.output_mapping + assert TeleopSessionLifecycle.HAND_LEFT_KEY in lifecycle.pipeline.output_mapping + assert TeleopSessionLifecycle.HAND_RIGHT_KEY in lifecycle.pipeline.output_mapping assert TeleopSessionLifecycle._CONTROLLER_LEFT_KEY not in lifecycle.pipeline.output_mapping # The right controller is always chained for button polling. assert TeleopSessionLifecycle._CONTROLLER_RIGHT_KEY in lifecycle.pipeline.output_mapping diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py new file mode 100644 index 000000000000..f0824a8c66d2 --- /dev/null +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -0,0 +1,1263 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused tests for the Isaac Lab host side of external OVXR presentation.""" + +from __future__ import annotations + +import builtins +import contextlib +import importlib +import math +import subprocess +import sys +import threading +from contextlib import contextmanager +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest +from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg + + +class _FakeLease: + def __init__(self): + self.ovrtx_renderer = object() + self.ovstage = object() + self.partition_calls = [] + self.validated_ordinals = [] + self.owner_lane_entries = 0 + self.auxiliary_render_product_paths = () + + @contextmanager + def ovstage_owner_lane(self): + self.owner_lane_entries += 1 + yield + + def configure_external_camera_scene_partitions(self, environment_root_path, camera_paths): + self.partition_calls.append((environment_root_path, camera_paths)) + + def validate_external_scene_partitions(self, ordinal): + self.validated_ordinals.append(ordinal) + + +class _FakePublisher: + def __init__(self): + self.calls = [] + self.fail = None + self.required = True + self.closed = False + + def publication_required(self, _publication_id): + return self.required + + def write_complete_scene(self, ordinal, publication_id): + self.calls.append((ordinal, publication_id, threading.get_native_id())) + if self.fail is not None: + raise self.fail + + def close(self): + self.closed = True + + +class _FakeOrdinalLedger: + def __init__(self, stage): + self.stage = stage + self.next_ordinal = 11 + self.reserved = [] + self.published = [] + self.abandoned = [] + self.publish_error = None + self.destroyed = False + + def reserve(self): + ordinal = self.next_ordinal + self.next_ordinal += 1 + self.reserved.append(ordinal) + return ordinal + + def publish(self, ordinal): + if self.publish_error is not None: + raise self.publish_error + self.published.append(ordinal) + + def abandon(self, ordinal): + self.abandoned.append(ordinal) + + def destroy(self): + self.destroyed = True + + +class _FakeBootstrap: + def __init__(self): + self.lease = _FakeLease() + self.publisher = _FakePublisher() + self.events = [] + self.closed = False + self.aborted = False + self.auxiliary_camera_cfgs = [] + + def capture_scene_publisher(self): + self.events.append("capture") + return self.publisher + + def offer_render_owner_transfer(self): + self.events.append("offer") + return "capability" + + def claim_render_owner_transfer(self, capability): + self.events.append(("claim", capability)) + + def bind_environment(self, env): + self.events.append(("bind", env)) + return self.lease + + def configure_auxiliary_camera(self, camera_cfg, *, max_fps=30.0): + self.auxiliary_camera_cfgs.append(camera_cfg) + + def validate_environment(self, env): + pass + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + self.closed = True + + def abort(self): + self.aborted = True + + +@pytest.fixture +def real_lease_type(): + from isaaclab_ov.renderers.render_lease import OvLibrariesRenderLease + + return OvLibrariesRenderLease + + +@pytest.fixture +def xr_module(monkeypatch, real_lease_type): + """Import the adapter with native renderer dependencies stubbed. + + The stub-bound adapter is removed from ``sys.modules`` and its package on teardown so + later tests import the real module. + """ + # The adapter needs a Capture release with OVXR session support. The Kit ``teleop`` extra + # selects an older release without it; skip rather than error in that environment. + pytest.importorskip("isaaccapture.ovxr_session_manager") + import isaaclab_teleop + + isaaclab_ov = ModuleType("isaaclab_ov") + isaaclab_ov.__path__ = [] + renderers = ModuleType("isaaclab_ov.renderers") + renderers.OVRTXRendererCfg = object + renderers.OVRTXRenderer = object + renderers.OvLibrariesRenderLease = object + monkeypatch.setitem(sys.modules, "isaaclab_ov", isaaclab_ov) + monkeypatch.setitem(sys.modules, "isaaclab_ov.renderers", renderers) + source = Path(__file__).parents[1] / "isaaclab_teleop/ovxr_session.py" + spec = importlib.util.spec_from_file_location("isaaclab_teleop.ovxr_session", source) + ovxr_session = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, ovxr_session) + spec.loader.exec_module(ovxr_session) + name = "isaaclab_teleop.ov_libraries_xr" + saved_module = sys.modules.pop(name, None) + saved_attribute = isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + try: + module = importlib.import_module(name) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + yield module + finally: + sys.modules.pop(name, None) + isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + if saved_module is not None: + sys.modules[name] = saved_module + if saved_attribute is not None: + isaaclab_teleop.ov_libraries_xr = saved_attribute + + +def _fake_ovxr(): + return SimpleNamespace(apply_renderer_settings=lambda _settings: None) + + +def test_stage_ordinal_ledger_tracks_one_application_reservation(xr_module): + class ImmediateQuery: + def __init__(self, fetch): + self._fetch = fetch + + def __enter__(self): + return self + + def __exit__(self, *_exc): + pass + + def wait(self): + pass + + def fetch(self): + return self._fetch() + + class ImmediateAdvance: + def wait(self): + pass + + class Stage: + def __init__(self): + self.floor = 10 + self.advances = [] + self.queries = 0 + + def get_attribute_write_floor(self): + self.queries += 1 + return ImmediateQuery(lambda: self.floor) + + def advance_write_floor(self, *, ordinal): + self.advances.append(ordinal) + self.floor = ordinal + return ImmediateAdvance() + + stage = Stage() + ledger = xr_module._StageOrdinalLedger(stage) + assert stage.queries == 1 + + assert ledger.reserve() == 11 + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + ledger.abandon(11) + + assert ledger.reserve() == 11 + ledger.publish(11) + assert stage.advances == [11] + assert stage.queries == 2 # Only the earlier abandon needs reconciliation. + + assert ledger.reserve() == 12 + stage.floor = 12 # The explicit-ordinal OVXR call sealed its own write. + assert ledger.resolve_external(12, expected_published=True) + assert stage.queries == 2 + + assert ledger.reserve() == 13 + assert not ledger.resolve_external(13, expected_published=None) + assert stage.queries == 3 + assert ledger.reserve() == 13 + stage.floor = 13 + with pytest.raises(RuntimeError, match="expected unsealed"): + ledger.resolve_external(13, expected_published=False) + assert stage.queries == 4 + # Even a contradicted response must not permit reuse of an observed sealed ordinal. + assert ledger.reserve() == 14 + ledger.abandon(14) + ledger.destroy() + with pytest.raises(RuntimeError, match="closed"): + ledger.reserve() + + +def test_runtime_render_prims_have_the_declared_paths_and_types(xr_module): + class FakePrim: + def __init__(self, type_name=""): + self.type_name = type_name + + def __bool__(self): + return bool(self.type_name) + + def IsValid(self): + return bool(self.type_name) + + def GetTypeName(self): + return self.type_name + + class FakeStage: + def __init__(self): + self.prims = {} + self.definitions = [] + + def GetPrimAtPath(self, path): + return self.prims.get(path, FakePrim()) + + def DefinePrim(self, path, type_name): + self.definitions.append((path, type_name)) + self.prims[path] = FakePrim(type_name) + + stage = FakeStage() + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) + + assert stage.definitions == [ + ("/OVXRGeneratedCameras/OVXR_LeftEyeCamera", "Camera"), + ("/OVXRGeneratedCameras/OVXR_RightEyeCamera", "Camera"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", "RenderProduct"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", "RenderProduct"), + ("/Render/Vars/LdrColor", "RenderVar"), + ("/Render/Vars/XRDepth", "RenderVar"), + ("/Render/OmniverseGlobalRenderSettings", "RenderSettings"), + ] + + +def test_runtime_render_prim_declaration_preserves_disabled_and_fails_on_conflict(xr_module): + class ExistingPrim: + def IsValid(self): + return True + + def GetTypeName(self): + return "Xform" + + stage = SimpleNamespace(GetPrimAtPath=lambda _path: ExistingPrim(), DefinePrim=lambda *_args: None) + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg(create_render_products=False)) + with pytest.raises(RuntimeError, match="incompatible type"): + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) + + +def _make_host(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + env = SimpleNamespace(scene=SimpleNamespace(stage=None), close=lambda: bootstrap.events.append("environment-close")) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda _env, _cfg: bootstrap.events.append("declare")) + env.unwrapped = env + host = xr_module._IsaacLabOvxrApplicationInterface(bootstrap, env, xr_module.OvLibrariesXrCfg()) + return host, bootstrap, env + + +def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monkeypatch): + host, bootstrap, env = _make_host(xr_module, monkeypatch) + + capability = host.offer_render_owner_transfer() + binding = host.attach_on_render_thread(capability) + with host.publication_guard(): + ordinal = host.publish_complete_scene("step-1") + + assert binding.renderer is bootstrap.lease.ovrtx_renderer + assert binding.stage is bootstrap.lease.ovstage + assert binding.ordinal_ledger.stage is bootstrap.lease.ovstage + assert bootstrap.events == ["capture", "offer", ("claim", "capability"), "declare", ("bind", env)] + assert ordinal == 11 + assert bootstrap.publisher.calls[0][:2] == (11, "step-1") + host.validate_first_published_scene(ordinal) + assert bootstrap.lease.validated_ordinals == [11] + with host.owner_lane(): + pass + assert bootstrap.lease.owner_lane_entries == 3 + assert host.get_target_frame_T_world("/World/target") == ("target", "/World/target") + + host.close_publication() + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.publisher.closed and bootstrap.closed + assert binding.ordinal_ledger.destroyed + assert bootstrap.events[-1] == "environment-close" + + +def test_host_smooths_committed_anchor_on_xr_clock_and_reseeds_reset(xr_module, monkeypatch): + """Use the real pose policy; substitute only frame time, scene I/O, and native sealing.""" + import numpy as np + from isaaclab_teleop import xr_anchor_manager + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode, XrCfg + from scipy.spatial.transform import Rotation + + from pxr import Usd, UsdGeom + + from isaaclab.sim.utils.stage import use_stage + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg( + anchor_prim_path="/World/Robot", + anchor_rotation_mode=XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, + anchor_rotation_smoothing_time=0.1, + ) + with use_stage(Usd.Stage.CreateInMemory()): + UsdGeom.Xform.Define(xr_anchor_manager.get_current_stage(), "/World/Robot") + manager = xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=True) + + bootstrap = _FakeBootstrap() + reference = np.eye(4) + bootstrap.get_world_transform = lambda _path: reference.copy() + env = SimpleNamespace(sim=SimpleNamespace(get_rendering_dt=lambda: 0.25)) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, env, xr_module.OvLibrariesXrCfg(), external_anchor_state=manager + ) + host._scene_publisher = bootstrap.publisher + host._ordinal_ledger = _FakeOrdinalLedger(None) + writes = [] + monkeypatch.setattr( + host, "_write_anchor_matrices", lambda matrices, ordinal: writes.append((matrices.copy(), ordinal)) + ) + frame_times = iter((10.0, 10.01, 10.02, 10.05, 20.0, 20.1, 20.11)) + monkeypatch.setattr(xr_module.time, "monotonic", lambda: next(frame_times)) + + def committed_yaw(): + return Rotation.from_matrix(writes[-1][0][0, :3, :3]).as_euler("xyz")[2] + + seed = host.publish_complete_scene("initial") + assert writes[-1][1] == seed == host._ordinal_ledger.published[-1] + host.prepare_frame_scene() + reference[:3, :3] = Rotation.from_euler("z", 90, degrees=True).as_matrix() + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(0.0) # Unpublished physics must not move the view. + host.publish_complete_scene("moved") + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.01 / 0.1))) + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.04 / 0.1))) + + manager.reset() + write_count = len(writes) + host.prepare_frame_scene() + assert len(writes) == write_count # An old reference cannot seed the new epoch. + reference[:3, :3] = Rotation.from_euler("z", 45, degrees=True).as_matrix() + seed = host.publish_complete_scene("reset") + assert writes[-1][1] == seed == host._ordinal_ledger.published[-1] + assert committed_yaw() == pytest.approx(0.0) + assert np.isfinite(manager.get_world_matrix()).all() # Input is usable before the next XR frame. + host.prepare_frame_scene() + reference[:3, :3] = Rotation.from_euler("z", 135, degrees=True).as_matrix() + host.publish_complete_scene("after-reset") + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.01 / 0.1))) + + +def test_host_lends_the_selected_auxiliary_product_without_rendering_it(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + bootstrap.lease.auxiliary_render_product_paths = ("/IsaacLabAuxiliaryRender/RenderProduct_robot_pov_cam",) + camera = object() + env = SimpleNamespace( + unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={"robot_pov_cam": camera})) + ) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, + env, + xr_module.OvLibrariesXrCfg( + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ), + ), + ) + + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + + assert binding.frame_consumers[0].render_product_paths == bootstrap.lease.auxiliary_render_product_paths + + +def test_host_rejects_a_missing_selected_auxiliary_camera(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={}))) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, + env, + xr_module.OvLibrariesXrCfg( + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ), + ), + ) + + with pytest.raises(RuntimeError, match="not present"): + host.attach_on_render_thread(host.offer_render_owner_transfer()) + # The selection fails before native Stage binding and partition export. + assert not any(isinstance(event, tuple) and event[0] == "bind" for event in bootstrap.events) + + +def test_headless_attach_does_not_import_the_desktop_mirror(xr_module, monkeypatch): + """``--headless`` must work without the optional desktop-mirror modules.""" + host, _bootstrap, _env = _make_host(xr_module, monkeypatch) + monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager.owned_mirror", None) + monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager.desktop_mirror", None) + assert not host._cfg.desktop_mirror_enabled + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + assert binding.frame_consumers == () + + +def test_selected_camera_poses_follow_motion_and_reset_before_publication(xr_module, monkeypatch): + host, bootstrap, env = _make_host(xr_module, monkeypatch) + names = ("head", "left_wrist", "right_wrist") + events = [] + + class PoseOnlyCamera: + live = 0 + cached = -1 + fail = False + + def refresh_world_poses(self): + if self.fail: + raise RuntimeError("pose unavailable") + self.cached = self.live + events.append(self) + + @property + def data(self): + pytest.fail("Publication must not fetch/render camera images") + + cameras = {name: PoseOnlyCamera() for name in (*names, "unselected")} + env.scene = SimpleNamespace(stage=None, sensors=cameras) + host._auxiliary_cameras = tuple( + xr_module.OvLibrariesXrCameraCfg(name, SimpleNamespace(width=720, height=450)) for name in names + ) + bootstrap.lease.auxiliary_render_product_paths = tuple(f"/Render/{name}" for name in names) + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + published = [] + bootstrap.publisher.write_complete_scene = lambda *_args: published.append( + tuple(cameras[name].cached for name in names) + ) + + for identity, positions in ((1, (0, 2, 3)), (("reset", 1, 1), (0, -2, -3)), (2, (0, 4, 5))): + for name, value in zip(names, positions, strict=True): + cameras[name].live = value + with host.publication_guard(): + host.publish_complete_scene(identity) + assert published[-1] == positions + assert events == [cameras[name] for name in names] * 3 + assert cameras["unselected"].cached == -1 + + bootstrap.publisher.required = False + with host.publication_guard(): + host.publish_complete_scene(2) + assert len(events) == 9 # Reused scene/cadence gaps do not resample poses. + + bootstrap.publisher.required = True + cameras["left_wrist"].fail = True + reserved = list(binding.ordinal_ledger.reserved) + with host.publication_guard(), pytest.raises(RuntimeError, match="pose unavailable"): + host.publish_complete_scene(3) + assert binding.ordinal_ledger.reserved == reserved + assert len(published) == 3 + + +def test_host_abandons_failed_write_and_failed_publish(xr_module, monkeypatch): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + bootstrap.publisher.fail = RuntimeError("write failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="write failed"): + host.publish_complete_scene("step-1") + assert host._ordinal_ledger.abandoned == [11] + + bootstrap.publisher.fail = None + host._ordinal_ledger.publish_error = RuntimeError("publish failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="publish failed"): + host.publish_complete_scene("step-2") + assert host._ordinal_ledger.abandoned == [11, 12] + + +@pytest.mark.parametrize("sealed", [False, True]) +def test_host_reconciles_failed_publication_without_reusing_generation(xr_module, monkeypatch, real_lease_type, sealed): + ledger_type = xr_module._StageOrdinalLedger + host, _bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + failure = RuntimeError("native publication wait failed") + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + # Use the real publication identity owner; a permanently-dirty fake cannot + # detect a retry returning the older ordinal after a failed seal. + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _update_transforms=lambda: None, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + host._scene_publisher = lease.capture_scene_publisher() + renderer.scene = SimpleNamespace(stage=stage, ordinal=10) + lease._bind(object(), stage, 1) + stage.advance_write_floor = lambda *, ordinal: SimpleNamespace(wait=lambda: setattr(stage, "floor", ordinal)) + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(): + assert host.publish_complete_scene("previous") == 11 + + def failed_advance(*, ordinal): + if sealed: + stage.floor = ordinal + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("changed") + assert caught.value is failure + with host.publication_guard(), pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("changed") + next_ordinal = ledger.reserve() + assert next_ordinal == (13 if sealed else 12) + ledger.abandon(next_ordinal) + host.close_on_render_thread() + lease.release() + + +def test_failed_bootstrap_retirement_retains_environment(xr_module, monkeypatch): + """Capture's independent environment callback cannot bypass failed lease retirement.""" + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + + def fail_close(): + raise RuntimeError("producer drain failed") + + monkeypatch.setattr(bootstrap, "close", fail_close) + with pytest.raises(RuntimeError, match="producer drain failed"): + host.close_on_render_thread() + with pytest.raises(RuntimeError, match="lease retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +def test_failed_publication_with_unknown_floor_retains_reservation_and_first_error(xr_module, monkeypatch): + ledger_type = xr_module._StageOrdinalLedger + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + stage = SimpleNamespace(unknown=False) + failure = RuntimeError("native publication wait failed") + + def fetch(): + if stage.unknown: + raise RuntimeError("floor read unavailable") + return 10 + + stage.get_attribute_write_floor = lambda: contextlib.nullcontext(SimpleNamespace(wait=lambda: None, fetch=fetch)) + + def failed_advance(*, ordinal): + stage.unknown = True + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is failure + assert any("floor read unavailable" in note for note in getattr(caught.value, "__notes__", [])) + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + # Even repeated cleanup cannot retire queries/Stage while the write is unresolved. + anchor_resources_closed = [] + host._anchor_resources.callback(lambda: anchor_resources_closed.append(True)) + for _ in range(2): + with pytest.raises(RuntimeError, match="unresolved OVStage ordinal"): + host.close_on_render_thread() + assert not bootstrap.closed + assert not anchor_resources_closed + with pytest.raises(RuntimeError, match="write retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +@pytest.mark.parametrize("failure", ["write", "publish"]) +def test_publication_retains_primary_cause_when_abandon_fails(xr_module, monkeypatch, failure): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + original_cause = ValueError("original publication cause") + first_error = RuntimeError("publication failed") + first_error.__cause__ = original_cause + if failure == "write": + bootstrap.publisher.fail = first_error + else: + host._ordinal_ledger.publish_error = first_error + + def fail_abandon(_ordinal): + raise RuntimeError("abandon failed") + + monkeypatch.setattr(host._ordinal_ledger, "abandon", fail_abandon) + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert any("abandon failed" in note for note in caught.value.__notes__) + + +@pytest.mark.parametrize("camera_enabled", [False, True]) +def test_session_delegates_lifecycle_to_isaacteleop_owner(xr_module, monkeypatch, camera_enabled): + calls = [] + + class FakeOwner: + owns_environment = True + shutdown_requested = False + + def __init__(self, host, config, **kwargs): + calls.append(("init", host, config, kwargs)) + self.host = host + + def start(self, timeout_s): + calls.append(("start", timeout_s)) + self.binding = self.host.attach_on_render_thread(self.host.offer_render_owner_transfer()) + return True + + def external_oxr_handles(self): + return "handles" + + def request_render(self, identity, *, force=False): + calls.append(("request", identity, force)) + return 17 + + def request_shutdown(self): + self.shutdown_requested = True + calls.append(("request_shutdown",)) + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + calls.append(("close",)) + + class FakeMirror: + status = SimpleNamespace(main_thread_fps=30.0, render_thread_fps=40.0) + + def __init__(self, config, *, camera_products=()): + self.config = config + + def record_host_frame(self): + calls.append(("timing",)) + + def set_status_text(self, text): + calls.append(("status", text)) + + mirror_module = importlib.import_module("isaaccapture.ovxr_session_manager.owned_mirror") + monkeypatch.setattr(mirror_module, "OwnedDesktopMirror", FakeMirror) + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", FakeOwner) + bootstrap = _FakeBootstrap() + assert xr_module.OvLibrariesXrCfg().extra_openxr_extensions == () + session = xr_module.OvLibrariesXrSession( + bootstrap, + xr_module.OvLibrariesXrCfg( + extra_openxr_extensions=("XR_EXT_debug_utils",), + desktop_mirror_enabled=True, + desktop_mirror_max_fps=20.0, + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ) + if camera_enabled + else (), + ), + ) + env = SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={"robot_pov_cam": object()})) + env.unwrapped = env + bootstrap.lease.auxiliary_render_product_paths = ("/Render/robot_pov_cam",) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + + assert session.start_render_worker( + env, + required_openxr_extensions=("XR_EXT_hand_tracking",), + max_blank_frames=0 if camera_enabled else 1, + timeout_s=4.0, + ) + + assert calls[0][0] == "init" + assert calls[0][2].session_mode == "ar" + assert calls[0][3]["required_openxr_extensions"] == ("XR_EXT_hand_tracking",) + assert calls[0][3]["max_blank_frames"] == (0 if camera_enabled else 1) + assert calls[0][2].extra_openxr_extensions == ("XR_EXT_debug_utils",) + assert calls[0][2].host_wait_render_output is True + consumers = session._owner.binding.frame_consumers + consumer = consumers[-1] + assert consumer.config.max_fps == 20.0 + assert consumer.config.initial_status_text == "Waiting for headset" + if camera_enabled: + assert consumers[0].panels[0].label == "robot_pov_cam" + else: + assert len(consumers) == 1 + assert calls[1] == ("start", 4.0) + assert session.external_oxr_handles() == "handles" + assert session.request_render("step") == 17 + assert session.request_render("reset", force=True) == 17 + assert calls[-2:] == [("request", "step", False), ("request", "reset", True)] + monkeypatch.setattr(session._physics_rate, "sample", lambda _count: 114.7) + session.report_host_timing(42, 1.0 / 120.0) + assert calls[-2:] == [ + ("timing",), + ("status", "Main 30.0 FPS Physics 114.7/120 Hz (0.96× RT) Render 40.0 FPS"), + ] + assert session.get_target_frame_T_world("/target") == ("target", "/target") + assert session.async_worker_owns_environment + assert not session.shutdown_requested + session.request_shutdown() + assert calls[-1] == ("request_shutdown",) + assert session.shutdown_requested + session._owner.shutdown_requested = True + assert session.shutdown_requested + session.close() + assert calls[-1] == ("close",) + + +@pytest.fixture +def bootstrap_context(xr_module, monkeypatch): + """Use the real bootstrap with only simulation/native construction replaced.""" + module = sys.modules["isaaclab_teleop.ovxr_session"] + events = [] + + class Context: + current = None + failure = None + cleanup_failure = None + + def __init__(self, _cfg): + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + events.append("context-close") + cls.current = None + if cls.cleanup_failure is not None: + raise cls.cleanup_failure + + def get_or_create_backend(self, _cfg): + if self.failure is not None: + raise self.failure + lease = SimpleNamespace( + release=lambda: events.append("lease-release"), defer_stage_until_owner_transfer=lambda: None + ) + return SimpleNamespace(acquire_ov_libraries_render_lease=lambda: lease) + + monkeypatch.setattr(module, "SimulationContext", Context) + monkeypatch.delattr(builtins, module._TERMINAL_MODE_SENTINEL, raising=False) + return module, Context, events + + +@pytest.mark.parametrize("route", ["before_owner", "before_worker"]) +def test_failed_start_releases_lease_before_environment_and_context(xr_module, bootstrap_context, route): + """An existing environment retains its context until the caller closes it.""" + module, context, events = bootstrap_context + bootstrap = module.OvxrSessionBootstrap(object()) + if route == "before_owner": + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + session.close() + else: + # Capture calls host.abort when validation fails before its worker exists. + host = xr_module._IsaacLabOvxrApplicationInterface.__new__(xr_module._IsaacLabOvxrApplicationInterface) + host._bootstrap = bootstrap + host.abort() + assert context.current is not None + events.append("environment-close") + context.clear_instance() + assert events == ["lease-release", "environment-close", "context-close"] + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +@pytest.mark.parametrize("cleanup_fails", [False, True]) +def test_bootstrap_construction_failure_restores_terminal_mode(bootstrap_context, error_type, cleanup_fails): + """Failed or interrupted construction restores process state and preserves its cause.""" + module, context, events = bootstrap_context + error = context.failure = error_type("renderer construction failed") + if cleanup_fails: + context.cleanup_failure = RuntimeError("context cleanup failed") + with pytest.raises(error_type) as caught: + module.OvxrSessionBootstrap(object()) + assert caught.value is error + assert events == ["context-close"] + assert context.current is None + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + if cleanup_fails: + assert any("context cleanup failed" in note for note in error.__notes__) + + +def test_desktop_mirror_rate_must_be_positive_and_finite(xr_module): + with pytest.raises(ValueError, match="desktop_mirror_max_fps"): + xr_module.OvLibrariesXrCfg(desktop_mirror_max_fps=0.0) + with pytest.raises(ValueError, match="desktop_mirror_max_fps"): + xr_module.OvLibrariesXrCfg(desktop_mirror_max_fps=float("nan")) + + +def test_auxiliary_camera_config_is_preconfigured(xr_module, monkeypatch): + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraCfg() + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraCfg(name="robot_pov_cam") + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraCfg("", object()) + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", None) + with pytest.raises(ValueError, match="max_fps"): + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", object(), max_fps=0.0) + with pytest.raises(ValueError, match="head_locked"): + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", object(), space="world") + + bootstrap = _FakeBootstrap() + camera_cfg = SimpleNamespace(width=720, height=450) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_args: bootstrap) + camera = xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", camera_cfg) + session = xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,))) + assert isinstance(session, xr_module.OvLibrariesXrSession) + assert bootstrap.auxiliary_camera_cfgs == [camera_cfg] + + +def test_caller_renderer_settings_precede_renderer_creation(xr_module, monkeypatch): + settings = " --/rtx/rendermode=MinimalRendering " + events = [] + bootstrap = _FakeBootstrap() + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda value: events.append(value)) + ) + + def create_bootstrap(*_args): + assert len(events) == 1 + assert events[0] == settings.strip() + return bootstrap + + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", create_bootstrap) + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(renderer_settings=settings)) + + +def test_prepare_rejects_existing_context_before_global_settings(xr_module, monkeypatch): + from isaaclab.sim import SimulationContext + + monkeypatch.setattr(SimulationContext, "instance", lambda: object()) + monkeypatch.setitem( + sys.modules, + "ovxr", + SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed existing renderer settings")), + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated second bootstrap")) + with pytest.raises(RuntimeError, match="must create the SimulationContext"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + +@pytest.mark.parametrize("terminal_mode", [False, None, 1]) +def test_prepare_rejects_extension_mode_before_global_settings(xr_module, monkeypatch, terminal_mode): + monkeypatch.setattr(builtins, "ISAAC_LAUNCHED_FROM_TERMINAL", terminal_mode, raising=False) + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(RuntimeError, match="extension mode"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + +def test_counter_rate_uses_application_counter_and_resets_cleanly(xr_module): + rate = xr_module._CounterRate() + assert rate.sample(10, now_ns=0) == 0.0 + assert rate.sample(15, now_ns=500_000_000) == pytest.approx(10.0) + assert rate.sample(20, now_ns=1_000_000_000) == pytest.approx(10.0) + assert rate.sample(0, now_ns=1_100_000_000) == 0.0 + + +def test_disabled_desktop_mirror_does_not_collect_or_format_status(xr_module): + session = xr_module.OvLibrariesXrSession( + _FakeBootstrap(), + xr_module.OvLibrariesXrCfg(desktop_mirror_enabled=False), + ) + session._owner = SimpleNamespace(report_host_timing=lambda: pytest.fail("unexpected mirror work")) + + session.report_host_timing(42, 1.0 / 120.0) + + +def test_config_rejects_unknown_session_mode(): + with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): + OvLibrariesXrCfg(session_mode="automatic") + + +@pytest.mark.parametrize("scale", [0.0, -1.0, float("nan"), float("inf")]) +def test_config_rejects_invalid_scene_scale(scale): + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + OvLibrariesXrCfg(meters_per_unit=scale) + + +@pytest.mark.parametrize("camera_paths", [("/Camera/mono",), ("/Camera/left", "")]) +def test_config_rejects_non_stereo_camera_paths(camera_paths): + with pytest.raises(ValueError, match="exactly two"): + OvLibrariesXrCfg(camera_paths=camera_paths) + + +def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): + cfg = xr_module.OvLibrariesXrCfg() + cfg.meters_per_unit = float("nan") + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + xr_module.OvLibrariesXrSession.prepare(object(), cfg) + + +@pytest.mark.parametrize("count", [-1, 1.5, True]) +def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, monkeypatch, count): + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + with pytest.raises(ValueError, match="max_blank_frames must be a non-negative integer"): + session.start_render_worker(SimpleNamespace(), max_blank_frames=count) + assert bootstrap.events == [] + + +def test_three_auxiliary_camera_configs_prepare_the_same_bootstrap(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + cameras = tuple( + xr_module.OvLibrariesXrCameraCfg( + name=name, camera_cfg=SimpleNamespace(width=640, height=480), max_fps=hz, space="head_locked" + ) + for name, hz in zip(("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"), (20, 10, 10), strict=True) + ) + session = xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=cameras)) + assert bootstrap.auxiliary_camera_cfgs == [camera.camera_cfg for camera in cameras] + assert session is not None + extra = xr_module.OvLibrariesXrCameraCfg("overview", SimpleNamespace(width=640, height=480)) + with pytest.raises(ValueError, match="at most three"): + xr_module.OvLibrariesXrCfg(auxiliary_cameras=(*cameras, extra)) + + +def test_prepare_snapshots_camera_selection_without_mutating_caller(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + camera_cfg = SimpleNamespace(width=640, height=480) + camera = xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", camera_cfg) + cfg = xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,)) + session = xr_module.OvLibrariesXrSession.prepare(object(), cfg) + cfg.auxiliary_cameras = () + assert session._cfg.auxiliary_cameras == (camera,) + assert session._cfg.auxiliary_cameras[0].camera_cfg is bootstrap.auxiliary_camera_cfgs[0] + + +@pytest.mark.parametrize("fail_during", ["camera", "session"]) +@pytest.mark.parametrize("abort_fails", [False, True]) +def test_prepare_aborts_partial_bootstrap_and_preserves_first_error(xr_module, monkeypatch, fail_during, abort_fails): + bootstrap = _FakeBootstrap() + first_error = RuntimeError("preparation failed") + original_cause = ValueError("invalid session input") + first_error.add_note("Original preparation diagnostic") + abort_error = RuntimeError("abort failed") + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + + def fail(*_args, **_kwargs): + raise first_error from original_cause + + def abort(): + bootstrap.aborted = True + if abort_fails: + raise abort_error + + monkeypatch.setattr(bootstrap, "abort", abort) + if fail_during == "camera": + monkeypatch.setattr(bootstrap, "configure_auxiliary_camera", fail) + else: + monkeypatch.setattr(xr_module.OvLibrariesXrSession, "__init__", fail) + + cfg = xr_module.OvLibrariesXrCfg( + auxiliary_cameras=(xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=640, height=480)),), + ) + with pytest.raises(RuntimeError) as caught: + xr_module.OvLibrariesXrSession.prepare(object(), cfg) + + assert bootstrap.aborted + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert caught.value.__notes__ == ["Original preparation diagnostic"] + ( + [f"Cleanup after failed XR preparation also failed: {abort_error!r}"] if abort_fails else [] + ) + + +def test_public_xr_exports_resolve_to_implemented_types(xr_module): + import isaaclab_teleop + + assert isaaclab_teleop.OvLibrariesXrCfg is xr_module.OvLibrariesXrCfg + assert isaaclab_teleop.OvLibrariesXrCameraCfg is xr_module.OvLibrariesXrCameraCfg + assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession + + +def test_poisoned_publication_still_retires_in_capture_order(xr_module, monkeypatch, real_lease_type): + """After a failed scene write, Capture's OvxrFramePump.close must still retire the pair and the env.""" + host, bootstrap, env = _make_host(xr_module, monkeypatch) + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + native = object() + + def failed_transforms(): + raise ValueError("geometry callback failed") + + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _update_transforms=failed_transforms, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + bootstrap.publisher = lease.capture_scene_publisher() + # Stand in for clone preparation and Stage binding of the reserved pair. + renderer.backend = SimpleNamespace(renderer=native, attached=True, _render_lease_owner=renderer) + renderer.scene = SimpleNamespace(stage=stage, ordinal=10, _render_lease_owner=renderer) + renderer._initialized_scene = True + controller._generation = 1 + lease._bind(native, stage, 1) + bootstrap.lease = lease + # Attach proves the eye-camera scene partitions; this fake Stage has no partition tokens. + monkeypatch.setattr(controller, "configure_camera_partitions", lambda *_args: "env_0") + + def close_bootstrap(): + lease.release() + bootstrap.closed = True + + bootstrap.close = close_bootstrap + host.attach_on_render_thread(host.offer_render_owner_transfer()) + with host.publication_guard(), pytest.raises(RuntimeError, match="recreate the exact pair"): + host.publish_complete_scene("step-1") + + # OvxrFramePump.close(): close publication, destroy the context in the owner lane, then retire. + with host.publication_guard(): + host.close_publication() + with host.publication_guard(): + with host.owner_lane(): + pass + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.closed + assert controller.active is None + assert bootstrap.events[-1] == "environment-close" + + +class _FakeAnchorState: + """Minimal :class:`OvLibrariesXrSession.ExternalAnchorState` with scripted epochs.""" + + external_anchor_paths = ("/World/XrAnchor",) + external_reference_path = "/World/Robot" + + def __init__(self): + self.epoch = (0, True) + self.commits = [] + self.accept = True + + @property + def external_epoch(self): + return self.epoch + + def commit_external_pose(self, reference, rendering_dt, epoch, commit): + if not self.accept or epoch != self.epoch[0]: + return False + import numpy as np + + self.commits.append((reference.copy(), rendering_dt, epoch)) + commit(reference[np.newaxis]) + self.epoch = (self.epoch[0], False) + return True + + +def _make_anchor_host(xr_module, monkeypatch, anchor): + import numpy as np + + bootstrap = _FakeBootstrap() + bootstrap.get_world_transform = lambda _path: np.eye(4) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrCfg(), external_anchor_state=anchor + ) + host._scene_publisher = bootstrap.publisher + host._ordinal_ledger = _FakeOrdinalLedger(None) + writes = [] + monkeypatch.setattr(host, "_write_anchor_matrices", lambda matrices, ordinal: writes.append(ordinal)) + return host, writes + + +def test_anchor_seed_is_sealed_with_scene_and_frames_use_the_xr_clock(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + times = iter((5.0, 5.25)) + monkeypatch.setattr(xr_module.time, "monotonic", lambda: next(times)) + + seed = host.publish_complete_scene("initial") + assert writes == [seed] == host._ordinal_ledger.published + assert anchor.commits[-1][1] == 0.0 # The seed has no elapsed interval. + host.prepare_frame_scene() + host.prepare_frame_scene() + assert [commit[1] for commit in anchor.commits] == [0.0, 0.0, 0.25] + assert writes == host._ordinal_ledger.published + + +def test_anchor_epoch_change_during_seed_fails_closed(xr_module, monkeypatch): + anchor = _FakeAnchorState() + anchor.accept = False + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + with pytest.raises(RuntimeError, match="anchor reset changed"): + host.publish_complete_scene("initial") + assert writes == [] + assert host._ordinal_ledger.abandoned == [11] + with pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("initial") + + +def test_anchor_state_readoption_requires_the_same_paths(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, _writes = _make_anchor_host(xr_module, monkeypatch, anchor) + host.publish_complete_scene("initial") + replacement = _FakeAnchorState() + host.set_external_anchor_state(replacement) + assert host._anchor_snapshot is None and host._last_anchor_frame_time is None + moved = _FakeAnchorState() + moved.external_anchor_paths = ("/World/Other",) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(moved) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(None) + + +def test_start_rejects_anchor_state_without_the_protocol_before_owner_creation(xr_module, monkeypatch): + """A policy lacking the external-anchor API must fail on the caller, before the worker claims the lease.""" + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + with pytest.raises(TypeError, match="ExternalAnchorState"): + session.start_render_worker(SimpleNamespace(), external_anchor_state=SimpleNamespace(reset=lambda: None)) + assert bootstrap.events == [] + assert isinstance(_FakeAnchorState(), xr_module.OvLibrariesXrSession.ExternalAnchorState) + + +@pytest.mark.parametrize("reference_path", [None, "World/Robot"]) +def test_start_rejects_relative_anchor_reference_before_owner_creation(xr_module, monkeypatch, reference_path): + """Publication reads the reference prim on the worker; a missing path must fail before the lease is claimed.""" + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + anchor = _FakeAnchorState() + anchor.external_reference_path = reference_path + with pytest.raises(ValueError, match="absolute prim path"): + session.start_render_worker(SimpleNamespace(), external_anchor_state=anchor) + assert bootstrap.events == [] + + +def test_config_imports_without_native_ovxr(): + """Launchers build the configuration in a fresh process where Capture and the native OV packages are absent.""" + script = ( + "import sys\n" + "for name in ('isaaccapture', 'ovxr', 'ovrtx', 'ovstage'):\n" + " sys.modules[name] = None\n" + "from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg\n" + "assert OvLibrariesXrCfg(session_mode='vr').session_mode == 'vr'\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py new file mode 100644 index 000000000000..201dae8ac90f --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -0,0 +1,198 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Native scene publication and renderer ownership transfer.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from dataclasses import dataclass +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1", + reason="native OVRTX transfer harness is explicitly opt-in", + ), +] + +_TASK = "Isaac-Cartpole" +"""Manager-based task with an ``ovphysx`` physics preset and no camera sensors.""" + +_TRANSFER_DEVICES = ("cuda:0", "cpu") +"""Simulation devices for the transfer case; CPU physics publishes host scene tensors.""" + +_GR1T2_TASK = "IsaacContrib-PickPlace-GR1T2-Abs" +"""Humanoid task whose registered entry point prepares it for kitless OvPhysX.""" + + +@dataclass +class _CaseResult: + offer_thread: int + claim_thread: int | None = None + renderer_id: int | None = None + stage_id: int | None = None + ordinal: int | None = None + error: BaseException | None = None + + +def _make_env_and_bootstrap(task: str, device: str): + import gymnasium as gym + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop import remove_camera_configs + from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap + + import isaaclab_tasks # noqa: F401 -- registers the task + from isaaclab_tasks.utils.hydra import resolve_presets + from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry + + # parse_env_cfg resolves every preset to its default, so select OvPhysX while resolving. + cfg = resolve_presets(load_cfg_from_registry(task, "env_cfg_entry_point"), selected=("ovphysx",)) + assert isinstance(cfg.sim.physics, OvPhysxCfg) + cfg.sim.device = device + cfg.scene.num_envs = 1 + # Like an --ovxr launch, keep task cameras off the leased renderer. + cfg = remove_camera_configs(cfg) + bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) + try: + return bootstrap, gym.make(task, cfg=cfg).unwrapped + except BaseException: + bootstrap.abort() + raise + + +def _run_case(task: str, device: str, *, transfer: bool) -> _CaseResult: + from isaaclab_teleop.ov_libraries_xr import _IsaacLabOvxrApplicationInterface + from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg + + bootstrap, env = _make_env_and_bootstrap(task, device) + result = _CaseResult(offer_thread=threading.get_native_id()) + host = _IsaacLabOvxrApplicationInterface(bootstrap, env, OvLibrariesXrCfg(auto_launch_cloudxr=False)) + capability = host.offer_render_owner_transfer() + ready, published = threading.Event(), threading.Event() + + def attach(): + binding = host.attach_on_render_thread(capability) + result.renderer_id, result.stage_id = id(binding.renderer), id(binding.stage) + return binding + + def update_stage(binding): + with host.publication_guard(), host.owner_lane(): + binding.renderer.update_from_stage(result.ordinal) + + def close(): + host.close_publication() + try: + host.close_on_render_thread() + finally: + host.close_environment_on_render_thread() + + if not transfer: + try: + binding = attach() + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-control", 0)) + update_stage(binding) + finally: + close() + return result + + with pytest.raises(RuntimeError, match="transfer is pending"): + _ = bootstrap._lease.ovstage + + def worker(): + try: + result.claim_thread = threading.get_native_id() + binding = attach() + ready.set() + if not published.wait(60): + raise TimeoutError("application did not complete its scene publication") + if result.error is None: + update_stage(binding) + with pytest.raises(RuntimeError, match="capability is invalid"): + bootstrap.claim_render_owner_transfer(capability) + except BaseException as exc: + if result.error is None: + result.error = exc + finally: + ready.set() + try: + close() + except BaseException as exc: + if result.error is None: + result.error = exc + + thread = threading.Thread(target=worker, name="ovrtx-native-transfer-worker") + thread.start() + try: + if not ready.wait(90): + raise TimeoutError("renderer claimant did not finish binding") + if result.error is not None: + raise result.error + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-transfer", 0)) + except BaseException as exc: + result.error = exc + finally: + published.set() + thread.join(timeout=90) + assert not thread.is_alive(), "native transfer worker timed out" + if result.error is not None: + raise result.error + assert result.claim_thread is not None and result.claim_thread != result.offer_thread + with pytest.raises(RuntimeError, match="(owner thread|released)"): + _ = bootstrap._lease.ovstage + return result + + +def _run_native_subprocess(task: str, device: str, transfer: bool) -> None: + # A native non-daemon thread cannot be bounded by Thread.join alone. + # Keep the native lifetime in a child that the standard subprocess timeout owns. + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), task, device, "transfer" if transfer else "same-thread"], + env={**os.environ, "ISAAC_LAB_OVRTX_USE_OVSTAGE": "1"}, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + print(completed.stdout) + assert completed.returncode == 0, completed.stderr + + +def test_same_thread_ovxr_session_native_control(): + """One-thread production scene publication remains viable.""" + _run_native_subprocess(_TASK, "cuda:0", False) + + +@pytest.mark.parametrize("device", _TRANSFER_DEVICES) +def test_cross_thread_preuse_owner_transfer_native(device): + """The application publishes while the render thread binds, renders and closes the pair.""" + _run_native_subprocess(_TASK, device, True) + + +@pytest.mark.parametrize("transfer", [False, True], ids=["same-thread", "transfer"]) +def test_gr1t2_cpu_ovphysx_native(transfer): + """The prepared GR1T2 scene publishes from CPU OvPhysX, with and without the owner transfer.""" + _run_native_subprocess(_GR1T2_TASK, "cpu", transfer) + + +if __name__ == "__main__": + if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": + raise RuntimeError("native transfer harness requires its explicit opt-in") + result = _run_case(sys.argv[1], sys.argv[2], transfer=sys.argv[3] == "transfer") + assert result.renderer_id is not None and result.stage_id is not None + assert result.ordinal is not None + print( + f"offer={result.offer_thread} claim={result.claim_thread} " + f"renderer={result.renderer_id} stage={result.stage_id} ordinal={result.ordinal}" + ) diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py new file mode 100644 index 000000000000..30be9dad763f --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""OVXR bootstrap retirement across the teleoperation/renderer boundary.""" + +import builtins +from types import SimpleNamespace + +import pytest + +from isaaclab.sim import SimulationCfg + +pytest.importorskip("ovrtx") +from isaaclab_ov.renderers import OVRTXRenderer, OVRTXRendererCfg + + +@pytest.mark.parametrize("operation", ["close", "abort"]) +def test_bootstrap_failed_transfer_cancellation_keeps_owner_retryable(bootstrap, operation): + """A wrong-thread close cannot destroy a pending lease or prevent its owner retiring it.""" + import threading + + from isaaclab_teleop import ovxr_session + + bootstrap.offer_render_owner_transfer() + context = ovxr_session.SimulationContext.instance() + errors = [] + + def wrong_owner(): + try: + getattr(bootstrap, operation)() + except RuntimeError as error: + errors.append(str(error)) + + thread = threading.Thread(target=wrong_owner) + thread.start() + thread.join(5) + assert not thread.is_alive() + assert len(errors) == 1 and "owner thread must cancel" in errors[0] + assert not bootstrap.closed + assert ovxr_session.SimulationContext.instance() is context + # The real owner must still be able to retire the real renderer lease. + getattr(bootstrap, operation)() + assert bootstrap.closed + assert ovxr_session.SimulationContext.instance() is (None if operation == "abort" else context) + + +@pytest.fixture +def bootstrap(monkeypatch): + """Run the real bootstrap constructor and renderer lease; only the physics-backed context is replaced.""" + from isaaclab_teleop import ovxr_session + + from isaaclab.app import get_settings_manager + from isaaclab.renderers import RenderContext + + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + monkeypatch.delattr(builtins, ovxr_session._TERMINAL_MODE_SENTINEL, raising=False) + renderer = OVRTXRenderer( + OVRTXRendererCfg(), sim=SimpleNamespace(device="cpu", get_scene_data_provider=lambda: None) + ) + + class Context: + current = None + + def __init__(self, cfg): + self.cfg = cfg + self.render_context = RenderContext([(renderer.cfg, renderer)]) + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + cls.current = None + + def get_or_create_backend(self, _cfg): + return renderer + + monkeypatch.setattr(ovxr_session, "SimulationContext", Context) + # SimulationContext resolves a bare "cuda" device from this setting. + settings = get_settings_manager() + previous_cuda_device = settings.get("/physics/cudaDevice") + settings.set("/physics/cudaDevice", 1) + try: + bootstrap = ovxr_session.OvxrSessionBootstrap(SimulationCfg(device="cuda:1", dt=0.01)) + yield bootstrap + bootstrap.abort() + assert renderer._render_lease.active is None + finally: + settings.set("/physics/cudaDevice", previous_cuda_device) + + +def _environment(bootstrap, *, num_envs=1, device="cuda:1", dt=0.01): + return SimpleNamespace( + sim=bootstrap.simulation_context, + scene=SimpleNamespace(stage=object()), + num_envs=num_envs, + cfg=SimpleNamespace(sim=SimulationCfg(device=device, dt=dt)), + ) + + +@pytest.mark.parametrize( + ("num_envs", "device", "dt", "error"), + [ + (1, "cuda", 0.01, None), + (2, "cuda:1", 0.01, "exactly one environment"), + (1, "cuda:0", 0.01, r"sim\.device='cuda:0'"), + (1, "cuda:1", 0.02, r"sim\.dt=0\.02"), + ], +) +def test_environment_validation_resolves_devices_and_requires_one_environment(bootstrap, num_envs, device, dt, error): + """The caller-side check resolves a bare CUDA device like SimulationContext and rejects late worker failures.""" + env = _environment(bootstrap, num_envs=num_envs, device=device, dt=dt) + if error is None: + bootstrap.validate_environment(env) + else: + with pytest.raises(RuntimeError, match=error): + bootstrap.validate_environment(env) + + +def test_binding_requires_the_claimed_owner_transfer(bootstrap): + """Binding before the claim names the missing claim, even when no camera deferred the Stage yet.""" + with pytest.raises(RuntimeError, match="requires a claimed owner transfer"): + bootstrap.bind_environment(_environment(bootstrap)) diff --git a/source/isaaclab_teleop/test/test_session_step_failure.py b/source/isaaclab_teleop/test/test_session_step_failure.py index ad33e1e7a5a2..8897c72b4c73 100644 --- a/source/isaaclab_teleop/test/test_session_step_failure.py +++ b/source/isaaclab_teleop/test/test_session_step_failure.py @@ -122,9 +122,9 @@ def _make_lifecycle(**kwargs) -> TeleopSessionLifecycle: return TeleopSessionLifecycle(cfg, **kwargs) -def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: +def _make_failing_lifecycle(error: Exception, **kwargs) -> TeleopSessionLifecycle: """Lifecycle with a live session mock whose step raises *error*.""" - lifecycle = _make_lifecycle() + lifecycle = _make_lifecycle(**kwargs) lifecycle._pipeline = object() session = MagicMock() session.step.side_effect = error @@ -133,6 +133,144 @@ def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: class TestStepFailureDiagnosis: + @pytest.mark.parametrize( + "error_type,replay", [(RuntimeError, False), (KeyboardInterrupt, False), (RuntimeError, True)] + ) + def test_failed_drain_retains_owner_and_original_step_error(self, error_type, replay): + step_error = ValueError("retarget worker failed") + drain_error = error_type("input cleanup failed") + lifecycle = _make_failing_lifecycle( + step_error, + use_kit_xr_bridge=False, + external_oxr_handles_provider=None if replay else lambda: object(), + ovxr_session=replay, + mcap_replay_path="recording.mcap" if replay else None, + ) + session = lifecycle._session + session.__exit__.side_effect = drain_error + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + pytest.raises(ValueError, match="retarget worker failed") as raised, + ): + lifecycle.step() + assert raised.value is step_error + assert raised.value.__cause__ is drain_error + assert lifecycle._session is session + assert any("input cleanup failed" in note for note in step_error.__notes__) + notes = tuple(step_error.__notes__) + lifecycle.stop(type(step_error), step_error, step_error.__traceback__) + assert tuple(step_error.__notes__) == notes + + def test_replay_step_failure_is_not_restarted_from_the_beginning(self): + error = ValueError("invalid recorded input") + lifecycle = _make_failing_lifecycle(error) + lifecycle._is_replay = True + session = lifecycle._session + with ( + patch.object(lifecycle, "_build_external_inputs", return_value=None), + pytest.raises(ValueError, match="invalid recorded input") as raised, + ): + lifecycle.step() + assert raised.value is error + session.__exit__.assert_called_once() + assert lifecycle._session is None + + def test_kit_replay_exit_error_is_suppressed_without_latching(self): + """Kit replay keeps the Kit teardown suppression; only the step failure ends the replay.""" + error = ValueError("invalid recorded input") + lifecycle = _make_failing_lifecycle(error, mcap_replay_path="recording.mcap") + session = lifecycle._session + session.__exit__.side_effect = RuntimeError("XR_ERROR_HANDLE_INVALID") + with ( + patch.object(lifecycle, "_build_external_inputs", return_value=None), + pytest.raises(ValueError, match="invalid recorded input") as raised, + ): + lifecycle.step() + assert raised.value is error + assert raised.value.__cause__ is None + session.__exit__.assert_called_once() + assert lifecycle._session is None + assert not lifecycle.session_cleanup_failed + + def test_stop_ar_exit_error_without_borrowed_handles_allows_restart(self): + """Kit "Stop AR" invalidates the handles; a failed exit is suppressed so Start AR can restart.""" + lifecycle = _make_failing_lifecycle(RuntimeError("XR_ERROR_INSTANCE_LOST")) + session = lifecycle._session + session.__exit__.side_effect = RuntimeError("XR_ERROR_HANDLE_INVALID") + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + session.__exit__.assert_called_once() + assert lifecycle._session is None + assert not lifecycle.session_cleanup_failed + with patch.object(lifecycle, "_try_start_session", return_value=False) as try_start: + assert lifecycle.step() is None + try_start.assert_called_once() + + @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) + def test_terminal_exit_failure_is_latched_across_callbacks_and_restart(self, error_type): + lifecycle = _make_lifecycle(use_kit_xr_bridge=False, external_oxr_handles_provider=lambda: object()) + lifecycle._pipeline = object() + error = error_type("terminal native cleanup failed") + + class TerminalSession: + def __init__(self): + self.entered = True + self.steps = 0 + self.exits = 0 + + def __exit__(self, *_exc): + self.exits += 1 + if self.entered: + self.entered = False + raise error + + def step(self, **_kwargs): + self.steps += 1 + raise AssertionError("must not step a terminal session") + + session = TerminalSession() + lifecycle._session = session + with pytest.raises(error_type) as raised: + lifecycle._on_pre_shutdown(None) + assert raised.value is error + assert lifecycle.session_cleanup_failed + for operation in (lifecycle.step, lifecycle.step, lifecycle.try_start_session, lifecycle.start, lifecycle.stop): + with pytest.raises(error_type) as raised: + operation() + assert raised.value is error + assert session.exits == 1 + assert session.steps == 0 + assert lifecycle.session_cleanup_failed + assert lifecycle._session is session + + def test_failed_submission_keeps_pending_host_reset_for_reentry(self): + lifecycle = _make_failing_lifecycle(ValueError("persistent pipeline error")) + lifecycle._pending_reset = True + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + assert lifecycle._pending_reset + + def test_ovxr_session_pipeline_failure_also_bounds_restart_rate(self): + lifecycle = _make_failing_lifecycle(ValueError("persistent pipeline error")) + lifecycle._external_oxr_handles_provider = lambda: object() + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + assert lifecycle._restart_holdoff_until > time.monotonic() + with patch.object(lifecycle, "_try_start_session") as try_start: + for _ in range(5): + assert lifecycle.step() is None + try_start.assert_not_called() + def test_pipeline_error_with_active_xr_sets_holdoff(self): lifecycle = _make_failing_lifecycle(ValueError("Found zero norm quaternions in `quat`.")) session = lifecycle._session diff --git a/source/isaaclab_teleop/test/test_target_frame_rebase.py b/source/isaaclab_teleop/test/test_target_frame_rebase.py index f4b523f68a70..b2b1f54b558a 100644 --- a/source/isaaclab_teleop/test/test_target_frame_rebase.py +++ b/source/isaaclab_teleop/test/test_target_frame_rebase.py @@ -178,3 +178,38 @@ def test_advance_selects_target_frame(configured, explicit, expected): assert step_kwargs["target_T_world"] is expected else: np.testing.assert_allclose(step_kwargs["target_T_world"], expected, atol=1e-6) + + +def test_external_replay_reads_resolved_target_frame(monkeypatch): + """A cloned target path reaches the scene reader without mutating the task template.""" + from types import SimpleNamespace + + from isaaclab_teleop import IsaacTeleopCfg + from isaaclab_teleop.entrypoints.teleop_replay_agent import _advance_replay_input + from isaaclab_teleop.isaac_teleop_device import create_isaac_teleop_device + + from pxr import Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + frame = UsdGeom.Xform.Define(stage, "/Lab/Cells/Cell0/Robot/base") + frame.AddTranslateOp().Set(Gf.Vec3d(4, 5, 6)) + cfg = IsaacTeleopCfg(target_frame_prim_path="{ENV_REGEX_NS}/Robot/base", sim_device="cpu") + lifecycle = MagicMock() + monkeypatch.setattr("isaaclab_teleop.isaac_teleop_device.TeleopSessionLifecycle", lambda *a, **k: lifecycle) + monkeypatch.setattr("isaaclab_teleop.isaac_teleop_device.XrAnchorManager", lambda *a, **k: MagicMock()) + device = create_isaac_teleop_device( + cfg, env_prim_path="/Lab/Cells/Cell0", use_kit_xr_bridge=False, ovxr_session=True + ) + device._dispatch_control_callbacks = lambda: None + + def read_target(path): + prim = stage.GetPrimAtPath(path) + assert prim.IsValid(), f"Target path did not resolve to the scene: {path}" + return np.asarray(UsdGeom.XformCache().GetLocalToWorldTransform(prim).GetInverse()).T + + presentation = SimpleNamespace(session=SimpleNamespace(get_target_frame_T_world=read_target)) + _advance_replay_input(device, presentation) + expected = np.eye(4) + expected[:3, 3] = [-4, -5, -6] + np.testing.assert_allclose(lifecycle.step.call_args.kwargs["target_T_world"], expected) + assert cfg.target_frame_prim_path == "{ENV_REGEX_NS}/Robot/base" diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py new file mode 100644 index 000000000000..71c60c0acaa0 --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -0,0 +1,1232 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""In-process teleoperation entrypoint, configuration, and control-loop contracts.""" + +import os +import signal +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +# Repository root: this file lives at source/isaaclab_teleop/test/.py +ROOT = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize( + ("kitless", "selection", "expected", "reject"), + [ + (True, None, None, False), + (True, " 1 ", " 1 ", False), + (True, "0", "0", True), + (True, "invalid", "invalid", True), + (False, None, None, False), + ], +) +@pytest.mark.parametrize("with_preparation", [False, True]) +def test_teleop_ovxr_validates_external_stage_without_changing_environment( + monkeypatch, kitless, selection, expected, reject, with_preparation +): + """The task resolver preserves the process setting and rejects incompatible explicit modes before launch.""" + from types import SimpleNamespace + + from isaaclab_teleop import ( + IsaacTeleopCfg, + se3_gamepad_teleop_cfg, + se3_keyboard_teleop_cfg, + se3_spacemouse_teleop_cfg, + ) + from isaaclab_teleop.device_selection import select_teleop_cfg + + from isaaclab.devices import DeviceCfg + + from isaaclab_tasks.utils import hydra + + monkeypatch.setattr(hydra, "enumerate_task_presets", lambda _task: None) + name = "ISAAC_LAB_OVRTX_USE_OVSTAGE" + if selection is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, selection) + preparations = [] + from isaaclab_ov.physics import OvPhysxCfg + + config = SimpleNamespace( + sim=SimpleNamespace(device="cpu", physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1), + terminations=SimpleNamespace(time_out=1), + isaac_teleop=IsaacTeleopCfg( + prepare_environment=(lambda selected: preparations.append(selected)) if with_preparation else None, + ), + ) + scans = [] + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + namespace = { + "__package__": "isaaclab_teleop.entrypoints", + "IsaacTeleopCfg": IsaacTeleopCfg, + "DeviceCfg": DeviceCfg, + "select_teleop_cfg": select_teleop_cfg, + "se3_keyboard_teleop_cfg": se3_keyboard_teleop_cfg, + "se3_spacemouse_teleop_cfg": se3_spacemouse_teleop_cfg, + "se3_gamepad_teleop_cfg": se3_gamepad_teleop_cfg, + "os": os, + "sys": sys, + "ManagerBasedRLEnvCfg": SimpleNamespace, + "resolve_task_config": lambda *_args, **_kwargs: (config, None), + "validate_launch": lambda *_args: (scans.append(os.environ.get(name)) or ((), ())), + "args_cli": SimpleNamespace( + task="Test", + device="cpu", + num_envs=1, + teleop_device=None, + sensitivity=1.0, + xr=False, + using_ovxr=kitless, + ovxr_pip_camera=None, + disable_external_cameras=False, + physics=None, + ), + } + for key, value in namespace.items(): + monkeypatch.setattr(agent, key, value, raising=False) + if reject: + with pytest.raises(ValueError, match="requires.*OVStage"): + agent._resolve_env_cfg() + assert scans == [] + else: + agent._resolve_env_cfg() + assert scans == [expected] + assert os.environ.get(name) == expected + + assert preparations == ([config] if with_preparation and not reject else []) + + +@pytest.fixture +def run_live_workflow(monkeypatch): + """Run the imported live owner with finite input, renderer, and simulation endpoints.""" + import contextlib + + # The live OVXR owner imports the session adapter, which needs a Capture release with OVXR + # session support. The Kit ``teleop`` extra selects an older release; skip rather than error there. + pytest.importorskip("isaaccapture.ovxr_session_manager") + import isaaclab_teleop + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + def run( + env, interface, session, *, conditioner=None, validator=None, step_hz=0.0, refresh=lambda: None, started=True + ): + class Device: + target_frame_prim_path = None + xr_render_anchor_path = None + required_openxr_extensions = () + external_anchor_state = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def advance(self): + return interface.advance() + + def reset(self): + return interface.reset() + + @property + def last_control_events(self): + return interface.control_events() + + camera = SimpleNamespace(bind=lambda _env: contextlib.nullcontext(), refresh=refresh, close=lambda: None) + session.external_oxr_handles = None + session.start_render_worker = lambda *_args, **_kwargs: started + vars(session).setdefault("close", lambda: None) + session.async_worker_owns_environment = False + vars(env).setdefault("close", lambda: None) + env.unwrapped = env + env.scene = SimpleNamespace(env_prim_paths=["/World/envs/env_0"]) + env.sim.device = "cpu" + env.sim.visualizers = () + cfg = SimpleNamespace( + sim=object(), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + action_conditioner=conditioner, + action_validator=validator, + ), + ) + monkeypatch.setattr( + agent, + "args_cli", + agent._build_parser().parse_args(["--task", "Test", "--ovxr", "--step_hz", str(step_hz)]), + raising=False, + ) + monkeypatch.setattr(agent, "has_kit", lambda: False) + monkeypatch.setattr(agent.gym, "make", lambda *_args, **_kwargs: SimpleNamespace(unwrapped=env)) + monkeypatch.setattr(agent, "create_teleop_device", lambda *_args, **_kwargs: Device()) + monkeypatch.setattr(isaaclab_teleop, "create_haptic_feedback_driver", lambda *_args: None) + monkeypatch.setattr(agent, "_install_ovxr_shutdown_handlers", lambda *_args: {}) + monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", lambda *_args, **_kwargs: camera) + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", lambda *_args, **_kwargs: session) + agent.run_teleoperation(cfg, cfg.isaac_teleop, True, ()) + + return run + + +def test_teleop_se3_agent_yields_only_when_ovxr_is_idle(monkeypatch): + """A paused kitless loop must yield without slowing Kit or active physics frames.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + sleeps = [] + monkeypatch.setattr(time, "sleep", sleeps.append) + monkeypatch.setattr(time, "perf_counter", lambda: 10.003) + yield_idle = agent._pace_ovxr_control_loop + + timing = {"iteration_started_s": 10.0, "action_period_s": 0.02} + yield_idle(using_ovxr=False, action_applied=False, **timing) + yield_idle(using_ovxr=True, action_applied=True, **timing) + assert sleeps == [] + + yield_idle(using_ovxr=True, action_applied=False, **timing) + assert sleeps == pytest.approx([0.017]) + yield_idle(using_ovxr=True, action_applied=False, iteration_started_s=9.9, action_period_s=0.02) + assert sleeps == pytest.approx([0.017]) # Slow input work needs no added delay. + + +@pytest.mark.parametrize( + "active,step_hz,expected_polls", + [(False, 0.0, 6), (True, 25.0, 3), (None, 25.0, 3)], +) +def test_kitless_loop_respects_control_rate_and_processes_reset( + monkeypatch, run_live_workflow, active, step_hz, expected_polls +): + """The real caller preserves idle pacing and applies an explicit active limit.""" + from types import SimpleNamespace + + import torch + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + clock = 0.0 + advances = 0 + resets = [] + refreshes = [] + publications = [] + applied = [] + seeded_from = [] + stepped_values = [] + sequence = active is None + anchor_ready = False + + def reset(): + nonlocal anchor_ready + resets.append(clock) + anchor_ready = False + + def request(identity, *, force=False): + nonlocal anchor_ready + publications.append((identity, {"force": force})) + anchor_ready = True + return 1 + + def sleep(duration): + nonlocal clock + clock += duration + + def advance(): + nonlocal advances, clock + assert anchor_ready, "Input cannot read an unseeded post-reset anchor" + advances += 1 + clock += 0.003 # Caller work consumes part of the independent 20ms action budget. + return torch.ones((1, 36)) if active or (sequence and advances != 4) else None + + def step(value): + applied.append(clock) + stepped_values.append(value.clone()) + + def condition(_env, value, state): + previous = state.get("frames", 0) + seeded_from.append(previous) + state["frames"] = previous + 1 + return value + previous + 2 + + sim = SimpleNamespace( + is_running=lambda: clock < (0.199 if sequence else 0.119), + get_physics_step_count=lambda: 0, + get_physics_dt=lambda: 0.005, + render=lambda: None, + ) + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: clock, sleep=sleep)) + run_live_workflow( + SimpleNamespace(sim=sim, reset=reset, step_dt=0.02, num_envs=1, step=step), + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace( + is_active=advances not in (2, 4) if sequence else active, + should_reset=advances == (5 if sequence else 3), + ), + ), + SimpleNamespace( + shutdown_requested=False, + wait_for_shutdown=sleep, + request_render=request, + report_host_timing=lambda *args: None, + ), + conditioner=condition, + validator=lambda value: None, + step_hz=step_hz, + refresh=lambda: refreshes.append(clock), + ) + + assert advances == (5 if sequence else expected_polls) + assert clock == pytest.approx( + 0.20 if sequence else 0.12 + ) # Work time is credited, rather than adding a full-period sleep. + assert len(applied) == (3 if sequence else advances if active else 0) + assert len(resets) == 2 # Initial reset plus the public control event. + assert len(refreshes) == 1 + assert len(publications) == (4 if sequence else advances + 1 if active else 2) + assert publications[-1][1] == {"force": True} + + if sequence: + # Both STOP branches clear an established ramp before the next accepted action. + assert seeded_from == [0, 0, 0] + for value in stepped_values: + torch.testing.assert_close(value, torch.full((1, 36), 3.0)) + elif active: + assert seeded_from == [0, 1, 2] + for index, value in enumerate(stepped_values): + torch.testing.assert_close(value, torch.full((1, 36), 3.0 + index)) + + +def test_paused_loop_retries_the_latest_unaccepted_publication(monkeypatch, run_live_workflow): + """A paused iteration retries the rejected latest state without another physics step.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + advances = 0 + steps = [] + requests = [] + sleeps = [] + reports = [] + session = SimpleNamespace(shutdown_requested=False) + + def advance(): + nonlocal advances + advances += 1 + return torch.ones(1) + + def request(identity, *, force): + requests.append((identity, force)) + if force: + return 1 + if len(requests) == 2: + return None + session.shutdown_requested = True + return 2 + + def sleep(duration): + sleeps.append((duration, len(requests))) + + session.request_render = request + session.report_host_timing = lambda step, dt: reports.append((step, dt, len(requests))) + session.wait_for_shutdown = sleep + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: 0.0, sleep=sleep)) + env = SimpleNamespace( + num_envs=1, + step_dt=0.02, + reset=lambda: None, + step=lambda action: steps.append(action), + sim=SimpleNamespace( + is_running=lambda: advances < 4, get_physics_step_count=lambda: len(steps), get_physics_dt=lambda: 0.005 + ), + ) + run_live_workflow( + env, + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace(is_active=advances == 1, should_reset=False), + ), + session, + ) + assert advances == 2 and len(steps) == 1 + assert len(requests) == 3 and requests[0][1] is True + assert requests[1:] == [((1, 1), False), ((1, 1), False)] + assert reports == [(1, 0.005, 2), (1, 0.005, 3)] + assert sleeps == [(0.02, 3)] # Pacing follows the retried publication. + + +def test_live_render_worker_startup_failure_skips_input_and_drains(run_live_workflow): + """A session that shuts down during worker startup runs no input and still closes before the environment.""" + events = [] + env = SimpleNamespace( + sim=SimpleNamespace(is_running=lambda: True), + reset=lambda: events.append("env-reset"), + close=lambda: events.append("env-close"), + ) + interface = SimpleNamespace(advance=lambda: events.append("advance"), reset=lambda: None) + session = SimpleNamespace(shutdown_requested=False, close=lambda: events.append("xr-close")) + run_live_workflow(env, interface, session, started=False) + assert events == ["xr-close", "env-close"] + + +@pytest.mark.parametrize("selector", [["--physics", "ovphysx"], ["physics=ovphysx"]]) +@pytest.mark.parametrize("ovxr", [False, True]) +def test_teleop_replay_agent_resolves_typed_presets_without_reapplying_physics(monkeypatch, selector, ovxr): + """Public CLI aliases reach task resolution without resetting backend settings.""" + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + + from isaaclab.managers import TerminationTermCfg + from isaaclab.sim import SimulationCfg + + from isaaclab_tasks.utils import parse_cfg + + source_path = ROOT / "source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py" + argv = [str(source_path), "--task", "Test", "--replay_file", "recording.mcap", *selector] + if ovxr: + argv.append("--ovxr") + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + monkeypatch.setattr(replay, "args_cli", None, raising=False) + monkeypatch.setattr(replay, "hydra_overrides", None, raising=False) + namespace = vars(replay) + replay._parse_args() + prepared_configs = [] + physics = OvPhysxCfg() + cfg = SimpleNamespace( + sim=SimulationCfg(physics=physics), + scene=SimpleNamespace(num_envs=1), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + prepare_environment=lambda selected: prepared_configs.append(selected) + ), + terminations=SimpleNamespace( + success=TerminationTermCfg(func=lambda env: False), + time_out=TerminationTermCfg(func=lambda env: False), + ), + ) + physics = cfg.sim.physics + resolutions = [] + + def resolve(task, agent, *, overrides): + resolutions.append((task, agent, overrides)) + return cfg, None + + monkeypatch.setattr(replay, "ManagerBasedRLEnvCfg", SimpleNamespace) + monkeypatch.setattr(parse_cfg, "resolve_task_config", resolve) + if not ovxr: + # Replay without --ovxr starts Kit, which cannot host standalone OvPhysX. + with pytest.raises(ValueError, match="Invalid backend combination"): + namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) + assert resolutions == [("test-task", None, ["physics=ovphysx"])] + assert not prepared_configs + return + prepared, success, cameras = namespace["_prepare_env_cfg"]( + "test-task", 1, None, overrides=namespace["hydra_overrides"] + ) + assert resolutions == [("test-task", None, ["physics=ovphysx"])] + assert prepared is cfg and success is not None and cameras == () + assert cfg.sim.physics is physics + assert namespace["args_cli"].require_kit is False + assert not namespace["args_cli"].start_on_replay + legacy_args = replay._build_parser().parse_args( + ["--task", "Test", "--replay_file", "recording.mcap", "--start_on_replay"] + ) + assert legacy_args.start_on_replay + assert not hasattr(namespace["args_cli"], "physics") + assert prepared_configs == [cfg] + assert "ISAAC_LAB_OVRTX_USE_OVSTAGE" not in os.environ + + +@pytest.mark.parametrize( + "backend,viewer,hard_reset", + [ + ("ovphysx", False, True), + ("ovphysx", True, True), + ("physx", False, True), + (None, False, True), + ("newton", False, False), + ("newton", True, False), + ], +) +def test_recorded_environment_reset_preserves_backend_policy_and_hand_cursor(monkeypatch, backend, viewer, hard_reset): + """The imported replay loop preserves backend reset order and the input cursor.""" + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + calls = [] + frame = -1 + + class Device: + target_frame_prim_path = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def reset(self): + calls.append("device-reset") + + def request_start(self): + calls.append("start") + + def advance(self): + nonlocal frame + frame += 1 + return torch.zeros(1) + + @property + def last_control_events(self): + return SimpleNamespace(is_active=False, should_reset=frame == 0) + + env = SimpleNamespace( + scene=SimpleNamespace(physics_backend=backend, env_prim_paths=["/World/envs/env_0"]), + sim=SimpleNamespace( + device="cpu", is_running=lambda: frame < 1, render=lambda: None, reset=lambda: calls.append("sim-reset") + ), + recorder_manager=SimpleNamespace(reset=lambda: calls.append("recorder-reset")), + reset=lambda: calls.append("env-reset"), + step_dt=0, + ) + presentation = ( + SimpleNamespace( + session=SimpleNamespace(shutdown_requested=False), + start=lambda *_args, **_kwargs: True, + publish=lambda *_args, **kwargs: calls.append("publish-reset") if kwargs.get("reset") else None, + ) + if viewer + else None + ) + monkeypatch.setattr( + replay, + "args_cli", + SimpleNamespace( + device="cpu", + replay_file="recording.mcap", + start_on_replay=True, + enable_debug_visualization=False, + max_replay_duration_s=1, + max_stage_load_wait_s=1, + ), + raising=False, + ) + monkeypatch.setattr(replay, "create_isaac_teleop_device", lambda *_args, **_kwargs: Device()) + monkeypatch.setattr(replay, "NvmlGpuStatsProvider", lambda: SimpleNamespace(summary=lambda: {})) + replay._run_single_replay(env, SimpleNamespace(), None, 1, 2, presentation) + initial = ["sim-reset"] if hard_reset else [] + initial += ["env-reset", "device-reset", "start"] + if viewer: + initial.append("publish-reset") + recorded = ["sim-reset"] if hard_reset else [] + recorded += ["recorder-reset", "env-reset", "device-reset", "start"] + if viewer: + recorded.append("publish-reset") + assert calls == initial + recorded + assert frame == 1 + + +def _load_teleop_agent(monkeypatch, *arguments): + from isaaclab_teleop.entrypoints import teleop_se3_agent as module + + monkeypatch.setattr(sys, "argv", ["teleop_se3_agent.py", *arguments]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + module._parse_args() + return module + + +def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch): + """Launch validation must not replace task-specific settings on the selected backend.""" + agent = _load_teleop_agent(monkeypatch) + from isaaclab_newton.physics import NewtonCfg + + from isaaclab.app.sim_launcher import validate_launch + + physics_cfg = NewtonCfg() + physics_cfg.solver_cfg.nconmax = 4096 + physics_cfg.solver_cfg.njmax = 1536 + env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)) + configs, _ = validate_launch(env_cfg, vars(agent.args_cli).copy()) + + assert not hasattr(agent.args_cli, "physics") + assert env_cfg.sim.physics is physics_cfg + assert any(cfg is physics_cfg for cfg in configs) + assert physics_cfg.solver_cfg.nconmax == 4096 + assert physics_cfg.solver_cfg.njmax == 1536 + + +@pytest.mark.parametrize( + "task_name,physics", + [ + ("IsaacContrib-PickPlace-GR1T2-Abs", "isaacsim_physx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "newton_mjwarp"), + ], +) +def test_teleop_launcher_prepares_humanoid_backends(monkeypatch, task_name, physics): + """The entrypoint runs the task's preparation hook for the selected standalone backend.""" + # Standalone backends run under --ovxr, which removes the task cameras before launch validation. + ovxr = () if physics == "isaacsim_physx" else ("--ovxr",) + agent = _load_teleop_agent(monkeypatch, "--task", task_name, "--physics", physics, *ovxr) + env_cfg, _teleop_cfg, _use_isaac_teleop, _selected_cameras = agent._resolve_env_cfg() + assert env_cfg._prepared_kitless_backend == (None if physics == "isaacsim_physx" else physics) + + +def test_task_action_hook_validates_before_and_after_conditioning(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + calls = [] + + def validate(action): + calls.append(("validate", action)) + return "rejected" if action in (3, 6) else None + + def condition(env, action, state): + calls.append(("condition", env, action, state)) + state["last"] = action + return action + 1 + + state = {} + action, issue = agent.prepare_teleop_action("env", 1, condition, validate, state) + assert (action, issue, state) == (2, None, {"last": 1}) + assert [call[0] for call in calls] == ["validate", "condition", "validate"] + + calls.clear() + action, issue = agent.prepare_teleop_action("env", 3, condition, validate, state) + assert (action, issue) == (3, "rejected") + assert [call[0] for call in calls] == ["validate"] + + # A conditioned action rejected by the second validation leaves the state uncommitted. + calls.clear() + action, issue = agent.prepare_teleop_action("env", 5, condition, validate, state) + assert (action, issue, state) == (6, "rejected", {"last": 1}) + assert [call[0] for call in calls] == ["validate", "condition", "validate"] + + +def test_teleop_agent_installs_late_graceful_shutdown_handlers(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + requests = [] + installed = {} + + class Session: + def request_shutdown(self): + requests.append("shutdown") + + def install(signum, handler): + installed[signum] = handler + return f"previous-{signum}" + + monkeypatch.setattr(agent.signal, "signal", install) + previous = agent._install_ovxr_shutdown_handlers(agent.OvxrPresentation(Session())) + + assert set(installed) == {signal.SIGINT, signal.SIGTERM} + installed[signal.SIGINT](signal.SIGINT, None) + assert requests == ["shutdown"] + + restored = {} + monkeypatch.setattr(agent.signal, "signal", lambda signum, handler: restored.update({signum: handler})) + agent._restore_ovxr_shutdown_handlers(previous) + assert restored == previous + + +def test_ovxr_presentation_starts_once_and_adopts_later_input_anchors(): + """A failed start leaves the worker unstarted; a started worker adopts each later device's anchor.""" + from unittest.mock import Mock + + from isaaclab_teleop.entrypoints._ovxr_presentation import OvxrPresentation + + session = Mock() + session.start_render_worker.return_value = False + presentation = OvxrPresentation(session, excluded_visual_meshes=("/Hidden",)) + presentation.input_device = SimpleNamespace(xr_render_anchor_path="/Anchor", external_anchor_state="first") + assert presentation.start("env", timeout_s=5.0, required_openxr_extensions=("XR_ext",)) is False + assert presentation.started is False + session.start_render_worker.assert_called_once_with( + "env", + tracking_anchor_path="/Anchor", + required_openxr_extensions=("XR_ext",), + excluded_visual_meshes=("/Hidden",), + external_anchor_state="first", + timeout_s=5.0, + ) + + session.start_render_worker.return_value = True + assert presentation.start("env", timeout_s=5.0) + presentation.input_device = SimpleNamespace(external_anchor_state="second") + assert presentation.start("env", timeout_s=5.0) + assert session.start_render_worker.call_count == 2 + session.set_external_anchor_state.assert_called_once_with("second") + + +def test_ovxr_presentation_retains_a_forced_reset_until_admitted(): + """A reset publication stays forced across declined requests; later changes publish unforced.""" + from isaaclab_teleop.entrypoints._ovxr_presentation import OvxrPresentation + + requests = [] + reports = [] + admissions = iter([None, 1, None, 2]) + step = 0 + + def request(identity, *, force): + requests.append((identity, force)) + return next(admissions) + + presentation = OvxrPresentation( + SimpleNamespace(request_render=request, report_host_timing=lambda step, _dt: reports.append(step)) + ) + env = SimpleNamespace(sim=SimpleNamespace(get_physics_step_count=lambda: step, get_physics_dt=lambda: 0.005)) + presentation.mark_changed(env, reset=True) + assert requests == [] and presentation.reset_pending + assert not presentation.publish(env, report_timing=False) + assert presentation.publish(env, report_timing=False) + assert not presentation.reset_pending + step = 1 + assert not presentation.publish(env, changed=True) + assert presentation.publish(env) + assert presentation.publish(env) # Nothing pending, so nothing is requested. + assert requests == [((1, 0), True), ((1, 0), True), ((1, 1), False), ((1, 1), False)] + assert reports == [1, 1, 1] # Only completed host frames report the physics rate. + + +@pytest.mark.parametrize("shutdown_fails", [False, True]) +@pytest.mark.parametrize("drained", [True, False]) +def test_ovxr_presentation_closes_only_after_input_drain(drained, shutdown_fails): + """An undrained input device keeps the session and environment borrowed and only requests shutdown.""" + from unittest.mock import Mock + + from isaaclab_teleop.entrypoints._ovxr_presentation import OvxrPresentation + + session = Mock() + if shutdown_fails: + session.request_shutdown.side_effect = RuntimeError("shutdown failed") + presentation = OvxrPresentation(session) + presentation.input_device = SimpleNamespace(session_cleanup_failed=not drained) + if drained: + presentation.close() + session.close.assert_called_once_with() + session.request_shutdown.assert_not_called() + return + with pytest.raises(RuntimeError, match="input did not drain") as caught: + presentation.close() + session.request_shutdown.assert_called_once_with() + session.close.assert_not_called() + assert any("shutdown failed" in note for note in getattr(caught.value, "__notes__", ())) is shutdown_fails + + +@pytest.mark.parametrize( + ("configs", "kit_sources"), + [ + pytest.param((), ("renderer",), id="kit-source"), + pytest.param(("visualizer",), (), id="local-visualizer"), + ], +) +def test_ovxr_rejects_kit_runtimes_and_local_visualizers(configs, kit_sources): + """Live and replay share one rejection for components that compete with the OVXR frame loop.""" + from isaaclab_teleop.entrypoints._ovxr_launch import reject_ovxr_runtime_conflicts + + from isaaclab.visualizers import VisualizerBackendCfg + + configs = tuple(VisualizerBackendCfg() if cfg == "visualizer" else cfg for cfg in configs) + reject_ovxr_runtime_conflicts((object(),), ()) + with pytest.raises(ValueError, match="no Kit runtime or local visualizer"): + reject_ovxr_runtime_conflicts(configs, kit_sources) + + +@pytest.mark.parametrize( + ("left_active", "right_active", "expected"), + ( + (False, True, "waiting for initial left hand tracking"), + (True, False, "waiting for initial right hand tracking"), + ), +) +def test_kitless_hand_guard_requires_each_hand_once_then_allows_held_dropout( + monkeypatch, left_active, right_active, expected +): + agent = _load_teleop_agent(monkeypatch) + + class OptionalHand: + def __init__(self, active): + self.is_none = not active + + lifecycle = SimpleNamespace( + HAND_LEFT_KEY="hand_left", + HAND_RIGHT_KEY="hand_right", + last_step_result={ + "hand_left": OptionalHand(left_active), + "hand_right": OptionalHand(right_active), + }, + ) + from isaaclab_teleop import IsaacTeleopDevice + + interface = IsaacTeleopDevice.__new__(IsaacTeleopDevice) + interface._session_lifecycle = lifecycle + initialized = set() + + assert agent.initial_hand_tracking_issue(interface, initialized) == expected + lifecycle.last_step_result = {"hand_left": OptionalHand(True), "hand_right": OptionalHand(True)} + # Numeric zero is not a sentinel: a present, valid fully-open hand frame is accepted. + assert agent.initial_hand_tracking_issue(interface, initialized) is None + assert initialized == {"left", "right"} + + lifecycle.last_step_result = {"hand_left": OptionalHand(False), "hand_right": OptionalHand(False)} + assert agent.initial_hand_tracking_issue(interface, initialized) is None + + initialized.clear() + assert agent.initial_hand_tracking_issue(interface, initialized) == "waiting for initial left/right hand tracking" + + +def test_kitless_hand_guard_renders_without_step_and_next_valid_frame_resumes(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + + class Sim: + def __init__(self): + self.render_count = 0 + + def render(self): + self.render_count += 1 + + class Env: + def __init__(self): + self.num_envs = 1 + self.sim = Sim() + self.actions = [] + + def step(self, action): + self.actions.append(action.clone()) + + env = Env() + haptics = {"update": 0, "stop": 0} + + class HapticDriver: + def update(self): + haptics["update"] += 1 + + def stop(self): + haptics["stop"] += 1 + + driver = HapticDriver() + valid = torch.arange(36, dtype=torch.float32) + + assert agent._apply_teleop_frame(env, valid, None, driver) + assert not agent._apply_teleop_frame(env, torch.zeros_like(valid), "missing left hand tracking", driver) + assert agent._apply_teleop_frame(env, valid, None, driver) + + assert env.sim.render_count == 1 + assert len(env.actions) == 2 + torch.testing.assert_close(env.actions[0], env.actions[1]) + assert haptics == {"update": 2, "stop": 1} + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize( + "flags, expected", + [ + ([], True), + (["--headless"], False), + ], +) +def test_ovxr_mirror_cli_reaches_session_construction(monkeypatch, entrypoint, flags, expected): + """The real entrypoint parser controls the config passed to XR session preparation.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") + import isaaclab_teleop + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + argv = ["--task", "Test", "--ovxr", *flags] + if entrypoint == "replay": + argv += ["--replay_file", "recording.mcap"] + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + module._parse_args(argv) + assert module.args_cli.headless is ("--headless" in flags) + cfg = SimpleNamespace(sim=object(), isaac_teleop=isaaclab_teleop.IsaacTeleopCfg()) + received = [] + + class PreparationReached(Exception): + pass + + def prepare(_sim, xr_cfg, *_args, **_kwargs): + assert isinstance(xr_cfg, isaaclab_teleop.OvLibrariesXrCfg) + received.append(xr_cfg.desktop_mirror_enabled) + raise PreparationReached + + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + with pytest.raises(PreparationReached): + module.run_teleoperation(cfg, cfg.isaac_teleop, True, ()) + else: + with pytest.raises(PreparationReached): + module._run_replay_batch(cfg, None, ()) + assert received == [expected] + assert isaaclab_teleop.OvLibrariesXrCfg().desktop_mirror_enabled is False + + +@pytest.mark.parametrize("entrypoint, flags", [("live", ["--xr", "--headless"]), ("replay", ["--headless"])]) +def test_headless_requires_ovxr(monkeypatch, capsys, entrypoint, flags): + """Kit runs headless by default, so only the OVXR desktop mirror accepts ``--headless``.""" + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + argv = ["--task", "Test", *flags] + if entrypoint == "replay": + argv += ["--replay_file", "recording.mcap"] + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + with pytest.raises(SystemExit): + module._parse_args(argv) + assert "--headless requires --ovxr" in capsys.readouterr().err + + +def _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, flags, calls): + """Run an entrypoint's ``main`` with one PiP camera until it prepares the OVXR session. + + Task resolution and launch are stubbed; CLI parsing, pre-launch preparation and session + construction are production code. ``calls`` records the kept cameras, launch and preparation. + + Returns: + The task configuration and the arguments passed to session preparation. + """ + from contextlib import contextmanager, suppress + + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_teleop.entrypoints import _ovxr_launch, teleop_replay_agent, teleop_se3_agent + + from isaaclab.sensors import CameraCfg + from isaaclab.sim import SimulationContext + + from isaaclab_tasks.utils import parse_cfg + + config = SimpleNamespace( + sim=SimpleNamespace(device="cpu", physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1, head_cam=CameraCfg(prim_path="/World/head_cam", width=640, height=480)), + terminations=SimpleNamespace(success=None, time_out=None), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + xr_camera_feeds=[isaaclab_teleop.XrCameraFeedCfg(camera_name="head_cam", offset_m=(0.1, -0.15))], + xr_camera_feed_layout=layout, + ), + ) + prepared = [] + + class SessionReached(RuntimeError): + pass + + @contextmanager + def launch(selected_cfg, _args, *, cleanup_context): + assert selected_cfg is config + calls.append("launch") + yield + + def prepare(sim_cfg, xr_cfg, renderer_cfg=None): + assert sim_cfg is config.sim + calls.append("prepare") + prepared.append((xr_cfg, renderer_cfg)) + raise SessionReached + + def remove_camera_configs(selected_cfg, *, keep=()): + calls.append(("keep", keep)) + return selected_cfg + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + monkeypatch.setattr(module, "ManagerBasedRLEnvCfg", SimpleNamespace) + resolver_owner = teleop_se3_agent if entrypoint == "live" else parse_cfg + monkeypatch.setattr(resolver_owner, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) + monkeypatch.setattr(module, "validate_launch", lambda *_args: ((), ())) + monkeypatch.setattr(module, "launch_simulation", launch) + monkeypatch.setattr(_ovxr_launch, "remove_camera_configs", remove_camera_configs) + # The Kit Capture release has no OVXR session; layout rejection never reaches the session. + with suppress(ModuleNotFoundError): + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + monkeypatch.setattr(SimulationContext, "instance", lambda: None) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + arguments = ["--task", "Test", "--ovxr", "--cloudxr_env", "none", "--ovxr_pip_camera", "head_cam", *flags] + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + else: + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + arguments += ["--replay_file", "recording.mcap"] + module._parse_args(arguments) + with suppress(SessionReached): + module.main() + return config, prepared + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize( + "placement, flags, position", + [ + # A head-locked task layout keeps its panel placement without the flag. + ("head_locked", [], (0.1, -0.15, -0.8)), + # The flag places a world layout in the VIEW space, at the requested position. + ("world", ["--ovxr_pip_head_locked", "--ovxr_pip_view_position", "0", "-0.1", "-0.6"], (0.0, -0.1, -0.6)), + ], +) +def test_ovxr_pip_cli_reaches_session_construction(monkeypatch, entrypoint, placement, flags, position): + """The PiP and mirror options of both workflows reach the OVXR session configuration.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop import XrCameraFeedLayoutCfg + + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(1.0, 0.0, 1.5)) + rates = ["--ovxr_pip_fps", "15", "--ovxr_desktop_mirror_fps", "20"] + calls = [] + config, prepared = _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, flags + rates, calls) + + assert calls == [("keep", ("head_cam",)), "launch", "prepare"] + [(xr_cfg, renderer_cfg)] = prepared + assert xr_cfg.desktop_mirror_enabled is True + assert xr_cfg.desktop_mirror_max_fps == 20.0 + [camera] = xr_cfg.auxiliary_cameras + # Configclasses copy their fields, so the panel holds a copy of the scene sensor configuration. + assert camera.name == "head_cam" and camera.camera_cfg.prim_path == config.scene.head_cam.prim_path + assert camera.camera_cfg.renderer_cfg == config.scene.head_cam.renderer_cfg + assert camera.max_fps == 15.0 + assert camera.position == pytest.approx(position) + assert isinstance(renderer_cfg, OVRTXRendererCfg) and renderer_cfg == config.scene.head_cam.renderer_cfg + # Forcing head-locked placement leaves the task layout unchanged. + assert config.isaac_teleop.xr_camera_feed_layout.placement == placement + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize("placement", ["viewer_start", "world"]) +def test_ovxr_pip_rejects_task_layout_that_is_not_head_locked_before_launch(monkeypatch, entrypoint, placement): + """Without ``--ovxr_pip_head_locked``, a task layout OVXR cannot present fails before launch.""" + from isaaclab_teleop import XrCameraFeedLayoutCfg + + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(1.0, 0.0, 1.5)) + calls = [] + with pytest.raises(ValueError, match=rf"{placement!r} placement.*--ovxr_pip_head_locked"): + _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, [], calls) + assert calls == [] + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoint): + """Both actual workflows finalize the selected config before OVXR allocates its context.""" + pytest.importorskip("ovrtx") + pytest.importorskip("isaaccapture.ovxr_session_manager") + from contextlib import contextmanager + + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_teleop import ovxr_session + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + from isaaclab.sim import SimulationCfg, SimulationContext + + from isaaclab_tasks.utils import parse_cfg + + calls = [] + + def prepare_environment(selected_cfg): + assert selected_cfg is config + calls.append("prepare") + selected_cfg.sim.dt = 0.007 + + config = SimpleNamespace( + sim=SimulationCfg(physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1), + terminations=SimpleNamespace(success=None, time_out=None), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg(prepare_environment=prepare_environment), + ) + + class ContextReached(RuntimeError): + pass + + class ContextAllocation: + def __init__(self, sim_cfg): + assert sim_cfg is config.sim + assert sim_cfg.dt == 0.007 + assert calls == ["prepare", "launch"] + calls.append("context") + raise ContextReached + + @staticmethod + def instance(): + return None + + @contextmanager + def launch(selected_cfg, _args, *, cleanup_context): + assert selected_cfg is config + assert not cleanup_context + calls.append("launch") + yield + + def validate_launch(selected_cfg, _args): + assert selected_cfg is config + return (), () + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + monkeypatch.setattr(module, "ManagerBasedRLEnvCfg", SimpleNamespace) + resolver_owner = teleop_se3_agent if entrypoint == "live" else parse_cfg + monkeypatch.setattr(resolver_owner, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) + monkeypatch.setattr(module, "validate_launch", validate_launch) + monkeypatch.setattr(module, "launch_simulation", launch) + monkeypatch.setattr(SimulationContext, "instance", lambda: None) + monkeypatch.setattr(ovxr_session, "SimulationContext", ContextAllocation) + monkeypatch.setitem(sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _settings: None)) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + arguments = ["--task", "Test", "--ovxr", "--cloudxr_env", "none"] + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + else: + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + arguments += ["--replay_file", "recording.mcap"] + module._parse_args(arguments) + + with pytest.raises(ContextReached): + module.main() + assert calls == ["prepare", "launch", "context"] + + +_UNSET_PHYSICS_TASK = "IsaacContrib-PickPlace-GR1T2-Abs" +_UNSET_PHYSICS_OVXR_ERROR = rf"task '{_UNSET_PHYSICS_TASK}' leaves sim.physics unset.*Run this task with --xr" + + +def _unset_task_physics(monkeypatch, module): + """Resolve the real task configuration, then leave ``sim.physics`` at its default ``None``.""" + real_resolve = module.resolve_task_config + + def resolve(*args, **kwargs): + env_cfg, agent_cfg = real_resolve(*args, **kwargs) + env_cfg.sim.physics = None + return env_cfg, agent_cfg + + monkeypatch.setattr(module, "resolve_task_config", resolve) + + +def test_live_unset_task_physics_keeps_simulation_default(monkeypatch): + """``physics=None`` must reach SimulationContext's PhysxCfg default and launch Kit for it.""" + from isaaclab.app.sim_launcher import validate_launch + + agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, "--disable_external_cameras") + _unset_task_physics(monkeypatch, agent) + env_cfg = agent._resolve_env_cfg()[0] + assert env_cfg.sim.physics is None + _configs, kit_sources = validate_launch(env_cfg, vars(agent.args_cli).copy()) + assert kit_sources + + +def test_live_ovxr_rejects_unset_task_physics_before_launch(monkeypatch): + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, "--ovxr") + _unset_task_physics(monkeypatch, agent) + with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): + agent._resolve_env_cfg() + + +@pytest.mark.parametrize("ovxr", [False, True]) +def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monkeypatch, ovxr): + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + from isaaclab_tasks.utils import parse_cfg + + argv = ["teleop_replay_agent.py", "--task", _UNSET_PHYSICS_TASK, "--replay_file", "recording.mcap"] + if ovxr: + argv.append("--ovxr") + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + monkeypatch.setattr(replay, "args_cli", None, raising=False) + monkeypatch.setattr(replay, "hydra_overrides", None, raising=False) + replay._parse_args() + _unset_task_physics(monkeypatch, parse_cfg) + if ovxr: + with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): + replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) + return + env_cfg, _success, _cameras = replay._prepare_env_cfg( + _UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides + ) + assert env_cfg.sim.physics is None + assert replay.args_cli.require_kit + + +def test_rejected_ovxr_frame_does_not_render_on_the_control_thread(monkeypatch, run_live_workflow): + """OVXR owns rendering, so a rejected frame holds the scene without a control-thread render or step.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + advances = 0 + renders = [] + steps = [] + requests = [] + waits = [] + + def advance(): + nonlocal advances + advances += 1 + return torch.ones((1, 2)) + + def request(identity, *, force): + requests.append(force) + return 1 + + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: 0.0, sleep=waits.append)) + env = SimpleNamespace( + num_envs=1, + step_dt=0.02, + reset=lambda: None, + step=steps.append, + sim=SimpleNamespace( + is_running=lambda: advances < 1, + get_physics_step_count=lambda: len(steps), + get_physics_dt=lambda: 0.005, + render=lambda: renders.append(True), + ), + ) + run_live_workflow( + env, + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace(is_active=True, should_reset=False), + ), + SimpleNamespace( + shutdown_requested=False, + request_render=request, + wait_for_shutdown=waits.append, + report_host_timing=lambda *args: None, + ), + validator=lambda _action: "rejected", + ) + assert advances == 1 + assert renders == [] and steps == [] + assert requests == [True] # Only the reset publication; the held scene is not republished. + assert waits == [0.02] # A rejected frame waits one action period, like a paused one. diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py new file mode 100644 index 000000000000..f01ab3d50af4 --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -0,0 +1,225 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Execute the real public entry without native startup to prove failure/cleanup contracts.""" + +import contextlib +import logging +import signal +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock + +import pytest + + +@pytest.fixture +def entry_namespace(monkeypatch): + """Use the imported entrypoint and restore all globals replaced by each test.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + namespace = vars(agent) + for name in ( + "args_cli", + "gym", + "_resolve_cloudxr_env", + "has_kit", + "create_teleop_device", + "_install_ovxr_shutdown_handlers", + "_restore_ovxr_shutdown_handlers", + "torch", + ): + monkeypatch.setitem(namespace, name, namespace.get(name)) + monkeypatch.setattr(agent, "_restore_ovxr_shutdown_handlers", Mock()) + return namespace + + +@pytest.mark.parametrize( + "stage,use_capture", [("environment", True), ("device", True), ("haptics", True), ("input", True), ("input", False)] +) +def test_live_entry_failure_closes_resources_and_reports_cause( + monkeypatch, caplog, entry_namespace, stage, use_capture +): + """Startup and live-loop failures are logged or propagated, and both release the resources.""" + import isaaclab_teleop + + failure = RuntimeError(f"{stage} failed") + camera = SimpleNamespace(close=Mock(), bind=lambda _env: contextlib.nullcontext()) + env = SimpleNamespace( + sim=SimpleNamespace(device="cpu", has_gui=False, visualizers=(), is_running=lambda: True), + scene=SimpleNamespace(env_prim_paths=["/World/envs/env_0"]), + reset=Mock(), + close=Mock(), + ) + env.unwrapped = env + interface = MagicMock() + interface.advance.side_effect = failure + monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", Mock(return_value=camera)) + monkeypatch.setattr( + isaaclab_teleop, + "create_haptic_feedback_driver", + Mock(side_effect=failure if stage == "haptics" else None, return_value=None), + ) + entry_namespace.update( + { + "args_cli": SimpleNamespace( + task="test-task", + device="cpu", + xr=False, + using_ovxr=False, + disable_external_cameras=True, + auto_launch_cloudxr=False, + enable_debug_visualization=False, + cloudxr_env="none", + ), + "gym": SimpleNamespace( + make=Mock( + side_effect=failure if stage == "environment" else None, return_value=SimpleNamespace(unwrapped=env) + ) + ), + "_resolve_cloudxr_env": lambda *_args: None, + "has_kit": lambda: False, + "create_teleop_device": Mock(side_effect=failure if stage == "device" else None, return_value=interface), + "_install_ovxr_shutdown_handlers": Mock(return_value={}), + "torch": SimpleNamespace(inference_mode=contextlib.nullcontext), + } + ) + cfg = SimpleNamespace( + isaac_teleop=SimpleNamespace( + teleoperation_active_default=False, + require_initial_bimanual_hand_tracking=False, + action_conditioner=None, + action_validator=None, + ) + ) + if not use_capture: + cfg = SimpleNamespace() # Native inputs must not read Capture-only action policy. + if stage == "haptics": + with pytest.raises(RuntimeError) as caught: + entry_namespace["run_teleoperation"](cfg, False, use_capture, ()) + assert caught.value is failure + else: + # Environment, device and loop failures keep the upstream log-and-return exit. + with caplog.at_level(logging.ERROR): + entry_namespace["run_teleoperation"](cfg, False, use_capture, ()) + assert f"{stage} failed" in caplog.text + camera.close.assert_called_once_with() + assert env.close.call_count == int(stage != "environment") + + +@pytest.mark.parametrize("ovxr", [True, False]) +def test_input_cleanup_failure_prevents_borrowed_owner_release(entry_namespace, ovxr): + env, camera, xr = Mock(), Mock(), Mock() + interface = SimpleNamespace(session_cleanup_failed=True) + presentation = None + if ovxr: + presentation = entry_namespace["OvxrPresentation"](xr) + presentation.input_device = interface + with pytest.raises(RuntimeError, match="drain"): + entry_namespace["_close_teleop_resources"](env, camera, presentation, {}, interface) + if ovxr: + xr.request_shutdown.assert_called_once_with() + xr.close.assert_not_called() + camera.close.assert_not_called() + env.close.assert_not_called() + + +@pytest.mark.parametrize( + ("cleanup_stage", "has_primary_error"), + [("xr", True), ("environment", True), ("camera", False), ("camera", True), ("signals", False)], +) +def test_cleanup_preserves_first_error_and_never_frees_undrained_stage( + entry_namespace, cleanup_stage, has_primary_error +): + """Cleanup errors must stay visible without replacing the initial cause or risking UAF.""" + events = [] + cleanup_error = RuntimeError("cleanup failed") + primary_error = RuntimeError("original failure") + + def close(name): + events.append(name) + if name == cleanup_stage: + raise cleanup_error + + xr = SimpleNamespace(close=lambda: close("xr"), async_worker_owns_environment=False) + camera = SimpleNamespace(close=lambda: close("camera")) + env = SimpleNamespace(close=lambda: close("environment")) + entry_namespace["_restore_ovxr_shutdown_handlers"] = lambda _: close("signals") + + def exercise(): + try: + if has_primary_error: + raise primary_error + finally: + entry_namespace["_close_teleop_resources"](env, camera, entry_namespace["OvxrPresentation"](xr), {}) + + with pytest.raises(RuntimeError) as caught: + exercise() + assert caught.value is (primary_error if has_primary_error else cleanup_error) + assert events == ( + ["xr", "camera", "signals"] if cleanup_stage in ("xr", "camera") else ["xr", "camera", "environment", "signals"] + ) + if has_primary_error: + assert any("cleanup failed" in note for note in primary_error.__notes__) + + +def test_shutdown_signal_stays_owned_until_xr_drain_finishes(entry_namespace): + """A second Ctrl-C requests shutdown again instead of interrupting native cleanup.""" + events = [] + original = signal.signal(signal.SIGINT, lambda *_args: events.append("shutdown")) + + def restore(handlers): + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + def close_xr(): + signal.raise_signal(signal.SIGINT) + events.append("xr") + + entry_namespace["_restore_ovxr_shutdown_handlers"] = restore + try: + entry_namespace["_close_teleop_resources"]( + SimpleNamespace(close=lambda: events.append("environment")), + SimpleNamespace(close=lambda: events.append("camera")), + entry_namespace["OvxrPresentation"](SimpleNamespace(close=close_xr, async_worker_owns_environment=False)), + {signal.SIGINT: original}, + ) + assert events == ["shutdown", "xr", "camera", "environment"] + assert signal.getsignal(signal.SIGINT) is original + finally: + signal.signal(signal.SIGINT, original) + + +@pytest.mark.parametrize("new_context,drain_failed", [(True, False), (False, False), (True, True)]) +def test_partial_construction_cleans_only_new_unowned_context(monkeypatch, entry_namespace, new_context, drain_failed): + """Failed env construction is cleaned, but preexisting/undrained contexts survive.""" + from isaaclab.app import launch_simulation + from isaaclab.physics import PhysicsCfg + from isaaclab.sim import SimulationContext + + previous = object() + state = {"context": previous} + clear = Mock(side_effect=lambda: state.update(context=None)) + monkeypatch.setattr(SimulationContext, "instance", lambda: state["context"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear) + camera = SimpleNamespace(close=Mock()) + xr = SimpleNamespace( + close=Mock(side_effect=RuntimeError("drain failed") if drain_failed else None), + async_worker_owns_environment=False, + ) + failure = RuntimeError("environment construction failed") + with pytest.raises(RuntimeError) as caught: + with launch_simulation(PhysicsCfg(), cleanup_context=False): + if new_context: + state["context"] = object() + try: + raise failure + finally: + entry_namespace["_close_teleop_resources"]( + None, camera, entry_namespace["OvxrPresentation"](xr), {}, previous_context=previous + ) + assert caught.value is failure + assert clear.call_count == int(new_context and not drain_failed) + if not new_context: + assert state["context"] is previous diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py new file mode 100644 index 000000000000..3ae057392b48 --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -0,0 +1,465 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Execute the real replay bootstrap without starting a simulator. + +Only device/simulator endpoints are isolated; the replay/reset function bodies +and Kit detection are production code. These tests do not qualify MCAP motion. +""" + +import builtins +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.mark.parametrize("layout_mode", ["horizontal", "manual", None]) +def test_ovxr_presentation_preserves_camera_layout(monkeypatch, layout_mode): + """The forced head-locked placement keeps task packing, and the viewer receives the CLI rates.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") + import isaaclab_teleop + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg + + from isaaclab.sensors import CameraCfg + + names = ["left", "right"] + config = SimpleNamespace( + scene=SimpleNamespace(**{name: CameraCfg(prim_path=f"/World/{name}", width=640, height=480) for name in names}), + sim=object(), + isaac_teleop=IsaacTeleopCfg( + xr_camera_feeds=[XrCameraFeedCfg(camera_name=name, panel_width_m=0.4) for name in names], + xr_camera_feed_layout=( + XrCameraFeedLayoutCfg( + mode=layout_mode, + placement="viewer_start", + center_offset_m=(0.1, -0.2), + panel_gap_m=0.1, + distance_m=1.2, + ) + if layout_mode is not None + else None + ), + ), + ) + original_layout = config.isaac_teleop.xr_camera_feed_layout + original_layout_fields = original_layout.to_dict() if original_layout is not None else None + original_feeds = [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop.entrypoints._ovxr_launch import resolve_ovxr_pip_cameras + + args = SimpleNamespace( + ovxr_pip_camera=names, + ovxr_pip_head_locked=True, + ovxr_pip_fps=15.0, + ovxr_pip_view_position=None, + task="Test", + ovxr_session_mode="ar", + cloudxr_env="none", + auto_launch_cloudxr=False, + headless=False, + ovxr_desktop_mirror_fps=20.0, + ) + # Environment preparation resolves the panels that the presentation receives. + cameras = resolve_ovxr_pip_cameras(config, args, OVRTXRendererCfg()) + prepare = Mock() + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + from isaaclab_teleop.entrypoints._ovxr_presentation import OvxrPresentation + + presentation = OvxrPresentation.prepare(config, args, application_name="replay", auxiliary_cameras=cameras) + assert presentation.session is prepare.return_value + session_config = prepare.call_args.args[1] + assert session_config.desktop_mirror_enabled is True + assert session_config.desktop_mirror_max_fps == 20.0 + assert [camera.max_fps for camera in session_config.auxiliary_cameras] == [15.0, 15.0] + # Panels copy the sensor configuration; an equal renderer configuration shares one renderer. + assert prepare.call_args.kwargs["renderer_cfg"] == config.scene.left.renderer_cfg + expected = ( + [(-0.15, -0.2, -1.2), (0.35, -0.2, -1.2)] + if layout_mode == "horizontal" + else [(0.0, 0.0, -0.8), (0.0, 0.0, -0.8)] + ) + for camera, position in zip(session_config.auxiliary_cameras, expected, strict=True): + assert camera.position == pytest.approx(position) + assert config.isaac_teleop.xr_camera_feed_layout is original_layout + if original_layout is not None: + assert original_layout.to_dict() == original_layout_fields + assert [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] == original_feeds + + +@pytest.mark.parametrize( + "kit_active,run_index,backend,viewer,hard_reset", + [ + (False, 0, "ovphysx", False, True), + (True, 0, "ovphysx", False, True), + (True, 1, "ovphysx", False, True), + (False, 0, "ovphysx", True, True), + (False, 0, "newton_mjwarp", True, False), + ], +) +def test_replay_bootstrap_waits_only_for_an_active_kit_application( + monkeypatch, kit_active, run_index, backend, viewer, hard_reset +): + """A viewer preserves backend reset policy; kitless replay never waits for Kit assets.""" + calls = [] + + class Device: + xr_render_anchor_path = None + external_anchor_state = None + + def __enter__(self): + calls.append("device-enter") + return self + + def reset(self): + calls.append("device-reset") + + def request_start(self): + calls.append("device-start") + + def __exit__(self, *args): + calls.append("device-exit") + + app = ModuleType("omni.kit.app") + app.get_app = lambda: object() + if kit_active: + monkeypatch.setitem(sys.modules, "omni.kit.app", app) + else: + monkeypatch.delitem(sys.modules, "omni.kit.app", raising=False) + wait_module = ModuleType("isaaclab_physx.renderers.isaac_rtx_renderer_utils") + wait_module.wait_for_stage_load = lambda *args, **kwargs: calls.append("kit-wait") + monkeypatch.setitem(sys.modules, wait_module.__name__, wait_module) + original_import = builtins.__import__ + + def import_with_kit_guard(name, *args, **kwargs): + if name == wait_module.__name__ and not kit_active: + raise AssertionError("Kit-only stage loader imported by a kitless replay") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_with_kit_guard) + + def render(): + calls.append("render") + + env = SimpleNamespace( + scene=SimpleNamespace(physics_backend=backend, env_prim_paths=["/Lab/Cells/Cell0"]), + sim=SimpleNamespace( + device="cpu", + render=render, + reset=lambda: calls.append("sim-reset"), + is_running=lambda: False, + get_physics_step_count=lambda: 0, + get_physics_dt=lambda: 0.005, + ), + reset=lambda: calls.append("env-reset"), + ) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + from isaacteleop.teleop_session_manager import RetargetingExecutionConfig + + cfg = SimpleNamespace(retargeting_execution=RetargetingExecutionConfig(mode="pipelined")) + constructed_modes = [] + + def create_device(selected, **kwargs): + assert kwargs["env_prim_path"] == "/Lab/Cells/Cell0" + constructed_modes.append(selected.retargeting_execution.mode) + assert selected.retargeting_execution.mode == "pipelined" + return Device() + + namespace = { + "args_cli": SimpleNamespace( + device="cpu", + replay_file="fixture.mcap", + enable_debug_visualization=False, + max_stage_load_wait_s=5, + replay_start_delay_s=0, + start_on_replay=False, + max_replay_duration_s=1, + ), + "NvmlGpuStatsProvider": lambda: SimpleNamespace(_available=True, summary=lambda: {}), + "create_isaac_teleop_device": create_device, + "_RENDERER_SETTLE_FRAMES": 30, + } + for name, value in namespace.items(): + monkeypatch.setattr(replay, name, value, raising=False) + presentation = None + if viewer: + from isaaclab_teleop.entrypoints._ovxr_presentation import OvxrPresentation + + session = SimpleNamespace( + start_render_worker=lambda *_args, **_kwargs: calls.append("viewer-start") or True, + request_render=lambda identity, *, force: calls.append(("publish", identity, force)) or 1, + report_host_timing=lambda *_args: None, + ) + presentation = OvxrPresentation(session) + replay._run_single_replay(env, cfg, None, run_index, 2, presentation) + expected = ["device-enter"] + if hard_reset: + expected.append("sim-reset") + expected.extend(["env-reset", "device-reset"]) + if viewer: + expected.extend(["viewer-start", ("publish", (1, 0), True)]) + if kit_active and run_index == 0: + expected.append("kit-wait") + assert calls == [*expected, "device-exit"] + assert env.sim.render is render + assert constructed_modes == ["pipelined"] + + +@pytest.mark.parametrize( + "stop_at,requested", + [ + ("startup", 2), + ("startup-second-signal", 2), + ("construction-new", 2), + ("construction-previous", 2), + ("after-first", 2), + ("after-first", 1), + ], +) +def test_presented_replay_preserves_startup_ownership_and_batch_completion(monkeypatch, stop_at, requested): + """Failed construction preserves prior owners; a partial batch never reports completion.""" + from isaaclab.sim import SimulationContext + + calls = [] + previous = object() + context_state = {"current": previous} + + def clear_context(): + calls.append("context-clear") + context_state["current"] = None + + monkeypatch.setattr(SimulationContext, "instance", lambda: context_state["current"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear_context) + handlers = {} + reports = [] + session = SimpleNamespace( + shutdown_requested=False, + async_worker_owns_environment=not stop_at.startswith("construction"), + request_shutdown=lambda: None, + ) + + class Presentation: + @classmethod + def prepare(cls, cfg, args, *, application_name, auxiliary_cameras): + assert application_name == "isaaclab-ovxr-replay-task" + assert auxiliary_cameras == () + return cls() + + def __init__(self): + self.session = session + self.started = False + + def close(self): + if stop_at == "startup-second-signal": + handlers[2](2, None) + calls.append("presentation-close") + + def install_signal(sig, handler): + handlers[sig] = handler + return + + def make_env(*args, **kwargs): + if stop_at.startswith("startup"): + handlers[1](1, None) + if stop_at.startswith("construction"): + if stop_at == "construction-new": + context_state["current"] = object() + raise RuntimeError("environment construction failed") + return SimpleNamespace(unwrapped=env) + + def replay(**kwargs): + calls.append("replay") + session.shutdown_requested = True + return SimpleNamespace(outcome="success", to_dict=lambda index: {"outcome": "success"}) + + env = SimpleNamespace(sim=SimpleNamespace(is_running=lambda: True)) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay_agent + + namespace = { + "args_cli": SimpleNamespace( + ovxr=True, + require_kit=False, + task="task", + replay_file="fixture.mcap", + num_replays=requested, + stats_output_file=None, + ), + "OvxrPresentation": Presentation, + "signal": SimpleNamespace(SIGINT=1, SIGTERM=2, signal=install_signal), + "gym": SimpleNamespace(make=make_env), + "_run_single_replay": replay, + "_extract_env_perf_cfg": lambda cfg: None, + "_print_stdout_summary": reports.append, + } + for name, value in namespace.items(): + monkeypatch.setattr(replay_agent, name, value, raising=False) + if stop_at.startswith("startup"): + with pytest.raises(KeyboardInterrupt): + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) + assert calls == ["presentation-close"] + elif stop_at.startswith("construction"): + with pytest.raises(RuntimeError, match="environment construction failed"): + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) + assert calls == ["presentation-close"] + (["context-clear"] if stop_at == "construction-new" else []) + assert context_state["current"] is (None if stop_at == "construction-new" else previous) + else: + result = replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) + assert result == (0 if requested == 1 else 1) + assert reports[0]["num_replays"] == 1 + assert reports[0]["requested_num_replays"] == requested + assert reports[0]["batch_complete"] is (requested == 1) + assert calls == ["replay", "presentation-close"] + assert handlers == {1: None, 2: None} + + +@pytest.mark.parametrize("start_on_replay", [False, True]) +def test_replay_control_timing_uses_recorded_events_or_explicit_legacy_start(monkeypatch, start_on_replay): + """The real control owners gate input, distinguish RESET from STOP, and retain the replay cursor.""" + import torch + from isaaclab_teleop.control_events import ControlEvents + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle + from isaaclab_teleop.teleop_message_processor import TeleopMessageProcessor + from isaacteleop.teleop_session_manager import DefaultTeleopStateManager + + processor = TeleopMessageProcessor(name="recorded_control") + state_manager = DefaultTeleopStateManager(name="recorded_state") + lifecycle = TeleopSessionLifecycle.__new__(TeleopSessionLifecycle) + lifecycle._message_processor = processor + lifecycle._last_control_events = ControlEvents() + lifecycle._last_control_event_frame_id = None + lifecycle._haptic_forces = {} + lifecycle._session = SimpleNamespace(last_context=None, last_step_info=None) + messages = {7: b"reset", 12: b"stop"} + if not start_on_replay: + messages.update({3: b"start", 10: b"start"}) + frame = -1 + applied, resets, recorder_resets = [], [], [] + + class Signal(list): + is_none = False + + class RecordedDevice(IsaacTeleopDevice): + def __init__(self): + self._session_lifecycle = lifecycle + self._anchor_manager = SimpleNamespace(reset=lambda: None, cleanup=lambda: None) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def advance(self): + nonlocal frame + frame += 1 + payload = messages.get(frame) + tracked = SimpleNamespace(data=[SimpleNamespace(payload=payload)] if payload is not None else []) + outputs = {name: Signal([False]) for name in ("kill", "run_toggle", "reset")} + processor._compute_fn({processor.INPUT_MESSAGES: [tracked]}, outputs, context=None) + events = state_manager._compute_execution_events( + { + state_manager.INPUT_KILL: outputs["kill"], + state_manager.INPUT_RUN_TOGGLE: outputs["run_toggle"], + state_manager.INPUT_RESET: outputs["reset"], + }, + context=None, + ) + lifecycle._session.last_context = SimpleNamespace(execution_events=events) + lifecycle._session.last_step_info = SimpleNamespace(returned_frame_id=frame) + lifecycle._update_control_events() + return torch.zeros(1) + + env = SimpleNamespace( + num_envs=1, + scene=SimpleNamespace(physics_backend="newton_mjwarp", env_prim_paths=["/World/envs/env_0"]), + sim=SimpleNamespace(device="cpu", render=lambda: None, is_running=lambda: frame < 15), + recorder_manager=SimpleNamespace(reset=lambda: recorder_resets.append(frame)), + reset=lambda: resets.append(frame), + ) + + def step(_actions): + applied.append(frame) + env.sim.render() + return None, None, torch.tensor([False]), torch.tensor([False]), {} + + env.step = step + monkeypatch.setattr(replay, "create_isaac_teleop_device", lambda *_args, **_kwargs: RecordedDevice()) + monkeypatch.setattr( + replay, + "NvmlGpuStatsProvider", + lambda: SimpleNamespace(sample=lambda: None, summary=lambda: {}), + ) + monkeypatch.setattr( + replay, + "args_cli", + SimpleNamespace( + device="cpu", + replay_file="recording.mcap", + start_on_replay=start_on_replay, + enable_debug_visualization=False, + max_replay_duration_s=10.0, + num_success_steps=1, + ), + raising=False, + ) + conditioning_frames = [] + + def condition(_env, action, state): + previous = state.get("frames", 0) + conditioning_frames.append(previous) + state["frames"] = previous + 1 + return action + + cfg = IsaacTeleopCfg(action_conditioner=condition) + stats = replay._run_single_replay(env, cfg, None, run_index=1, total_runs=2) + assert applied == ([2, 3, 4, 5, 6, 9, 10, 11] if start_on_replay else [5, 6, 10, 11]) + assert resets == [-1, 0, 7] + assert recorder_resets == [0, 7] + assert frame == 12 # STOP terminates; RESET retains the cursor and permits the later START. + assert conditioning_frames == ([0, 1, 2, 3, 4, 0, 1, 2] if start_on_replay else [0, 1, 0, 1]) + assert stats.active_iterations == len(applied) + assert stats.outcome == "incomplete" + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + (["--ovxr_pip_head_locked"], "OVXR presentation options require --ovxr"), + (["--ovxr", "--ovxr_pip_fps", "0"], "--ovxr_pip_fps must be finite and positive"), + ( + [ + "--ovxr", + "--ovxr_pip_camera", + "a", + "b", + "--ovxr_pip_head_locked", + "--ovxr_pip_view_position", + "0", + "0", + "-1", + ], + "A PiP position override requires one camera", + ), + ], +) +def test_replay_and_live_share_ovxr_presentation_validation(monkeypatch, capsys, arguments, message): + """Replay accepts the live OVXR presentation options and rejects the same invalid selections.""" + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + for module, required in ( + (teleop_replay_agent, ["--task", "Test", "--replay_file", "recording.mcap"]), + (teleop_se3_agent, ["--task", "Test"]), + ): + monkeypatch.setattr(sys, "argv", ["agent.py", *required, *arguments]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + with pytest.raises(SystemExit): + module._parse_args() + assert message in capsys.readouterr().err diff --git a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py new file mode 100644 index 000000000000..d67948027848 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py @@ -0,0 +1,206 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused tests for kitless XR anchor construction.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import numpy as np +import pytest +from isaaclab_teleop import xr_anchor_manager +from isaaclab_teleop.xr_cfg import XrCfg + +from pxr import Usd, UsdGeom + +from isaaclab.sim.utils.stage import use_stage + + +def test_dynamic_external_anchor_commits_pose_before_input_and_preserves_reset(monkeypatch): + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode + from scipy.spatial.transform import Rotation + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg( + anchor_prim_path="/World/Robot", + anchor_pos=(1.0, 2.0, -0.95), + fixed_anchor_height=True, + anchor_rotation_mode=XrAnchorRotationMode.FOLLOW_PRIM, + ) + with use_stage(Usd.Stage.CreateInMemory()): + UsdGeom.Xform.Define(xr_anchor_manager.get_current_stage(), "/World/Robot") + manager = xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=True) + with pytest.raises(RuntimeError, match="committed post-reset"): + manager.get_world_matrix() + reference = np.eye(4) + reference[:3, 3] = (3, 4, 5) + committed = [] + epoch, seed = manager.external_epoch + assert seed + assert manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + np.testing.assert_allclose(committed[-1][0, :3, 3], (4, 6, 4.05)) + reference[:3, 3] = (8, 9, 12) + reference[:3, :3] = Rotation.from_euler("z", 90, degrees=True).as_matrix() + manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + expected = committed[-1][0].copy() + expected[:3, :3] = expected[:3, :3] @ manager._OXR_TO_USD_ROTATION + np.testing.assert_allclose(manager.get_world_matrix(), expected, atol=1e-7) + np.testing.assert_allclose(expected[:3, 3], (9, 11, 4.05)) + before = manager.get_world_matrix() + np.testing.assert_array_equal(manager.get_world_matrix(), before) + manager.toggle_anchor_rotation() + reference[:3, :3] = np.eye(3) + manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + np.testing.assert_allclose(manager.get_world_matrix()[:3, :3], before[:3, :3], atol=1e-7) + manager.reset() + assert not manager.commit_external_pose(reference, 0.02, epoch, lambda _: pytest.fail("old epoch sealed")) + with pytest.raises(RuntimeError, match="committed post-reset"): + manager.get_world_matrix() + manager.commit_external_pose(reference, 0.02, manager.external_epoch[0], lambda m: committed.append(m.copy())) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], (9, 11, 11.05)) + with pytest.raises(RuntimeError, match="write failed"): + manager.commit_external_pose( + reference, 0.02, manager.external_epoch[0], lambda _: (_ for _ in ()).throw(RuntimeError("write failed")) + ) + with pytest.raises(RuntimeError, match="publication failed"): + manager.get_world_matrix() + + manager.reset() + with pytest.raises(RuntimeError, match="publication failed"): + manager.commit_external_pose(reference, 0.02, epoch, lambda _: pytest.fail("failed pair revived")) + + +def test_standalone_replay_without_kit_xr_preserves_anchor_fallback(monkeypatch): + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) + cfg = XrCfg(anchor_prim_path="/World/Robot", anchor_pos=(1.0, 2.0, 3.0)) + manager = xr_anchor_manager.XrAnchorManager(cfg) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + + +def test_ovxr_session_does_not_borrow_an_available_kit_xr_core(monkeypatch): + core = MagicMock() + monkeypatch.setattr(xr_anchor_manager, "XRCore", core) + with use_stage(Usd.Stage.CreateInMemory()): + manager = xr_anchor_manager.XrAnchorManager(XrCfg(), ovxr_session=True) + assert manager.xr_core is None + core.get_singleton.assert_not_called() + + +@pytest.mark.parametrize("capability", ["none", "live-handles", "recorded-input-viewer"]) +def test_public_device_exposes_dynamic_anchor_only_for_ovxr_session(monkeypatch, capability): + from isaaclab_teleop import isaac_teleop_device + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(isaac_teleop_device, "CommandHandler", MagicMock()) + lifecycle = MagicMock() + monkeypatch.setattr(isaac_teleop_device, "TeleopSessionLifecycle", lifecycle) + cfg = IsaacTeleopCfg( + pipeline_builder=lambda: None, + xr_cfg=XrCfg(anchor_prim_path="/World/Robot"), + ) + kwargs = dict(use_kit_xr_bridge=False) + if capability == "live-handles": + kwargs["external_oxr_handles_provider"] = lambda: None + elif capability == "recorded-input-viewer": + kwargs.update(ovxr_session=True, mcap_replay_path="recording.mcap") + with use_stage(Usd.Stage.CreateInMemory()): + device = isaac_teleop_device.IsaacTeleopDevice(cfg, **kwargs) + assert (device.external_anchor_state is not None) == (capability != "none") + lifecycle.assert_called_once() + assert lifecycle.call_args.kwargs["ovxr_session"] == (capability != "none") + + +def test_replay_reuses_real_stage_scoped_anchors_without_kit(monkeypatch, caplog): + """A second device must not recreate or overwrite the same USD anchor.""" + stage = Usd.Stage.CreateInMemory() + cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0)) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + with use_stage(stage): + first = xr_anchor_manager.XrAnchorManager(cfg) + first.cleanup() + second = xr_anchor_manager.XrAnchorManager(cfg) + second.cleanup() + assert "Failed to create XR" not in caplog.text + prim = stage.GetPrimAtPath("/World/XRAnchor") + assert prim.IsValid() + np.testing.assert_allclose( + UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(0).ExtractTranslation(), cfg.anchor_pos + ) + + +def test_replay_refreshes_configured_static_anchor_on_the_same_stage(monkeypatch): + from isaaclab.sim.utils.transforms import resolve_prim_pose + + stage = Usd.Stage.CreateInMemory() + world = UsdGeom.Xform.Define(stage, "/World") + world.AddTranslateOp().Set((2.0, -1.0, 0.5)) + world.AddRotateZOp().Set(90.0) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + first_cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0)) + next_cfg = XrCfg(anchor_pos=(-1.0, 0.5, 2.0), anchor_rot=(0.0, 0.0, 1.0, 0.0)) + with use_stage(stage): + first = xr_anchor_manager.XrAnchorManager(first_cfg, ovxr_session=True) + observed_pos, observed_rot = resolve_prim_pose(stage.GetPrimAtPath(first.anchor_headset_path)) + np.testing.assert_allclose(observed_pos, first_cfg.anchor_pos, atol=1e-7) + assert abs(np.dot(observed_rot, first_cfg.anchor_rot)) == pytest.approx(1.0) + first.cleanup() + second = xr_anchor_manager.XrAnchorManager(next_cfg, ovxr_session=True) + observed_pos, observed_rot = resolve_prim_pose(stage.GetPrimAtPath(second.anchor_headset_path)) + np.testing.assert_allclose(observed_pos, next_cfg.anchor_pos, atol=1e-7) + assert abs(np.dot(observed_rot, next_cfg.anchor_rot)) == pytest.approx(1.0) + np.testing.assert_allclose(second.get_world_matrix()[:3, 3], next_cfg.anchor_pos) + second.cleanup() + + +def test_ovxr_session_rejects_an_unusable_stage_anchor(monkeypatch): + """An invalid anchor must not silently misalign tracking and rendering.""" + stage = Usd.Stage.CreateInMemory() + UsdGeom.Scope.Define(stage, "/World/XRAnchor") + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + with use_stage(stage), pytest.raises(ValueError, match="is not transformable"): + xr_anchor_manager.XrAnchorManager(XrCfg(), ovxr_session=True) + + +def test_standalone_replay_keeps_warning_for_an_unusable_anchor(monkeypatch, caplog): + stage = Usd.Stage.CreateInMemory() + UsdGeom.Scope.Define(stage, "/World/XRAnchor") + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0)) + with use_stage(stage): + manager = xr_anchor_manager.XrAnchorManager(cfg) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + manager.cleanup() + assert "is not transformable" in caplog.text + + +def test_kit_anchor_clock_is_lazy_until_subsequent_smoothed_update(monkeypatch): + from isaaclab_teleop import xr_anchor_utils + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode + + cfg = XrCfg(anchor_prim_path=None) + core = MagicMock() + clock = MagicMock() + clock.instance.side_effect = AssertionError("unused simulation clock") + monkeypatch.setattr(xr_anchor_utils, "SimulationContext", clock) + stage = Usd.Stage.CreateInMemory() + UsdGeom.Xform.Define(stage, "/XRAnchor") + with use_stage(stage): + synchronizer = xr_anchor_utils.XrAnchorSynchronizer(core, cfg, "/XRAnchor") + synchronizer.sync_headset_to_anchor() + core.set_world_transform_matrix.assert_called_once() + clock.instance.assert_not_called() + + cfg.anchor_rotation_mode = XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED + policy = xr_anchor_utils._XrAnchorPosePolicy(cfg) + dt = MagicMock(return_value=0.02) + policy.update(np.eye(4), dt) + dt.assert_not_called() + moved = np.eye(4) + moved[0, 3] = 2.0 + policy.update(moved, dt) + dt.assert_called_once_with() diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index 67aa9e45bb26..09f38fe284d8 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -33,6 +33,76 @@ def test_layout_preserves_positional_constructor(): assert camera_feed._PanelDescriptor(**descriptor).use_scene_partition is False +def test_ovxr_feed_selection_preserves_order_shared_renderer_and_rate_limits(): + from isaaclab_ov.renderers import OVRTXRendererCfg + + feeds = [ + XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=12.0), + XrCameraFeedCfg(camera_name="wrist_cam", max_update_hz=0.0, offset_m=(0.1, -0.2), distance_m=0.9), + ] + cfg = _teleop_env_cfg(feeds, camera=_camera_cfg()) + cfg.scene.wrist_cam = _camera_cfg() + original = cfg.scene.robot_pov_cam.renderer_cfg + renderer = OVRTXRendererCfg() + with pytest.raises(ValueError, match="missing.*CameraCfg"): + camera_feed.select_ovxr_camera_feeds(cfg, ["robot_pov_cam", "missing"], renderer) + assert cfg.scene.robot_pov_cam.renderer_cfg is original + + selected = camera_feed.select_ovxr_camera_feeds(cfg, ["wrist_cam", "robot_pov_cam"], renderer) + cameras = camera_feed.resolve_ovxr_camera_configs( + selected, XrCameraFeedLayoutCfg(placement="head_locked"), max_fps=20.0 + ) + assert [camera.name for camera in cameras] == ["wrist_cam", "robot_pov_cam"] + assert [camera.max_fps for camera in cameras] == [20.0, 12.0] + # Panel configurations copy the sensor configuration; equal renderer configurations share one renderer. + assert all(camera.camera_cfg.renderer_cfg == renderer for camera in cameras) + assert all(camera.space == "head_locked" for camera in cameras) + assert cameras[0].position == (0.1, -0.2, -0.9) + with pytest.raises(ValueError, match="position override requires one camera"): + camera_feed.resolve_ovxr_camera_configs( + selected, + XrCameraFeedLayoutCfg(placement="head_locked"), + max_fps=20.0, + position_override=(0.1, 0.2, -0.8), + ) + overridden = camera_feed.resolve_ovxr_camera_configs( + selected[:1], + XrCameraFeedLayoutCfg(placement="head_locked"), + max_fps=20.0, + position_override=(0.1, 0.2, -0.8), + ) + assert overridden[0].position == (0.1, 0.2, -0.8) + + +@pytest.mark.parametrize("placement", ["viewer_start", "world"]) +def test_ovxr_rejects_unsupported_panel_placement(placement): + selected = [("robot_pov_cam", _camera_cfg(), XrCameraFeedCfg(camera_name="robot_pov_cam"))] + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(0.0, 0.8, 1.6)) + with pytest.raises(ValueError, match="require head_locked"): + camera_feed.resolve_ovxr_camera_configs(selected, layout, max_fps=20.0) + + +@pytest.mark.parametrize( + "field,value,mode", + [ + ("max_update_hz", -1.0, "manual"), + ("max_update_hz", float("nan"), "manual"), + ("max_update_hz", float("inf"), "manual"), + ("distance_m", 0.0, "manual"), + ("distance_m", -0.5, "manual"), + ("offset_m", (0.0, 0.0, 1.0), "manual"), + ("offset_m", (float("nan"), 0.0), "horizontal"), + ], +) +def test_ovxr_rejects_invalid_feed_before_layout_conversion(field, value, mode): + feed = XrCameraFeedCfg(camera_name="robot_pov_cam", **{field: value}) + selected = [(feed.camera_name, _camera_cfg(), feed)] + with pytest.raises(ValueError, match=field): + camera_feed.resolve_ovxr_camera_configs( + selected, XrCameraFeedLayoutCfg(mode=mode, placement="head_locked"), max_fps=20.0 + ) + + @pytest.fixture def isolated_settings(monkeypatch): values = {ISAAC_RTX_SHOW_ALL_PARTITIONS_BY_DEFAULT_SETTING: True} @@ -729,3 +799,38 @@ def test_isaac_teleop_default_has_no_camera_feeds(): cfg = IsaacTeleopCfg(pipeline_builder=lambda: None) assert cfg.xr_camera_feeds == [] + + +def test_ovxr_automatic_panels_use_shared_layout_without_mutating_feeds(): + feeds = [XrCameraFeedCfg(camera_name=name, panel_width_m=0.4) for name in ("left", "right")] + selected = [(feed.camera_name, _camera_cfg(), feed) for feed in feeds] + layout = XrCameraFeedLayoutCfg( + mode="horizontal", + placement="head_locked", + center_offset_m=(0.1, -0.2), + panel_gap_m=0.1, + distance_m=1.2, + ) + panels = camera_feed.resolve_ovxr_camera_configs(selected, layout, max_fps=20.0) + assert panels[0].position == pytest.approx((-0.15, -0.2, -1.2)) + assert panels[1].position == pytest.approx((0.35, -0.2, -1.2)) + assert feeds[0].offset_m == XrCameraFeedCfg().offset_m + + +def test_remove_camera_configs_keeps_selected_sensors_but_drops_their_observations(): + """OVXR keeps selected PiP sensors for its renderer while no observation requests a second render.""" + from isaaclab_teleop import remove_camera_configs + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg( + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", device="cpu", num_envs=1, overrides=["presets=rgb"] + ) + assert hasattr(cfg.observations.policy, "left_wrist_cam") + remove_camera_configs(cfg, keep=("robot_pov_cam", "left_wrist_cam")) + assert hasattr(cfg.scene, "robot_pov_cam") and hasattr(cfg.scene, "left_wrist_cam") + assert not hasattr(cfg.scene, "right_wrist_cam") + for name in ("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"): + assert not hasattr(cfg.observations.policy, name) + assert hasattr(cfg.observations.policy, "robot_joint_pos") diff --git a/source/isaaclab_teleop/test/test_xr_scene_partition.py b/source/isaaclab_teleop/test/test_xr_scene_partition.py new file mode 100644 index 000000000000..de6c86c563ab --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_scene_partition.py @@ -0,0 +1,109 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from isaaclab_ov.stage import export_stage_to_string +from isaaclab_teleop.xr_scene_partition import partition_visual_meshes + +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + +def _scene(): + stage = Usd.Stage.CreateInMemory() + env = UsdGeom.Xform.Define(stage, "/World/envs/env_0").GetPrim() + env.CreateAttribute("primvars:omni:scenePartition", Sdf.ValueTypeNames.Token).Set("env_0") + UsdGeom.Mesh.Define(stage, "/Asset/head/mesh") + UsdGeom.Mesh.Define(stage, "/Asset/body/mesh") + visual = UsdGeom.Xform.Define(stage, "/World/envs/env_0/Robot/visuals").GetPrim() + visual.GetReferences().AddInternalReference("/Asset") + visual.SetInstanceable(True) + collision = UsdGeom.Mesh.Define(stage, "/World/envs/env_0/Robot/collisions/head").GetPrim() + UsdPhysics.CollisionAPI.Apply(collision) + UsdGeom.Camera.Define(stage, "/World/envs/env_0/Robot/Camera") + return stage + + +@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) +def test_partition_export_preserves_source_visibility_physics_and_camera(settings_path): + stage = _scene() + if settings_path is not None: + stage.DefinePrim(settings_path, "RenderSettings") + stage.SetMetadata("renderSettingsPrimPath", settings_path) + original = stage.GetRootLayer().ExportToString() + session = stage.GetSessionLayer().ExportToString() + head = "/World/envs/env_0/Robot/visuals/head/mesh" + with partition_visual_meshes(stage, [head]): + prim = stage.GetPrimAtPath(head) + assert UsdGeom.PrimvarsAPI(prim).FindPrimvarWithInheritance("omni:scenePartition").Get() == "xr_excluded" + assert UsdGeom.Imageable(prim).ComputeVisibility() == "inherited" + for path in ("/World/envs/env_0/Robot/visuals/body/mesh", "/World/envs/env_0/Robot/collisions/head"): + assert ( + UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).FindPrimvarWithInheritance("omni:scenePartition").Get() + == "env_0" + ) + assert stage.GetPrimAtPath("/World/envs/env_0/Robot/Camera").IsA(UsdGeom.Camera) + layer = Sdf.Layer.CreateAnonymous() + layer.ImportFromString( + export_stage_to_string( + stage, + ("/World/envs/env_0",), + source_paths=("/World/envs/env_0",), + preserve_partition_visibility=True, + ) + ) + exported = Usd.Stage.Open(layer) + settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) + assert settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault").Get() is False + assert ( + UsdGeom.PrimvarsAPI(exported.GetPrimAtPath(head)).FindPrimvarWithInheritance("omni:scenePartition").Get() + == "xr_excluded" + ) + assert stage.GetRootLayer().ExportToString() == original + assert stage.GetSessionLayer().ExportToString() == session + assert stage.GetPrimAtPath(head).IsInstanceProxy() + + +@pytest.mark.parametrize("pattern", ["/missing", "/World/envs/env_0/Robot/collisions/head"]) +def test_reject_missing_or_collision_mesh_before_edit(pattern): + stage = _scene() + original = stage.GetSessionLayer().ExportToString() + with pytest.raises(ValueError): + with partition_visual_meshes(stage, [pattern]): + pass + assert stage.GetSessionLayer().ExportToString() == original + + +def test_export_failure_removes_temporary_layer(): + stage = _scene() + before = stage.GetSessionLayer().ExportToString() + with pytest.raises(RuntimeError): + with partition_visual_meshes(stage, ["/World/envs/env_0/Robot/visuals/head/mesh"]): + raise RuntimeError("export failed") + assert stage.GetSessionLayer().ExportToString() == before + + +@pytest.mark.parametrize("conflict", ["instance", "partition", "show_all"]) +def test_stronger_session_opinion_fails_closed_and_is_preserved(conflict): + stage = _scene() + head = "/World/envs/env_0/Robot/visuals/head/mesh" + with Usd.EditContext(stage, stage.GetSessionLayer()): + visual = stage.GetPrimAtPath("/World/envs/env_0/Robot/visuals") + if conflict == "instance": + visual.SetInstanceable(True) + elif conflict == "partition": + visual.SetInstanceable(False) + UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(head)).CreatePrimvar( + "omni:scenePartition", Sdf.ValueTypeNames.Token, UsdGeom.Tokens.constant + ).Set("env_0") + else: + settings = stage.DefinePrim("/Render/IsaacLabSettings", "RenderSettings") + settings.CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ).Set(True) + before = stage.GetSessionLayer().ExportToString() + with pytest.raises(RuntimeError, match="Stronger source opinion"): + with partition_visual_meshes(stage, [head]): + pytest.fail("A stronger source opinion must not silently defeat the exclusion") + assert stage.GetSessionLayer().ExportToString() == before diff --git a/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst b/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst new file mode 100644 index 000000000000..79924c3b7569 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst @@ -0,0 +1,2 @@ +* Fixed headless Kit XR sessions remaining idle by continuing scene publication and + application updates without a desktop viewport. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py index 66bdbbba80dd..11b27512ccab 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py @@ -170,6 +170,9 @@ def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraC def step(self, dt: float) -> None: """Advance visualizer/UI updates for one simulation step. + Headless XR still publishes scene updates and pumps the Kit application. + Ordinary headless image capture renders only when an image is requested. + Args: dt: Simulation time-step in seconds. """ @@ -178,9 +181,9 @@ def step(self, dt: float) -> None: self._app_pumped_this_step = False self._sim_time += dt self._step_counter += 1 - # Headless mode: skip the app update and camera panel refresh; rendering is - # triggered on demand by render_rgb_array() / render_tiled_rgb_array(). - if self._runtime_headless: + # Headless capture renders on demand, but XR needs continuous app updates + # to create its session and submit frames even without a desktop viewport. + if self._runtime_headless and not self._sim.get_setting("/isaaclab/xr/enabled"): return provider = self._sim.get_scene_data_provider() self._fabric.update_transforms(provider) diff --git a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py index 1f393616d38f..d3bd335c0960 100644 --- a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py +++ b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py @@ -58,12 +58,13 @@ def kit_timeline(monkeypatch): return timeline -@pytest.mark.parametrize("headless", [False, True]) -def test_viewport_pose_publication_is_deferred_for_headless_capture(monkeypatch, kit_app, headless): +@pytest.mark.parametrize(("headless", "xr"), [(False, False), (True, False), (True, True)]) +def test_viewport_pose_publication_is_deferred_only_for_headless_capture(monkeypatch, kit_app, headless, xr): visualizer = KitVisualizer(VisualizerCfg(headless=headless, backend=KitBackendCfg(origin_type="asset"))) visualizer._is_initialized = True visualizer._fabric = MagicMock() visualizer._sim = sim = MagicMock(render_generation=12) + sim.get_setting.side_effect = lambda key: xr if key == "/isaaclab/xr/enabled" else None provider = sim.get_scene_data_provider.return_value monkeypatch.setattr(visualizer, "is_training_paused", lambda: True) tracking = MagicMock() @@ -75,8 +76,8 @@ def test_viewport_pose_publication_is_deferred_for_headless_capture(monkeypatch, visualizer.step(0.1) - assert tracking.call_count == int(not headless) - if headless: + assert tracking.call_count == int(not headless or xr) + if headless and not xr: visualizer._fabric.update_transforms.assert_not_called() visualizer._fabric.update_geometries.assert_not_called() settings.set.assert_not_called()