From 3795b51222d557d76b1877e6745e8ed6322d90ef Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Fri, 30 Oct 2026 18:16:16 -0400 Subject: [PATCH 001/145] Add renderer leases and the OVXR host adapter --- .../changelog.d/renderer-lease.added.rst | 4 + .../changelog.d/renderer-lease.changed.rst | 3 + .../changelog.d/renderer-lease.minor | 0 .../isaaclab_ov/cloner/replicate.py | 8 +- .../isaaclab_ov/renderers/__init__.pyi | 2 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 188 ++--- .../isaaclab_ov/renderers/ovrtx_usd.py | 7 +- .../renderers/publication_buffers.py | 62 ++ .../isaaclab_ov/renderers/render_lease.py | 414 ++++++++++ .../isaaclab_ov/renderers/visual_materials.py | 5 + source/isaaclab_ov/isaaclab_ov/stage.py | 11 +- .../isaaclab_ov/test/test_ovrtx_clone_plan.py | 4 +- .../test/test_ovrtx_deformable_bindings.py | 8 +- .../test/test_ovrtx_renderer_contract.py | 322 +++++++- source/isaaclab_ov/test/test_ovrtx_usd.py | 18 +- .../test/test_ovrtx_visual_material_sync.py | 4 +- .../changelog.d/kitless-ovxr.added.rst | 4 + .../changelog.d/kitless-ovxr.minor | 0 source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 59 ++ source/isaaclab_teleop/docs/README.md | 4 + .../isaaclab_teleop/__init__.pyi | 11 +- .../isaaclab_teleop/ov_libraries_xr.py | 689 +++++++++++++++++ .../isaaclab_teleop/ovxr_session.py | 217 ++++++ .../test/test_ov_libraries_xr.py | 707 ++++++++++++++++++ .../isaaclab_teleop/test/test_ovxr_session.py | 57 ++ 25 files changed, 2696 insertions(+), 112 deletions(-) create mode 100644 source/isaaclab_ov/changelog.d/renderer-lease.added.rst create mode 100644 source/isaaclab_ov/changelog.d/renderer-lease.changed.rst create mode 100644 source/isaaclab_ov/changelog.d/renderer-lease.minor create mode 100644 source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py create mode 100644 source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py create mode 100644 source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst create mode 100644 source/isaaclab_teleop/changelog.d/kitless-ovxr.minor create mode 100644 source/isaaclab_teleop/docs/OVXR_ADAPTER.md create mode 100644 source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py create mode 100644 source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py create mode 100644 source/isaaclab_teleop/test/test_ov_libraries_xr.py create mode 100644 source/isaaclab_teleop/test/test_ovxr_session.py diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst new file mode 100644 index 00000000000..886b2872dc0 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst @@ -0,0 +1,4 @@ +* Added application-owned OVRTX/OVStage borrowing for external consumers, with + private-stage selection, capability-checked render-thread ownership transfer, scene publication, and ordered cleanup. +* Supported CPU scene transforms and camera buffers alongside GPU rendering. +* Published dirty visual materials with leased scene updates, including material-only changes. diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst b/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst new file mode 100644 index 00000000000..c12b3b953e6 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst @@ -0,0 +1,3 @@ +* Preserved authored ``showAllPartitionsByDefault`` values in render-stage exports, + including exports without an external lease. Scenes that relied on export forcing + all partitions visible should explicitly author ``showAllPartitionsByDefault=True``. diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.minor b/source/isaaclab_ov/changelog.d/renderer-lease.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py index f350bd6b262..387d1531621 100644 --- a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py +++ b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py @@ -177,6 +177,7 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderers = sim.get_backends(OVRTXRendererCfg) if not renderers: return + leased = [renderer for renderer in renderers if renderer._render_lease.active is not None] initialized = next((renderer for renderer in renderers if renderer.backend is not None), None) if initialized is not None: use_ovstage = initialized._use_ovstage @@ -185,13 +186,16 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], # Default to native OVRTX cloning to preserve camera-task throughput on both physics backends. # Keep sharing opt-in until OVPhysX cold binding and shared-stage update performance are fixed. with_ovphysx = sim.backend_manager.clone_context_type is OvPhysxReplicateContext - value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0").strip() + # An explicit lease selects its private OVStage without changing process globals. + value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1" if leased else "0").strip() if value not in {"0", "1"}: raise ValueError(f"Invalid ISAAC_LAB_OVRTX_USE_OVSTAGE: {value!r}. Expected 0 or 1.") share_ovstage = os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1" use_ovstage = share_ovstage or value == "1" if share_ovstage and not with_ovphysx: raise ValueError("Shared OVStage cloning requires OVPhysX.") + if leased and (not use_ovstage or share_ovstage): + raise RuntimeError("The render lease requires a private rendering OVStage") if share_ovstage and sim.backend_manager.backend is not None and sim.backend_manager.backend.owns_scene: raise RuntimeError("Configure OVRTX cameras before the first OVPhysX reset to share its OVStage.") if use_ovstage: @@ -230,6 +234,8 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderer.backend = sim.get_or_create_backend(backend_cfg) renderer.scene = scene if use_ovstage else renderer.backend renderer._use_ovstage = use_ovstage + if renderer in leased: + renderer._render_lease.reserve_pair() if renderer.cfg.async_rendering and use_ovstage: logger.warning("Asynchronous OVRTX rendering is unsupported with ovstage; rendering synchronously.") diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi index d430a333224..a018004f429 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi +++ b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi @@ -7,9 +7,11 @@ __all__ = [ "OVRTXRenderer", "OVRTXRendererCfg", "OVRTXBackendCfg", + "OvLibrariesRenderLease", "Renderer", ] from .ovrtx_renderer import OVRTXRenderer from .ovrtx_renderer import OVRTXRenderer as Renderer from .ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg +from .render_lease import OvLibrariesRenderLease diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index a9c7152d275..6c34a2cca16 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -14,6 +14,10 @@ its own pending ``step_async`` render operation and publishes it at the next ``read_output``. The ovrtx binding write operations retain their input buffers until ovrtx has consumed them. +- **render_lease.py**: External borrowing, ownership transfer and scene publication ordering. + +- **publication_buffers.py**: Producer-buffer retention and native write completion. + - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. - **ovrtx_usd.py**: USD helpers for OVRTX: render var config, camera injection, etc. @@ -30,10 +34,9 @@ import tempfile import weakref from builtins import ExceptionGroup -from collections import deque from collections.abc import Callable, Iterable, Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import TYPE_CHECKING, Any logger = logging.getLogger(__name__) @@ -73,8 +76,14 @@ from isaaclab.sim import PreviewSurfaceCfg, SimulationContext, use_stage from isaaclab.utils.warp import ProxyArray -from isaaclab_ov.cloner import ovrtx_replicate -from isaaclab_ov.renderers.ovrtx_annotator_utils import ( +from ..cloner import ovrtx_replicate +from ..stage import ( + export_stage_to_string, + points_tensor_from_warp, + xform_tensor_from_numpy, + xform_tensor_from_warp, +) +from .ovrtx_annotator_utils import ( IdentifierMap, build_instance_id_to_labels_and_semantics, build_semantic_id_to_labels, @@ -82,10 +91,10 @@ decode_stable_id_map, decode_stable_id_semantic_id_map, ) -from isaaclab_ov.renderers.ovrtx_compat import OVRTX_USES_NAMED_SEGMENTATION_MAPS -from isaaclab_ov.renderers.ovrtx_compat import OVRTX_VERSION as _OVRTX_VERSION -from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg -from isaaclab_ov.renderers.ovrtx_renderer_kernels import ( +from .ovrtx_compat import OVRTX_USES_NAMED_SEGMENTATION_MAPS +from .ovrtx_compat import OVRTX_VERSION as _OVRTX_VERSION +from .ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg +from .ovrtx_renderer_kernels import ( accumulate_mesh_normals, create_camera_transforms_kernel, create_marker_transforms_kernel, @@ -93,19 +102,15 @@ generate_random_colors_from_ids_kernel, normalize_mesh_normals, ) -from isaaclab_ov.renderers.ovrtx_shader_cache import redirect_shader_cache -from isaaclab_ov.renderers.ovrtx_usd import ( +from .ovrtx_shader_cache import redirect_shader_cache +from .ovrtx_usd import ( _RTX_MINIMAL_MODES, build_render_product_as_string, render_var_prim_names_by_source, ) -from isaaclab_ov.renderers.visual_materials import OVRTXVisualMaterialWriter -from isaaclab_ov.stage import ( - export_stage_to_string, - points_tensor_from_warp, - xform_tensor_from_numpy, - xform_tensor_from_warp, -) +from .publication_buffers import _AsyncWriteBuffers +from .render_lease import OvLibrariesRenderLease, RenderLeaseController +from .visual_materials import OVRTXVisualMaterialWriter if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline @@ -113,7 +118,7 @@ from isaaclab.renderers.base_renderer import VisualMaterialBatch - from isaaclab_ov.stage import OvstageBackend + from ..stage import OvstageBackend from isaaclab.renderers.camera_render_spec import CameraRenderSpec @@ -213,6 +218,7 @@ class OVRTXBackend: def __init__(self, cfg: OVRTXBackendCfg): self.cfg = cfg self.attached = False + self._render_lease_owner: OVRTXRenderer | None = None self.render_products: set[str] = set() # Resolve the wheel's native dependency here, including callers without a viewer. dependency = Path(ovstage.__file__).parent / "bin/plugins/libosdCPU.so.3.6.0" @@ -243,12 +249,16 @@ def __init__(self, cfg: OVRTXBackendCfg): def detach(self) -> None: """Release the borrowed stage before it is rebuilt or destroyed.""" + if self._render_lease_owner is not None: + raise RuntimeError("Cannot detach a backend with an active render lease") if self.attached: self.renderer.detach_ovstage() self.attached = False def close(self) -> None: """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward.""" + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy a backend with an active render lease") if self.renderer is not None: # The SDK destroys bindings and detaches only if attached, including partial initialization. self.renderer.destroy() @@ -334,6 +344,8 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): super().__init__(cfg, sim=sim) + self._closed = False + self._render_lease = RenderLeaseController(self) self._warp_device = wp.get_device(self._sim.device) self._device = str(self._warp_device) self._camera_render_data: list[OVRTXCameraRenderData] = [] @@ -375,8 +387,27 @@ def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ... self._visual_material_writer_ref = weakref.ref(writer) return writer + @contextlib.contextmanager + def _publish_visual_materials(self) -> Iterator[None]: + """Publish dirty materials and complete their reads before buffers may change.""" + writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None + try: + if writer is not None: + writer.publish() + yield + finally: + if writer is not None: + primary_active = sys.exc_info()[0] is not None + try: + writer.drain() + except Exception as exc: + if not primary_active: + raise + logger.warning("Error draining material writes after a failed scene update: %s", exc, exc_info=True) + def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" + self._render_lease.prepare_stage(num_envs) logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -447,6 +478,11 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: self.scene.resources.callback(self.close) self._initialized_scene = True self._needs_transform_initialization = not self._use_ovstage and self.backend.cfg.read_gpu_transforms + self._render_lease.bind_stage() + + def acquire_ov_libraries_render_lease(self) -> OvLibrariesRenderLease: + """Reserve the native renderer/scene pair before stage preparation.""" + return self._render_lease.acquire() def _setup_xform_bindings(self, stage: Usd.Stage) -> None: """Bind SDP body paths and preserve their authored scales when writing world poses.""" @@ -556,6 +592,7 @@ def _setup_geometry_bindings(self) -> None: def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: """Register a camera's product and bindings in the prepared scene.""" + self._render_lease.reject_ordinary_operation("create ordinary camera render data") if str(wp.get_device(spec.device)) != self._device: raise ValueError("Cameras sharing an OVRTX renderer must use the simulation device.") camera_paths = list(spec.camera_prim_paths) @@ -574,7 +611,10 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: render_data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) scope, product_path = render_data.render_scope_name, render_data.render_product_path usd = build_render_product_as_string( - spec, render_data, device_id=self._warp_device.ordinal, enable_shadows=self.cfg.enable_shadows + spec, + render_data, + device_id=self._warp_device.ordinal if self._warp_device.is_cuda else None, + enable_shadows=self.cfg.enable_shadows, ) try: # Registration mutates the shared scene; finish renders using its existing products first. @@ -741,7 +781,7 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Wait for rendering on the consuming stream, or on the host on Linux unless :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` is ``1``. Consume the zero-copy view inside the context. Unmapping records the consuming stream; native release waits for its queued - reads and the last view to be dropped. + reads and the last view to be dropped. CPU consumers wait on the host and release without a CUDA stream. Args: render_var: OVRTX ``RenderVarOutput`` to map (looked up from ``frame.render_vars``). @@ -749,6 +789,14 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Yields: The render var's contents as a Warp array, valid for the duration of the context. """ + if not self._warp_device.is_cuda: + mapping = render_var.map(device=Device.CPU) + try: + mapping.wait() + yield wp.from_dlpack(mapping) + finally: + mapping.unmap() + return gpu_side_sync = _gpu_side_render_var_sync_enabled() # OVRTX uses 0 for no synchronization and 1 for Torch's legacy default stream (CUDA handle 0). stream = self._warp_device.stream.cuda_stream or 1 @@ -982,6 +1030,11 @@ def _process_render_products( ) def update_transforms(self) -> None: + """Publish ordinary scene data when the native pair is not leased.""" + self._render_lease.reject_ordinary_operation("update ordinary transforms") + self._update_transforms() + + def _update_transforms(self) -> None: """Write changed SDP transforms to OVRTX.""" binding = self._object_xform_binding if binding is None: @@ -989,7 +1042,7 @@ def update_transforms(self) -> None: timestamp = self._sdp.backend.transforms_timestamp if self._transforms_timestamp == timestamp: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage output = self._transform_writes.acquire(stream) if not self._sdp.get_transforms(output, allow_passthrough=not asynchronous, scales=self._object_scales): @@ -1005,15 +1058,24 @@ def update_transforms(self) -> None: tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._transform_writes.submit(binding, matrices, stream) else: - binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + matrices, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._transforms_timestamp = timestamp def update_geometries(self) -> None: + """Publish ordinary scene data when the native pair is not leased.""" + self._render_lease.reject_ordinary_operation("update ordinary geometries") + self._update_geometries() + + def _update_geometries(self) -> None: """Write changed SDP geometry to OVRTX.""" for callback in self.geometry_callbacks.values(): callback(self) @@ -1032,7 +1094,7 @@ def update_geometries(self) -> None: binding = self._geometry_points_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage if asynchronous: output = self._geometry_writes.acquire(stream) @@ -1053,12 +1115,16 @@ def update_geometries(self) -> None: tensors=[points_tensor_from_warp(array) for array in points], is_array=True, semantic=ovstage.AttributeSemantic.POINT, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._geometry_writes.submit(binding, points, stream) else: - binding.write(points, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + points, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._geometry_timestamp = timestamp def update_camera( @@ -1069,10 +1135,11 @@ def update_camera( intrinsics: ProxyArray, ) -> None: """Update camera poses using the camera's reusable conversion buffers.""" + self._render_lease.reject_ordinary_operation("update ordinary cameras") binding = render_data.camera_xform_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None matrices = render_data.camera_writes.acquire(stream) wp.launch( create_camera_transforms_kernel, len(matrices), [positions, orientations, matrices], device=self._device @@ -1085,7 +1152,7 @@ def update_camera( tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() else: operation = render_data.camera_writes.submit(binding, matrices, stream) @@ -1093,13 +1160,14 @@ def update_camera( operation.wait() def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsics: wp.array, parameters: wp.array): - """Publish calibration columns from GPU memory into the renderer-owned scene.""" + """Publish calibration columns from their producing device into the renderer-owned scene.""" # A runtime calibration change is a scene write for this camera's bindings, so its # in-flight renders must finish first. Other cameras keep their pipelines. + self._render_lease.reject_ordinary_operation("update ordinary camera intrinsics") errors = self.drain_pending_renders((render_data,)) if errors: raise ExceptionGroup("OVRTX renders failed before calibration update", errors) - stream = wp.get_stream(parameters.device).cuda_stream or 1 + stream = (wp.get_stream(parameters.device).cuda_stream or 1) if parameters.device.is_cuda else None if self._use_ovstage: self.scene.stage.write_attributes( render_data.camera_xform_binding, @@ -1126,6 +1194,7 @@ def render(self, render_data: OVRTXCameraRenderData) -> None: def log_mesh(self, name: str, *args, **kwargs) -> None: """Publish a runtime mesh through the same scene used by every camera.""" + self._render_lease.reject_ordinary_operation("log runtime meshes") if name not in self.meshes: self.meshes[name] = OVRTXMesh(self, f"/IsaacLabMeshes/{name.lstrip('/')}") self.meshes[name].update(*args, **kwargs) @@ -1147,15 +1216,13 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: """ if not render_data: return + self._render_lease.reject_ordinary_operation("render an ordinary camera batch") if not self._initialized_scene: raise RuntimeError("Scene not initialized. Call initialize() first.") if self.backend.renderer is None or not self._camera_render_data: return products = self.backend.render_products.copy() - material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None - try: - if material_writer is not None: - material_writer.publish() + with self._publish_visual_materials(): if self._needs_transform_initialization: # OVRTX 0.6 initializes GPU world transforms on its first step using # authored poses. Flush that initialization before capturing runtime @@ -1177,18 +1244,6 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: else: result = self.backend.renderer.step(render_products=products, delta_time=_RENDER_DELTA_TIME) self._process_render_products(render_data, result) - finally: - if material_writer is not None: - # When another exception is already propagating, log the drain failure instead of - # replacing it. Losing it silently would hide a failed material write. - primary_active = sys.exc_info()[0] is not None - try: - material_writer.drain() - except Exception as e: - if not primary_active: - raise - logger.warning("Error draining material writes after a failed render: %s", e, exc_info=True) - if self._use_ovstage: # Stage writes and material reads finish before rendering. Consume only after advancing # the ordinal so a failed readback cannot leave later writes at the sealed floor. @@ -1221,6 +1276,7 @@ def _remove_usd_reference(self, reference: int) -> None: def close(self) -> None: """Complete borrowed-buffer reads and release bindings; the registry owns the native engine.""" + self._render_lease.reject_ordinary_operation("close the renderer") try: with contextlib.ExitStack() as resources: resources.callback(self._bindings.close) @@ -1245,6 +1301,7 @@ def close(self) -> None: self.meshes.clear() self.markers.clear() self.geometry_callbacks.clear() + self._closed = True class OVRTXMesh: @@ -1577,42 +1634,3 @@ def close(self) -> None: raise ExceptionGroup("OVRTX renders failed before marker cleanup", errors) self._transforms = self._selection = self._stage = None self._local_transforms = None - - -_BufferT = TypeVar("_BufferT", wp.array, SceneDataFormat.TransposedMatrix44d) - - -class _AsyncWriteBuffers(Generic[_BufferT]): - """Retain GPU buffers until native writes finish; reuse them in submission order.""" - - def __init__(self, buffers: Iterable[_BufferT] = ()): - self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( - (buffer, None, None) for buffer in buffers - ) - - def acquire(self, stream: wp.Stream) -> _BufferT: - """Wait until the next buffer is writable on the given stream.""" - buffer, operation, producer = self._writes[0] - if operation is not None: - operation.wait() - if producer != stream: - stream.wait_stream(producer) - return buffer - - def submit( - self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream - ) -> Operation[bool]: - """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" - operation = binding.write_async(values, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) - self._writes[0] = (self._writes[0][0], operation, stream) - self._writes.rotate(-1) - return operation - - def close(self) -> None: - """Complete every pending write before releasing storage, including on failure.""" - with contextlib.ExitStack() as writes: - writes.callback(self._writes.clear) - for _, operation, producer in self._writes: - if operation is not None: - writes.callback(wp.synchronize_stream, producer) - writes.callback(operation.wait) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 85032bcbd42..30094ae61f2 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -144,7 +144,7 @@ def build_render_product_as_string( spec: CameraRenderSpec, render_data: OVRTXCameraRenderData, *, - device_id: int, + device_id: int | None = None, enable_shadows: bool = False, ) -> str: """Build a complete render product USD layer as a string. @@ -160,6 +160,7 @@ def build_render_product_as_string( render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. + When None, leave native GPU selection automatic for CPU consumers. enable_shadows: Whether lights cast shadows. Defaults to False. Only honored for the ``simple_shading_*`` data types, which are the ones that select RTX Minimal mode. @@ -210,6 +211,8 @@ def build_render_product_as_string( for _, name, source in render_var_configs ) + device_ids_line = "" if device_id is None else f"uint[] deviceIds = [{device_id}]" + layer = f'''#usda 1.0 (defaultPrim = "{render_data.render_scope_name}") @@ -219,7 +222,7 @@ def RenderProduct "RenderProduct" ( prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] ) {{ rel camera = [<{render_data.camera_paths[0]}>] - uint[] deviceIds = [{device_id}] + {device_ids_line} {bg_type_line} {render_mode_block} token[] omni:rtx:waitForEvents = ["AllLoadingFinished", "OnlyOnFirstRequest"] diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py b/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py new file mode 100644 index 00000000000..950990d3bd2 --- /dev/null +++ b/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py @@ -0,0 +1,62 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Producer storage retained until native scene publication completes.""" + +from __future__ import annotations + +import contextlib +from collections import deque +from collections.abc import Iterable +from typing import TYPE_CHECKING, Generic, TypeVar + +import warp as wp +from ovrtx import DataAccess + +from isaaclab.scene_data import SceneDataFormat + +if TYPE_CHECKING: + from ovrtx import AttributeBinding, Operation +_BufferT = TypeVar("_BufferT", wp.array, SceneDataFormat.TransposedMatrix44d) + + +class _AsyncWriteBuffers(Generic[_BufferT]): + """Retain buffers until native writes finish; reuse them in submission order.""" + + def __init__(self, buffers: Iterable[_BufferT] = ()): + self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( + (buffer, None, None) for buffer in buffers + ) + + def acquire(self, stream: wp.Stream | None) -> _BufferT: + """Wait until the next buffer is writable on the given stream.""" + buffer, operation, producer = self._writes[0] + if operation is not None: + operation.wait() + if stream is not None and producer is not None and producer != stream: + stream.wait_stream(producer) + return buffer + + def submit( + self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream | None + ) -> Operation[bool]: + """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" + operation = binding.write_async( + values, data_access=DataAccess.ASYNC, cuda_stream=(stream.cuda_stream or 1) if stream is not None else None + ) + self._writes[0] = (self._writes[0][0], operation, stream) + self._writes.rotate(-1) + return operation + + def close(self) -> None: + """Complete every pending write before releasing storage, including on failure.""" + with contextlib.ExitStack() as writes: + writes.callback(self._writes.clear) + for _, operation, producer in self._writes: + if operation is not None: + if producer is not None: + writes.callback(wp.synchronize_stream, producer) + writes.callback(operation.wait) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py new file mode 100644 index 00000000000..4a56beb3bd5 --- /dev/null +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -0,0 +1,414 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Exclusive borrowing, thread handoff and publication ordering for OVRTX/OVStage.""" + +from __future__ import annotations + +import contextlib +import os +import threading +from collections.abc import Hashable, Iterator +from enum import Enum, auto +from typing import TYPE_CHECKING, cast + +if TYPE_CHECKING: + from ovrtx import Renderer + from ovstage import Stage as OvStage + + from .ovrtx_renderer import OVRTXRenderer +_NO_PHYSICS_STEP_ID = object() + + +class _OvLibrariesRenderLeaseState(Enum): + """Internal lifecycle state for a renderer/scene pair lease.""" + + UNBOUND = auto() + BOUND = auto() + RELEASED = auto() + + +class OvLibrariesRenderLease: + """Exclusive non-owning lease over one exact attached OVRTX/OVStage pair. + + The lease is acquired before stage preparation. Stage preparation then + materializes the renderer-owned scene without requiring a camera sensor and + binds this object to the exact live Python ``Renderer`` and ``Stage`` + instances. External consumers must release the lease before the renderer is + closed; they must never detach or destroy either borrowed object. + """ + + def __init__(self) -> None: + raise RuntimeError("OvLibrariesRenderLease must be acquired from OVRTXRenderer") + + def __enter__(self) -> OvLibrariesRenderLease: + self._validate_active() + return self + + def __exit__(self, *_exc: object) -> None: + self.release() + + @property + def ovrtx_renderer(self) -> Renderer: + """Return the exact retained application renderer.""" + self.validate_live() + return cast("Renderer", self._renderer) + + @property + def ovstage(self) -> OvStage: + """Return the exact retained renderer-attached OVStage.""" + self.validate_live() + return cast("OvStage", self._stage) + + def offer_owner_transfer(self) -> object: + """Freeze one unbound lease and return its one-shot owner capability.""" + owner = cast("RenderLeaseController", self._controller) + return owner.offer_owner_transfer(self) + + def claim_owner_transfer(self, capability: object) -> None: + cast("RenderLeaseController", self._controller).claim_owner_transfer(self, capability) + + def cancel_owner_transfer(self, capability: object) -> None: + cast("RenderLeaseController", self._controller).cancel_owner_transfer(self, capability) + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application-thread scene publisher before render ownership transfers.""" + return cast("RenderLeaseController", self._controller).capture_scene_publisher(self) + + @contextlib.contextmanager + def ovstage_owner_lane(self) -> Iterator[None]: + """Guard one non-reentrant operation on the render-owner thread. + + The application's publication guard must separately exclude concurrent + producer writes while this operation uses the borrowed stage. + """ + self.validate_live() + owner = cast("RenderLeaseController", self._controller) + with owner.owner_lane("external renderer operation"): + yield + + def release(self) -> None: + """Release the borrowing intent without destroying either pair member.""" + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + return + owner = cast("RenderLeaseController", self._controller) + owner.release(self) + self._renderer = None + self._stage = None + self._state = _OvLibrariesRenderLeaseState.RELEASED + + def validate_live(self) -> None: + """Verify that this lease still refers to its live renderer and stage.""" + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OvLibrariesRenderLease is not bound to a prepared Stage") + owner = cast("RenderLeaseController", self._controller) + owner.validate_live( + self, + cast("Renderer", self._renderer), + cast("OvStage", self._stage), + self._generation, + ) + + @classmethod + def _create_unbound(cls, owner: RenderLeaseController) -> OvLibrariesRenderLease: + lease = object.__new__(cls) + lease._controller = owner + lease._state = _OvLibrariesRenderLeaseState.UNBOUND + lease._renderer = None + lease._stage = None + lease._generation = 0 + return lease + + def _bind(self, renderer: Renderer, stage: OvStage, generation: int) -> None: + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("OvLibrariesRenderLease can only bind once") + self._renderer = renderer + self._stage = stage + self._generation = generation + self._state = _OvLibrariesRenderLeaseState.BOUND + + def _validate_active(self) -> None: + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + raise RuntimeError("OvLibrariesRenderLease is released") + owner = cast("RenderLeaseController", self._controller) + owner.validate_active(self) + + +class OvLibrariesScenePublisher: + """Application-thread writer for complete provider-derived OVStage batches.""" + + def __init__(self, owner: RenderLeaseController, lease: OvLibrariesRenderLease) -> None: + self._controller = owner + self._lease = lease + self._application_thread = threading.current_thread() + self._lock = threading.Lock() + self._closed = False + + def publication_required(self, physics_step_id: Hashable) -> bool: + self._require_application_thread() + return self._controller.producer_publication_required(self._lease, physics_step_id) + + def write_complete_scene(self, ordinal: int, physics_step_id: Hashable) -> None: + """Write, but do not publish, one complete batch at ``ordinal``.""" + self._require_application_thread() + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + self._controller.write_producer_scene(self._lease, ordinal, physics_step_id) + + def close(self) -> None: + """Stop scene publication and wait for an in-flight batch to finish.""" + with self._lock: + self._closed = True + + def _require_application_thread(self) -> None: + if threading.current_thread() is not self._application_thread: + raise RuntimeError("OVStage scene publisher must run on its captured application thread") + + +class RenderLeaseController: + """Coordinate external borrowing without owning the renderer's native resources. + + Lease state, thread transfer and publication ordering live here. The renderer + retains scene/camera storage and supplies the component-write operations. + """ + + def __init__(self, renderer: OVRTXRenderer) -> None: + self._renderer = renderer + self._owner_thread = threading.current_thread() + self._transfer_lock = threading.Lock() + self._transfer_capability: object | None = None + self._transfer_claimed = False + self.active: OvLibrariesRenderLease | None = None + self._publisher: OvLibrariesScenePublisher | None = None + self._generation = 0 + self._lane_operation: str | None = None + self._last_step_id: Hashable = _NO_PHYSICS_STEP_ID + self._poisoned = False + + def acquire(self) -> OvLibrariesRenderLease: + """Reserve the exact OVRTX/OVStage pair before stage preparation.""" + self.require_owner_thread("acquire a render lease") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise RuntimeError("OvLibrariesRenderLease requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") + if os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1": + raise RuntimeError("The render lease requires a private rendering OVStage") + if self._renderer._closed: + raise RuntimeError("Cannot acquire OvLibrariesRenderLease from a closed renderer") + if self.active is not None: + raise RuntimeError("An OvLibrariesRenderLease is already active") + if self._renderer.backend is not None or self._renderer._initialized_scene or self._generation != 0: + raise RuntimeError("OvLibrariesRenderLease must be acquired before clone preparation") + lease = OvLibrariesRenderLease._create_unbound(self) + self.active = lease + return lease + + def require_owner_thread(self, operation: str) -> None: + """Reject lease operations outside the renderer construction thread.""" + if self._transfer_capability is not None: + raise RuntimeError("OVRTXRenderer owner transfer is pending") + if threading.current_thread() is not self._owner_thread: + raise RuntimeError(f"OVRTXRenderer owner thread must {operation}") + + def offer_owner_transfer(self, lease: OvLibrariesRenderLease) -> object: + self.require_owner_thread("offer an owner transfer") + with self._transfer_lock: + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + or self._transfer_capability is not None + or self._transfer_claimed + ): + raise RuntimeError("OVRTXRenderer is not eligible for a pre-use owner transfer") + capability = object() + self._transfer_capability = capability + return capability + + def claim_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + if ( + capability is None + or capability is not self._transfer_capability + or self._transfer_claimed + or self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + ): + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._owner_thread = threading.current_thread() + self._transfer_claimed = True + self._transfer_capability = None + + def cancel_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + # Cancellation is the one original-owner operation allowed while an offer is pending. + if threading.current_thread() is not self._owner_thread: + raise RuntimeError("OVRTXRenderer owner thread must cancel an owner transfer") + if capability is None or capability is not self._transfer_capability or self.active is not lease: + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._transfer_capability = None + + def capture_scene_publisher(self, lease: OvLibrariesRenderLease) -> OvLibrariesScenePublisher: + """Create the scene publisher before render ownership transfers.""" + self.require_owner_thread("capture an OVStage scene publisher") + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._publisher is not None + ): + raise RuntimeError("OVRTXRenderer is not eligible to capture an OVStage scene publisher") + publisher = OvLibrariesScenePublisher(self, lease) + self._publisher = publisher + return publisher + + @contextlib.contextmanager + def owner_lane(self, operation: str) -> Iterator[None]: + """Reject nested operations and access outside the render-owner thread. + + This lane is not a cross-thread lock. The external consumer must share + its application publication guard with the captured scene publisher. + """ + self.require_owner_thread(f"enter the OVStage owner lane for {operation}") + if self._lane_operation is not None: + raise RuntimeError( + f"OVStage owner lane is already active for {self._lane_operation}; nested writers are forbidden" + ) + self._lane_operation = operation + try: + yield + finally: + self._lane_operation = None + + def validate_active(self, lease: OvLibrariesRenderLease) -> None: + self.require_owner_thread("access a render lease") + if self.active is not lease: + raise RuntimeError("OvLibrariesRenderLease is not active for this renderer") + if self._renderer._closed: + raise RuntimeError("OvLibrariesRenderLease owner is closed or no longer uses OVStage") + + def validate_live( + self, + lease: OvLibrariesRenderLease, + ovrtx_renderer: Renderer, + ovstage_instance: OvStage, + generation: int, + ) -> None: + """Validate retained Python-object identity for the live attached pair.""" + self.validate_active(lease) + if self._poisoned: + raise RuntimeError("OvLibrariesRenderLease owner pair is poisoned") + if generation != self._generation: + raise RuntimeError("OvLibrariesRenderLease has a stale Stage generation") + if ovrtx_renderer is not self._renderer.backend.renderer or ovstage_instance is not self._renderer.scene.stage: + raise RuntimeError("OvLibrariesRenderLease no longer matches its exact renderer-owned pair") + if not self._renderer._initialized_scene or not self._renderer.backend.attached: + raise RuntimeError("OvLibrariesRenderLease pair is no longer initialized and attached") + + def release(self, lease: OvLibrariesRenderLease) -> None: + """Retire the lease after borrower destruction and rebase the dormant ordinal.""" + self.validate_active(lease) + if self._lane_operation is not None: + raise RuntimeError("Cannot release OvLibrariesRenderLease while the owner lane is active") + if lease._state is _OvLibrariesRenderLeaseState.BOUND and (not self._poisoned): + with self.owner_lane("lease-release ordinal rebase"): + self._renderer.scene.ordinal = self._next_ordinal() + self._last_step_id = _NO_PHYSICS_STEP_ID + if self._publisher is not None: + self._publisher.close() + self._publisher = None + self._transfer_claimed = False + self.active = None + if self._renderer.backend is not None: + self._renderer.backend._render_lease_owner = None + if self._renderer.scene is not None: + self._renderer.scene._render_lease_owner = None + + def reject_ordinary_operation(self, operation: str) -> None: + """Reject ordinary OVRTX operations while an external lease is active.""" + if self.active is not None or ( + self._renderer.backend is not None and self._renderer.backend._render_lease_owner is not None + ): + raise RuntimeError(f"Cannot {operation} while an OvLibrariesRenderLease is active") + + def _next_ordinal(self) -> int: + """Return ``floor + 1`` from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None: + raise RuntimeError("Cannot query an OVStage floor before scene materialization") + with self._renderer.scene.stage.get_attribute_write_floor() as floor_query: + floor_query.wait() + floor = int(floor_query.fetch()) + if floor >= (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + return floor + 1 + + def producer_publication_required(self, lease: OvLibrariesRenderLease, physics_step_id: Hashable) -> bool: + """Check publication on the application thread without entering the render thread.""" + if self.active is not lease or self._publisher is None: + raise RuntimeError("OVStage scene publisher is not active for this lease") + if self._poisoned or lease._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OVStage scene publisher pair is unavailable") + try: + hash(physics_step_id) + except TypeError as exc: + raise TypeError("physics_step_id must be hashable") from exc + writer_ref = self._renderer._visual_material_writer_ref + writer = writer_ref() if writer_ref is not None else None + return self._last_step_id != physics_step_id or (writer is not None and writer.has_pending_updates) + + def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, physics_step_id: Hashable) -> None: + """Write a complete scene batch without advancing its reserved ordinal.""" + if ordinal <= 0: + raise ValueError("OVStage publication ordinal must be nonzero") + if not self.producer_publication_required(lease, physics_step_id): + raise RuntimeError("OVStage scene publication is not required for this complete state") + self._renderer.scene.ordinal = ordinal + try: + with self._renderer._publish_visual_materials(): + self._renderer._update_transforms() + self._renderer._update_geometries() + except Exception as exc: + self._poisoned = True + raise RuntimeError("Producer scene publication failed; recreate the exact pair") from exc + self._last_step_id = physics_step_id + + def prepare_stage(self, num_envs: int) -> None: + """Validate leased preparation; ordinary rendering is immediately allowed.""" + lease = self.active + if lease is not None: + self.require_owner_thread("prepare the leased Stage") + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + if num_envs != 1: + raise RuntimeError("The render lease requires exactly one environment") + if not self._renderer._use_ovstage or self._renderer.backend._render_lease_owner is not self._renderer: + raise RuntimeError("The render lease requires its reserved private OVStage renderer") + + def bind_stage(self) -> None: + """Bind the prepared native pair after renderer initialization succeeds.""" + lease = self.active + if lease is not None: + self._generation += 1 + lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) + + def reserve_pair(self) -> None: + """Reserve a newly acquired private native pair during clone preparation.""" + renderer = self._renderer + if renderer.backend._render_lease_owner is not None or renderer.scene._render_lease_owner is not None: + raise RuntimeError("The native pair already has a render lease") + with renderer.scene.stage.get_attribute_write_floor() as floor: + floor.wait() + if int(floor.fetch()) != 0: + raise RuntimeError("The render lease requires an unused private OVStage") + renderer.backend._render_lease_owner = renderer + renderer.scene._render_lease_owner = renderer diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 3138cf964e8..38b0505d6f9 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -79,6 +79,11 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, self._addresses.append((channel, address, attribute_name, rows)) self._resources = resources.pop_all() + @property + def has_pending_updates(self) -> bool: + """Whether material changes require a scene publication even without a physics step.""" + return bool(self._dirty_channels) + def __call__(self, material_offsets: dict[str, Any] | None = None, env_ids: Any | None = None) -> None: """Mark channels dirty; OVRTX currently copies each dirty channel's full device buffer.""" del env_ids diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index e9f9f67a0d5..230d57d4f8f 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -175,6 +175,7 @@ def __init__(self, cfg: OvstageBackendCfg): self.clone_env_paths: list[str] = [] self.population_env_paths: list[str] = [] self.clone_positions: np.ndarray | None = None + self._render_lease_owner = None self.resources = contextlib.ExitStack() """Release borrowers before paths and stage, in reverse registration order.""" self.reset() @@ -225,6 +226,8 @@ def commit(self) -> int: def close(self) -> None: """Release borrowers, then the path dictionary and stage.""" + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy or reset a stage with an active render lease") self.resources.close() self.stage = self.paths = None @@ -270,13 +273,15 @@ def export_stage_to_string( export_session.subLayerPaths = [stage.GetSessionLayer().identifier] export_stage = Usd.Stage.Open(stage.GetRootLayer(), export_session) export_stage.SetEditTarget(export_session) - # Perspective cameras see every partition; sensor cameras retain their explicit partition. + # Preserve authored partition visibility; default to every partition when absent. settings_path = export_stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" settings = UsdRender.Settings.Define(export_stage, settings_path) settings.GetPrim().AddAppliedSchema("OmniRtxSettingsGlobalCommonAPI_1") - settings.GetPrim().CreateAttribute( + show_all = settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ).Set(True) + ) + if not show_all.HasAuthoredValueOpinion(): + show_all.Set(True) export_stage.SetMetadata("renderSettingsPrimPath", settings_path) prim_paths = [] if len(clone_env_paths) > 1: diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index 91d0bbc7d63..1d85cb0d570 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -94,6 +94,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: clone_positions=None, render_products=set(), cfg=SimpleNamespace(read_gpu_transforms=False), + _render_lease_owner=None, ) renderer.scene = renderer.backend renderer.backend.renderer = SimpleNamespace( @@ -104,7 +105,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: write_array_attribute=lambda *args, **kwargs: None, write_attribute=lambda *args, **kwargs: None, ) - renderer._warp_device = SimpleNamespace(ordinal=0) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True) renderer.scene.next_camera_id = 0 return renderer @@ -284,6 +285,7 @@ def test_create_render_data_writes_combined_stage_on_the_simulation_device(tmp_p class Device: ordinal = 1 + is_cuda = True def __str__(self): return "cuda:1" diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 529ab0a469b..edc27f8e13b 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -42,11 +42,11 @@ def _simulation(monkeypatch): def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) - renderer.backend = SimpleNamespace(renderer=MagicMock()) + renderer.backend = SimpleNamespace(renderer=MagicMock(), _render_lease_owner=None) renderer.scene = renderer.backend renderer.scene.query = OvstageBackend.query.__get__(renderer.scene) renderer._device = "cpu" - renderer._warp_device = SimpleNamespace(ordinal=0, stream=SimpleNamespace(cuda_stream=99)) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True, stream=SimpleNamespace(cuda_stream=99)) return renderer, renderer.backend.renderer @@ -57,7 +57,7 @@ def test_geometry_bindings_follow_mixed_sdp_points_and_pointer_swaps(mode): renderer.cfg.async_rendering = mode == "async" use_ovstage = mode == "ovstage" renderer._use_ovstage = use_ovstage - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=42)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=42)) points = { path: wp.array(np.arange(count * 3, dtype=np.float32).reshape(count, 3), dtype=wp.vec3f, device="cpu") for path, count in ( @@ -189,7 +189,7 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc stage = Usd.Stage.CreateInMemory() UsdGeom.Xform.Define(stage, paths[0]).AddScaleOp().Set(Gf.Vec3d(2, 3, 4) if scaled else Gf.Vec3d(1)) renderer.scene.clone_copies = [(paths[0], [])] - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=0)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=0)) renderer._use_ovstage = use_ovstage renderer.scene.ordinal = 5 writes = [] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 9c7b174f0a9..2e3fa03f601 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -81,6 +81,7 @@ def _make_ovrtx_camera_render_data() -> OVRTXCameraRenderData: def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) renderer.backend = OVRTXBackend.__new__(OVRTXBackend) + renderer.backend._render_lease_owner = None renderer.scene = renderer.backend from isaaclab_ov.stage import OvstageBackend @@ -1041,7 +1042,7 @@ def test_ovrtx_render_var_sync_rejects_non_boolean_values(monkeypatch, value): @pytest.mark.parametrize( ("gpu_side", "cuda_stream", "expected_stream", "consumer_fails"), - [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False)], + [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False), (False, None, None, True)], ) def test_ovrtx_map_render_var_orders_the_read_against_render_completion( monkeypatch, gpu_side, cuda_stream, expected_stream, consumer_fails @@ -1049,15 +1050,17 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( """Order reads after rendering and release after reads, including when the consumer raises.""" events = [] mapping = types.SimpleNamespace( - wait=lambda: events.append("wait"), unmap=lambda *, stream: events.append(("release", stream)) + wait=lambda: events.append("wait"), unmap=lambda *, stream=None: events.append(("release", stream)) ) render_var = types.SimpleNamespace(map=MagicMock(return_value=mapping)) sentinel = object() monkeypatch.setattr(ovrtx_renderer_module, "_gpu_side_render_var_sync_enabled", lambda: gpu_side) - monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: sentinel) + monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: events.append("wrap") or sentinel) renderer = _make_ovrtx_renderer_without_backend() - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=cuda_stream)) + renderer._warp_device = types.SimpleNamespace( + is_cuda=cuda_stream is not None, stream=types.SimpleNamespace(cuda_stream=cuda_stream) + ) with pytest.raises(ValueError, match="consumer failed") if consumer_fails else contextlib.nullcontext(): with renderer._map_render_var_to_dlpack(render_var) as array: assert array is sentinel @@ -1065,8 +1068,11 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( if consumer_fails: raise ValueError("consumer failed") - assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) - assert events == ([] if gpu_side else ["wait"]) + ["consume", ("release", expected_stream)] + if cuda_stream is None: + assert render_var.map.call_args.kwargs == {"device": ovrtx_renderer_module.Device.CPU} + else: + assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) + assert events == ([] if gpu_side else ["wait"]) + ["wrap", "consume", ("release", expected_stream)] @pytest.mark.parametrize( @@ -1188,7 +1194,7 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs assert not vars(renderer).keys() & {"_camera_xform_binding", "_camera_xform_query", "_camera_paths_list"} monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(wp, "synchronize_stream", lambda stream: None) - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=99)) + renderer._warp_device = types.SimpleNamespace(is_cuda=True, stream=types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(ovrtx_renderer_module, "xform_tensor_from_warp", lambda value: value) positions = wp.array([[1, 2, 3], [4, 5, 6]], dtype=wp.vec3f, device="cpu") rotations = np.array([[1, 2, 3, 4], [0, 0, 0, 1]], dtype=np.float32) @@ -1212,6 +1218,7 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs renderer.update_camera_intrinsics(cameras[1], wp.zeros(2, dtype=wp.mat33f, device="cpu"), parameters) if use_ovstage: assert renderer.scene.stage.write_attributes.call_args.args[0] is cameras[1].camera_xform_binding + assert all(desc.cuda_stream is None for desc in renderer.scene.stage.write_attributes.call_args.args[1]) bound_paths = [call.args[0] for call in renderer.scene.paths.create_path_list_from_strings.call_args_list] assert bound_paths == [ entry @@ -1270,6 +1277,7 @@ def query(paths): events.append(f"release_query:{paths}") renderer.scene = OvstageBackend.__new__(OvstageBackend) + renderer.scene._render_lease_owner = None SimulationContext.instance()._backend_registry.insert( 0, (OvstageBackendCfg(scene_key=renderer.cfg), renderer.scene) ) @@ -1412,3 +1420,303 @@ def attach(native_stage): assert events.count("disable-camera-reads") == (needs_fallback and stage_mode != "legacy") if stage_mode != "legacy": assert events[-2:] == ["commit", "attach"] + + +@pytest.fixture +def leased_renderer(monkeypatch): + """Exercise real deferred resource acquisition with native SDK endpoints doubled.""" + import isaaclab_ov.stage as stage_module + + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + native, stage, paths = MagicMock(), MagicMock(), MagicMock() + floor = stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 0 + + def advance(*, ordinal, **kwargs): + return types.SimpleNamespace(wait=lambda: setattr(floor.fetch, "return_value", ordinal)) + + stage.advance_write_floor.side_effect = advance + monkeypatch.setattr(ovrtx_renderer_module, "Renderer", lambda _cfg: native) + monkeypatch.setattr(ovrtx_renderer_module, "redirect_shader_cache", lambda _cfg: None) + monkeypatch.setattr(stage_module, "create_ovstage", lambda _name: contextlib.nullcontext(stage)) + monkeypatch.setattr(stage_module.ovstage, "PathDictionary", lambda _: contextlib.nullcontext(paths)) + monkeypatch.setattr(stage_module.ovstage.population, "open_usd_from_string", lambda *args, **kwargs: None) + renderer = SimulationContext.instance().get_or_create_backend(OVRTXRendererCfg()) + assert renderer.backend is renderer.scene is None + return renderer + + +def _prepare_leased_renderer(renderer, *, bind=True): + from isaaclab_ov.cloner import OvrtxReplicateContext + + OvrtxReplicateContext.prepare(renderer._sim, {}) + if bind: + renderer.prepare_stage(renderer._sim.stage, 1) + + +def test_render_lease_lease_borrows_registry_pair_and_blocks_shared_users(leased_renderer, monkeypatch): + renderer = leased_renderer + shared = SimulationContext.instance().get_or_create_backend(renderer.cfg) + assert shared is renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="already active"): + shared.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + with pytest.raises(RuntimeError, match="Lease"): + shared.render_batch((object(),)) + shared.render_batch(()) + with monkeypatch.context() as patch: + patch.setattr(ovrtx_renderer_module, "OVRTXMesh", lambda *_: pytest.fail("constructed a mesh while leased")) + with pytest.raises(RuntimeError, match="log runtime meshes"): + shared.log_mesh("blocked") + assert "blocked" not in shared.meshes + with pytest.raises(RuntimeError, match="active render lease"): + SimulationContext.instance().close_backend(renderer.backend) + with pytest.raises(RuntimeError, match="active render lease"): + renderer.scene.reset() + native = lease.ovrtx_renderer + assert native is shared.backend.renderer + assert lease.ovstage is shared.scene.stage + lease.release() + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene._render_lease_owner is None + native.destroy.assert_not_called() + SimulationContext.instance().close_backend(renderer.backend) + native.destroy.assert_called_once_with() + + +def test_render_lease_rejects_acquisition_after_clone_preparation(leased_renderer): + renderer = leased_renderer + _prepare_leased_renderer(renderer, bind=False) + with pytest.raises(RuntimeError, match="before clone preparation"): + renderer.acquire_ov_libraries_render_lease() + assert renderer.backend._render_lease_owner is None + + +def test_render_lease_lease_rejects_calibration_before_scene_access(leased_renderer, monkeypatch): + """An ordinary calibration write cannot bypass the external owner's publication lane.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + camera = _make_ovrtx_camera_render_data() + intrinsics = wp.zeros(2, dtype=wp.mat33f, device="cpu") + parameters = wp.zeros((5, 2), dtype=wp.float32, device="cpu") + drain = MagicMock(return_value=[]) + monkeypatch.setattr(renderer, "drain_pending_renders", drain) + monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) + try: + with pytest.raises(RuntimeError, match="Lease"): + renderer.update_camera_intrinsics(camera, intrinsics, parameters) + drain.assert_not_called() + assert renderer.backend is renderer.scene is None + finally: + lease.release() + + +def test_render_lease_owner_transfer_is_single_use(leased_renderer): + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(None) + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(token) + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._render_lease._owner_thread is thread + assert renderer.backend is renderer.scene is None + + +def test_render_lease_complete_publication_deduplicates_and_poison_fails_closed(leased_renderer, monkeypatch): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + renderer.scene.stage.advance_write_floor.reset_mock() + transforms = MagicMock() + monkeypatch.setattr(renderer, "_update_transforms", transforms) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + assert publisher.publication_required("step-1") + publisher.write_complete_scene(18, "step-1") + assert not publisher.publication_required("step-1") + transforms.assert_called_once_with() + renderer.scene.stage.advance_write_floor.assert_not_called() + transforms.side_effect = ValueError("write failed") + with pytest.raises(RuntimeError, match="scene publication failed"): + publisher.write_complete_scene(19, "step-2") + with pytest.raises(RuntimeError, match="poisoned"): + _ = lease.ovstage + publisher.close() + lease.release() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA material buffers") +def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch): + """Real material composition and the lease publisher must reach native writes, including before physics changes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + context = renderer._sim.render_context + _prepare_leased_renderer(renderer) + material = types.SimpleNamespace( + channels=("color",), + is_per_env=False, + num_instances=1, + _material_paths=("/Looks/Feedback",), + _shader_paths=("/Looks/Feedback/Shader",), + _input_names={"color": "diffuseColor"}, + _offsets={}, + _values={"color": torch.tensor([[1.0, 0.0, 0.0]], device="cuda")}, + ) + context.register_visual_material(material) + context.finalize_consumers([]) + stage = renderer.scene.stage + try: + for ordinal, color in ((23, [0.0, 1.0, 0.0]), (24, [0.0, 0.0, 1.0])): + context.write_visual_materials([material], {"color": torch.tensor([color], device="cuda")}) + assert publisher.publication_required("unchanged-physics") + stage.write_attribute.reset_mock() + completion = MagicMock() + stage.write_attribute.return_value = completion + publisher.write_complete_scene(ordinal, "unchanged-physics") + writes = [c for c in stage.write_attribute.call_args_list if c.args[1] == "inputs:diffuseColor"] + assert len(writes) == 1 + assert writes[0].kwargs["ordinal"] == ordinal + torch.testing.assert_close(writes[0].kwargs["tensors"].cpu(), torch.tensor([color])) + assert completion.wait.called + assert not publisher.publication_required("unchanged-physics") + finally: + lease.release() + for writer in context._visual_material_writers: + writer.close() + + +@pytest.mark.parametrize("use_ovstage, asynchronous", [(True, False), (False, False), (False, True)]) +def test_cpu_scene_publication_uses_host_tensors_without_a_cuda_stream(use_ovstage, asynchronous): + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = use_ovstage + renderer.cfg = OVRTXRendererCfg(async_rendering=asynchronous) + renderer._warp_device = wp.get_device("cpu") + renderer._device = "cpu" + renderer._object_xform_binding = MagicMock() + renderer._geometry_points_binding = MagicMock() + renderer._transforms_timestamp = renderer._geometry_timestamp = None + renderer._object_scales = None + renderer._geometry_paths = ["/World/Object"] + matrices = wp.array(np.eye(4, dtype=np.float64)[None], dtype=wp.mat44d, device="cpu") + points = wp.array([[1.0, 2.0, 3.0]], dtype=wp.vec3f, device="cpu") + renderer._transform_writes = _AsyncWriteBuffers([types.SimpleNamespace(matrices=matrices)]) + renderer._geometry_writes = _AsyncWriteBuffers([wp.empty_like(points)]) + renderer._geometry_offsets = {"/World/Object": 0} + + def get_points(*, output=None, offsets=None): + if output is None: + return {"/World/Object": points} + wp.copy(output, points) + return None + + renderer._sdp = types.SimpleNamespace( + backend=types.SimpleNamespace(transforms_timestamp=1, geometry_timestamp=1), + get_transforms=lambda *_args, **_kwargs: True, + get_geometry_points=get_points, + ) + stage = MagicMock() + renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) + renderer.update_transforms() + renderer.update_geometries() + bindings = (renderer._object_xform_binding, renderer._geometry_points_binding) + writes = [getattr(binding, "write_async" if asynchronous else "write") for binding in bindings] + calls = stage.write_attribute.call_args_list if use_ovstage else [write.call_args for write in writes] + assert len(calls) == 2 and all(c.kwargs["cuda_stream"] is None for c in calls) + if use_ovstage: + assert [c.args[1] for c in calls] == ["omni:xform", "points"] + assert all(c.kwargs["ordinal"] == 11 for c in calls) + else: + np.testing.assert_array_equal(calls[0].args[0].numpy(), matrices.numpy()) + np.testing.assert_array_equal(calls[1].args[0][0].numpy(), points.numpy()) + # A repeated publication of unchanged state must not duplicate either write. + renderer.update_transforms() + renderer.update_geometries() + assert stage.write_attribute.call_count == 2 if use_ovstage else all(write.call_count == 1 for write in writes) + camera = _make_ovrtx_camera_render_data() + camera.camera_xform_binding = MagicMock() + camera.camera_writes = _AsyncWriteBuffers([wp.empty(2, dtype=wp.mat44d, device="cpu")]) + positions = wp.array([[1.0, 2.0, 3.0]] * 2, dtype=wp.vec3f, device="cpu") + orientations = wp.array([[0.0, 0.0, 0.0, 1.0]] * 2, dtype=wp.quatf, device="cpu") + update = renderer.update_camera + update(camera, positions, orientations, None) + call = stage.write_attribute.call_args if use_ovstage else camera.camera_xform_binding.write_async.call_args + assert call.kwargs["cuda_stream"] is None + if use_ovstage: + assert call.args == (camera.camera_xform_binding, "omni:xform") + assert call.kwargs["ordinal"] == 11 + else: + # Reusing the camera ring waits for previous writes without a CUDA stream. + update(camera, positions, orientations, None) + update(camera, positions, orientations, None) + camera.camera_writes.close() + renderer._transform_writes.close() + renderer._geometry_writes.close() + + +@pytest.mark.parametrize("cuda_ordinal", [None, 2]) +def test_camera_registration_authors_valid_render_device_selection(monkeypatch, cuda_ordinal): + """CPU camera consumers leave GPU selection automatic; CUDA consumers retain their device.""" + from pxr import Usd + + class AuthoredProduct(Exception): + pass + + renderer = _make_ovrtx_renderer_without_backend() + renderer.cfg = OVRTXRendererCfg() + + class Device: + is_cuda = cuda_ordinal is not None + ordinal = cuda_ordinal + + def __str__(self): + return f"cuda:{cuda_ordinal}" if self.is_cuda else "cpu" + + renderer._warp_device = Device() + renderer._device = str(renderer._warp_device) + monkeypatch.setattr(ovrtx_renderer_module.wp, "get_device", lambda _device: renderer._warp_device) + renderer.scene.clone_env_paths = [] + renderer.drain_pending_renders = lambda: [] + spec = types.SimpleNamespace( + cfg=_make_camera_cfg(["rgb"]), + device=renderer._device, + num_instances=1, + camera_prim_paths=["/World/envs/env_0/Camera"], + render_settings={}, + ) + build_product = ovrtx_renderer_module.build_render_product_as_string + + def consume_product(spec, render_data, **kwargs): + stage = Usd.Stage.CreateInMemory() + assert stage.GetRootLayer().ImportFromString(build_product(spec, render_data, **kwargs)) + product = stage.GetPrimAtPath(render_data.render_product_path) + devices = product.GetAttribute("deviceIds") + if cuda_ordinal is None: + assert not devices.HasAuthoredValueOpinion() + else: + assert list(devices.Get()) == [cuda_ordinal] + # Native population is outside this parser/selection regression. + raise AuthoredProduct + + monkeypatch.setattr(ovrtx_renderer_module, "build_render_product_as_string", consume_product) + with pytest.raises(AuthoredProduct): + renderer.create_render_data(spec) diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 522a4d5d479..51c6abd7d96 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -435,11 +435,11 @@ def test_export_stage_homogeneous_keeps_only_env0_prototype(): def test_export_stage_without_keep_env_roots_trims_non_source_env_roots(): - """The ovstage clone path also trims the non-source env roots themselves. + """Whole-environment cloning trims the non-source environment roots themselves. ``ovstage.Stage.clone`` requires every target path to not already exist, so the exported stage - must not retain env roots that the clone will recreate. This is the only difference from the - legacy ``renderer.clone_usd`` path, which keeps the roots as placeholders. + must not retain environment roots that the clone will recreate. Asset-level clone plans + retain typed parent roots on both renderer paths. """ num_envs = 4 stage = _make_multi_env_stage(num_envs) @@ -528,14 +528,20 @@ def test_export_stage_heterogeneous_keeps_multiple_sources(): assert 'def Camera "Camera"' not in exported -@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) -def test_export_stage_preserves_authored_scene(settings_path): +@pytest.mark.parametrize( + "settings_path, authored_show_all", + [(None, None), ("/Render/AuthoredSettings", False), ("/Render/AuthoredSettings", True)], +) +def test_export_stage_preserves_authored_scene(settings_path, authored_show_all): """Renderer settings and clone trimming are confined to the exported scene.""" num_envs = 4 stage = _make_multi_env_stage(num_envs) if settings_path is not None: settings = UsdRender.Settings.Define(stage, settings_path) settings.CreateResolutionAttr(Gf.Vec2i(1280, 720)) + settings.GetPrim().CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ).Set(authored_show_all) stage.SetMetadata("renderSettingsPrimPath", settings_path) before = stage.ExportToString() layer = Sdf.Layer.CreateAnonymous() @@ -548,7 +554,7 @@ def test_export_stage_preserves_authored_scene(settings_path): assert exported.HasAuthoredMetadata("renderSettingsPrimPath") settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) show_all = settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault") - assert show_all and show_all.Get() is True + assert show_all and show_all.Get() is (True if authored_show_all is None else authored_show_all) if settings_path is not None: assert settings.GetPath() == settings_path assert settings.GetAttribute("resolution").Get() == (1280, 720) diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 6c7b0c9efc2..ca892860c01 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -23,6 +23,7 @@ if not _MISSING_MODULES: from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXRendererCfg + from isaaclab_ov.renderers.render_lease import RenderLeaseController from ovrtx import DataAccess from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -121,7 +122,8 @@ def destroy_path_list(self, path_list): def _renderer(*, use_ovstage: bool = False): events: list[str] = [] renderer = OVRTXRenderer.__new__(OVRTXRenderer) - renderer.backend = SimpleNamespace(render_products=set()) + renderer._render_lease = RenderLeaseController(renderer) + renderer.backend = SimpleNamespace(render_products=set(), _render_lease_owner=None) renderer.scene = renderer.backend renderer._initialized_scene = True renderer._needs_transform_initialization = False diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst new file mode 100644 index 00000000000..646b66b93b0 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst @@ -0,0 +1,4 @@ +* Added an OVXR application adapter and session bootstrap that borrow the simulation-owned + OVRTX renderer and OVStage, publish scene updates, and coordinate render-thread ownership + and shutdown with Isaac Capture. + Publication failures close admission, and failed lease retirement retains borrowed resources. diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor b/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md new file mode 100644 index 00000000000..4b10b2b20a8 --- /dev/null +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -0,0 +1,59 @@ +# OVXR application adapter + +`isaaclab_teleop.OvLibrariesXrSession` connects an Isaac Lab environment to +Isaac Capture's OVXR session owner. It uses the simulation's existing OVRTX +renderer and OVStage through `isaaclab_ov.renderers.OvLibrariesRenderLease`. + +## Ownership + +- `isaaclab_ov` owns the renderer, stage, scene publication and their lifetime. + Its lease validates access and enforces render-thread ownership. The adapter's + publication guard serializes application writes with Capture's frame operations. + It does not create an XR session or select teleoperation behavior. +- `isaaclab_teleop.ovxr_session.OvxrSessionBootstrap` creates the simulation + context and reserves the lease before environment construction. It binds the + completed environment and reads current target-frame transforms. +- `isaaclab_teleop.ov_libraries_xr.OvLibrariesXrSession` supplies the application + adapter to Isaac Capture. It declares XR render products, publishes scene and + anchor updates, transfers render-thread ownership and coordinates shutdown. +- Isaac Capture owns the OVXR session and render worker. Its borrowed OpenXR + handles are available to the teleoperation input session. + +## Lifecycle + +Prepare the adapter before constructing the environment so that both use the +same simulation context and native renderer/scene pair. Start the render worker +after the environment exists. Publish simulation updates through the session; +the worker owns rendering. Stop input consumers before closing the XR session, +and retire the lease before destroying the simulation's renderer or stage. +If startup fails before the worker takes ownership, the caller closes the +environment after releasing the lease. If input retirement fails, request +shutdown and retain the borrowed resources until their retirement is proved. + +The lease currently requires one environment and a private rendering OVStage. +Ownership transfer is single-use. Failed retirement must preserve resources +for the owning thread to finish cleanup; a failed drain is not successful +shutdown. Importing the adapter requires the compatible Capture OVXR packages. + +A failed scene write or seal stops further publication. Recreate the session +after retiring the failed pair; retrying the same generation cannot return an +older scene as if the failed update succeeded. During shutdown, publication +storage drains before lease guards are removed. The adapter retains the +environment if lease retirement cannot establish completion. + +## Backend implementation + +The ordinary renderer remains in `isaaclab_ov.renderers.ovrtx_renderer`. +`render_lease.RenderLeaseController` owns the lease state, thread-transfer +capability and per-owner operation guards; it borrows the renderer's native resources +and delegates component writes back to that renderer. `publication_buffers` +retains producer storage until native writes complete. Public renderer and lease +imports remain available from `isaaclab_ov.renderers`. + +Acquiring a lease selects a private rendering OVStage without changing process +environment variables. An explicit OVStage disable or shared-stage selection is +rejected. Without an active lease, the controller permits +ordinary renderer operations and does not start an XR session. While a consumer +holds the lease, ordinary camera rendering, scene writes and renderer destruction +are rejected to prevent concurrent access to the borrowed resources. The same +generic borrowing interface is available to applications outside teleoperation. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 266f5470dda..df4a0adfc7f 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -24,6 +24,10 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. ## Architecture +The [OVXR application adapter](OVXR_ADAPTER.md) connects a simulation-owned +renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; +`isaaclab_ov` supplies the generic renderer/scene lease and lifetime checks. + `IsaacTeleopDevice` composes three focused collaborators: | Component | Responsibility | diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index d810345978e..ce01c7f56cf 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,6 +15,8 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", + "OvLibrariesXrConfig", + "OvLibrariesXrSession", "SupportsControlEvents", "SystemCheckItem", "SystemCheckResult", @@ -42,8 +44,12 @@ __all__ = [ from .camera_feed import XrCameraFeedSession from .control_events import TELEOP_CONTROL_CHANNEL_UUID, ControlEvents, SupportsControlEvents, poll_control_events from .device_pipelines import ( - se2_gamepad_teleop_cfg, se2_keyboard_teleop_cfg, se2_spacemouse_teleop_cfg, - se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, + se2_gamepad_teleop_cfg, + se2_keyboard_teleop_cfg, + se2_spacemouse_teleop_cfg, + se3_gamepad_teleop_cfg, + se3_keyboard_teleop_cfg, + se3_spacemouse_teleop_cfg, ) from .haptic_feedback import ( ControllerHapticFeedbackCfg, @@ -62,6 +68,7 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device +from .ov_libraries_xr import OvLibrariesXrConfig, OvLibrariesXrSession from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py new file mode 100644 index 00000000000..bae4e0a0c4a --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -0,0 +1,689 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Lab host adapter for IsaacTeleop-owned OVXR presentation.""" + +from __future__ import annotations + +import contextlib +import logging +import math +import re +import threading +import time +from collections.abc import Hashable, Iterable +from dataclasses import dataclass, replace +from typing import TYPE_CHECKING, Any, Literal + +from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger +from isaaclab_ov.renderers import OVRTXRendererCfg + +from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation + +if TYPE_CHECKING: + import numpy as np + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + from isaacteleop.oxr import OpenXRSessionHandles + + from isaaclab.sim import SimulationCfg + + from .xr_anchor_manager import XrAnchorManager + +logger = logging.getLogger(__name__) + +_DEFAULT_OVXR_CAMERA_PATHS = ( + "/OVXRGeneratedCameras/OVXR_LeftEyeCamera", + "/OVXRGeneratedCameras/OVXR_RightEyeCamera", +) +_DEFAULT_OVXR_RENDER_PRODUCT_PATHS = ( + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", +) +_DEFAULT_OVXR_RENDER_VAR_PATHS = ( + "/Render/Vars/LdrColor", + "/Render/Vars/XRDepth", +) +_DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" + + +def _apply_hidden_visual_prims(env: Any, patterns: Iterable[str]) -> tuple[str, ...]: + """Author persistent application-stage visibility before materialization. + + This affects every view of the application scene, not just the XR eyes. + Use scene partitions when source visibility must remain unchanged. + """ + patterns = tuple(patterns) + if not patterns: + return () + + from pxr import UsdGeom + + unwrapped = getattr(env, "unwrapped", env) + stage = unwrapped.scene.stage + if stage is None: + raise RuntimeError("Configured hidden visual prims require an environment Stage") + + compiled = [(pattern, re.compile(pattern)) for pattern in patterns] + matches: dict[str, list[Any]] = {pattern: [] for pattern in patterns} + for prim in stage.Traverse(): + if not prim.IsA(UsdGeom.Imageable): + continue + path = str(prim.GetPath()) + for pattern, expression in compiled: + if expression.fullmatch(path): + matches[pattern].append(prim) + + missing = [pattern for pattern, prims in matches.items() if not prims] + if missing: + raise RuntimeError(f"Hidden visual prim patterns matched no imageable prims: {missing}") + + hidden_paths: list[str] = [] + hidden_path_set: set[str] = set() + for pattern in patterns: + for prim in matches[pattern]: + path = str(prim.GetPath()) + if path in hidden_path_set: + continue + UsdGeom.Imageable(prim).MakeInvisible() + hidden_paths.append(path) + hidden_path_set.add(path) + logger.info("Hid %d prim(s) from all camera views (%s)", len(hidden_paths), list(patterns)) + return tuple(hidden_paths) + + +def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrConfig) -> None: + """Declare type-only OVXR prims for the application Stage's lifetime.""" + if not cfg.create_render_products: + return + + unwrapped = getattr(env, "unwrapped", env) + stage = unwrapped.scene.stage + if stage is None: + raise RuntimeError("OVXR runtime prim declaration requires an environment Stage") + + render_products = cfg.render_products or _DEFAULT_OVXR_RENDER_PRODUCT_PATHS + declarations = ( + *((path, "Camera") for path in cfg.camera_paths), + *((path, "RenderProduct") for path in render_products), + *((path, "RenderVar") for path in _DEFAULT_OVXR_RENDER_VAR_PATHS), + (_DEFAULT_OVXR_RENDER_SETTINGS_PATH, "RenderSettings"), + ) + expected_types: dict[str, str] = {} + for path, type_name in declarations: + if not path.startswith("/") or path == "/": + raise ValueError(f"OVXR runtime prim path must be absolute and non-root: {path!r}") + previous_type = expected_types.setdefault(path, type_name) + if previous_type != type_name: + raise ValueError(f"OVXR runtime prim path {path!r} has conflicting requested types") + + for path, type_name in expected_types.items(): + existing = stage.GetPrimAtPath(path) + existing_type = str(existing.GetTypeName()) if existing and existing.IsValid() else "" + if existing_type and existing_type != type_name: + raise RuntimeError( + f"OVXR runtime prim {path!r} already has incompatible type {existing_type!r}; expected {type_name!r}" + ) + stage.DefinePrim(path, type_name) + + +class _StageOrdinalLedger(StageOrdinalLedger): + """Reserve explicit OVStage ordinals for the application publication lane.""" + + def __init__(self, stage: Any) -> None: + self._stage = stage + self._state_lock = threading.Lock() + self._active_ordinal: int | None = None + self._closed = False + self._floor = self._read_floor() + + def _read_floor(self) -> int: + with self._stage.get_attribute_write_floor() as query: + query.wait() + floor = int(query.fetch()) + if floor < 0 or floor > (1 << 64) - 1: + raise RuntimeError(f"OVStage returned an invalid write floor: {floor}") + return floor + + def _require_active(self, ordinal: int) -> None: + if self._active_ordinal != ordinal: + raise RuntimeError( + f"OVStage ordinal {ordinal} is not the active application reservation ({self._active_ordinal=})" + ) + + def reserve(self) -> int: + """Reserve exactly ``floor + 1`` while the application publication guard is held.""" + with self._state_lock: + if self._closed: + raise RuntimeError("OVStage ordinal ledger is closed") + if self._active_ordinal is not None: + raise RuntimeError(f"OVStage ordinal {self._active_ordinal} is still unresolved") + if self._floor == (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + self._active_ordinal = self._floor + 1 + return self._active_ordinal + + def publish(self, ordinal: int) -> None: + """Seal one application-written reservation and mark it resolved.""" + with self._state_lock: + self._require_active(ordinal) + self._stage.advance_write_floor(ordinal=ordinal).wait() + self._floor = ordinal + self._active_ordinal = None + + def resolve_external(self, ordinal: int, *, expected_published: bool | None) -> bool: + """Resolve an external write and verify its documented publication result.""" + with self._state_lock: + self._require_active(ordinal) + if expected_published is True: + published = True + else: + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + published = floor == ordinal + # Record native side effects even when the response contradicts the + # expected result. A reported error cannot make a sealed ordinal reusable. + if published: + self._floor = ordinal + if expected_published is not None and published != expected_published: + self._active_ordinal = None + state = "sealed" if published else "unsealed" + expected = "sealed" if expected_published else "unsealed" + raise RuntimeError(f"OVStage ordinal {ordinal} is {state} after an external write; expected {expected}") + self._active_ordinal = None + return published + + def abandon(self, ordinal: int) -> None: + """Reconcile a failed writer before releasing its reservation.""" + with self._state_lock: + self._require_active(ordinal) + # A publication wait can fail after the native Stage has sealed it. + # Query only on the failure/abandon path; an unreadable or unrelated + # floor leaves the reservation unresolved so it cannot be reused. + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + self._floor = floor + self._active_ordinal = None + + def destroy(self) -> None: + """Close the ledger after every application and external write is resolved.""" + with self._state_lock: + if self._active_ordinal is not None: + raise RuntimeError(f"Cannot close with unresolved OVStage ordinal {self._active_ordinal}") + self._closed = True + + +@dataclass +class OvLibrariesXrConfig: + """Isaac Lab configuration for an IsaacTeleop-owned OVXR session.""" + + application_name: str = "isaaclab-kitless-teleop" + """Application identifier passed to the OpenXR runtime.""" + session_mode: Literal["ar", "vr"] = "ar" + """OpenXR presentation mode: augmented or virtual reality.""" + renderer_settings: str = "--/app/hydra/aperture/conform=stretch" + """Process-global OVXR renderer settings applied before renderer construction.""" + cloudxr_env_file: str | None = None + """CloudXR environment-file path; None leaves runtime launch to the caller.""" + auto_launch_cloudxr: bool = True + """Whether the session owner launches the CloudXR runtime automatically.""" + render_products: tuple[str, ...] = () + """Stereo RenderProduct paths; an empty selection uses OVXR's default paths.""" + camera_paths: tuple[str, ...] = _DEFAULT_OVXR_CAMERA_PATHS + """Exactly two non-empty USD camera paths, ordered left eye then right eye.""" + create_render_products: bool = True + """Whether to declare and configure the selected stereo rendering prims.""" + meters_per_unit: float = 1.0 + """Finite positive scene scale [m per USD unit].""" + depth_near_z: float = 0.0 + """Near depth distance [m]; both depth distances zero select provider defaults.""" + depth_far_z: float = 0.0 + """Far depth distance [m]; explicit distances must satisfy 0 < near < far.""" + tracking_anchor_path: str = "" + """USD tracking-anchor path; an empty path uses the stage origin.""" + host_wait_render_output: bool = True + """Whether OVXR waits for render output before returning from frame rendering.""" + extra_openxr_extensions: tuple[str, ...] = () + """Additional OpenXR extension names requested when creating the session.""" + + def __post_init__(self) -> None: + if self.session_mode not in ("ar", "vr"): + raise ValueError("session_mode must be 'ar' or 'vr'") + if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: + raise ValueError("meters_per_unit must be finite and positive") + if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( + not math.isfinite(self.depth_near_z) + or not math.isfinite(self.depth_far_z) + or self.depth_near_z <= 0.0 + or self.depth_far_z <= self.depth_near_z + ): + raise ValueError("depth_near_z and depth_far_z must both be zero or finite with 0 < near < far") + if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): + raise ValueError("Kitless stereo requires exactly two non-empty OVXR camera paths") + + +class _IsaacLabOvxrApplicationInterface(OvxrApplicationInterface): + """Supply only the renderer/Stage/coordinator and simulation publication.""" + + def __init__( + self, + bootstrap: OvxrSessionBootstrap, + env: Any, + cfg: OvLibrariesXrConfig, + hidden_visual_prims: Iterable[str], + external_anchor_state: XrAnchorManager | None = None, + ) -> None: + self._bootstrap = bootstrap + self._env = env + self._cfg = cfg + self._hidden_visual_prims = tuple(hidden_visual_prims) + self._publication_lock = threading.Lock() + self._scene_publisher = None + self._lease = None + self._ordinal_ledger = None + self._last_published_ordinal: int | None = None + self._publication_closed = False + self._external_anchor = external_anchor_state + self._anchor_snapshot: tuple[int, np.ndarray] | None = None + self._last_anchor_frame_time: float | None = None + self._anchor_resources = contextlib.ExitStack() + self._anchor_query = None + self._anchor_stage = None + self._anchor_retirement_failed = False + + def offer_render_owner_transfer(self) -> object: + self._scene_publisher = self._bootstrap.capture_scene_publisher() + return self._bootstrap.offer_render_owner_transfer() + + def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + + self._bootstrap.claim_render_owner_transfer(capability) + _apply_hidden_visual_prims(self._env, self._hidden_visual_prims) + _declare_ovxr_runtime_prims(self._env, self._cfg) + self._lease = self._bootstrap.bind_environment(self._env) + with self._lease.ovstage_owner_lane(): + self._ordinal_ledger = _StageOrdinalLedger(self._lease.ovstage) + if self._external_anchor is not None: + import ovstage + + self._anchor_stage = self._lease.ovstage + try: + paths = self._anchor_resources.enter_context(ovstage.PathDictionary(self._anchor_stage)) + path_list = paths.create_path_list_from_strings(self._external_anchor.external_anchor_paths) + self._anchor_resources.callback(paths.destroy_path_list, path_list) + self._anchor_query = self._anchor_resources.enter_context( + self._anchor_stage.query_from_path_list(path_list) + ) + self._anchor_query.wait() + except BaseException as setup_error: + try: + self._anchor_resources.close() + except BaseException as cleanup_error: + self._anchor_retirement_failed = True + setup_error.add_note(f"Anchor query cleanup also failed: {cleanup_error!r}") + raise + return OvxrSessionBinding( + renderer=self._lease.ovrtx_renderer, + stage=self._lease.ovstage, + ordinal_ledger=self._ordinal_ledger, + prepare_frame_scene=self.prepare_frame_scene if self._external_anchor is not None else None, + ) + + def publication_guard(self) -> contextlib.AbstractContextManager[None]: + return self._publication_lock + + def owner_lane(self) -> contextlib.AbstractContextManager[None]: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + return self._lease.ovstage_owner_lane() + + def publish_complete_scene(self, publication_id: Hashable) -> int: + if self._publication_closed or self._scene_publisher is None or self._ordinal_ledger is None: + raise RuntimeError("IsaacLab scene publication is not available") + anchor_epoch = None + if self._external_anchor is not None: + # A failed seed can follow the publisher recording its identity. + # Check the terminal policy before dedup may return an older seal. + anchor_epoch = self._external_anchor.external_epoch + if not self._scene_publisher.publication_required(publication_id): + if self._last_published_ordinal is None: + raise RuntimeError("OVStage publication deduplicated before a sealed ordinal exists") + return self._last_published_ordinal + snapshot = None + seed = False + if self._external_anchor is not None: + epoch, seed = anchor_epoch + reference = self._bootstrap.get_world_transform(self._external_anchor.external_reference_path) + snapshot = (epoch, reference) + ordinal = self._ordinal_ledger.reserve() + try: + self._scene_publisher.write_complete_scene(ordinal, publication_id) + except BaseException as write_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + write_error.add_note(f"Abandon after failed scene write also failed: {abandon_error!r}") + raise + try: + if seed: + epoch, reference = snapshot + + def commit(matrices: np.ndarray) -> None: + self._write_anchor_matrices(matrices, ordinal) + self._ordinal_ledger.publish(ordinal) + + # Reset seeding must be sealed with this scene before input resumes. + # The first pose initializes the filter without an elapsed interval. + if not self._external_anchor.commit_external_pose(reference, 0.0, epoch, commit): + raise RuntimeError("XR anchor reset changed during complete scene publication") + self._last_anchor_frame_time = None + else: + self._ordinal_ledger.publish(ordinal) + except BaseException as publish_error: + # Component writes already consumed this generation. Do not let a + # retry deduplicate it to an older seal after publication failed. + self.close_publication() + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publish_error.add_note(f"Abandon after failed scene publication also failed: {abandon_error!r}") + raise + self._last_published_ordinal = ordinal + self._anchor_snapshot = snapshot + return ordinal + + def _write_anchor_matrices(self, matrices: np.ndarray, ordinal: int) -> None: + import numpy as np + import ovstage + from isaaclab_ov.stage import xform_tensor_from_numpy + + # SDP uses column vectors; the Stage's omni:xform stores USD row matrices. + rows = np.ascontiguousarray(matrices.transpose(0, 2, 1), dtype=np.float64) + tensor = xform_tensor_from_numpy(rows) + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:xform", + ordinal=ordinal, + tensors=tensor, + semantic=ovstage.AttributeSemantic.MATRIX, + is_array=False, + ).wait() + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:resetXformStack", + ordinal=ordinal, + tensors=np.ones(len(rows), dtype=np.bool_), + is_array=False, + ).wait() + + def prepare_frame_scene(self) -> None: + """Advance the committed reference with elapsed XR-frame time [s]. + + The first begun frame after reset establishes the monotonic clock with + zero elapsed time. Scene publication never advances this frame clock. + """ + if self._anchor_snapshot is None: + return + epoch, reference = self._anchor_snapshot + now = time.monotonic() + dt = 0.0 if self._last_anchor_frame_time is None else now - self._last_anchor_frame_time + + def commit(matrices: np.ndarray) -> None: + ordinal = self._ordinal_ledger.reserve() + try: + self._write_anchor_matrices(matrices, ordinal) + # OVXR reads sealed anchor state at bind; bind reserves the next ordinal. + self._ordinal_ledger.publish(ordinal) + except BaseException as publication_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publication_error.add_note( + f"Abandon after failed anchor publication also failed: {abandon_error!r}" + ) + raise + + if self._external_anchor.commit_external_pose(reference, dt, epoch, commit): + self._last_anchor_frame_time = now + + def set_external_anchor_state(self, state: XrAnchorManager | None) -> None: + """Replace a replay device's policy while preserving native query ownership.""" + if self._external_anchor is not None: + # A new replay manager must not revive a failed borrowed native pair. + _ = self._external_anchor.external_epoch + if (state is None) != (self._external_anchor is None) or ( + state is not None and state.external_anchor_paths != self._external_anchor.external_anchor_paths + ): + raise ValueError("A reused XR environment must preserve its anchor paths") + self._external_anchor = state + self._anchor_snapshot = None + self._last_anchor_frame_time = None + + def validate_first_published_scene(self, ordinal: int) -> None: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + with self._lease.ovstage_owner_lane(): + self._lease.validate_live() + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + return self._bootstrap.get_target_frame_T_world(prim_path) + + def close_publication(self) -> None: + if self._publication_closed: + return + self._publication_closed = True + if self._scene_publisher is not None: + self._scene_publisher.close() + + def close_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + # Unresolved writes may still use the anchor queries and Stage. + # Retain their resources; retirement requires proven completion. + self._ordinal_ledger.destroy() + self._ordinal_ledger = None + try: + self._anchor_resources.close() + except BaseException: + self._anchor_retirement_failed = True + raise + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its borrowed Stage") + self._bootstrap.close() + + def close_environment_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + raise RuntimeError("OVStage write retirement is unproved; retaining its environment") + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its environment") + if not self._bootstrap.closed: + raise RuntimeError("Render lease retirement is unproved; retaining its environment") + self._env.close() + + def abort(self) -> None: + # The caller owns an environment even when worker startup failed. + # Release the lease before it closes that environment and its context. + self._bootstrap.close() + + +class OvLibrariesXrSession: + """Thin Isaac Lab adapter around IsaacTeleop's OVXR session.""" + + def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrConfig) -> None: + self._bootstrap = bootstrap + self._cfg = cfg + self._owner = None + self._host = None + self._closed = False + + @classmethod + def prepare( + cls, + sim_cfg: SimulationCfg, + cfg: OvLibrariesXrConfig, + renderer_cfg: OVRTXRendererCfg | None = None, + ) -> OvLibrariesXrSession: + """Snapshot session selection, create simulation and reserve its OVRTX lease. + + Renderer settings are process-global on the stock renderer interface; + use one configured session per process. No supported readback exists + here to restore a previous renderer's settings on close. + """ + cfg = replace(cfg) + validate_ovxr_session_preparation() + import ovxr + + # Stock settings are applied before Renderer construction. Preserve explicit caller choices. + ovxr.apply_renderer_settings(cfg.renderer_settings.strip()) + bootstrap = OvxrSessionBootstrap(sim_cfg, renderer_cfg) + try: + return cls(bootstrap, cfg) + except BaseException as preparation_error: + try: + bootstrap.abort() + except BaseException as abort_error: + # Cleanup follows the failure; retain its original cause. + preparation_error.add_note(f"Cleanup after failed XR preparation also failed: {abort_error!r}") + raise + + def start_render_worker( + self, + env: Any, + *, + tracking_anchor_path: str | None = None, + required_openxr_extensions: Iterable[str] = (), + hidden_visual_prims: Iterable[str] = (), + max_blank_frames: int = 1, + timeout_s: float = 30.0, + external_anchor_state: XrAnchorManager | None = None, + ) -> bool: + """Give IsaacTeleop the application pair and report whether XR started. + + Args: + env: Constructed environment sharing the prepared simulation context. + tracking_anchor_path: Override the configured USD tracking-anchor path. + required_openxr_extensions: Extensions required by the input pipeline. + hidden_visual_prims: Patterns hidden persistently from every scene view. + max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. + timeout_s: Maximum worker startup wait [s]. + external_anchor_state: Dynamic anchor policy seeded from committed scene publications. + """ + self._require_open() + if self._owner is not None: + raise RuntimeError("OVXR async render worker is already started") + if isinstance(max_blank_frames, bool) or not isinstance(max_blank_frames, int) or max_blank_frames < 0: + raise ValueError("max_blank_frames must be a non-negative integer") + from isaaccapture.ovxr_session_manager.ovxr_session import ( + OvxrSession, + OvxrSessionConfig, + ) + + from .session_lifecycle import cloudxr_eula_accepted + + anchor_path = self._cfg.tracking_anchor_path if tracking_anchor_path is None else tracking_anchor_path + host = _IsaacLabOvxrApplicationInterface( + self._bootstrap, env, self._cfg, hidden_visual_prims, external_anchor_state + ) + self._host = host + self._owner = OvxrSession( + host, + OvxrSessionConfig( + application_name=self._cfg.application_name, + session_mode=self._cfg.session_mode, + cloudxr_env_file=self._cfg.cloudxr_env_file, + auto_launch_cloudxr=self._cfg.auto_launch_cloudxr, + accept_eula=cloudxr_eula_accepted(), + render_products=self._cfg.render_products, + camera_paths=self._cfg.camera_paths, + create_render_products=self._cfg.create_render_products, + meters_per_unit=self._cfg.meters_per_unit, + depth_near_z=self._cfg.depth_near_z, + depth_far_z=self._cfg.depth_far_z, + tracking_anchor_path=anchor_path, + floor_aligned_anchor=False, + host_wait_render_output=self._cfg.host_wait_render_output, + extra_openxr_extensions=self._cfg.extra_openxr_extensions, + ), + required_openxr_extensions=required_openxr_extensions, + max_blank_frames=max_blank_frames, + ) + return self._owner.start(timeout_s=timeout_s) + + def external_oxr_handles(self) -> OpenXRSessionHandles | None: + """Return IsaacTeleop's typed external handles once its owner is ready.""" + self._require_open() + return None if self._owner is None else self._owner.external_oxr_handles() + + def request_render(self, physics_step_id: Hashable, *, force: bool = False) -> int | None: + """Offer one complete application generation for IsaacTeleop to render. + + Return the admitted ordinal, or ``None`` when render demand is already + satisfied. Retry the latest dirty generation until admitted, including + while paused. ``force`` preserves reset/lifecycle discontinuities. + """ + self._require_started() + return self._owner.request_render(physics_step_id, force=force) + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + self._require_started() + return self._owner.get_target_frame_T_world(prim_path) + + def close(self) -> None: + """Close IsaacTeleop's owner before releasing the application environment.""" + if self._closed: + return + if self._owner is None: + self._bootstrap.close() + else: + self._owner.close() + self._closed = True + + @property + def async_worker_owns_environment(self) -> bool: + """Whether IsaacTeleop accepted environment teardown ownership.""" + return self._owner is not None and self._owner.owns_environment + + @property + def shutdown_requested(self) -> bool: + """Whether the OVXR session requested application shutdown.""" + return self._owner is not None and self._owner.shutdown_requested + + def request_shutdown(self) -> None: + """Request orderly OVXR shutdown.""" + if self._owner is not None: + self._owner.request_shutdown() + + def set_external_anchor_state(self, state: XrAnchorManager | None) -> None: + """Adopt the current replay device's anchor policy between runs.""" + self._require_started() + with self._host.publication_guard(): + self._host.set_external_anchor_state(state) + + def wait_for_shutdown(self, timeout_s: float) -> bool: + """Wait up to timeout_s [s]; return whether application cancellation was requested.""" + self._require_started() + return self._owner.wait_for_shutdown(timeout_s) + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR session is closed") + + def _require_started(self) -> None: + self._require_open() + if self._owner is None: + raise RuntimeError("OVXR session is not started") + + def __enter__(self) -> OvLibrariesXrSession: + self._require_open() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py new file mode 100644 index 00000000000..e3d9a060f52 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -0,0 +1,217 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Lifecycle bootstrap for an application-owned OVRTX OVXR pair.""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING, Any + +from isaaclab_ov.renderers import OvLibrariesRenderLease, OVRTXRenderer, OVRTXRendererCfg + +from isaaclab.sim import SimulationContext + +_TERMINAL_MODE_SENTINEL = "ISAAC_LAUNCHED_FROM_TERMINAL" + +if TYPE_CHECKING: + import numpy as np + from isaaclab_ov.renderers.render_lease import OvLibrariesScenePublisher + + from isaaclab.sim import SimulationCfg + + +def validate_ovxr_session_preparation() -> None: + """Reject ineligible preparation before the caller changes renderer globals.""" + if SimulationContext.instance() is not None: + raise RuntimeError("OVXR bootstrap must create the SimulationContext immediately before the environment") + if hasattr(builtins, _TERMINAL_MODE_SENTINEL) and getattr(builtins, _TERMINAL_MODE_SENTINEL) is not True: + raise RuntimeError("OVXR bootstrap cannot reuse a context in extension mode") + + +class OvxrSessionBootstrap: + """Reserve an OVRTX/OVStage lease before a real environment is constructed.""" + + def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None = None) -> None: + validate_ovxr_session_preparation() + + self._sim: SimulationContext | None = None + self._lease: OvLibrariesRenderLease | None = None + self._owner_transfer_capability: object | None = None + self._bound = False + self._closed = False + self._installed_terminal_mode_sentinel = False + self._target_frame_readers: dict[str, tuple[Any, Any, Any]] = {} + + if not hasattr(builtins, _TERMINAL_MODE_SENTINEL): + setattr(builtins, _TERMINAL_MODE_SENTINEL, True) + self._installed_terminal_mode_sentinel = True + + try: + sim = SimulationContext(sim_cfg) + # The simulation registry owns the renderer, including cloning and + # cleanup; the XR session only borrows its exact native pair. + renderer = sim.get_or_create_backend(renderer_cfg or OVRTXRendererCfg()) + if not isinstance(renderer, OVRTXRenderer): + raise TypeError(f"OVRTXRendererCfg resolved to unexpected renderer {type(renderer).__name__}") + lease = renderer.acquire_ov_libraries_render_lease() + except BaseException as preparation_error: + sim = SimulationContext.instance() + try: + if sim is not None: + sim.clear_instance() + except BaseException as cleanup_error: + preparation_error.add_note(f"Cleanup after failed XR construction also failed: {cleanup_error!r}") + finally: + self._restore_terminal_mode_sentinel() + raise + + self._sim = sim + self._lease = lease + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application scene publisher before render ownership transfers.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Scene publisher is available only before OVXR Stage binding") + return self._lease.capture_scene_publisher() + + @property + def simulation_context(self) -> SimulationContext: + """Return the exact pre-created simulation context.""" + self._require_open() + assert self._sim is not None + return self._sim + + def bind_environment(self, env: Any) -> OvLibrariesRenderLease: + """Prepare and bind the completed environment Stage to the reserved pair.""" + self._require_open() + if self._bound: + raise RuntimeError("OVXR bootstrap is already bound") + try: + unwrapped = getattr(env, "unwrapped", env) + if unwrapped.sim is not self._sim: + raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") + scene = unwrapped.scene + if scene.stage is None: + raise RuntimeError("Environment has no completed InteractiveScene Stage") + num_envs = unwrapped.num_envs + if not isinstance(num_envs, int) or num_envs <= 0: + raise RuntimeError(f"Environment has invalid num_envs: {num_envs!r}") + assert self._sim is not None and self._lease is not None + self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) + self._lease.validate_live() + self._bound = True + return self._lease + finally: + self._restore_terminal_mode_sentinel() + + def offer_render_owner_transfer(self) -> object: + """Offer the exact unbound OVXR lease before Stage binding.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Owner transfer is available only before OVXR Stage binding") + capability = self._lease.offer_owner_transfer() + self._owner_transfer_capability = capability + return capability + + def claim_render_owner_transfer(self, capability: object) -> OvLibrariesRenderLease: + """Claim the offered lease on the worker that will bind, use, and close it.""" + self._require_open() + assert self._lease is not None + self._lease.claim_owner_transfer(capability) + self._owner_transfer_capability = None + return self._lease + + def get_world_transform(self, prim_path: str) -> np.ndarray: + """Read target-to-world transform [4, 4], float64, with translation in scene units.""" + self._require_open() + if not self._bound or self._sim is None: + raise RuntimeError("Target-frame transforms require a bound OVXR environment") + if not isinstance(prim_path, str) or not prim_path.startswith("/"): + raise ValueError(f"Target-frame prim path must be absolute: {prim_path!r}") + + import numpy as np + import warp as wp + + from isaaclab.scene_data import SceneDataFormat + + provider = self._sim.get_scene_data_provider() + cached = self._target_frame_readers.get(prim_path) + if cached is None or cached[0] is not provider: + transform_paths = list(provider.backend.transform_paths) + match_count = transform_paths.count(prim_path) + if match_count != 1: + raise RuntimeError( + f"Target-frame prim path {prim_path!r} matched {match_count} live backend transforms" + ) + mapping = provider.create_mapping([prim_path]) + output = SceneDataFormat.Matrix44() + cached = (provider, mapping, output) + self._target_frame_readers[prim_path] = cached + + provider, mapping, output = cached + if not provider.get_transforms(output, mapping=mapping, allow_passthrough=False, count=1): + raise RuntimeError(f"No SceneData transform conversion is available for target frame {prim_path!r}") + if output.matrices is None: + raise RuntimeError(f"SceneData returned no matrix buffer for target frame {prim_path!r}") + + wp.synchronize_device(str(output.matrices.device)) + world_T_target = np.asarray(output.matrices.numpy()[0], dtype=np.float64) + if world_T_target.shape != (4, 4) or not np.all(np.isfinite(world_T_target)): + raise RuntimeError(f"Target-frame transform {prim_path!r} is missing or non-finite") + if not np.allclose(world_T_target[3], (0.0, 0.0, 0.0, 1.0), atol=1e-5, rtol=0.0): + raise RuntimeError(f"Target-frame transform {prim_path!r} is not homogeneous") + return world_T_target.copy() + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + import numpy as np + + world_T_target = self.get_world_transform(prim_path) + try: + target_T_world = np.linalg.inv(world_T_target) + except np.linalg.LinAlgError as exc: + raise RuntimeError(f"Target-frame transform {prim_path!r} is singular") from exc + if not np.all(np.isfinite(target_T_world)): + raise RuntimeError(f"Inverse target-frame transform {prim_path!r} is non-finite") + return target_T_world.astype(np.float32) + + @property + def closed(self) -> bool: + """Whether the lease and its retained publication resources retired successfully.""" + return self._closed + + def close(self) -> None: + """Release after every borrower is destroyed; retain ownership for retry on failure.""" + if self._closed: + return + # Keep ownership and the bootstrap retryable until retirement succeeds. + if self._lease is not None: + if self._owner_transfer_capability is not None: + self._lease.cancel_owner_transfer(self._owner_transfer_capability) + self._owner_transfer_capability = None + self._lease.release() + self._target_frame_readers.clear() + self._restore_terminal_mode_sentinel() + self._closed = True + + def abort(self) -> None: + """Release a failed construction; retain its context if lease retirement fails.""" + if self._closed: + return + self.close() + if not self._bound and self._sim is not None and SimulationContext.instance() is self._sim: + self._sim.clear_instance() + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR bootstrap is closed") + + def _restore_terminal_mode_sentinel(self) -> None: + if self._installed_terminal_mode_sentinel: + if hasattr(builtins, _TERMINAL_MODE_SENTINEL): + delattr(builtins, _TERMINAL_MODE_SENTINEL) + self._installed_terminal_mode_sentinel = False diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py new file mode 100644 index 00000000000..a20d347df44 --- /dev/null +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -0,0 +1,707 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused tests for the Isaac Lab host side of external OVXR presentation.""" + +from __future__ import annotations + +import builtins +import contextlib +import importlib +import sys +import threading +from contextlib import contextmanager +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + + +class _FakeLease: + def __init__(self): + self.ovrtx_renderer = object() + self.ovstage = object() + self.owner_lane_entries = 0 + + @contextmanager + def ovstage_owner_lane(self): + self.owner_lane_entries += 1 + yield + + def validate_live(self): + pass + + +class _FakePublisher: + def __init__(self): + self.calls = [] + self.fail = None + self.required = True + self.closed = False + + def publication_required(self, _publication_id): + return self.required + + def write_complete_scene(self, ordinal, publication_id): + self.calls.append((ordinal, publication_id, threading.get_native_id())) + if self.fail is not None: + raise self.fail + + def close(self): + self.closed = True + + +class _FakeOrdinalLedger: + def __init__(self, stage): + self.stage = stage + self.next_ordinal = 11 + self.reserved = [] + self.published = [] + self.abandoned = [] + self.publish_error = None + self.destroyed = False + + def reserve(self): + ordinal = self.next_ordinal + self.next_ordinal += 1 + self.reserved.append(ordinal) + return ordinal + + def publish(self, ordinal): + if self.publish_error is not None: + raise self.publish_error + self.published.append(ordinal) + + def abandon(self, ordinal): + self.abandoned.append(ordinal) + + def destroy(self): + self.destroyed = True + + +class _FakeBootstrap: + def __init__(self): + self.lease = _FakeLease() + self.publisher = _FakePublisher() + self.events = [] + self.closed = False + + def capture_scene_publisher(self): + self.events.append("capture") + return self.publisher + + def offer_render_owner_transfer(self): + self.events.append("offer") + return "capability" + + def claim_render_owner_transfer(self, capability): + self.events.append(("claim", capability)) + + def bind_environment(self, env): + self.events.append(("bind", env)) + return self.lease + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + self.closed = True + + +@pytest.fixture +def real_lease_type(): + from isaaclab_ov.renderers.render_lease import OvLibrariesRenderLease + + return OvLibrariesRenderLease + + +@pytest.fixture +def xr_module(monkeypatch, real_lease_type): + """Import the adapter with native renderer dependencies stubbed.""" + isaaclab_ov = ModuleType("isaaclab_ov") + isaaclab_ov.__path__ = [] + renderers = ModuleType("isaaclab_ov.renderers") + renderers.OVRTXRendererCfg = object + renderers.OVRTXRenderer = object + renderers.OvLibrariesRenderLease = object + monkeypatch.setitem(sys.modules, "isaaclab_ov", isaaclab_ov) + monkeypatch.setitem(sys.modules, "isaaclab_ov.renderers", renderers) + source = Path(__file__).parents[1] / "isaaclab_teleop/ovxr_session.py" + spec = importlib.util.spec_from_file_location("isaaclab_teleop.ovxr_session", source) + ovxr_session = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, ovxr_session) + spec.loader.exec_module(ovxr_session) + sys.modules.pop("isaaclab_teleop.ov_libraries_xr", None) + module = importlib.import_module("isaaclab_teleop.ov_libraries_xr") + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + return module + + +def _fake_ovxr(): + return SimpleNamespace(apply_renderer_settings=lambda _settings: None) + + +def test_stage_ordinal_ledger_tracks_one_application_reservation(xr_module): + class ImmediateQuery: + def __init__(self, fetch): + self._fetch = fetch + + def __enter__(self): + return self + + def __exit__(self, *_exc): + pass + + def wait(self): + pass + + def fetch(self): + return self._fetch() + + class ImmediateAdvance: + def wait(self): + pass + + class Stage: + def __init__(self): + self.floor = 10 + self.advances = [] + self.queries = 0 + + def get_attribute_write_floor(self): + self.queries += 1 + return ImmediateQuery(lambda: self.floor) + + def advance_write_floor(self, *, ordinal): + self.advances.append(ordinal) + self.floor = ordinal + return ImmediateAdvance() + + stage = Stage() + ledger = xr_module._StageOrdinalLedger(stage) + assert stage.queries == 1 + + assert ledger.reserve() == 11 + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + ledger.abandon(11) + + assert ledger.reserve() == 11 + ledger.publish(11) + assert stage.advances == [11] + assert stage.queries == 2 # Only the earlier abandon needs reconciliation. + + assert ledger.reserve() == 12 + stage.floor = 12 # The explicit-ordinal OVXR call sealed its own write. + assert ledger.resolve_external(12, expected_published=True) + assert stage.queries == 2 + + assert ledger.reserve() == 13 + assert not ledger.resolve_external(13, expected_published=None) + assert stage.queries == 3 + assert ledger.reserve() == 13 + stage.floor = 13 + with pytest.raises(RuntimeError, match="expected unsealed"): + ledger.resolve_external(13, expected_published=False) + assert stage.queries == 4 + # Even a contradicted response must not permit reuse of an observed sealed ordinal. + assert ledger.reserve() == 14 + ledger.abandon(14) + ledger.destroy() + with pytest.raises(RuntimeError, match="closed"): + ledger.reserve() + + +def test_runtime_render_prims_have_the_declared_paths_and_types(xr_module): + class FakePrim: + def __init__(self, type_name=""): + self.type_name = type_name + + def __bool__(self): + return bool(self.type_name) + + def IsValid(self): + return bool(self.type_name) + + def GetTypeName(self): + return self.type_name + + class FakeStage: + def __init__(self): + self.prims = {} + self.definitions = [] + + def GetPrimAtPath(self, path): + return self.prims.get(path, FakePrim()) + + def DefinePrim(self, path, type_name): + self.definitions.append((path, type_name)) + self.prims[path] = FakePrim(type_name) + + stage = FakeStage() + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig()) + + assert stage.definitions == [ + ("/OVXRGeneratedCameras/OVXR_LeftEyeCamera", "Camera"), + ("/OVXRGeneratedCameras/OVXR_RightEyeCamera", "Camera"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", "RenderProduct"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", "RenderProduct"), + ("/Render/Vars/LdrColor", "RenderVar"), + ("/Render/Vars/XRDepth", "RenderVar"), + ("/Render/OmniverseGlobalRenderSettings", "RenderSettings"), + ] + + +def test_runtime_render_prim_declaration_preserves_disabled_and_fails_on_conflict(xr_module): + class ExistingPrim: + def IsValid(self): + return True + + def GetTypeName(self): + return "Xform" + + stage = SimpleNamespace(GetPrimAtPath=lambda _path: ExistingPrim(), DefinePrim=lambda *_args: None) + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig(create_render_products=False)) + with pytest.raises(RuntimeError, match="incompatible type"): + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig()) + + +def test_hidden_visual_prims_fullmatch_imageables_and_fail_closed(xr_module, monkeypatch): + class FakePrim: + def __init__(self, path): + self.path = path + self.visible = True + + def GetPath(self): + return self.path + + def IsA(self, _schema): + return True + + class FakeImageable: + def __init__(self, prim): + self.prim = prim + + def MakeInvisible(self): + self.prim.visible = False + + pxr = ModuleType("pxr") + pxr.UsdGeom = SimpleNamespace(Imageable=FakeImageable) + monkeypatch.setitem(sys.modules, "pxr", pxr) + head = FakePrim("/World/envs/env_0/Robot/head_yaw_link") + body = FakePrim("/World/envs/env_0/Robot/base_link") + stage = SimpleNamespace(Traverse=lambda: iter((head, body))) + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + + assert xr_module._apply_hidden_visual_prims(env, ("/World/envs/env_.*/Robot/head_.*",)) == (head.path,) + assert not head.visible and body.visible + with pytest.raises(RuntimeError, match="matched no imageable prims"): + xr_module._apply_hidden_visual_prims(env, ("/World/missing/.*",)) + + +def _make_host(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + env = SimpleNamespace(scene=SimpleNamespace(stage=None), close=lambda: bootstrap.events.append("environment-close")) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr( + xr_module, + "_apply_hidden_visual_prims", + lambda _env, _patterns: bootstrap.events.append("hide"), + ) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda _env, _cfg: bootstrap.events.append("declare")) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, + env, + xr_module.OvLibrariesXrConfig(), + ("/hidden",), + ) + return host, bootstrap, env + + +def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monkeypatch): + host, bootstrap, env = _make_host(xr_module, monkeypatch) + + capability = host.offer_render_owner_transfer() + binding = host.attach_on_render_thread(capability) + with host.publication_guard(): + ordinal = host.publish_complete_scene("step-1") + + assert binding.renderer is bootstrap.lease.ovrtx_renderer + assert binding.stage is bootstrap.lease.ovstage + assert binding.ordinal_ledger.stage is bootstrap.lease.ovstage + assert bootstrap.events == ["capture", "offer", ("claim", "capability"), "hide", "declare", ("bind", env)] + assert ordinal == 11 + assert bootstrap.publisher.calls[0][:2] == (11, "step-1") + host.validate_first_published_scene(ordinal) + with host.owner_lane(): + pass + assert bootstrap.lease.owner_lane_entries == 3 + assert host.get_target_frame_T_world("/World/target") == ("target", "/World/target") + + host.close_publication() + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.publisher.closed and bootstrap.closed + assert binding.ordinal_ledger.destroyed + assert bootstrap.events[-1] == "environment-close" + + +def test_host_abandons_failed_write_and_failed_publish(xr_module, monkeypatch): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + bootstrap.publisher.fail = RuntimeError("write failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="write failed"): + host.publish_complete_scene("step-1") + assert host._ordinal_ledger.abandoned == [11] + + bootstrap.publisher.fail = None + host._ordinal_ledger.publish_error = RuntimeError("publish failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="publish failed"): + host.publish_complete_scene("step-2") + assert host._ordinal_ledger.abandoned == [11, 12] + + +@pytest.mark.parametrize("sealed", [False, True]) +def test_host_reconciles_failed_publication_without_reusing_generation(xr_module, monkeypatch, real_lease_type, sealed): + ledger_type = xr_module._StageOrdinalLedger + host, _bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + failure = RuntimeError("native publication wait failed") + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + # Use the real publication identity owner; a permanently-dirty fake cannot + # detect a retry returning the older ordinal after a failed seal. + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _geometry_publication=None, + _update_transforms=lambda: None, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + host._scene_publisher = lease.capture_scene_publisher() + renderer.scene = SimpleNamespace(stage=stage, ordinal=10) + lease._bind(object(), stage, 1) + stage.advance_write_floor = lambda *, ordinal: SimpleNamespace(wait=lambda: setattr(stage, "floor", ordinal)) + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(): + assert host.publish_complete_scene("previous") == 11 + + def failed_advance(*, ordinal): + if sealed: + stage.floor = ordinal + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("changed") + assert caught.value is failure + with host.publication_guard(), pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("changed") + next_ordinal = ledger.reserve() + assert next_ordinal == (13 if sealed else 12) + ledger.abandon(next_ordinal) + host.close_on_render_thread() + lease.release() + + +def test_failed_bootstrap_retirement_retains_environment(xr_module, monkeypatch): + """Capture's independent environment callback cannot bypass failed lease retirement.""" + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + + def fail_close(): + raise RuntimeError("producer drain failed") + + monkeypatch.setattr(bootstrap, "close", fail_close) + with pytest.raises(RuntimeError, match="producer drain failed"): + host.close_on_render_thread() + with pytest.raises(RuntimeError, match="lease retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +def test_failed_publication_with_unknown_floor_retains_reservation_and_first_error(xr_module, monkeypatch): + ledger_type = xr_module._StageOrdinalLedger + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + stage = SimpleNamespace(unknown=False) + failure = RuntimeError("native publication wait failed") + + def fetch(): + if stage.unknown: + raise RuntimeError("floor read unavailable") + return 10 + + stage.get_attribute_write_floor = lambda: contextlib.nullcontext(SimpleNamespace(wait=lambda: None, fetch=fetch)) + + def failed_advance(*, ordinal): + stage.unknown = True + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is failure + assert any("floor read unavailable" in note for note in getattr(caught.value, "__notes__", [])) + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + # Even repeated cleanup cannot retire queries/Stage while the write is unresolved. + anchor_resources_closed = [] + host._anchor_resources.callback(lambda: anchor_resources_closed.append(True)) + for _ in range(2): + with pytest.raises(RuntimeError, match="unresolved OVStage ordinal"): + host.close_on_render_thread() + assert not bootstrap.closed + assert not anchor_resources_closed + with pytest.raises(RuntimeError, match="write retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +@pytest.mark.parametrize("failure", ["write", "publish"]) +def test_publication_retains_primary_cause_when_abandon_fails(xr_module, monkeypatch, failure): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + original_cause = ValueError("original publication cause") + first_error = RuntimeError("publication failed") + first_error.__cause__ = original_cause + if failure == "write": + bootstrap.publisher.fail = first_error + else: + host._ordinal_ledger.publish_error = first_error + + def fail_abandon(_ordinal): + raise RuntimeError("abandon failed") + + monkeypatch.setattr(host._ordinal_ledger, "abandon", fail_abandon) + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert any("abandon failed" in note for note in caught.value.__notes__) + + +def test_session_delegates_lifecycle_to_isaacteleop_owner(xr_module, monkeypatch): + calls = [] + + class FakeOwner: + owns_environment = True + shutdown_requested = False + + def __init__(self, host, config, **kwargs): + calls.append(("init", host, config, kwargs)) + + def start(self, timeout_s): + calls.append(("start", timeout_s)) + return True + + def external_oxr_handles(self): + return "handles" + + def request_render(self, identity, *, force=False): + calls.append(("request", identity, force)) + return 17 + + def request_shutdown(self): + self.shutdown_requested = True + calls.append(("request_shutdown",)) + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + calls.append(("close",)) + + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", FakeOwner) + bootstrap = _FakeBootstrap() + assert xr_module.OvLibrariesXrConfig().extra_openxr_extensions == () + session = xr_module.OvLibrariesXrSession( + bootstrap, xr_module.OvLibrariesXrConfig(extra_openxr_extensions=("XR_EXT_debug_utils",)) + ) + env = SimpleNamespace() + assert session.start_render_worker( + env, required_openxr_extensions=("XR_EXT_hand_tracking",), max_blank_frames=0, timeout_s=4.0 + ) + assert calls[0][0] == "init" + assert calls[0][2].session_mode == "ar" + assert calls[0][3]["required_openxr_extensions"] == ("XR_EXT_hand_tracking",) + assert calls[0][3]["max_blank_frames"] == 0 + assert calls[0][2].extra_openxr_extensions == ("XR_EXT_debug_utils",) + assert calls[0][2].host_wait_render_output is True + assert calls[1] == ("start", 4.0) + assert session.external_oxr_handles() == "handles" + assert session.request_render("step") == 17 + assert session.request_render("reset", force=True) == 17 + assert calls[-2:] == [("request", "step", False), ("request", "reset", True)] + assert session.get_target_frame_T_world("/target") == ("target", "/target") + assert session.async_worker_owns_environment + assert not session.shutdown_requested + session.request_shutdown() + assert calls[-1] == ("request_shutdown",) + assert session.shutdown_requested + session._owner.shutdown_requested = True + assert session.shutdown_requested + session.close() + assert calls[-1] == ("close",) + + +@pytest.fixture +def bootstrap_context(xr_module, monkeypatch): + """Use the real bootstrap with only simulation/native construction replaced.""" + module = sys.modules["isaaclab_teleop.ovxr_session"] + events = [] + + class Context: + current = None + failure = None + cleanup_failure = None + + def __init__(self, _cfg): + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + events.append("context-close") + cls.current = None + if cls.cleanup_failure is not None: + raise cls.cleanup_failure + + def get_or_create_backend(self, _cfg): + if self.failure is not None: + raise self.failure + lease = SimpleNamespace(release=lambda: events.append("lease-release")) + return SimpleNamespace(acquire_ov_libraries_render_lease=lambda: lease) + + monkeypatch.setattr(module, "SimulationContext", Context) + monkeypatch.delattr(builtins, module._TERMINAL_MODE_SENTINEL, raising=False) + return module, Context, events + + +@pytest.mark.parametrize("route", ["before_owner", "before_worker"]) +def test_failed_start_releases_lease_before_environment_and_context(xr_module, bootstrap_context, route): + """An existing environment retains its context until the caller closes it.""" + module, context, events = bootstrap_context + bootstrap = module.OvxrSessionBootstrap(object()) + if route == "before_owner": + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + session.close() + else: + # Capture calls host.abort when validation fails before its worker exists. + host = xr_module._IsaacLabOvxrApplicationInterface.__new__(xr_module._IsaacLabOvxrApplicationInterface) + host._bootstrap = bootstrap + host.abort() + assert context.current is not None + events.append("environment-close") + context.clear_instance() + assert events == ["lease-release", "environment-close", "context-close"] + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +@pytest.mark.parametrize("cleanup_fails", [False, True]) +def test_bootstrap_construction_failure_restores_terminal_mode(bootstrap_context, error_type, cleanup_fails): + """Failed or interrupted construction restores process state and preserves its cause.""" + module, context, events = bootstrap_context + error = context.failure = error_type("renderer construction failed") + if cleanup_fails: + context.cleanup_failure = RuntimeError("context cleanup failed") + with pytest.raises(error_type) as caught: + module.OvxrSessionBootstrap(object()) + assert caught.value is error + assert events == ["context-close"] + assert context.current is None + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + if cleanup_fails: + assert any("context cleanup failed" in note for note in error.__notes__) + + +def test_caller_renderer_settings_precede_renderer_creation(xr_module, monkeypatch): + settings = " --/rtx/rendermode=MinimalRendering " + events = [] + bootstrap = _FakeBootstrap() + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda value: events.append(value)) + ) + + def create_bootstrap(*_args): + assert len(events) == 1 + assert events[0] == settings.strip() + return bootstrap + + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", create_bootstrap) + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(renderer_settings=settings)) + + +def test_config_rejects_unknown_session_mode(xr_module): + with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): + xr_module.OvLibrariesXrConfig(session_mode="automatic") + + +@pytest.mark.parametrize("scale", [0.0, -1.0, float("nan"), float("inf")]) +def test_config_rejects_invalid_scene_scale(xr_module, scale): + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + xr_module.OvLibrariesXrConfig(meters_per_unit=scale) + + +def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): + cfg = xr_module.OvLibrariesXrConfig() + cfg.meters_per_unit = float("nan") + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + xr_module.OvLibrariesXrSession.prepare(object(), cfg) + + +@pytest.mark.parametrize("count", [-1, 1.5, True]) +def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, monkeypatch, count): + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + with pytest.raises(ValueError, match="max_blank_frames must be a non-negative integer"): + session.start_render_worker(SimpleNamespace(), max_blank_frames=count) + assert bootstrap.events == [] + + +def test_public_xr_exports_resolve_to_implemented_types(xr_module): + import isaaclab_teleop + + assert isaaclab_teleop.OvLibrariesXrConfig is xr_module.OvLibrariesXrConfig + assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py new file mode 100644 index 00000000000..ec31863ca6b --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""OVXR bootstrap retirement across the teleoperation/renderer boundary.""" + +from types import SimpleNamespace + +import pytest + +from isaaclab.sim import SimulationContext + +pytest.importorskip("ovrtx") +from isaaclab_ov.renderers import OVRTXRenderer, OVRTXRendererCfg + + +@pytest.mark.parametrize("operation", ["close", "abort"]) +def test_bootstrap_failed_transfer_cancellation_keeps_owner_retryable(monkeypatch, operation): + """A wrong-thread close cannot destroy a pending lease or prevent its owner retiring it.""" + import threading + + from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap + + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + sim = SimpleNamespace(device="cpu", get_scene_data_provider=lambda: None) + monkeypatch.setattr(SimulationContext, "_instance", sim) + renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=sim) + lease = renderer.acquire_ov_libraries_render_lease() + bootstrap = OvxrSessionBootstrap.__new__(OvxrSessionBootstrap) + bootstrap._lease = lease + bootstrap._owner_transfer_capability = lease.offer_owner_transfer() + bootstrap._closed = bootstrap._bound = False + bootstrap._installed_terminal_mode_sentinel = False + bootstrap._target_frame_readers = {} + bootstrap._sim = sim = SimulationContext.instance() + cleared = [] + monkeypatch.setattr(sim, "clear_instance", lambda: cleared.append(True), raising=False) + errors = [] + + def wrong_owner(): + try: + getattr(bootstrap, operation)() + except RuntimeError as error: + errors.append(str(error)) + + thread = threading.Thread(target=wrong_owner) + thread.start() + thread.join(5) + assert not thread.is_alive() + assert len(errors) == 1 and "owner thread must cancel" in errors[0] + assert not cleared + # The real owner must still be able to retire the real renderer lease. + getattr(bootstrap, operation)() + assert renderer._render_lease.active is None + assert cleared == ([True] if operation == "abort" else []) From a0cedbc2ec219fcb38b2d1a9e5c281edcebea958 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Fri, 30 Oct 2026 18:16:18 -0400 Subject: [PATCH 002/145] Add live and recorded OVXR teleoperation --- .../source/_static/css/environment-browser.js | 2 +- docs/source/how-to/cloudxr_teleoperation.rst | 2 +- docs/source/how-to/index.rst | 12 + docs/source/how-to/kitless_xr.md | 15 + docs/source/how-to/kitless_xr_source.md | 8 + .../kitless-ovxr-lifecycle.fixed.rst | 2 + source/isaaclab/isaaclab/app/sim_launcher.py | 13 +- .../isaaclab/isaaclab/utils/warp/__init__.py | 18 +- .../app/test_launch_simulation_require_kit.py | 37 + .../test/cli/test_teleop_entrypoints.py | 29 + .../test/utils/warp/test_proxy_array.py | 21 +- .../changelog.d/kitless-ovxr-model.fixed.rst | 1 + .../isaaclab_newton/physics/newton_backend.py | 7 +- .../test_newton_manager_abstraction.py | 48 + .../changelog.d/kitless-teleop.fixed.rst | 2 + .../config/so101/stack_ik_abs_env_cfg.py | 26 +- .../test/contrib/stack/test_so101_pose_ik.py | 48 + .../contrib/test_so101_teleop_pipeline.py | 113 ++- .../test/core/test_preset_cli.py | 30 + .../changelog.d/ovxr-live-replay.added.rst | 3 + .../changelog.d/ovxr-live-replay.changed.rst | 7 + .../changelog.d/ovxr-live-replay.fixed.rst | 2 + .../changelog.d/ovxr-live-replay.major | 0 source/isaaclab_teleop/docs/README.md | 35 +- source/isaaclab_teleop/docs/kitless_xr.md | 109 +++ .../isaaclab_teleop/docs/kitless_xr_source.md | 63 ++ .../isaaclab_teleop/action_policy.py | 45 + .../entrypoints/teleop_replay_agent.py | 469 +++++++--- .../entrypoints/teleop_se3_agent.py | 795 +++++++++++----- .../isaaclab_teleop/isaac_teleop_cfg.py | 33 +- .../isaaclab_teleop/isaac_teleop_device.py | 157 +++- .../isaaclab_teleop/session_lifecycle.py | 231 +++-- .../isaaclab_teleop/xr_anchor_manager.py | 204 +++- .../isaaclab_teleop/xr_anchor_utils.py | 312 ++++--- .../isaaclab_teleop/isaaclab_teleop/xr_cfg.py | 31 + .../test/test_cloudxr_lifecycle.py | 193 +++- .../test/test_control_events.py | 102 +- .../test/test_hand_debug_viz.py | 79 +- .../test/test_openxr_device_constructors.py | 5 + .../test/test_ov_libraries_xr.py | 69 ++ .../test/test_session_step_failure.py | 104 +++ .../test/test_target_frame_rebase.py | 47 + .../test/test_teleop_entrypoint_units.py | 868 ++++++++++++++++++ .../test/test_teleop_failure_exit.py | 205 +++++ .../test/test_teleop_replay_bootstrap.py | 346 +++++++ .../test/test_teleop_scripts_smoke.py | 61 +- .../test/test_xr_anchor_manager_contract.py | 344 +++++++ .../changelog.d/headless-xr.fixed.rst | 2 + .../kit/kit_visualizer.py | 9 +- .../test_kit_visualizer_scene_partitioning.py | 8 +- 50 files changed, 4537 insertions(+), 835 deletions(-) create mode 100644 docs/source/how-to/kitless_xr.md create mode 100644 docs/source/how-to/kitless_xr_source.md create mode 100644 source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst create mode 100644 source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst create mode 100644 source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-live-replay.major create mode 100644 source/isaaclab_teleop/docs/kitless_xr.md create mode 100644 source/isaaclab_teleop/docs/kitless_xr_source.md create mode 100644 source/isaaclab_teleop/isaaclab_teleop/action_policy.py create mode 100644 source/isaaclab_teleop/test/test_teleop_entrypoint_units.py create mode 100644 source/isaaclab_teleop/test/test_teleop_failure_exit.py create mode 100644 source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py create mode 100644 source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py create mode 100644 source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index edb297ec0bf..022715b3db3 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -129,7 +129,7 @@ ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka-IK-Rel", "", "", "", "", "environment-previews/franka_instance_randomize_stack.jpg"], ["IsaacContrib-Stack-Cube-RedGreen-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-RedGreenBlue-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], - ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], + ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-SO101-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/ur10_stack_surface_gripper.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index 066f2f4f01d..3e7ceae2f4e 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -192,7 +192,7 @@ Complete these steps first: #. ``isaacteleop`` ships as part of the ``teleop`` extra, so no separate pip install step is required — but note that it comes from the extra rather than from the ``isaaclab_teleop`` package metadata, so installing ``isaaclab_teleop`` on its own does **not** pull it in. - For building from source or plugin development, see the `Isaac Capture GitHub + For Capture APIs and plugin development, see the `Isaac Capture GitHub `_. #. Configure the firewall to allow CloudXR traffic. The required ports depend on the diff --git a/docs/source/how-to/index.rst b/docs/source/how-to/index.rst index fa71e2db08c..805f7d7a188 100644 --- a/docs/source/how-to/index.rst +++ b/docs/source/how-to/index.rst @@ -510,6 +510,12 @@ How-to Guides :guide-tag:`PhysX` :guide-tag:`Isaac Sim` + .. container:: guide-entry + + :doc:`Kitless XR with Isaac Capture ` + + Install Isaac Capture and OVXR packages for live teleoperation and MCAP replay. + .. container:: guide-entry :doc:`Setting up Haply teleoperation ` @@ -556,3 +562,9 @@ How-to Guides This collection is a work in progress. If a question is not answered here, open an issue on the `Isaac Lab GitHub repository `_. + +.. toctree:: + :hidden: + + kitless_xr + kitless_xr_source diff --git a/docs/source/how-to/kitless_xr.md b/docs/source/how-to/kitless_xr.md new file mode 100644 index 00000000000..bcf93859f77 --- /dev/null +++ b/docs/source/how-to/kitless_xr.md @@ -0,0 +1,15 @@ +--- +myst: + heading_anchors: 2 +--- + + + +(kitless-xr)= + +```{include} ../../../source/isaaclab_teleop/docs/kitless_xr.md +``` diff --git a/docs/source/how-to/kitless_xr_source.md b/docs/source/how-to/kitless_xr_source.md new file mode 100644 index 00000000000..7903c9ebc59 --- /dev/null +++ b/docs/source/how-to/kitless_xr_source.md @@ -0,0 +1,8 @@ + + +```{include} ../../../source/isaaclab_teleop/docs/kitless_xr_source.md +``` diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst new file mode 100644 index 00000000000..4274dd1920f --- /dev/null +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst @@ -0,0 +1,2 @@ +* Allowed asynchronous XR owners to drain before releasing the simulation context. +* Supported empty CPU Warp arrays in Torch conversion. diff --git a/source/isaaclab/isaaclab/app/sim_launcher.py b/source/isaaclab/isaaclab/app/sim_launcher.py index d832f83b1d0..bf8d08fd98a 100644 --- a/source/isaaclab/isaaclab/app/sim_launcher.py +++ b/source/isaaclab/isaaclab/app/sim_launcher.py @@ -412,12 +412,19 @@ def validate_launch( # noqa: C901 def launch_simulation( cfg, launcher_args: argparse.Namespace | dict | None = None, + *, + cleanup_context: bool = True, ) -> Generator[PhysicsCfg | None, None, None]: """Context manager that launches the appropriate simulation runtime for *cfg*. Resolves configured consumers before starting their required runtimes. Each runtime starts once and closes on exit. Cameras are auto-enabled for Kit-renderer sensors. + On exit, stops and releases a simulation context created inside the scope before closing + its runtimes. An existing context owned by the caller is preserved. Applications + with asynchronous resource owners can set ``cleanup_context=False`` and take + responsibility for context cleanup, including failed construction and drain. + Resolve viewer and physics choices on the same config used to construct the simulation:: sim_cfg = SimulationCfg() @@ -437,6 +444,10 @@ def launch_simulation( a tool that reaches a Kit-only extension API. This is additive -- it can only turn a kitless launch into a Kit one, never the reverse, so a config that already needs Kit still launches it when the key is absent or ``False``. + cleanup_context: Automatically release a context created inside this scope. + Disable only when the application owns its cleanup and must retain it + if asynchronous consumers cannot prove they have drained. This controls + context release only; configured runtime launchers still close on exit. """ launcher_args = {} if launcher_args is None else launcher_args launcher_args = vars(launcher_args) if isinstance(launcher_args, argparse.Namespace) else launcher_args @@ -500,7 +511,7 @@ def launch_simulation( finally: # Release runtime consumers before a launcher can terminate the process. context = SimulationContext.instance() - if context is not None and context is not previous_context: + if cleanup_context and context is not None and context is not previous_context: SimulationContext.clear_instance() except KeyboardInterrupt: exit_code = 130 diff --git a/source/isaaclab/isaaclab/utils/warp/__init__.py b/source/isaaclab/isaaclab/utils/warp/__init__.py index 2b2eef4ecef..68105780ea2 100644 --- a/source/isaaclab/isaaclab/utils/warp/__init__.py +++ b/source/isaaclab/isaaclab/utils/warp/__init__.py @@ -37,8 +37,9 @@ def _wp_to_torch_with_proxyarray(a, requires_grad=None): The shim routes :class:`ProxyArray` arguments to their cached ``.torch`` view (a zero-copy :class:`torch.Tensor` of the same underlying memory) and - emits a one-shot :class:`DeprecationWarning`. For any other input type, - the original :func:`warp.to_torch` handles the call as before. + emits a one-shot :class:`DeprecationWarning`. Empty CPU arrays use DLPack + because NumPy rejects their null-pointer array interface. Other native + arrays keep the original :func:`warp.to_torch` conversion. """ global _WP_TO_TORCH_WARNED if isinstance(a, ProxyArray): @@ -51,6 +52,19 @@ def _wp_to_torch_with_proxyarray(a, requires_grad=None): stacklevel=2, ) return a.torch + if isinstance(a, wp.array) and a.size == 0 and a.device is not None and a.device.is_cpu: + # Preserve Warp's structured-array rejection before the DLPack fallback. + if isinstance(a.dtype, wp._src.codegen.Struct): # noqa: SLF001 + return _WP_TO_TORCH_ORIGINAL(a, requires_grad=requires_grad) + import torch + + if requires_grad is None: + requires_grad = a.requires_grad + tensor = torch.from_dlpack(wp.to_dlpack(a)) + tensor.requires_grad = requires_grad + if requires_grad and a.requires_grad: + tensor.grad = torch.from_dlpack(wp.to_dlpack(a.grad)) + return tensor return _WP_TO_TORCH_ORIGINAL(a, requires_grad=requires_grad) diff --git a/source/isaaclab/test/app/test_launch_simulation_require_kit.py b/source/isaaclab/test/app/test_launch_simulation_require_kit.py index 32d754ba16e..e8f44054ce1 100644 --- a/source/isaaclab/test/app/test_launch_simulation_require_kit.py +++ b/source/isaaclab/test/app/test_launch_simulation_require_kit.py @@ -256,3 +256,40 @@ def test_legacy_recording_reuses_the_display_producer(kit_branch_taken, visualiz assert cfg.sim.visualizer_cfgs[0].headless == (not visualizers) assert args.get("enable_cameras", False) == (source == "kit") assert kit_branch_taken == [True] * (source == "kit") + + +@pytest.mark.parametrize( + "application_owns_cleanup,drain_failed", [(False, False), (False, True), (True, False), (True, True)] +) +def test_context_cleanup_respects_application_drain_ownership(monkeypatch, application_owns_cleanup, drain_failed): + """An outer launcher must not release a context retained by a failed native drain.""" + from isaaclab.sim import SimulationContext + + state = {"context": None} + released = [] + + def clear(): + released.append(state["context"]) + state["context"] = None + + monkeypatch.setattr(SimulationContext, "instance", lambda: state["context"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear) + context = object() + + def run(): + with launch_simulation(PhysicsCfg(), cleanup_context=not application_owns_cleanup): + state["context"] = context + if drain_failed: + raise RuntimeError("consumer drain unproved") + if application_owns_cleanup: + # The already-owning environment/worker performs normal release. + SimulationContext.clear_instance() + + if drain_failed: + with pytest.raises(RuntimeError, match="consumer drain unproved"): + run() + else: + run() + retained = application_owns_cleanup and drain_failed + assert released == ([] if retained else [context]) + assert state["context"] is (context if retained else None) diff --git a/source/isaaclab/test/cli/test_teleop_entrypoints.py b/source/isaaclab/test/cli/test_teleop_entrypoints.py index 210e42c92da..94691d8f266 100644 --- a/source/isaaclab/test/cli/test_teleop_entrypoints.py +++ b/source/isaaclab/test/cli/test_teleop_entrypoints.py @@ -57,6 +57,35 @@ def test_teleop_workflow_help_exposes_task_preset_selectors(source_checkout_root assert "physics=NAME" in result.stdout assert "renderer=NAME" in result.stdout assert "presets=NAME[,NAME,...]" in result.stdout + if command == "run": + assert re.search(r"^\s+--ovxr(?:\s|$)", result.stdout, re.MULTILINE) + + +def test_ovxr_omitted_launch_options_match_documented_defaults(source_checkout_root, monkeypatch): + """The shortened user command retains the profile, launch policy and visualizers.""" + from isaaclab.app import validate_launch + + monkeypatch.delenv("LIVESTREAM", raising=False) + script = source_checkout_root / "scripts/environments/teleoperation/teleop_se3_agent.py" + + def parse(arguments): + monkeypatch.setattr(sys, "argv", [str(script), *arguments]) + return importlib.import_module("isaaclab_teleop.entrypoints.teleop_se3_agent")._parse_args() + + implicit = parse(["--ovxr"]) + explicit = parse(["--ovxr", "--viz", "none", "--auto_launch_cloudxr", "--cloudxr_env", "cloudxrjs"]) + assert implicit.auto_launch_cloudxr is explicit.auto_launch_cloudxr is True + assert implicit.cloudxr_env == explicit.cloudxr_env + assert implicit.cloudxr_env.endswith(".env") + left, right = vars(implicit).copy(), vars(explicit).copy() + from isaaclab.sim import SimulationCfg + + left_cfg, right_cfg = SimulationCfg(), SimulationCfg() + validate_launch(left_cfg, left) + validate_launch(right_cfg, right) + assert left_cfg.visualizer_cfgs == right_cfg.visualizer_cfgs == [] + assert parse(["--ovxr", "--no-auto_launch_cloudxr"]).auto_launch_cloudxr is False + assert parse(["--ovxr", "--cloudxr_env", "none"]).cloudxr_env is None @pytest.mark.parametrize(("command", "script_parts"), TELEOP_WORKFLOWS.items()) diff --git a/source/isaaclab/test/utils/warp/test_proxy_array.py b/source/isaaclab/test/utils/warp/test_proxy_array.py index 247644674b7..8e96e56eb5e 100644 --- a/source/isaaclab/test/utils/warp/test_proxy_array.py +++ b/source/isaaclab/test/utils/warp/test_proxy_array.py @@ -461,7 +461,7 @@ def test_proxy_array_warning_is_one_shot(self): assert not deprecation, "shim must not re-warn after the first call" def test_requires_grad_forwarded_to_raw_wp_array(self): - """``wp.to_torch(wp.array)`` still produces a torch view and forwards ``requires_grad``.""" + """Native and empty CPU views preserve dtype, shape and gradient selection.""" import isaaclab.utils.warp # noqa: F401 # ensure shim is installed arr = wp.array([1.0, 2.0], dtype=wp.float32, device="cpu") @@ -469,3 +469,22 @@ def test_requires_grad_forwarded_to_raw_wp_array(self): assert isinstance(t, torch.Tensor) assert t.shape == (2,) assert t.requires_grad is False + + empty_ids = wp.from_torch(torch.arange(4, dtype=torch.int32)[:0]) + ids = wp.to_torch(empty_ids) + assert ids.shape == (0,) and ids.dtype == torch.int32 + assert ids.data_ptr() == 0 and ids.stride() == (1,) + + for inherited, override, expected, has_grad in ( + (False, None, False, False), + (True, None, True, True), + (True, False, False, False), + (False, True, True, False), + ): + empty = wp.zeros((0, 3), dtype=wp.float32, device="cpu", requires_grad=inherited) + view = wp.to_torch(empty, requires_grad=override) + assert view.shape == (0, 3) and view.dtype == torch.float32 + assert view.requires_grad is expected + assert (view.grad is not None) is has_grad + if has_grad: + assert view.grad.shape == (0, 3) and view.grad.dtype == torch.float32 diff --git a/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst b/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst new file mode 100644 index 00000000000..10743010474 --- /dev/null +++ b/source/isaaclab_newton/changelog.d/kitless-ovxr-model.fixed.rst @@ -0,0 +1 @@ +* Recorded Newton CUDA graphs on a nonblocking stream to support concurrent renderer and input uploads. diff --git a/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py b/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py index 8a4f23bcbae..051d2305a3b 100644 --- a/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py +++ b/source/isaaclab_newton/isaaclab_newton/physics/newton_backend.py @@ -667,11 +667,12 @@ def capture_graph(device: wp.DeviceLike, fn: Callable[[], None], *, relaxed: boo Returns: The captured graph, replayed with :func:`warp.capture_launch`. """ - stream = wp.get_stream(device) + # A blocking capture stream forbids concurrent legacy-stream transfers, + # including renderer or input uploads. ScopedStream retains the incoming + # dependency; graph replay still uses the caller's stream. + stream = wp.stream_from_torch(torch.cuda.Stream(device=wp.device_to_torch(device))) mode = wp.CaptureMode.THREAD_LOCAL if relaxed: - # RTX uses the legacy CUDA stream. A nonblocking stream avoids implicit synchronization. - stream = wp.stream_from_torch(torch.cuda.Stream(device=wp.device_to_torch(device))) mode = wp.CaptureMode.RELAXED with _paused_gc(), wp.ScopedStream(stream): with wp.ScopedCapture(stream=stream, capture_mode=mode) as capture: diff --git a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py index 2362bdd36b3..fdb8f266fcd 100644 --- a/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py +++ b/source/isaaclab_newton/test/physics/test_newton_manager_abstraction.py @@ -20,6 +20,7 @@ import functools import logging +import threading from functools import partial as P from types import SimpleNamespace @@ -27,6 +28,7 @@ import isaaclab_newton.physics.newton_manager as newton_manager_module import numpy as np import pytest +import torch import warp as wp from isaaclab_newton.assets.articulation import articulation as articulation_module from isaaclab_newton.assets.rigid_object import rigid_object as rigid_object_module @@ -767,6 +769,52 @@ def test_step_captures_only_when_the_solver_supports_it(monkeypatch, supported): assert [event for event in events if event[0] == "check"] == [("check", supported)] * 2 +@pytest.mark.parametrize("device", test_devices(DeviceScope.CUDA)) +def test_graph_capture_allows_concurrent_legacy_stream_copy(device): + """A render/UI copy must not invalidate physics capture on the same GPU.""" + source = wp.full(4, 7.0, device=device) + result = wp.zeros(4, device=device) + image = torch.full((4,), 23.0, device=device) + copied = torch.zeros_like(image) + # Setup is complete before capture. No allocations or implicit dependency + # on captured physics work are needed by the independent image consumer. + wp.synchronize_device(device) + torch.cuda.synchronize(device) + entered = threading.Event() + finished = threading.Event() + errors = [] + + def copy_image(): + try: + assert entered.wait(5), "capture did not start" + with torch.cuda.device(device), torch.cuda.stream(torch.cuda.default_stream(device)): + copied.copy_(image) + torch.cuda.default_stream(device).synchronize() + except BaseException as error: + errors.append(error) + finally: + finished.set() + + def record_physics(): + wp.copy(result, source) + entered.set() + assert finished.wait(5), "concurrent image copy did not complete" + + worker = threading.Thread(target=copy_image) + worker.start() + try: + graph = newton_backend.capture_graph(device, record_physics) + finally: + entered.set() + worker.join(5) + assert not errors, errors + assert not worker.is_alive() + np.testing.assert_array_equal(result.numpy(), [0.0] * 4) # Recording did not advance physics. + wp.capture_launch(graph) + np.testing.assert_array_equal(result.numpy(), [7.0] * 4) + np.testing.assert_array_equal(copied.cpu().numpy(), [23.0] * 4) + + # --------------------------------------------------------------------------- # Manager state-refresh boundaries (no SimulationContext required) # --------------------------------------------------------------------------- diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst new file mode 100644 index 00000000000..404ba0c707a --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst @@ -0,0 +1,2 @@ +* Fixed standalone OvPhysX selection for SO101 teleoperation while retaining the Kit default and Newton settings. +* Updated SO101 clutch construction and nominal gripper reset poses for the supported Isaac Capture API. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py index d491fbc2a01..4381e241ce8 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py @@ -6,6 +6,7 @@ from dataclasses import MISSING import numpy as np +from isaaclab_ov.physics import OvPhysxCfg from isaaclab.actuators import ImplicitActuatorCfg from isaaclab.utils import configclass, replace @@ -37,8 +38,15 @@ # the home orientation target. Capture 1.7 seeds both channels from this transform on reset and # applies controller motion relative to each engage, retaining the commanded pose on re-clutch. # The base rebase is handled upstream by ``target_frame_prim_path``; no live EE pose is read here. -_BASE_T_GRIPPER_HOME = np.eye(4, dtype=np.float32) -_BASE_T_GRIPPER_HOME[:3, 3] = (0.01918, -0.18852, 0.18887) +_BASE_T_GRIPPER_HOME = np.array( + [ + [0.00000711, 1.0, -0.00000682, 0.02061444], + [-0.71735179, 0.00000985, 0.69671114, -0.18883384], + [0.69671114, -0.00000006, 0.71735179, 0.19226513], + [0.0, 0.0, 0.0, 1.0], + ], + dtype=np.float32, +) def _build_so101_stack_pipeline(): @@ -181,6 +189,16 @@ class SO101CubeStackEnvCfg(stack_joint_pos_env_cfg.SO101CubeStackEnvCfg): def __post_init__(self): # post init of parent super().__post_init__() + # The clutch resets to a configured full pose, so the arm must return to the same + # nominal joints on every reset. Keep the reset event active, without joint noise. + self.events.randomize_joint_state.params["std"] = 0.0 + self.sim.physics = preset( + default=self.sim.physics.default, + isaacsim_physx=self.sim.physics.isaacsim_physx, + physx=self.sim.physics.physx, + newton_mjwarp=self.sim.physics.newton_mjwarp, + ovphysx=OvPhysxCfg(), + ) # Switch to a stiffer PD controller so the arm tracks IK targets well. Replace the robot # configured by the parent so its task-specific seated root and stack joint pose are retained. @@ -201,7 +219,9 @@ def __post_init__(self): variants={ "Robot": "robot", "Sensor": "sensors", - "Physics": preset(default="physx", isaacsim_physx="physx", physx="physx", newton_mjwarp="physics"), + "Physics": preset( + default="physx", isaacsim_physx="physx", physx="physx", ovphysx="physx", newton_mjwarp="physics" + ), }, ), actuators={ diff --git a/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py b/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py index 01fbd819f85..2ad44d7e586 100644 --- a/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py +++ b/source/isaaclab_tasks/test/contrib/stack/test_so101_pose_ik.py @@ -132,3 +132,51 @@ def test_env_cfg_arm_action_is_pose_and_ordering_matches_pipeline(): "arm_action", "gripper_action", ] + + +def test_ik_reset_restores_nominal_joints_without_changing_joint_control_randomization(): + """The IK task's configured reset event restores its nominal joints without noise.""" + from types import SimpleNamespace + + from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import SO101CubeStackEnvCfg + from isaaclab_tasks.contrib.stack.config.so101.stack_joint_pos_env_cfg import ( + _SO101_STACK_INIT_JOINT_POS, + ) + from isaaclab_tasks.contrib.stack.config.so101.stack_joint_pos_env_cfg import ( + SO101CubeStackEnvCfg as JointPositionCfg, + ) + + cfg = SO101CubeStackEnvCfg() + assert JointPositionCfg().events.randomize_joint_state.params["std"] == 0.02 + defaults = torch.tensor([list(_SO101_STACK_INIT_JOINT_POS.values())]) + written = {} + + class Robot: + data = SimpleNamespace( + default_joint_pos=SimpleNamespace(torch=defaults), + default_joint_vel=SimpleNamespace(torch=torch.zeros_like(defaults)), + soft_joint_pos_limits=SimpleNamespace(torch=torch.tensor([[[-10.0, 10.0]] * defaults.shape[1]])), + ) + + def find_joints(self, names): + return [list(_SO101_STACK_INIT_JOINT_POS).index(name) for name in names], names + + def set_joint_position_target_index(self, target, env_ids): + written["target"] = target.clone() + + def set_joint_velocity_target_index(self, target, env_ids): + written["velocity_target"] = target.clone() + + def write_joint_position_to_sim_index(self, position, env_ids): + written["position"] = position.clone() + + def write_joint_velocity_to_sim_index(self, velocity, env_ids): + written["velocity"] = velocity.clone() + + env = SimpleNamespace(cfg=cfg, scene={"robot": Robot()}) + event = cfg.events.randomize_joint_state + event.func(env, torch.tensor([0]), **event.params) + torch.testing.assert_close(written["position"], defaults) + torch.testing.assert_close(written["target"], defaults) + torch.testing.assert_close(written["velocity"], torch.zeros_like(defaults)) + torch.testing.assert_close(written["velocity_target"], torch.zeros_like(defaults)) diff --git a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py index e90483653d8..f6088a4f2aa 100644 --- a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py +++ b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py @@ -3,48 +3,93 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Run the SO-101 Capture graph with encoded controller snapshots and verify its 8D action contract.""" +"""Exercise the real SO-101 Capture graph after hardware controller conversion, without XR/GPU.""" import numpy as np import pytest +from scipy.spatial.transform import Rotation pytest.importorskip("isaacteleop") -from isaacteleop.retargeting_engine.interface import ComputeContext, ExecutionEvents, ExecutionState, TensorGroup -from isaacteleop.schema import ControllerInputState, ControllerPose, ControllerSnapshot, Point, Pose, Quaternion +from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource +from isaacteleop.retargeting_engine.interface import ( + ComputeContext, + ExecutionEvents, + ExecutionState, + OptionalTensorGroup, + TensorGroup, +) +from isaacteleop.retargeting_engine.interface.retargeter_core_types import ExecutionCache, GraphTime +from isaacteleop.retargeting_engine.tensor_types import ControllerInput, ControllerInputIndex, TransformMatrix -from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import _build_so101_stack_pipeline +from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import ( + _BASE_T_GRIPPER_HOME, + _build_so101_stack_pipeline, +) -def test_so101_pipeline_tracks_engage_relative_pose_and_gripper(): - """Pose/closedness occupy the declared slots, and clutching holds the pose across release and re-engage.""" +def test_so101_pipeline_full_pose_clutch_and_action_order(): + """Home, motion, re-clutch and reset survive the real transform/retarget/flatten graph.""" pipeline = _build_so101_stack_pipeline() - assert pipeline.output_types()["action"].types[0].shape == (8,) - inputs = { - leaf.name: {name: TensorGroup(group_type) for name, group_type in leaf.input_spec().items()} - for leaf in pipeline.get_leaf_nodes() - } - inputs["world_T_anchor"]["value"][0] = np.eye(4, dtype=np.float32) - inputs["controllers"]["deviceio_controller_left"][0] = None - - def step(position, orientation, squeeze=1.0, trigger=1.0, reset=False): - inputs["controllers"]["deviceio_controller_right"][0] = ControllerSnapshot( - grip_pose=ControllerPose(Pose(Point(*position), Quaternion(*orientation)), True), - inputs=ControllerInputState(False, False, False, False, 0.0, 0.0, squeeze, trigger), + controllers = next(node for node in pipeline.get_leaf_nodes() if isinstance(node, ControllersSource)) + anchor = TensorGroup(TransformMatrix()) + anchor[0] = np.eye(4, dtype=np.float32) + + def step(position, rotation, *, running=True, trigger=0.0, reset=False): + controller = TensorGroup(ControllerInput()) + # Populate the same complete typed input as ControllersSource, replacing hardware polling + # only. Cached source output still traverses the real frame transform and all retargeters. + for index in ControllerInputIndex: + if index in (ControllerInputIndex.GRIP_POSITION, ControllerInputIndex.AIM_POSITION): + controller[index] = np.asarray(position, dtype=np.float32) + elif index in (ControllerInputIndex.GRIP_ORIENTATION, ControllerInputIndex.AIM_ORIENTATION): + controller[index] = rotation.as_quat().astype(np.float32) + elif index in (ControllerInputIndex.GRIP_IS_VALID, ControllerInputIndex.AIM_IS_VALID): + controller[index] = True + else: + controller[index] = 0.0 + controller[ControllerInputIndex.SQUEEZE_VALUE] = float(running) + controller[ControllerInputIndex.TRIGGER_VALUE] = trigger + context = ComputeContext( + graph_time=GraphTime(sim_time_ns=0, real_time_ns=0), + execution_events=ExecutionEvents( + reset=reset, execution_state=ExecutionState.RUNNING if running else ExecutionState.STOPPED + ), + ) + cache = ExecutionCache({"world_T_anchor": {"value": anchor}}, context) + cache.cache( + id(controllers), + {ControllersSource.LEFT: OptionalTensorGroup(ControllerInput()), ControllersSource.RIGHT: controller}, ) - context = ComputeContext(execution_events=ExecutionEvents(reset=reset, execution_state=ExecutionState.RUNNING)) - action = pipeline.execute_pipeline(inputs, context)["action"][0] - assert action.shape == (8,) and action.dtype == np.float32 - return action.copy() - - home = [0.01918, -0.18852, 0.18887, 0.0, 0.0, 0.0, 1.0, 1.0] - # The first squeeze emits the configured home even with a rotated controller. - np.testing.assert_allclose(step((0.2, 0.3, 0.4), (0.0, 0.0, 2**-0.5, 2**-0.5)), home, atol=1e-6) - # Relative to the initial Z quarter-turn, this controller quaternion gives [1, 2, 3, 4] / sqrt(30). - moved = [0.11918, 0.01148, 0.48887, *np.array([1, 2, 3, 4]) / np.sqrt(30), 1.0] - np.testing.assert_allclose(step((0.3, 0.5, 0.7), np.array([3, 1, 7, 1]) / np.sqrt(60)), moved, atol=1e-6) - # Release at another pose, then re-engage there: the commanded arm pose stays held. - held = [*moved[:7], 0.0] - np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), squeeze=0.0, trigger=0.0), held, atol=1e-6) - np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), trigger=0.0), held, atol=1e-6) - np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), reset=True), home, atol=1e-6) + action = np.from_dlpack(pipeline.execute_pipeline_with_cache(cache)["action"][0]).copy() + assert action.shape == (8,) + assert np.isfinite(action).all() + return action + + def check(action, position, rotation, gripper): + np.testing.assert_allclose(action[:3], position, atol=1e-6) + np.testing.assert_allclose(Rotation.from_quat(action[3:7]).as_matrix(), rotation.as_matrix(), atol=1e-6) + assert action[7] == pytest.approx(gripper) + + home_position = _BASE_T_GRIPPER_HOME[:3, 3] + home_rotation = Rotation.from_matrix(_BASE_T_GRIPPER_HOME[:3, :3]) + origin = Rotation.from_euler("xyz", [0.2, -0.4, 0.3]) + position = np.array([0.3, -0.1, 0.5]) + reset_rotation = home_rotation + engaged_rotation = home_rotation + check(step(position, origin, running=False, reset=True), home_position, reset_rotation, 0.0) + check(step(position, origin), home_position, engaged_rotation, 0.0) + delta = Rotation.from_rotvec([0.0, 0.0, 0.3]) + displacement = np.array([0.02, -0.03, 0.01]) + moved_position = home_position + displacement + moved_rotation = delta * engaged_rotation + check(step(position + displacement, delta * origin, trigger=1.0), moved_position, moved_rotation, 1.0) + other = Rotation.from_euler("xyz", [-0.5, 0.2, 0.8]) + check(step(-position, other, running=False), moved_position, moved_rotation, 0.0) + resumed_rotation = moved_rotation + check(step(-position, other), moved_position, resumed_rotation, 0.0) + # Re-engaging composes from the held full pose. + delta2 = Rotation.from_rotvec([0.2, 0.0, 0.0]) + check(step(-position + displacement, delta2 * other), moved_position + displacement, delta2 * resumed_rotation, 0.0) + check(step(position, origin, running=False, reset=True), home_position, reset_rotation, 0.0) + check(step(-position, other), home_position, home_rotation, 0.0) diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index e7e9f9e6e78..cd50665f7b7 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -269,3 +269,33 @@ def load(task, entry): for phrase in expected_phrases: assert phrase in flat, f"Missing phrase: {phrase!r}" + + +@pytest.mark.parametrize("physics", [None, "isaacsim_physx", "ovphysx", "newton_mjwarp"]) +def test_teleop_tasks_resolve_public_physics_selectors(physics): + """Resolve registered tasks through the public flag parser before backend preparation.""" + from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_physx.physics import PhysxCfg + + import isaaclab_tasks # noqa: F401 -- registers the public task IDs + from isaaclab_tasks.utils import parse_env_cfg + from isaaclab_tasks.utils.hydra import setup_preset_cli + + task_name = "IsaacContrib-Stack-Cube-SO101-IK-Abs-v0" + arguments = ["--task", task_name] + if physics is not None: + arguments.extend(["--physics", physics]) + _, selections = setup_preset_cli(_make_parser(), arguments) + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=selections) + expected_type = {None: PhysxCfg, "isaacsim_physx": PhysxCfg, "ovphysx": OvPhysxCfg, "newton_mjwarp": NewtonCfg} + assert isinstance(cfg.sim.physics, expected_type[physics]) + assert cfg.scene.num_envs == 1 and cfg.sim.device == "cpu" + if "SO101" in task_name: + assert cfg.scene.robot.spawn.variants["Physics"] == ("physics" if physics == "newton_mjwarp" else "physx") + if physics == "newton_mjwarp": + assert isinstance(cfg.sim.physics.solver_cfg, MJWarpSolverCfg) + if "SO101" in task_name: + assert cfg.sim.physics.solver_cfg.iterations == 100 + assert cfg.sim.physics.solver_cfg.impratio == 10.0 + assert cfg.sim.physics.solver_cfg.nconmax == 600 diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst new file mode 100644 index 00000000000..075c72fe464 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -0,0 +1,3 @@ +* Added kitless OVXR teleoperation and MCAP replay with independently paced rendering, + backend-derived anchors, and ordered shutdown. +* Preserved recorded replay controls by default, with explicit ``--start_on_replay`` for recordings without start events. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst new file mode 100644 index 00000000000..6dac77682f0 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -0,0 +1,7 @@ +* **Breaking:** Replay follows recorded START and RESET controls by default. Add + ``--start_on_replay`` for recordings made without START events that should begin + immediately. +* **Breaking:** Teleoperation context exit reports session or runtime cleanup failures + to the caller instead of silently suppressing them. Handle the reported failure; + resources whose asynchronous users have not retired must remain alive until + cleanup succeeds. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst new file mode 100644 index 00000000000..cab1837f07c --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst @@ -0,0 +1,2 @@ +* Applied the same filtered tracking pose to separate render anchors in Kit and OVXR. +* Deferred anchor USD imports until use so Kit can initialize its own USD provider. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.major b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.major new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index df4a0adfc7f..d64137cfd11 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -76,7 +76,8 @@ class MyEnvCfg(ManagerBasedRLEnvCfg): Create a function that builds your Isaac Capture retargeting pipeline. The builder should return an `OutputCombiner` with an `"action"` key containing the flattened action tensor (typically via -`TensorReorderer`). Optionally return a list of retargeters to expose in the tuning UI: +`TensorReorderer`). A setup factory may also return retargeters for the tuning UI; +unpack those before assigning the two configuration callbacks: ```python from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource @@ -95,6 +96,12 @@ def my_pipeline_builder(): ### 3. Run Teleoperation +The upstream device factory also supports keyboard, gamepad and SpaceMouse input +without Kit when the selected task and visualizers do not require it. Kit XR uses +`--xr` and its own installation selection; it continues updating a headless Kit +application without requiring a desktop viewport. Anchor helpers defer native +USD imports until use so that the selected runtime initializes its own provider. + The existing teleop scripts automatically detect `isaac_teleop` in the environment config: ```bash @@ -112,7 +119,11 @@ reference tasks, camera selection, layout, placement, renderer, disable, and kit ```python from isaaclab_teleop import IsaacTeleopCfg, IsaacTeleopDevice -cfg = IsaacTeleopCfg(pipeline_builder=my_pipeline_builder) +pipeline, retargeters = my_pipeline_builder() +cfg = IsaacTeleopCfg( + pipeline_builder=lambda: pipeline, + retargeters_to_tune=lambda: retargeters, +) with IsaacTeleopDevice(cfg) as device: device.add_callback("RESET", env.reset) @@ -166,20 +177,12 @@ contend for the GIL at the start of the step. ## Run with Docker -Teleoperation with Isaac Lab runs in a **single container**. Build the image yourself and run a single container. **Do not use Docker Compose** for this workflow (no multi-container setup). Everything runs inside one container with Isaac Lab. - -Inside the container: install Isaac Capture once (`uv pip install 'isaacteleop[retargeters,cloudxr]~=1.0.0' --extra-index-url https://pypi.nvidia.com`), then start the CloudXR runtime with `--accept-eula` so there is no interactive EULA prompt, and run your teleop script. Example: - -```bash -uv run python -m isaacteleop.cloudxr --accept-eula & -source ~/.cloudxr/run/cloudxr.env -uv run python scripts/tools/record_demos.py --task IsaacContrib-PickPlace-Locomanipulation-G1-Abs --num_demos 5 --dataset_file ./datasets/dataset.hdf5 --xr --visualizer kit -``` - -In the Isaac Sim UI, set the AR panel to **System OpenXR Runtime** and click **Start XR**. For the full flow and options, see the [CloudXR teleoperation how-to](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html) and [Isaac Capture Quick Start](https://nvidia.github.io/IsaacCapture/main/getting_started/quick_start.html). - -For a fully headless experience, omit `--visualizer kit` when running Docker, and the XR teleop session will -run automatically. +For Kit XR, follow the canonical +[CloudXR installation guide](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#install-isaac-capture) +and its [Docker instructions](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#run-with-docker). +They cover the current `teleop` dependency selection, container setup, runtime +startup and client connection. For OVXR, use the separate +[kitless XR installation and runtime guide](kitless_xr.md). ## Dependencies diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md new file mode 100644 index 00000000000..665441e43a6 --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -0,0 +1,109 @@ +# Kitless XR teleoperation + +OVXR presents Isaac Lab's OVRTX renderer and OVStage scene while Isaac Capture +provides tracking and retargeting. This workflow supports Linux x86-64 with +Newton or OVPhysX physics and one environment. + +## Prerequisites + +Before installing, complete the workstation and streaming setup in the +[CloudXR prerequisites](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#prerequisites). +Use the OVXR package selection below for this workflow; the Kit installation +commands in that guide belong to Kit XR. + +- Follow [CloudXR network setup](https://docs.nvidia.com/cloudxr-sdk/latest/requirement/network_setup.html) + for connectivity and firewall requirements between the workstation and client. +- Follow [XR client setup](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#connect-an-xr-device) + for a supported headset or the CloudXR.js browser client, including certificate setup. + The [Isaac Capture Quick Start](https://nvidia.github.io/IsaacCapture/main/teleop/quick_start.html#connect-an-xr-headset) + also covers the desktop IWER client. +- Review the [CloudXR license](https://github.com/NVIDIA/IsaacCapture/blob/main/deps/cloudxr/CLOUDXR_LICENSE) + and complete the first-run EULA acceptance described in + [CloudXR startup](https://nvidia.github.io/IsaacCapture/main/references/cloudxr.html#start-the-service) + before launching without an interactive terminal. + +## Installation + +Use Python 3.12 and the Isaac Capture wheel selected by the installation +guide. +Configure the `ovxr` dependency selection using +[Installing kitless XR](kitless_xr_source.md), then install and +check that selection: + +```bash +uv sync --python 3.12 --locked --extra ovxr +uv pip check +``` + +The runtime commands below use this prepared `ovxr` selection. + +The live and MCAP workflows are implemented in `isaaclab_teleop.entrypoints`. +The command-line entrypoints load registered downstream tasks before parsing +and restore the caller's arguments on return. The scripts under +`scripts/environments/teleoperation/` delegate to these package entrypoints. + +## Live teleoperation + +```bash +uv run --extra ovxr isaaclab teleop run \ + --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --num_envs 1 \ + --device cpu --physics ovphysx --ovxr +``` + +Use `--physics ovphysx --device cpu` for CPU OVPhysX. `--ovxr` selects OVStage +and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported +CloudXR client, then use Play, Pause and Reset as in Kit-based teleoperation. + +The SO101 IK task resets its joints to the configured nominal pose without +joint-position noise, giving the clutch a repeatable reset pose. Its configured +home transform supplies the clutch calibration. This applies to its Kit and +kitless backends. The separate joint-control task retains its randomized reset. + +For desktop IWER, select **VR** and set **Reference Space** to **local-floor** +under **Teleoperator adjustment** before connecting. This keeps the emulated +head and controllers relative to the floor. Between trials, end the previous XR +session and reload the client page before reconnecting; recheck both settings. +A blank or below-scene main view with a working camera panel warrants checking +this client origin before changing the renderer. + +`--ovxr` and Kit's `--xr` select separate XR runtimes and are mutually exclusive. +An explicit `ISAAC_LAB_OVRTX_USE_OVSTAGE=0` is incompatible with OVXR. + +No local window opens by default. Headset rendering still uses the GPU and works +with `DISPLAY` unset. The teleoperation CLI does not need a `--headless` flag. +Renderer startup has a bounded attachment timeout; it does not bound waiting for a +client to establish its OpenXR session. Installing Capture and resolving +Python packages do not warm application-specific RTX shaders or Newton/Warp +physics kernels. First runtime preparation and the first physics step can take +longer than subsequent runs. Retain startup logs and distinguish stage/render +preparation from client connection and active simulation; do not treat a warm +retry as proof that an earlier timeout was fixed. + +Use `--step_hz 20` to cap live control at 20 updates per second while XR rendering +runs independently. Time spent reading input, stepping physics and publishing +the scene counts toward that interval. Slow steps do not cause catch-up steps. +Omit the option or use zero for unthrottled control. Paused polling uses the +environment action interval when no explicit rate is set. This option requires +`--ovxr` and does not change physics dt or decimation. + +## MCAP replay + +```bash +uv run --extra ovxr python scripts/environments/teleoperation/teleop_replay_agent.py \ + --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --num_envs 1 \ + --device cpu --physics ovphysx --ovxr \ + --replay_file /absolute/path/to/session.mcap +``` + +MCAP supplies tracker and control input; the connected headset supplies the +view. A recorded STOP, task success/failure, or `--max_replay_duration_s` ends +the run. Playback waits for recorded START messages, including after a recorded +RESET. Reset does not rewind recorded input. For legacy captures started locally +without recorded START messages, add `--start_on_replay` to start automatically +at startup and after RESET. `--max_stage_load_wait_s` bounds +renderer startup separately; the replay duration starts after warmup and +includes any lazy first-step physics initialization. Deadlines are checked +between steps. Application render-call timing statistics are unavailable when +OVXR owns rendering independently. + +HDF5 demonstration recording and replay continue to use the Kit/Isaac Sim path. diff --git a/source/isaaclab_teleop/docs/kitless_xr_source.md b/source/isaaclab_teleop/docs/kitless_xr_source.md new file mode 100644 index 00000000000..9d95e824dbd --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr_source.md @@ -0,0 +1,63 @@ +# Installing kitless XR + +Use Python 3.12 on Linux x86-64. The Isaac Capture wheel includes the +OVXR session integration. + +Complete the [CloudXR, client, network and license prerequisites](kitless_xr.md#prerequisites) +before starting a session. + +From the Isaac Lab root, select the Capture version in an `ovxr` project +extra. Lab already configures NVIDIA's public Python index and allows prereleases: + +```bash +uv add --optional ovxr --frozen \ + "isaaccapture[ovxr,retargeters,ui,cloudxr]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +uv add --optional ovxr --frozen \ + "isaaclab-dev[ov] ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-teleop ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-mimic ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "ipywidgets>=8.1.5 ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +``` + +The `ov` extra supplies the compatible OVRTX, OVStage, OVPhysX and PhysX USD +schemas. Keep its provider requirements and the workspace package metadata unchanged. + +Add these mutually exclusive selections to `tool.uv.conflicts`, preserving existing entries: + +```toml +[tool.uv] +conflicts = [ + [{ extra = "ovxr" }, { extra = "teleop" }], + [{ extra = "ovxr" }, { extra = "isaacsim" }], + [{ extra = "ovxr" }, { extra = "viser" }], + [{ extra = "ovxr" }, { extra = "all" }], +] +``` + +The `teleop` extra selects Capture packages for Kit and non-Kit input +workflows. Use the separate `ovxr` selection for the XR commands here. +Isaac Sim and Viser require older WebSocket versions than this Capture +release, so these selections cannot be combined. The `all` extra +includes Viser. + +Resolve the selected environment and check its dependencies: + +```bash +uv lock --refresh-package isaaccapture --upgrade-package isaaccapture \ + --upgrade-package numpy==2.3.5 --upgrade-package packaging==23.2 +uv sync --python 3.12 --locked --extra ovxr +uv run --extra ovxr python -c 'import importlib.metadata as m; print(m.version("isaaccapture"), m.version("ovxr"))' +uv pip check +``` + +The lock selects compatible NumPy 2.3.5 and packaging 23.2 versions without +changing project-wide overrides. OVPhysX declares `packaging<24`; the explicit +selection above satisfies its metadata in this non-Kit selection. Stop applications before changing selections. +The Kit and OVXR provider selections are different; preserve the source metadata +and lockfile when changing between them, and check the resulting dependency graph. +Repeat this selection if a later lock upgrade replaces it. Isaac Capture's +`ovxr` extra selects OVXR and its compatible OVRTX/OVStage dependencies; see its +OVXR guide for application version overrides. + +Continue with [live teleoperation](kitless_xr.md#live-teleoperation). diff --git a/source/isaaclab_teleop/isaaclab_teleop/action_policy.py b/source/isaaclab_teleop/isaaclab_teleop/action_policy.py new file mode 100644 index 00000000000..81ed2dc357e --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/action_policy.py @@ -0,0 +1,45 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Task-owned action validation shared by live and replay.""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .isaac_teleop_device import IsaacTeleopDevice + + +def prepare_teleop_action( + env: Any, + action: Any, + conditioner: Callable[[Any, Any, dict[str, Any]], Any] | None, + validator: Callable[[Any], str | None] | None, + state: dict[str, Any], +) -> tuple[Any, str | None]: + """Validate and condition one task-owned action without stepping physics.""" + issue = validator(action) if validator is not None else None + if issue is None and conditioner is not None: + action = conditioner(env, action, state) + issue = validator(action) if validator is not None else None + return action, issue + + +def initial_hand_tracking_issue( + teleop_interface: IsaacTeleopDevice, initialized_sides: set[str], *, required: bool = True +) -> str | None: + """Wait for initial hand presence without changing later tracking-loss behavior.""" + required_sides = {"left", "right"} + if not required or required_sides.issubset(initialized_sides): + return None + presence = teleop_interface.hand_tracking_presence + if presence is None: + return "hand-tracking presence is unavailable" + initialized_sides.update(side for side, present in zip(("left", "right"), presence, strict=True) if present) + missing = sorted(required_sides - initialized_sides) + return f"waiting for initial {'/'.join(missing)} hand tracking" if missing else None diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index c7c04685689..8d3b82289fe 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -7,11 +7,17 @@ This is the non-interactive counterpart to ``teleop_se3_agent.py``. It builds a teleop environment, attaches an :class:`~isaaclab_teleop.IsaacTeleopDevice` -configured in :class:`isacteleop.teleop_session_manager.SessionMode.REPLAY`, +configured in :class:`isaacteleop.teleop_session_manager.SessionMode.REPLAY`, and pumps the simulation loop until the recorded operator presses STOP (or ``--max_replay_duration_s`` elapses, or the simulator is closed). The user-journey teleop script remains ``teleop_se3_agent.py``. +OVXR viewer: + ``--ovxr`` presents replay through an independently owned OVXR session while + tracker and control input remain sourced from MCAP. See + ``source/isaaclab_teleop/docs/kitless_xr.md`` for installation and launch + instructions. + Inputs: ``--replay_file`` is an MCAP capture produced by Isaac Capture's ``McapRecordingConfig`` path (typically written by ``record_demos.py @@ -19,7 +25,7 @@ messages (head / hands / controllers) plus the ``_teleop_control`` ``MessageChannelTracker`` that captured the operator's START / STOP / RESET gestures. TeleopCore's - :class:`~isacteleop.deviceio_session.ReplaySession` re-emits all of + :class:`~isaacteleop.deviceio_session.ReplaySession` re-emits all of them on the same monotonic-time cadence they were recorded on, so :func:`~isaaclab_teleop.poll_control_events` returns the same edges here that ``record_demos.py``'s loop saw at recording time. @@ -30,7 +36,10 @@ :func:`~isaaclab_teleop.poll_control_events`, gates ``env.step()`` on ``ctrl.is_active`` (so pre-START operator setup frames are rendered without stepping), and handles mid-demo ``ctrl.should_reset`` events - by running the full sim/env/teleop reset cycle. + by running the full sim/env/teleop reset cycle. Recorded controls govern + playback by default. For legacy captures started locally without recorded + START messages, pass ``--start_on_replay`` to start at replay startup and + resume after each recorded RESET. End-of-replay termination: Four distinct signals end the current run by breaking out of the @@ -64,28 +73,27 @@ Stats output: The wall-clock deltas between successive ``env.sim.render()`` calls - produced during the active window are recorded as a per-rendered-frame + produced during the active window are recorded as a per-render-call interval series (N gated renders produce N-1 samples). Pre-START render-only frames, warmup ticks, and post-quit render-only spin-down are excluded so the numbers reflect the steady-state replay workload rather than agent bookkeeping overhead. Per-run samples are summarised into mean / p50 / p90 / p95 / p99 / min / max / stddev (under ``cpu_frame_time_ms``) plus - derived FPS metrics (under ``fps``). Both blocks are reported on - a **per-render** basis -- ``cpu_frame_time_ms`` is the wall time - between rendered frames and ``fps`` is the render rate, matching - what the simulator's HUD shows and what the headset wearer / - spectator actually perceives during real-time teleop. The two + derived call-rate metrics (under the legacy ``fps`` key). Both blocks measure + successive application ``env.sim.render()`` calls, not completed renderer + frames, encoded video, or client/headset presentation. The two blocks stay self-consistent: ``fps.mean`` equals ``1000 / cpu_frame_time_ms.mean`` (harmonic mean of FPS = total frames / total step time). Each interval is the wall-clock delta between successive ``env.sim.render`` calls, so a run needs at least two - rendered frames during the active window to contribute. The agent + render calls during the active window to contribute. The agent asserts this per-run inside :func:`_run_single_replay`: if a run stepped the env but produced no usable intervals (0 or 1 renders -- typically the sim is not rendering, or the active window closed too quickly), it raises ``RuntimeError`` immediately, so the remaining replays never start and no stdout summary or JSON file - is written. Partial reports are never emitted: CI relies on + is written. OVXR owns rendering independently, so application render-call + timing statistics remain empty for OVXR replays. Partial reports are never emitted: CI relies on "no JSON output" as an unambiguous failure signal. Each active iteration also emits one ``GpuStatsProvider.sample()`` @@ -115,8 +123,10 @@ { "schema_version": 1, "task": "IsaacContrib-PickPlace-GR1T2-Abs", - "replay_file": "/tmp/pickplace_gr1t2.mcap", + "replay_file": "/absolute/path/to/session.mcap", "num_replays": 5, + "requested_num_replays": 5, + "batch_complete": true, "outcomes": {"success": 4, "failure": 1, "incomplete": 0, "timeout": 0}, "success_rate": 0.8, "env_cfg": { @@ -147,7 +157,7 @@ "mean": ..., "p50": ..., "p90": ..., "p95": ..., "p99": ..., "min": ..., "max": ..., "stddev": ..., "n": ... }, - "fps": { # per-render rate (matches simulator HUD) + "fps": { # application render-call rate, not headset presentation "mean": ..., "min_instantaneous": ..., "max_instantaneous": ... }, "gpu_stats": { @@ -252,6 +262,7 @@ import json import logging import os +import signal import statistics import sys import time @@ -265,14 +276,15 @@ import torch from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab.app import add_launcher_args, launch_simulation +from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks from isaaclab.devices.openxr import remove_camera_configs from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.envs.utils._manual_success import _prepare_success_term +from isaaclab.utils.version import has_kit import isaaclab_tasks # noqa: F401 -from isaaclab_tasks.utils import parse_env_cfg +from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli from .. import ( CLOUDXR_AVP_ENV, @@ -282,17 +294,17 @@ create_isaac_teleop_device, poll_control_events, ) +from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action logger = logging.getLogger(__name__) args_cli: argparse.Namespace """Parsed command line, set by :func:`_parse_args`.""" hydra_overrides: list[str] -"""Unrecognized arguments, forwarded as Hydra task config overrides.""" def _build_parser() -> argparse.ArgumentParser: - """Build the command-line parser, including the simulation launcher arguments.""" + """Build workflow and simulation launcher arguments.""" parser = argparse.ArgumentParser( description=( "Replay a captured Isaac Capture MCAP session against an Isaac Lab environment. " @@ -301,12 +313,21 @@ def _build_parser() -> argparse.ArgumentParser: ) parser.add_argument("--task", type=str, required=True, help="Name of the task.") parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.") + parser.add_argument( + "--ovxr", action="store_true", help="Present recorded motion through kitless OVXR; input stays MCAP." + ) + parser.add_argument("--ovxr_session_mode", choices=("ar", "vr"), default="ar") parser.add_argument( "--replay_file", type=str, required=True, help="Absolute path to the Isaac Capture MCAP capture to replay.", ) + parser.add_argument( + "--start_on_replay", + action="store_true", + help="Start automatically at startup and after RESET for legacy captures without recorded START messages.", + ) parser.add_argument( "--num_success_steps", type=int, @@ -368,9 +389,9 @@ def _build_parser() -> argparse.ArgumentParser: type=float, default=300.0, help=( - "Safety cap on how long to wait for omni.usd to finish loading the stage before" - " proceeding anyway. Hit only when something is misconfigured (missing asset, slow" - " Nucleus, etc.); a warning is logged and replay continues. Default is 300s." + "Safety cap for initial stage readiness. Kit stage loading warns and continues" + " at this cap; OVXR render-worker bootstrap fails and drains on expiry." + " Default is 300s." ), ) parser.add_argument( @@ -415,13 +436,14 @@ def _build_parser() -> argparse.ArgumentParser: def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse ``argv`` (default ``sys.argv[1:]``) into :data:`args_cli` and :data:`hydra_overrides`.""" + """Parse workflow arguments, leaving task overrides in ``sys.argv``.""" global args_cli, hydra_overrides if argv is not None: sys.argv = [sys.argv[0], *argv] parser = _build_parser() - # forward unrecognized args as Hydra-style task config overrides - args_cli, hydra_overrides = parser.parse_known_args() + + args_cli, hydra_overrides = setup_preset_cli(parser) + sys.argv = [sys.argv[0]] + hydra_overrides # Enable external camera rendering by default so the replay mimics the production teleop env (perf # parity with live camera rendering); ``--disable_external_cameras`` turns it off. The @@ -429,10 +451,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # the launcher; this selects a camera-rendering experience that provides RTX/DLSS. # Everywhere else we read ``args_cli.disable_external_cameras`` directly. args_cli.enable_cameras = not args_cli.disable_external_cameras - # the stage-load wait and the RTX settings use Kit APIs directly - args_cli.require_kit = True + # Let the resolved task/preset select Kit; only Kit paths use its stage-load APIs. + args_cli.require_kit = False # Resolve CLI profile names once; runtime construction receives a path or None. + if args_cli.ovxr and args_cli.cloudxr_env is None: + args_cli.cloudxr_env = "cloudxrjs" if args_cli.cloudxr_env is not None: if not args_cli.cloudxr_env.strip() or args_cli.cloudxr_env.lower() == "none": args_cli.cloudxr_env = None @@ -459,7 +483,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: class _RunStats: """Per-replay performance + outcome record. - ``active_frame_times_ms`` is the per-rendered-frame series: wall-clock + ``active_frame_times_ms`` is the per-render-call series: wall-clock deltas between successive :meth:`SimulationContext.render` calls produced from inside ``env.step`` during the active window (post-START, pre-terminator). Pre-START render-only frames, warmup @@ -482,6 +506,8 @@ class _RunStats: outcome: str = "incomplete" # "success" | "failure" | "incomplete" | "timeout" active_frame_times_ms: list[float] = field(default_factory=list) active_iterations: int = 0 + rejected_action_frames: int = 0 + last_action_rejection: str | None = None active_render_calls: int = 0 active_duration_s: float = 0.0 success_step_count: int = 0 # final consecutive-success counter at terminator @@ -495,6 +521,8 @@ def to_dict(self, run_index: int) -> dict: "run_index": run_index, "outcome": self.outcome, "active_iterations": self.active_iterations, + "rejected_action_frames": self.rejected_action_frames, + "last_action_rejection": self.last_action_rejection, "active_duration_s": round(self.active_duration_s, 6), "success_step_count": self.success_step_count, "cpu_frame_time_ms": _compute_frame_stats(self.active_frame_times_ms), @@ -558,7 +586,7 @@ def _compute_frame_stats(samples_ms: list[float]) -> dict: def _compute_fps_stats(samples_ms: list[float]) -> dict: - """Compute FPS stats from per-rendered-frame intervals (in ms). + """Compute render-call rate stats from application intervals (in ms). All three fields are derived from the same ``samples_ms`` series that feeds ``cpu_frame_time_ms`` and stay self-consistent with it: @@ -569,7 +597,7 @@ def _compute_fps_stats(samples_ms: list[float]) -> dict: ``n / active_duration_s`` (dragged down by inter-step bookkeeping). ``min_instantaneous`` and ``max_instantaneous`` are derived from - the slowest / fastest individual rendered frame respectively. + the slowest / fastest individual render-call interval respectively. """ n = len(samples_ms) if n == 0: @@ -811,6 +839,8 @@ def _build_report(args, env_cfg, all_runs: list[_RunStats]) -> dict: "task": args.task, "replay_file": args.replay_file, "num_replays": len(all_runs), + "requested_num_replays": args.num_replays, + "batch_complete": len(all_runs) == args.num_replays, "outcomes": outcomes_count, "success_rate": success_rate, "env_cfg": env_perf_cfg, @@ -991,8 +1021,7 @@ def _maybe_launch_cloudxr(cloudxr_env_path: str | None, auto_launch: bool): Mirrors the gating in :meth:`TeleopSessionLifecycle._ensure_cloudxr_runtime` (``--cloudxr_env`` set, ``--auto_launch_cloudxr`` enabled, ``ISAACLAB_CXR_SKIP_AUTOLAUNCH=1`` env var not set) so behavior parity - is preserved, including ``ISAACLAB_CXR_ACCEPT_EULA=1`` for the CloudXR - license, which is read through the same shared helper. + is preserved. Args: cloudxr_env_path: Resolved CloudXR ``.env`` file path, or ``None`` @@ -1026,10 +1055,7 @@ def _maybe_launch_cloudxr(cloudxr_env_path: str | None, auto_launch: bool): def _prepare_env_cfg( - task: str, - num_envs: int, - device: str, - overrides: Sequence[str] = (), + task: str, num_envs: int, device: str, overrides: Sequence[str] = () ) -> tuple[ManagerBasedRLEnvCfg, object | None]: """Build and tweak an env config suitable for non-interactive replay. @@ -1042,7 +1068,7 @@ def _prepare_env_cfg( ``--num_success_steps``. This matches record_demos.py's pattern of manually counting consecutive success steps before resetting. * The ``time_out`` term is cleared for the same reason it is cleared in - :mod:`isaaclab_teleop.entrypoints.record_demos` and + :file:`scripts/tools/record_demos.py` and :file:`scripts/imitation_learning/robomimic/play.py`: a recorded trajectory often exceeds ``episode_length_s`` (pick-place is 20s by default; a successful operator demo can easily run 25-30s). With the @@ -1063,13 +1089,17 @@ def _prepare_env_cfg( task: Registered task name to load the env config for. num_envs: Number of parallel environments. device: Simulation device (e.g. ``"cuda:0"``). - overrides: Hydra-style ``key=value`` overrides forwarded to :func:`parse_env_cfg`. + overrides: Hydra-style ``key=value`` overrides passed to :func:`resolve_task_config`. Returns: Tuple ``(env_cfg, success_term)``. ``success_term`` is ``None`` when the env doesn't define a ``success`` termination term. """ - env_cfg = parse_env_cfg(task, device=device, num_envs=num_envs, overrides=overrides) + overrides = list(overrides) + env_cfg, _ = resolve_task_config(task, "", overrides=overrides) + if device is not None: + env_cfg.sim.device = device + env_cfg.scene.num_envs = num_envs env_cfg.env_name = task.split(":")[-1] if not isinstance(env_cfg, ManagerBasedRLEnvCfg): raise ValueError( @@ -1084,29 +1114,60 @@ def _prepare_env_cfg( ) if hasattr(env_cfg.terminations, "time_out"): env_cfg.terminations.time_out = None - # Keep camera configs when external cameras are enabled (defaulted on) so the replay - # renders them for production parity; otherwise strip them for a lighter headless replay. - if args_cli.disable_external_cameras: + if args_cli.ovxr: + if args_cli.num_envs != 1 or args_cli.xr: + raise ValueError("--ovxr requires one environment and no Kit XR or local visualizer") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") + env_cfg = remove_camera_configs(env_cfg) + elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) - return env_cfg, success_term + if env_cfg.sim.physics is None: + from isaaclab.physics import PhysicsCfg + env_cfg.sim.physics = PhysicsCfg() + configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) + if args_cli.ovxr: + from isaaclab.visualizers import VisualizerBackendCfg + + if kit_sources or any(isinstance(cfg, VisualizerBackendCfg) for cfg in configs): + raise ValueError("--ovxr requires no Kit runtime or local visualizer") + args_cli.require_kit = bool(kit_sources) + if env_cfg.isaac_teleop is not None and env_cfg.isaac_teleop.prepare_environment is not None: + env_cfg.isaac_teleop.prepare_environment(env_cfg) + return (env_cfg, success_term) + + +def _replay_requires_hard_sim_reset(physics_backend: str | None) -> bool: + """Return whether replay must rebuild the simulator before an environment reset.""" + # Scene identity comes from the constructed manager, so preset aliases all + # resolve to NewtonManager.backend_name == "newton" before this policy runs. + return physics_backend is None or not physics_backend.startswith("newton") + + +def _reset_replay_environment(env: object, *, hard_sim_reset: bool, reset_recorder: bool) -> None: + """Reset replay-owned environment state in backend-safe order.""" + if hard_sim_reset: + env.sim.reset() + if reset_recorder: + env.recorder_manager.reset() + env.reset() -def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice) -> None: + +def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice, *, hard_sim_reset: bool) -> None: """Run the full env+teleop reset cycle used by ``record_demos.py``. - Mirrors :func:`isaaclab_teleop.entrypoints.record_demos.handle_reset` (sans the + Mirrors :func:`scripts.tools.record_demos.handle_reset` (sans the instruction-display update, which the headless replay agent doesn't - own). ``env.sim.reset()`` does the hard physics reinit that keeps Pink - IK seeded against fresh articulation views; see the initial-reset note - in :func:`main`. ``env.recorder_manager.reset()`` is a no-op when no - recorders are configured (the default for this script), but kept for - parity with record_demos.py so future recorder additions don't have to - re-derive the call sequence. + own). The optional hard physics reset keeps Pink seeded against fresh + articulation views on PhysX backends. Newton instead retains the solver + constructed before replay. ``env.recorder_manager.reset()`` is a no-op + when no recorders are configured (the default for this script), but kept + for parity with record_demos.py so future recorder additions don't have + to re-derive the call sequence. """ print("Resetting environment...") - env.sim.reset() - env.recorder_manager.reset() - env.reset() + _reset_replay_environment(env, hard_sim_reset=hard_sim_reset, reset_recorder=True) teleop_interface.reset() @@ -1118,7 +1179,7 @@ def _process_success_condition( ) -> tuple[int, bool]: """Track consecutive success steps and decide whether to reset. - Mirrors :func:`isaaclab_teleop.entrypoints.record_demos.process_success_condition` + Mirrors :func:`scripts.tools.record_demos.process_success_condition` minus the recorder-export side effects, which this script does not own. Returns: @@ -1139,22 +1200,127 @@ def _process_success_condition( _RENDERER_SETTLE_FRAMES: int = 30 -"""Number of additional render frames pumped after the USD stage finishes loading. +"""Additional Kit render ticks after asynchronous USD asset loading. -The stage-load status flips to ``count_loading == 0`` as soon as every referenced asset -has been resolved, but the renderer pipeline (shader compilation, articulation-view -binding, material warm-up) typically needs a few more event-loop ticks to converge. Thirty -frames at the default render cadence is ~0.5 s on most machines and is deterministic -per-machine -- unlike a wall-clock delay it does not have to be tuned for hardware. +This legacy Kit warmup is not a kitless renderer-completion or FPS guarantee. """ +class _ReplayPresentation: + """Own a batch's viewer independently of each recorded-input session.""" + + def __init__(self, env_cfg): + from isaaclab_ov.renderers import OVRTXRendererCfg + + from .. import OvLibrariesXrConfig, OvLibrariesXrSession + + self.session = OvLibrariesXrSession.prepare( + env_cfg.sim, + OvLibrariesXrConfig( + application_name=f"isaaclab-ovxr-replay-{args_cli.task}", + session_mode=args_cli.ovxr_session_mode, + cloudxr_env_file=args_cli.cloudxr_env, + auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + ), + OVRTXRendererCfg(), + ) + self.xr_cfg = env_cfg.isaac_teleop.xr_cfg + self.device = None + self.started = False + self.reset_generation = 0 + self.pending = None + self.force = False + + def start(self, env, device): + if not self.started: + if not self.session.start_render_worker( + env, + tracking_anchor_path=device.xr_render_anchor_path, + hidden_visual_prims=self.xr_cfg.hidden_visual_prims, + external_anchor_state=device.external_anchor_state, + timeout_s=args_cli.max_stage_load_wait_s, + ): + raise RuntimeError("OVXR viewer shut down before replay startup") + self.started = True + else: + self.session.set_external_anchor_state(device.external_anchor_state) + self.publish(env, changed=True, reset=True) + + def publish(self, env, *, changed=False, reset=False): + if reset: + self.reset_generation += 1 + self.force = True + if changed or reset: + self.pending = (self.reset_generation, env.sim.get_physics_step_count()) + if self.pending is not None and self.session.request_render(self.pending, force=self.force) is not None: + self.pending = None + self.force = False + + def close(self): + if self.device is not None and self.device.session_cleanup_failed: + error = RuntimeError("Replay input did not drain; retaining its borrowed environment") + try: + self.session.request_shutdown() + except Exception as shutdown_error: + error.add_note(f"Requesting XR shutdown also failed: {shutdown_error!r}") + raise error + self.session.close() + + +def _warmup_replay(env, run_index: int, presentation) -> None: + """Wait for Kit assets or pump the independently owned viewer before input.""" + # Only an active Kit application owns the asynchronous USD loader. + # Kitless environments finish their stage construction before this call. + if run_index == 0: + if has_kit(): + from isaaclab_physx.renderers.isaac_rtx_renderer_utils import wait_for_stage_load + + print("Waiting for USD stage to finish loading...") + wait_for_stage_load(args_cli.max_stage_load_wait_s, settle_frames=_RENDERER_SETTLE_FRAMES) + + # Optional extra wall-clock buffer on top of the deterministic + # wait. Useful as an escape hatch when the deterministic check + # is not enough (e.g. very slow shader compilation paths). + if args_cli.replay_start_delay_s > 0: + print( + f"Additional warmup buffer: rendering for {args_cli.replay_start_delay_s:.1f}s" + " before consuming MCAP frames." + ) + buffer_start_s = time.monotonic() + while env.sim.is_running() and time.monotonic() - buffer_start_s < args_cli.replay_start_delay_s: + if presentation is None: + env.sim.render() + else: + if presentation.session.shutdown_requested: + raise RuntimeError("OVXR viewer closed before replay started") + presentation.publish(env) + time.sleep(0.01) + + +def _advance_replay_input(device, presentation): + """Read recorded input using the viewer owner's target transform when needed.""" + target_path = device.target_frame_prim_path if presentation is not None else None + if target_path is None: + return device.advance() + return device.advance(target_T_world=presentation.session.get_target_frame_T_world(target_path)) + + +def _render_replay_idle(env, presentation) -> None: + """Keep the selected presentation responsive without advancing physics.""" + if presentation is None: + env.sim.render() + else: + presentation.publish(env) + time.sleep(env.step_dt) + + def _run_single_replay( env: gym.Env, isaac_teleop_cfg, success_term: object | None, run_index: int, total_runs: int, + presentation: _ReplayPresentation | None = None, ) -> _RunStats: """Run a single replay pass against ``env`` and return the per-run stats. @@ -1205,13 +1371,16 @@ def _run_single_replay( teleop_interface = create_isaac_teleop_device( isaac_teleop_cfg, env_prim_path=env.scene.env_prim_paths[0], - sim_device=args_cli.device, + sim_device=env.sim.device, callbacks={}, cloudxr_env_file=None, auto_launch_cloudxr=False, mcap_replay_path=args_cli.replay_file, enable_debug_visualization=args_cli.enable_debug_visualization, + ovxr_session=presentation is not None, ) + if presentation is not None: + presentation.device = teleop_interface if run_index == 0: print(f"Using teleop device: {teleop_interface}") @@ -1243,45 +1412,28 @@ def _patched_sim_render(): env.sim.render = _orig_sim_render with _patched_sim_render(), teleop_interface: - # Mirror the reset sequence used by ``record_demos.py``: ``sim.reset()`` - # does a hard physics reinit (re-binds articulation views, plays the - # timeline) that ``env.reset()`` alone does not perform. Pink IK reads - # ``data.joint_pos.torch`` every step to seed Pinocchio's configuration - # and to compute ``target = curr + delta``; if the articulation view is - # stale, every IK call produces zero-delta arm targets while the - # hand-finger path (which bypasses IK) keeps tracking. See PR #5507. - env.sim.reset() - env.reset() + # PhysX backends need the hard reset that refreshes articulation views + # before Pink reads them (PR #5507). Newton's constructed solver must + # instead remain alive across the ordinary environment reset. + _reset_replay_environment( + env, + hard_sim_reset=_replay_requires_hard_sim_reset(env.scene.physics_backend), + reset_recorder=False, + ) teleop_interface.reset() - - # Deterministic warmup is only required on the first run -- once the - # stage has been streamed in and the renderer settled, subsequent - # runs share the same stage and only need the per-run ``env.sim.reset`` - # above. Skipping it on later runs saves the renderer-settle frames - # at the cost of doing nothing measurable (the wait returns - # immediately on a fully-loaded stage anyway). - if run_index == 0: - from isaaclab_physx.renderers.isaac_rtx_renderer_utils import wait_for_stage_load - - print("Waiting for USD stage to finish loading...") - wait_for_stage_load(args_cli.max_stage_load_wait_s, settle_frames=_RENDERER_SETTLE_FRAMES) - - # Optional extra wall-clock buffer on top of the deterministic - # wait. Useful as an escape hatch when the deterministic check - # is not enough (e.g. very slow shader compilation paths). - if args_cli.replay_start_delay_s > 0: - print( - f"Additional warmup buffer: rendering for {args_cli.replay_start_delay_s:.1f}s" - " before consuming MCAP frames." - ) - buffer_start_s = time.monotonic() - while env.sim.is_running() and time.monotonic() - buffer_start_s < args_cli.replay_start_delay_s: - env.sim.render() + # Locally started legacy captures have no recorded START message. + if args_cli.start_on_replay: + teleop_interface.request_start() + if presentation is not None: + presentation.start(env, teleop_interface) + _warmup_replay(env, run_index, presentation) print( f"Replay {run_index + 1}/{total_runs} started; replaying MCAP from {args_cli.replay_file}" f" (max_replay_duration_s={args_cli.max_replay_duration_s:.1f})." ) + action_state: dict = {} + initialized_hand_tracking: set[str] = set() teleop_active = False teleop_was_active = False # only terminate on STOP after a real START success_step_count = 0 @@ -1312,7 +1464,11 @@ def _patched_sim_render(): stats.outcome = "timeout" break - action = teleop_interface.advance() + if presentation is not None and presentation.session.shutdown_requested: + print("OVXR viewer requested shutdown before replay completed.") + stats.outcome = "incomplete" + break + action = _advance_replay_input(teleop_interface, presentation) ctrl = poll_control_events(teleop_interface) # Track active state from the replayed _teleop_control @@ -1338,30 +1494,61 @@ def _patched_sim_render(): # ``ctrl.is_active`` does not flip. Recordings # without an operator STOP are terminated instead by # the success / failure / wall-clock terminators - # below. - if prev_active and not teleop_active and teleop_was_active: + # below. RESET also pauses the control graph, but must + # reach the reset handler instead of ending the replay. + if prev_active and not teleop_active and teleop_was_active and not ctrl.should_reset: print("Replay end observed (STOP edge); ending run.") break if ctrl.should_reset: - _handle_reset(env, teleop_interface) + action_state.clear() + initialized_hand_tracking.clear() + _handle_reset( + env, + teleop_interface, + hard_sim_reset=_replay_requires_hard_sim_reset(env.scene.physics_backend), + ) + if args_cli.start_on_replay: + teleop_interface.request_start() success_step_count = 0 + if presentation is not None: + presentation.publish(env, changed=True, reset=True) continue # Gate stepping on the active state (mirrors # teleop_se3_agent.py:309-328). Pre-START operator # setup frames render only; the recorded START flips # us into the stepping branch. + if not teleop_active: + action_state.clear() if action is None or not teleop_active: - env.sim.render() + _render_replay_idle(env, presentation) + continue + + issue = initial_hand_tracking_issue( + teleop_interface, + initialized_hand_tracking, + required=isaac_teleop_cfg.require_initial_bimanual_hand_tracking, + ) + if issue is None: + action, issue = prepare_teleop_action( + env, + action, + isaac_teleop_cfg.action_conditioner, + isaac_teleop_cfg.action_validator, + action_state, + ) + if issue is not None: + stats.rejected_action_frames += 1 + stats.last_action_rejection = issue + _render_replay_idle(env, presentation) continue # Gate the render shim to this env.step call so # only renders fired from within it contribute # interval samples. iter_start_s = time.perf_counter() - if active_start_s is None: - active_start_s = iter_start_s + active_start_s = iter_start_s if active_start_s is None else active_start_s actions = action.repeat(env.num_envs, 1) sampling_active[0] = True try: @@ -1370,6 +1557,8 @@ def _patched_sim_render(): sampling_active[0] = False last_active_end_s = time.perf_counter() stats.active_iterations += 1 + if presentation is not None: + presentation.publish(env, changed=True) # Snapshot GPU state right after env.step so the # sample reflects the active workload (post-render @@ -1391,7 +1580,7 @@ def _patched_sim_render(): # meaningful replay. End the run with a failure # outcome so the batch's exit code reflects it. if bool(terminated.any().item()) or bool(truncated.any().item()): - print("Replay failure: env terminated/truncated mid-trajectory; ending run.") + print("Replay environment terminated; ending run.") stats.outcome = "failure" break @@ -1440,7 +1629,8 @@ def _patched_sim_render(): # Fail loud before returning so the next replay never starts and # the batch produces no partial report. See _assert_run_measured. - _assert_run_measured(stats, run_index) + if presentation is None: + _assert_run_measured(stats, run_index) return stats @@ -1454,10 +1644,11 @@ def main() -> int: multi-run batches start essentially instantly. Per-replay control flow (see :func:`_run_single_replay` for details): - * Pre-loop warmup: ``wait_for_stage_load`` polls + * First-run Kit warmup: ``wait_for_stage_load`` polls ``omni.usd.UsdContext.get_stage_loading_status`` until the simulator reports zero pending assets, then renders a fixed number of - settle frames (only on ``run_index == 0``). An optional + settle frames only when Kit is active and ``run_index == 0``. + Kitless runs do not import Kit's asset loader. An optional ``--replay_start_delay_s`` buffer can be appended for hardware that needs more grace. ``advance()`` is not called during warmup so ``ReplaySession.update`` does not consume MCAP @@ -1505,16 +1696,18 @@ def main() -> int: persisted to disk; a one-line-per-run stdout summary is always printed. See the module docstring for the full JSON schema. - Resource cleanup is wrapped in a ``try/finally`` so that ``env.close()`` - always runs, even when device construction or any subsequent setup - raises -- otherwise the USD stage would leak across CI runs. + Cleanup drains replay input before the optional OVXR owner releases its + borrowed environment. Without that owner the batch closes the environment + directly. Failed input drain retains borrowed resources and raises rather + than releasing a stage that a worker may still use. Repeated shutdown signals + request owner shutdown without interrupting its drain. Returns: - The host process exit code, mapped from the worst-of-N outcome - across the multi-run batch: ``0`` if every run's - ``success_term`` fired, ``2`` if any run hit - ``--max_replay_duration_s``, otherwise ``1`` (any failure or - incomplete run). + ``1`` if the requested batch did not finish. For a complete batch, + return the worst-of-N outcome: ``0`` if every run met task success, + ``2`` if any run hit + ``--max_replay_duration_s``, otherwise ``1`` (any failure or incomplete + run). The report retains each completed run's outcome in either case. """ if args_cli.num_replays < 1: raise ValueError(f"--num_replays must be >= 1; got {args_cli.num_replays}") @@ -1532,7 +1725,7 @@ def main() -> int: "MCAP replay requires env_cfg.isaac_teleop to be set." ) - with launch_simulation(env_cfg, args_cli): + with launch_simulation(env_cfg, args_cli, cleanup_context=not args_cli.ovxr): exit_code = _run_replay_batch(env_cfg, success_term) return exit_code @@ -1543,8 +1736,14 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None Returns: The host process exit code, see :func:`main`. """ + from isaaclab.sim import SimulationContext + + previous_context = SimulationContext.instance() env: gym.Env | None = None cloudxr_launcher = None + presentation = None + previous_handlers = {} + shutting_down = False all_runs: list[_RunStats] = [] try: @@ -1555,9 +1754,24 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None # ``_run_single_replay``). This is what lets ``--num_replays > 1`` # work in XR-active mode without losing the OpenXR runtime # IPC between runs. - cloudxr_launcher = _maybe_launch_cloudxr(args_cli.cloudxr_env, args_cli.auto_launch_cloudxr) + if args_cli.ovxr: + presentation = _ReplayPresentation(env_cfg) + + def request_shutdown(_signum, _frame): + nonlocal shutting_down + if not presentation.started and not shutting_down: + # Begin normal unwinding during construction or peer-gated + # startup; subsequent signals must not interrupt owner drain. + shutting_down = True + raise KeyboardInterrupt + shutting_down = True + presentation.session.request_shutdown() + + previous_handlers = {sig: signal.signal(sig, request_shutdown) for sig in (signal.SIGINT, signal.SIGTERM)} + else: + cloudxr_launcher = _maybe_launch_cloudxr(args_cli.cloudxr_env, args_cli.auto_launch_cloudxr) - if not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr: + if args_cli.require_kit and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings apply_isaac_rtx_global_settings(IsaacRtxRendererGlobalSettingsCfg(antialiasing_mode="DLSS")) @@ -1571,18 +1785,33 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None success_term=success_term, run_index=run_idx, total_runs=args_cli.num_replays, + presentation=presentation, ) all_runs.append(run_stats) print(f"Replay {run_idx + 1}/{args_cli.num_replays} outcome: {run_stats.outcome}") - if not env.sim.is_running(): + if not env.sim.is_running() or (presentation is not None and presentation.session.shutdown_requested): # The simulator was closed externally mid-batch; stop the # outer loop rather than spawning a fresh device against # a dead app. break finally: - if env is not None: - env.close() - print("Environment closed") + shutting_down = True + try: + if presentation is not None: + presentation.close() + if env is not None: + if presentation is None or not presentation.session.async_worker_owns_environment: + env.close() + print("Environment closed") + elif presentation is None or not presentation.session.async_worker_owns_environment: + # Construction may have created a context before returning env. + # Only run after proven input/viewer drain; preserve older contexts. + context = SimulationContext.instance() + if context is not None and context is not previous_context: + SimulationContext.clear_instance() + finally: + for sig, handler in previous_handlers.items(): + signal.signal(sig, handler) if cloudxr_launcher is not None: try: cloudxr_launcher.stop() @@ -1594,6 +1823,8 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None _print_stdout_summary(report) if args_cli.stats_output_file is not None: _write_json_report(args_cli.stats_output_file, report) + if len(all_runs) != args_cli.num_replays: + return 1 return _exit_code_for_outcomes(all_runs) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index e0a26de2024..20594dfbbed 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -18,18 +18,19 @@ import argparse import logging +import math +import os +import signal import sys +import time from collections.abc import Callable -from contextlib import ExitStack import gymnasium as gym import torch -from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg -from isaaclab.app import add_launcher_args, launch_simulation +from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks from isaaclab.devices import DeviceCfg -from isaaclab.devices.openxr import remove_camera_configs from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.utils.string import string_to_callable @@ -48,16 +49,23 @@ se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, ) +from ..action_policy import ( + initial_hand_tracking_issue as _required_hand_tracking_issue, +) +from ..action_policy import prepare_teleop_action from ..device_selection import create_teleop_device, select_teleop_cfg logger = logging.getLogger(__name__) +_RENDERER_ATTACH_TIMEOUT_S = 180.0 +"""Allow cold RTX preparation before failing attachment; client connection waits separately.""" + args_cli: argparse.Namespace """Parsed command line, set by :func:`_parse_args`.""" def _build_parser() -> argparse.ArgumentParser: - """Build the command-line parser, including the simulation launcher arguments.""" + """Build workflow and simulation launcher arguments.""" parser = argparse.ArgumentParser(description="Teleoperation for Isaac Lab environments.") parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.") parser.add_argument( @@ -65,13 +73,18 @@ def _build_parser() -> argparse.ArgumentParser: type=str, default=None, help=( - "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," - " spacemouse, gamepad). When omitted, the Isaac Capture pipeline is used if configured in the env," - " otherwise keyboard. Keyboard, gamepad and spacemouse run through Isaac Capture (the teleop extra)." + "Teleop device name: a task device or keyboard, spacemouse, gamepad. When omitted, use the task's" + " Isaac Capture pipeline, otherwise keyboard. Built-in devices run through Isaac Capture." ), ) parser.add_argument("--task", type=str, default=None, help="Name of the task.") parser.add_argument("--sensitivity", type=float, default=1.0, help="Sensitivity factor.") + parser.add_argument( + "--step_hz", + type=float, + default=0.0, + help="Maximum host-control rate with --ovxr; 0 leaves active control unthrottled. XR rendering is independent.", + ) parser.add_argument( "--cloudxr_env", type=str, @@ -79,7 +92,7 @@ def _build_parser() -> argparse.ArgumentParser: help=( "Path to a CloudXR .env file, or a shorthand: 'cloudxrjs' (Quest/Pico), 'avp' (Apple Vision Pro)," " or 'standalone' (headless, no XR client). Set to 'none' to disable CloudXR auto-launch entirely." - " When unset, defaults to 'cloudxrjs' with --xr and 'standalone' without --xr." + " When unset, defaults to 'cloudxrjs' with --xr or --ovxr and 'standalone' otherwise." ), ) parser.add_argument( @@ -109,24 +122,37 @@ def _build_parser() -> argparse.ArgumentParser: " and improve XR performance)." ), ) + parser.add_argument( + "--ovxr", + dest="using_ovxr", + action="store_true", + default=False, + help="Use the OVXR OVRTX/OVStage/OVXR path instead of Kit XR.", + ) + parser.add_argument( + "--ovxr_session_mode", + choices=("ar", "vr"), + default="ar", + help="OVXR session mode. Defaults to 'ar'; pass 'vr' for an opaque background.", + ) add_launcher_args(parser) return parser def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse ``argv`` (default ``sys.argv[1:]``) into :data:`args_cli`; Hydra task overrides stay in ``sys.argv``.""" + """Parse workflow arguments, leaving task overrides in ``sys.argv``.""" global args_cli if argv is not None: sys.argv = [sys.argv[0], *argv] parser = _build_parser() args_cli, hydra_args = setup_preset_cli(parser) - - # Enable external camera rendering by default (``--disable_external_cameras`` turns it off). The - # ``--enable_cameras`` CLI flag was removed in Isaac Lab 3.0 (see #6656), so pass the intent to - # the launcher; this selects a camera-rendering experience that provides RTX/DLSS support. - # Everywhere else we read ``args_cli.disable_external_cameras`` directly. - args_cli.enable_cameras = not args_cli.disable_external_cameras + if args_cli.xr and args_cli.using_ovxr: + parser.error("--xr and --ovxr are mutually exclusive OpenXR frame-loop owners") + if not math.isfinite(args_cli.step_hz) or args_cli.step_hz < 0.0: + parser.error("--step_hz must be finite and non-negative; 0 disables active-control pacing") + if args_cli.step_hz > 0.0 and not args_cli.using_ovxr: + parser.error("--step_hz requires --ovxr") # Call an external callback if requested. if args_cli.external_callback: @@ -137,7 +163,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # Resolve CLI profile names once; runtime construction receives a path or None. if args_cli.cloudxr_env is None: - args_cli.cloudxr_env = "cloudxrjs" if args_cli.xr else "standalone" + args_cli.cloudxr_env = "cloudxrjs" if args_cli.xr or args_cli.using_ovxr else "standalone" if not args_cli.cloudxr_env.strip() or args_cli.cloudxr_env.lower() == "none": args_cli.cloudxr_env = None else: @@ -149,19 +175,93 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: return args_cli -def main() -> None: # noqa: C901 +def _pace_ovxr_control_loop( + using_ovxr: bool, + action_applied: bool, + *, + iteration_started_s: float, + action_period_s: float, + step_hz: float = 0.0, + wait_for_shutdown: Callable[[float], bool] | None = None, +) -> None: + """Limit host control independently of the XR frame clock. + + Credit elapsed input, physics and publication work; an overrun adds no wait + or catch-up steps. With no explicit limit, active control remains unpaced + and paused polling retains the environment's action interval. """ - Run teleoperation with an Isaac Lab manipulation environment. + if not using_ovxr or (action_applied and step_hz == 0.0): + return + period_s = 1.0 / step_hz if step_hz > 0.0 else action_period_s + remaining_s = period_s - (time.perf_counter() - iteration_started_s) + if remaining_s > 0.0: + if wait_for_shutdown is None: + time.sleep(remaining_s) + else: + wait_for_shutdown(remaining_s) - Creates the environment, sets up teleoperation interfaces and callbacks, - and runs the main simulation loop until the application is closed. - Returns: - None +def _rtx_rendering_requested(args: argparse.Namespace) -> bool: + """Return whether the CLI selects a renderer that actually drives RTX rendering. + + The RTX/DLSS global settings are only meaningful when something renders through RTX. + That happens when the Kit visualizer is enabled (``--viz kit``), when external cameras + are rendered (on by default; see ``--disable_external_cameras``), or in XR mode (``--xr``). + A pure-headless session with none of these renders nothing. + + This reads the resolved namespace intent rather than any Kit/carb runtime state so the + check keeps working as these scripts grow support for other rendering backends. """ - # Resolve the task configuration through Hydra so CLI presets are applied. - env_cfg, _ = resolve_task_config(args_cli.task, "") - env_cfg.sim.device = args_cli.device + visualizers = getattr(args, "visualizer", None) or [] + external_cameras = not getattr(args, "disable_external_cameras", False) + return external_cameras or ("kit" in visualizers) or bool(getattr(args, "xr", False)) + + +def _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop: bool): + """Return ``(update, stop)`` callables driving controller haptics, or no-ops. + + Keeps haptics opt-in without branching in the main loop: both callables are + no-ops unless the active device is an Isaac Capture device and the env declares + a ``haptic_feedback`` config. ``update`` renders the current contact force; + ``stop`` zeroes it so a stale pulse does not persist while teleop is paused. + """ + noop = lambda: None # noqa: E731 + if not use_isaac_teleop: + return noop, noop + from .. import create_haptic_feedback_driver + + driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) + if driver is None: + return noop, noop + return driver.update, driver.stop + + +def _apply_teleop_frame(env, action, issue: str | None, haptic_update, haptic_stop) -> bool: + """Step one valid action, or render-only while an unsafe tracker frame is held.""" + if issue is not None: + env.sim.render() + haptic_stop() + return False + env.step(action.repeat(env.num_envs, 1)) + haptic_update() + return True + + +def _ovxr_publication_identity( + physics_step_id: int, *, reset_generation: int | None = None +) -> int | tuple[str, int, int]: + """Return a dedup identity that also represents a reset at an unchanged physics step.""" + if reset_generation is None: + return physics_step_id + return ("reset", reset_generation, physics_step_id) + + +def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool]: + """Resolve task configuration and teleop ownership before runtime launch.""" + overrides = list(sys.argv[1:]) + env_cfg, _ = resolve_task_config(args_cli.task, "", overrides=overrides) + if args_cli.device is not None: + env_cfg.sim.device = args_cli.device env_cfg.scene.num_envs = args_cli.num_envs env_cfg.env_name = args_cli.task if not isinstance(env_cfg, ManagerBasedRLEnvCfg): @@ -202,224 +302,477 @@ def main() -> None: # noqa: C901 if isinstance(teleop_cfg, IsaacTeleopCfg): env_cfg.isaac_teleop = teleop_cfg - # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR - # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders - # normally, so gate on --xr alone. - if args_cli.xr: - # Keep camera configs when external cameras are enabled (defaulted on); otherwise - # strip them so the XR headset view is the sole render product. - if args_cli.disable_external_cameras: - env_cfg = remove_camera_configs(env_cfg) + use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) + if args_cli.using_ovxr and not use_isaac_teleop: + raise ValueError("--ovxr requires a task configured with env_cfg.isaac_teleop") + if args_cli.using_ovxr and args_cli.num_envs != 1: + raise ValueError("--ovxr currently requires --num_envs 1") + if args_cli.using_ovxr: + stage_mode = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + if stage_mode.strip() != "1": + raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") + if args_cli.using_ovxr or args_cli.disable_external_cameras: + from .. import remove_camera_configs + + env_cfg = remove_camera_configs(env_cfg) + if env_cfg.sim.physics is None: + from isaaclab.physics import PhysicsCfg + + env_cfg.sim.physics = PhysicsCfg() + configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) + if args_cli.using_ovxr: + from isaaclab.visualizers import VisualizerBackendCfg + + if kit_sources or any(isinstance(cfg, VisualizerBackendCfg) for cfg in configs): + raise ValueError("--ovxr requires no Kit runtime or local visualizer") + if use_isaac_teleop and teleop_cfg.prepare_environment is not None: + teleop_cfg.prepare_environment(env_cfg) + return (env_cfg, teleop_cfg, use_isaac_teleop) + + +def _install_ovxr_shutdown_handlers(ov_libraries_xr_session) -> dict: + """Make owner-script signals request the same orderly shutdown as OVUI.""" + if ov_libraries_xr_session is None: + return {} + + def request_shutdown(_signum, _frame) -> None: + ov_libraries_xr_session.request_shutdown() + + return {signum: signal.signal(signum, request_shutdown) for signum in (signal.SIGINT, signal.SIGTERM)} + + +def _restore_ovxr_shutdown_handlers(previous_handlers: dict) -> None: + for signum, handler in previous_handlers.items(): + signal.signal(signum, handler) + + +def _close_teleop_resources( + env, + camera_feed_session, + ov_libraries_xr_session, + previous_shutdown_handlers, + teleop_interface=None, + *, + previous_context=None, +) -> None: + """Preserve the first failure and drain XR before freeing its borrowed environment.""" + failure = sys.exception() + cleanup_failure = None + xr_drained = True + camera_drained = True + + def report_cleanup_error(error: Exception) -> None: + nonlocal cleanup_failure + logger.exception("Teleoperation cleanup failed") + if failure is not None: + failure.add_note(f"Cleanup also failed: {error!r}") + elif cleanup_failure is None: + cleanup_failure = error + else: + cleanup_failure.add_note(f"Cleanup also failed: {error!r}") + + try: + input_drain_failed = getattr(teleop_interface, "session_cleanup_failed", False) is True + if input_drain_failed: + xr_drained = False + report_cleanup_error( + RuntimeError("Isaac Capture cleanup failed; borrowed XR/environment drain is unproved") + ) + if ov_libraries_xr_session is not None: + try: + ov_libraries_xr_session.request_shutdown() + except Exception as error: + report_cleanup_error(error) + elif ov_libraries_xr_session is not None: + try: + ov_libraries_xr_session.close() + except Exception as error: + xr_drained = False + report_cleanup_error(error) + if not input_drain_failed: + # A successfully bound feed exits before Capture. This is also the + # cleanup path for preparation that never reached bind(). + try: + camera_feed_session.close() + except Exception as error: + camera_drained = False + report_cleanup_error(error) + if env is not None and xr_drained: + if ov_libraries_xr_session is None or not ov_libraries_xr_session.async_worker_owns_environment: + try: + env.close() + except Exception as error: + report_cleanup_error(error) + elif env is None and xr_drained and camera_drained: + from isaaclab.sim import SimulationContext + + if ov_libraries_xr_session is None or not ov_libraries_xr_session.async_worker_owns_environment: + context = SimulationContext.instance() + if context is not None and context is not previous_context: + try: + SimulationContext.clear_instance() + except Exception as error: + report_cleanup_error(error) + finally: + # Keep repeated SIGINT/SIGTERM on the owner throughout this cleanup attempt. + try: + _restore_ovxr_shutdown_handlers(previous_shutdown_handlers) + except Exception as error: + report_cleanup_error(error) + if cleanup_failure is not None: + raise cleanup_failure + if env is not None and xr_drained: + print("Environment closed") - with launch_simulation(env_cfg, args_cli), ExitStack() as cleanup: - run_teleoperation(env_cfg, teleop_cfg, cleanup) +def run_teleoperation( # noqa: C901 + env_cfg: ManagerBasedRLEnvCfg, + teleop_cfg: IsaacTeleopCfg | DeviceCfg, + use_isaac_teleop: bool, +) -> None: + """Run teleoperation through the selected Kit or OVXR runtime.""" -def run_teleoperation( - env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, cleanup: ExitStack -) -> None: # noqa: C901 - """Create the environment and teleop device, then run the teleoperation loop until the app is closed.""" - from .. import XrCameraFeedSession, poll_control_events + from isaaclab.sim import SimulationContext - use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) + from .. import XrCameraFeedSession + previous_context = SimulationContext.instance() camera_feed_session = XrCameraFeedSession.prepare( env_cfg, enabled=args_cli.xr and use_isaac_teleop, - camera_rendering_enabled=not args_cli.disable_external_cameras, + camera_rendering_enabled=not args_cli.disable_external_cameras and not args_cli.using_ovxr, ) - cleanup.callback(camera_feed_session.close) - - # Apply the RTX/DLSS global settings when Kit runs an RTX render pipeline (Kit visualizer, - # external cameras, or XR). - if has_kit() and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): - from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings - - apply_isaac_rtx_global_settings( - IsaacRtxRendererGlobalSettingsCfg( - antialiasing_mode="DLSS", - carb_settings=( - {"/rtx/dldenoiser/responsiveDenoising": True} - if camera_feed_session.requires_responsive_denoising - else None - ), - ), - ) + env = None + teleop_interface = None + ov_libraries_xr_session = None + previous_shutdown_handlers = {} try: - # create environment - env = gym.make(args_cli.task, cfg=env_cfg).unwrapped - # check environment name (for reach , we don't allow the gripper) - if "Reach" in args_cli.task: - logger.warning( - f"The environment '{args_cli.task}' does not support gripper control. The device command will be" - " ignored." + # Apply the RTX/DLSS global settings when an RTX render pipeline will run (Kit visualizer, + # external cameras, or XR). ``apply_isaac_rtx_global_settings`` uses ``omni.replicator``, + # which some experiences do not preload, so ensure it is loaded first. + if has_kit() and _rtx_rendering_requested(args_cli): + from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg + from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings + + # The renderer's public helper loads Replicator when its settings need it. + apply_isaac_rtx_global_settings( + IsaacRtxRendererGlobalSettingsCfg( + antialiasing_mode="DLSS", + carb_settings=( + {"/rtx/dldenoiser/responsiveDenoising": True} + if camera_feed_session.requires_responsive_denoising + else None + ), + ), ) - except Exception as e: - logger.error(f"Failed to create environment: {e}") - return - - # Flags for controlling teleoperation flow - should_reset_recording_instance = False - teleoperation_active = True - - # Callback handlers - def reset_recording_instance() -> None: - """ - Reset the environment to its initial state. - - Sets a flag to reset the environment on the next simulation step. - Returns: - None - """ - nonlocal should_reset_recording_instance - should_reset_recording_instance = True - print("Reset triggered - Environment will reset on next step") + if args_cli.using_ovxr: + from isaaclab_ov.renderers import OVRTXRendererCfg - def start_teleoperation() -> None: - """ - Activate teleoperation control of the robot. + from .. import OvLibrariesXrConfig, OvLibrariesXrSession - Enables the application of teleoperation commands to the environment. + ov_libraries_xr_session = OvLibrariesXrSession.prepare( + env_cfg.sim, + OvLibrariesXrConfig( + application_name=f"isaaclab-ovxr-teleop-{args_cli.task}", + session_mode=args_cli.ovxr_session_mode, + cloudxr_env_file=args_cli.cloudxr_env, + auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + ), + renderer_cfg=OVRTXRendererCfg(), + ) - Returns: - None - """ - nonlocal teleoperation_active - teleoperation_active = True - print("Teleoperation activated") - - def stop_teleoperation() -> None: - """ - Deactivate teleoperation control of the robot. - - Disables the application of teleoperation commands to the environment. - - Returns: - None - """ - nonlocal teleoperation_active - teleoperation_active = False - print("Teleoperation deactivated") - - # Create device config if not already in env_cfg - teleoperation_callbacks: dict[str, Callable[[], None]] = { - "R": reset_recording_instance, - "START": start_teleoperation, - "STOP": stop_teleoperation, - "RESET": reset_recording_instance, - } - - # For XR devices (hand tracking or Isaac Capture), default to inactive. Without - # --xr, teleop is started locally (see ``request_start`` below) rather than by - # a headset, so it still begins running -- but it flows through the same state - # machine, so keyboard/host pause/resume keeps working. - if use_isaac_teleop or args_cli.xr: - teleoperation_active = env_cfg.isaac_teleop.teleoperation_active_default if use_isaac_teleop else False - else: - # Always active for other devices + try: + # create environment + env = gym.make(args_cli.task, cfg=env_cfg).unwrapped + # check environment name (for reach , we don't allow the gripper) + if "Reach" in args_cli.task: + logger.warning( + f"The environment '{args_cli.task}' does not support gripper control. The device command will be" + " ignored." + ) + + except Exception: + logger.exception("Failed to create environment") + raise + + # Flags for controlling teleoperation flow + should_reset_recording_instance = False teleoperation_active = True + reset_publication_generation = 0 + + # Callback handlers + def reset_recording_instance() -> None: + """Request an environment reset at the next simulation step.""" + nonlocal should_reset_recording_instance + should_reset_recording_instance = True + print("Reset triggered - Environment will reset on next step") + + def start_teleoperation() -> None: + """Resume applying teleoperation commands to the environment.""" + nonlocal teleoperation_active + teleoperation_active = True + print("Teleoperation activated") + + def stop_teleoperation() -> None: + """Pause applying teleoperation commands to the environment.""" + nonlocal teleoperation_active + teleoperation_active = False + print("Teleoperation deactivated") + + # Create device config if not already in env_cfg + teleoperation_callbacks: dict[str, Callable[[], None]] = { + "R": reset_recording_instance, + "START": start_teleoperation, + "STOP": stop_teleoperation, + "RESET": reset_recording_instance, + } + + # For XR devices (hand tracking or Isaac Capture), default to inactive. Without + # --xr, teleop is started locally (see ``request_start`` below) rather than by + # a headset, so it still begins running -- but it flows through the same state + # machine, so keyboard/host pause/resume keeps working. + if use_isaac_teleop or args_cli.xr: + teleoperation_active = env_cfg.isaac_teleop.teleoperation_active_default if use_isaac_teleop else False + else: + # Always active for other devices + teleoperation_active = True + + # Use the upstream device factory for Capture and native inputs alike. + from .. import poll_control_events + + try: + teleop_interface = create_teleop_device( + teleop_cfg, + env_prim_path=env.scene.env_prim_paths[0], + sim_device=env.sim.device, + callbacks=teleoperation_callbacks, + visualizers=env.sim.visualizers, + cloudxr_env_file=(None if args_cli.using_ovxr else args_cli.cloudxr_env), + auto_launch_cloudxr=args_cli.auto_launch_cloudxr and not args_cli.using_ovxr, + enable_debug_visualization=args_cli.enable_debug_visualization, + use_kit_xr_bridge=args_cli.xr, + external_oxr_handles_provider=( + ov_libraries_xr_session.external_oxr_handles if ov_libraries_xr_session is not None else None + ), + haptic_cfg=getattr(env_cfg, "haptic_feedback", None), + ) + except Exception: + logger.exception("Failed to create teleop device") + raise - try: - teleop_interface = create_teleop_device( - teleop_cfg, - env_prim_path=env.scene.env_prim_paths[0], - sim_device=args_cli.device, - callbacks=teleoperation_callbacks, - visualizers=env.sim.visualizers, - cloudxr_env_file=args_cli.cloudxr_env, - auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - enable_debug_visualization=args_cli.enable_debug_visualization, - use_kit_xr_bridge=args_cli.xr, - haptic_cfg=getattr(env_cfg, "haptic_feedback", None), - ) - except Exception as e: - logger.error(f"Failed to create teleop device: {e}") - env.close() - return - - print(f"Using teleop device: {teleop_interface}") - - haptic_driver = None - if use_isaac_teleop: - from .. import create_haptic_feedback_driver - - haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - - if use_isaac_teleop: - # the device reads these from the focused visualizer window - print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") - - def run_loop(): - """Inner function to run the teleop loop with access to nonlocal variables.""" - nonlocal should_reset_recording_instance, teleoperation_active + if teleop_interface is None: + logger.error("Failed to create teleop interface") + raise RuntimeError("Failed to create teleop interface") - # reset environment - env.reset() - teleop_interface.reset() + print(f"Using teleop device: {teleop_interface}") - # Without --xr there is no headset to send START, so start locally ([B]/[P] can - # still pause/resume). The reset() above is a host reset (a pure pulse), so it does - # not cancel this start. - if use_isaac_teleop and not args_cli.xr: - teleop_interface.request_start() + # Optional controller haptics: no-ops unless the env declares a + # ``haptic_feedback`` config and the device can render it (Isaac Capture). + haptic_update, haptic_stop = _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop) - stack_name = "Isaac Capture" if use_isaac_teleop else "native" - print(f"{stack_name} teleoperation started. Press 'R' to reset the environment.") + require_initial_bimanual_tracking = ( + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking if use_isaac_teleop else False + ) + initialized_hand_tracking: set[str] = set() + skipped_tracking_frames = 0 + action_conditioner = env_cfg.isaac_teleop.action_conditioner if use_isaac_teleop else None + action_validator = env_cfg.isaac_teleop.action_validator if use_isaac_teleop else None + action_state: dict = {} + + if use_isaac_teleop: + print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") + + def run_loop(): + """Inner function to run the teleop loop with access to nonlocal variables.""" + nonlocal should_reset_recording_instance, skipped_tracking_frames, teleoperation_active + nonlocal reset_publication_generation + + # reset environment + env.reset() + teleop_interface.reset() + initialized_hand_tracking.clear() + action_state.clear() + pending_publication_identity = ( + _ovxr_publication_identity( + env.sim.get_physics_step_count(), reset_generation=reset_publication_generation + ) + if ov_libraries_xr_session is not None + else None + ) + pending_publication_force = ov_libraries_xr_session is not None + + # Without --xr there is no headset to send START, so start locally ([B]/[P] can + # still pause/resume). The reset() above is a host reset (a pure pulse), so it does + # not cancel this start. + if use_isaac_teleop and not (args_cli.xr or args_cli.using_ovxr): + teleop_interface.request_start() + + stack_name = "Isaac Capture" if use_isaac_teleop else "native" + print(f"{stack_name} teleoperation started. Press 'R' to reset the environment.") + + # simulate environment + while env.sim.is_running() and not ( + ov_libraries_xr_session is not None and ov_libraries_xr_session.shutdown_requested + ): + iteration_started_s = time.perf_counter() if ov_libraries_xr_session is not None else 0.0 + try: + # run everything in inference mode + with torch.inference_mode(): + # Reset invalidates the tracking anchor. Publish the new epoch + # before reading input, retaining force across deferred admissions. + if pending_publication_force: + admitted = ov_libraries_xr_session.request_render(pending_publication_identity, force=True) + if admitted is None: + ov_libraries_xr_session.wait_for_shutdown(env.step_dt) + continue + pending_publication_identity = None + pending_publication_force = False + # get device command + target_frame_prim_path = ( + teleop_interface.target_frame_prim_path + if ov_libraries_xr_session is not None and use_isaac_teleop + else None + ) + if target_frame_prim_path is None: + action = teleop_interface.advance() + else: + target_T_world = ov_libraries_xr_session.get_target_frame_T_world(target_frame_prim_path) + action = teleop_interface.advance(target_T_world=target_T_world) + + if use_isaac_teleop: + ctrl = poll_control_events(teleop_interface) + if ctrl.is_active is not None: + teleoperation_active = ctrl.is_active + if ctrl.should_reset: + should_reset_recording_instance = True + + if not teleoperation_active: + action_state.clear() + + # action is None when Isaac Capture session hasn't started yet + # (e.g. waiting for user to click "Start AR") + if action is None: + if ov_libraries_xr_session is None: + env.sim.render() + haptic_stop() + action_applied = False + elif teleoperation_active: + issue = ( + _required_hand_tracking_issue(teleop_interface, initialized_hand_tracking) + if require_initial_bimanual_tracking + else None + ) + if issue is None: + action, issue = prepare_teleop_action( + env, action, action_conditioner, action_validator, action_state + ) + action_applied = _apply_teleop_frame(env, action, issue, haptic_update, haptic_stop) + if issue is not None: + skipped_tracking_frames += 1 + if skipped_tracking_frames <= 5 or skipped_tracking_frames % 200 == 0: + logger.warning( + "Rendering without a physics step for unsafe teleop frame (%s); skipped=%d", + issue, + skipped_tracking_frames, + ) + else: + action_state.clear() + if ov_libraries_xr_session is None: + env.sim.render() + # not stepping: zero haptics so a paused grip stops buzzing + haptic_stop() + action_applied = False + + state_changed = action_applied + force_publication = False + publication_identity = env.sim.get_physics_step_count() + if should_reset_recording_instance: + env.reset() + teleop_interface.reset() + initialized_hand_tracking.clear() + action_state.clear() + camera_feed_session.refresh() + should_reset_recording_instance = False + if ov_libraries_xr_session is not None: + state_changed = True + reset_publication_generation += 1 + force_publication = True + publication_identity = _ovxr_publication_identity( + env.sim.get_physics_step_count(), reset_generation=reset_publication_generation + ) + print("Environment reset complete") + + if ov_libraries_xr_session is not None: + if state_changed: + pending_publication_identity = publication_identity + pending_publication_force = force_publication + if pending_publication_identity is not None: + admitted = ov_libraries_xr_session.request_render( + pending_publication_identity, force=pending_publication_force + ) + if admitted is not None: + pending_publication_identity = None + pending_publication_force = False + + _pace_ovxr_control_loop( + ov_libraries_xr_session is not None, + action_applied, + iteration_started_s=iteration_started_s, + action_period_s=env.step_dt, + step_hz=args_cli.step_hz, + wait_for_shutdown=( + ov_libraries_xr_session.wait_for_shutdown + if ov_libraries_xr_session is not None + else None + ), + ) + except Exception as e: + logger.exception("Error during simulation step: %s", e) + raise + + # Install after environment/device initialization because embedded native + # libraries may replace inherited signal dispositions during startup. + previous_shutdown_handlers = _install_ovxr_shutdown_handlers(ov_libraries_xr_session) + # IsaacTeleop requires a context manager, native devices do not. + if use_isaac_teleop: + if ov_libraries_xr_session is not None: + # Capture releases borrowed handles before the outer cleanup + # closes OVXR and then the environment. + with teleop_interface, camera_feed_session.bind(env): + render_worker_started = ov_libraries_xr_session.start_render_worker( + env, + tracking_anchor_path=teleop_interface.xr_render_anchor_path, + required_openxr_extensions=teleop_interface.required_openxr_extensions, + hidden_visual_prims=env_cfg.isaac_teleop.xr_cfg.hidden_visual_prims, + external_anchor_state=teleop_interface.external_anchor_state, + timeout_s=_RENDERER_ATTACH_TIMEOUT_S, + ) + if render_worker_started is not False: + run_loop() + else: + with teleop_interface, camera_feed_session.bind(env): + run_loop() + else: + with camera_feed_session.bind(env): + run_loop() + finally: + _close_teleop_resources( + env, + camera_feed_session, + ov_libraries_xr_session, + previous_shutdown_handlers, + teleop_interface, + previous_context=previous_context, + ) - # simulate environment - while env.sim.is_running(): - try: - # run everything in inference mode - with torch.inference_mode(): - # get device command - action = teleop_interface.advance() - - if use_isaac_teleop: - ctrl = poll_control_events(teleop_interface) - if ctrl.is_active is not None: - teleoperation_active = ctrl.is_active - if ctrl.should_reset: - should_reset_recording_instance = True - - # action is None when Isaac Capture session hasn't started yet - # (e.g. waiting for user to click "Start AR") - if action is None or not teleoperation_active: - env.sim.render() - if haptic_driver is not None: - haptic_driver.stop() - else: - # process actions - actions = action.repeat(env.num_envs, 1) - # apply actions - env.step(actions) - # render controller haptics from post-step contact forces - if haptic_driver is not None: - haptic_driver.update() - - if should_reset_recording_instance: - env.reset() - teleop_interface.reset() - camera_feed_session.refresh() - should_reset_recording_instance = False - print("Environment reset complete") - except Exception as e: - logger.error(f"Error during simulation step: {e}") - break - - # Run the teleoperation loop - # Isaac Capture requires a context manager, native devices don't - if use_isaac_teleop: - with teleop_interface, camera_feed_session.bind(env): - run_loop() - else: - with camera_feed_session.bind(env): - run_loop() - # close the simulator - env.close() - print("Environment closed") +def main() -> None: + """Resolve the task and run teleoperation with its selected runtime.""" + resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop = _resolve_env_cfg() + with launch_simulation(resolved_env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): + run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop) def run_teleop_se3_agent_cli(argv: list[str] | None = None) -> int: diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py index 675a32e8576..4514f9b3702 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_cfg.py @@ -10,7 +10,7 @@ from collections.abc import Callable from dataclasses import MISSING, field from pathlib import Path -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from isaaclab.utils import configclass @@ -38,6 +38,8 @@ from isaacteleop.retargeting_engine.interface import BaseRetargeter, OutputCombiner from isaacteleop.teleop_session_manager import PluginConfig, RetargetingExecutionConfig + from isaaclab.envs import ManagerBasedRLEnvCfg + @configclass class XrCameraFeedCfg: @@ -246,6 +248,35 @@ def build_pipeline(): ``"START"`` command is received from xr_core via the message bus. """ + require_initial_bimanual_hand_tracking: bool = False + """Wait for both hands to be present before the first environment step. + + An environment or device reset clears the initialization latch. Later + tracking loss follows the configured retargeting pipeline's behavior. + """ + + prepare_environment: Callable[[ManagerBasedRLEnvCfg], None] | None = None + """Optional task-owned preparation of the resolved environment configuration. + + Live and replay launchers call this before creating any simulation resources, + including the OVXR bootstrap. Preparation must tolerate repeated calls from + the task constructor. Keep this callable's imports safe before simulator startup. + """ + + action_conditioner: Callable[[Any, Any, dict[str, Any]], Any] | None = None + """Optional task-owned pre-step action conditioner. + + The callable receives ``(env, action, state)``. The launcher retains the + state dictionary across active frames and clears it on pause or reset. + """ + + action_validator: Callable[[Any], str | None] | None = None + """Optional task-owned fail-closed action validator. + + Return ``None`` for a valid action or a concise reason to render without a + physics step. Validation runs both before and after conditioning. + """ + retargeters_to_tune: Callable[[], list[BaseRetargeter]] | None = None """Optional callable returning retargeters to expose in the tuning UI. diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index 25749b3fcaa..113f8a3808f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -127,6 +127,9 @@ def __init__( enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, visualizers: Sequence[BaseVisualizer] = (), + *, + external_oxr_handles_provider: Callable[[], object | None] | None = None, + ovxr_session: bool = False, ): """Initialize the Isaac Capture device. @@ -144,6 +147,17 @@ def __init__( standalone (``False``) with ``isaacteleop`` owning its own OpenXR session through the CloudXR runtime -- teleop I/O with no Kit XR rendering. Typically wired to the ``--xr`` CLI flag. + external_oxr_handles_provider: Optional callback returning handles + borrowed from an application-owned OVXR context. + Supplying this capability delegates XR rendering/anchoring to + that owner. Static anchors use the configured pose; dynamic + anchors use the owner's committed physics pose and the configured + height/rotation policy. The owner must commit a scene pose before + input is read initially and after reset. + ovxr_session: Use application-owned presentation and anchoring + without borrowing live input handles. Static and dynamic anchors + follow the same policy as live OVXR, allowing recorded input to + drive a separately presented scene. mcap_record_path: Optional MCAP file path to record the live teleop session into. Mutually exclusive with *mcap_replay_path*. Debug-grade only -- the produced file @@ -168,14 +182,19 @@ def __init__( """ self._cfg = cfg self._target_world_matrix = None + self._ovxr_session = ovxr_session or external_oxr_handles_provider is not None - self._anchor_manager = XrAnchorManager(cfg.xr_cfg) + # Borrowed OVXR handles imply external anchor ownership. + self._anchor_manager = XrAnchorManager( + cfg.xr_cfg, ovxr_session=ovxr_session or external_oxr_handles_provider is not None + ) self._command_handler = CommandHandler() self._session_lifecycle = TeleopSessionLifecycle( cfg, cloudxr_env_file=cloudxr_env_file, auto_launch_cloudxr=auto_launch_cloudxr, use_kit_xr_bridge=use_kit_xr_bridge, + external_oxr_handles_provider=external_oxr_handles_provider, mcap_record_path=mcap_record_path, mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, @@ -231,36 +250,94 @@ def __str__(self) -> str: def __enter__(self) -> IsaacTeleopDevice: """Enter the context manager and prepare the Isaac Capture session. - Binds the configured target frame, builds the retargeting pipeline, and acquires OpenXR handles - from Kit's XR bridge extension. If the handles are not yet available - (e.g. the user has not clicked "Start AR"), session creation is deferred - and will be retried automatically on each :meth:`advance` call. + Builds the retargeting pipeline and starts the selected live or replay + input session. Kit sessions bind the configured target frame through + Fabric. External OVXR callers supply target transforms to :meth:`advance` + when rebasing is needed. If live OpenXR handles are not yet available, + session creation is deferred and retried on each :meth:`advance` call. Returns: Self for context manager protocol. Raises: - ValueError: If the configured target frame does not exist. + ValueError: If the configured Kit target frame does not exist. """ - if self._cfg.target_frame_prim_path is not None: - import usdrt - from usdrt import Rt - - stage = usdrt.Usd.Stage.Attach(get_current_stage_id()) - prim = stage.GetPrimAtPath(self._cfg.target_frame_prim_path) - if not prim.IsValid(): - raise ValueError(f"No target frame at {self._cfg.target_frame_prim_path!r}.") - self._target_world_matrix = Rt.Xformable(prim).GetFabricHierarchyWorldMatrixAttr() - self._session_lifecycle.start() + try: + if not self._ovxr_session and self._cfg.target_frame_prim_path is not None: + import usdrt + from usdrt import Rt + + stage = usdrt.Usd.Stage.Attach(get_current_stage_id()) + prim = stage.GetPrimAtPath(self._cfg.target_frame_prim_path) + if not prim.IsValid(): + raise ValueError(f"No target frame at {self._cfg.target_frame_prim_path!r}.") + self._target_world_matrix = Rt.Xformable(prim).GetFabricHierarchyWorldMatrixAttr() + self._session_lifecycle.start() + except BaseException as error: + # A failed __enter__ has no automatic __exit__; unwind any runtime, + # pipeline or UI acquired before the failure without replacing it. + self.__exit__(type(error), error, error.__traceback__) + raise return self def __exit__(self, exc_type, exc_val, exc_tb): - """Exit the context manager and clean up the Isaac Capture session.""" + """Exit the context manager and clean up the Isaac Capture session. + + Cleanup failures propagate when no body exception is active; otherwise + they are attached as notes to that original exception. A failed native + input drain retains its resources, as indicated by + :attr:`session_cleanup_failed`. + """ self._target_world_matrix = None - self._anchor_manager.cleanup() - self._session_lifecycle.stop(exc_type, exc_val, exc_tb) + error = exc_val + try: + self._session_lifecycle.stop(exc_type, exc_val, exc_tb) + except Exception as cleanup_error: + if error is None: + error = cleanup_error + else: + error.add_note(f"Isaac Capture cleanup failed: {cleanup_error!r}") + try: + self._anchor_manager.cleanup() + except Exception as cleanup_error: + if error is None: + error = cleanup_error + else: + error.add_note(f"XR anchor cleanup failed: {cleanup_error!r}") + if exc_val is None and error is not None: + raise error return False + @property + def target_frame_prim_path(self) -> str | None: + """Concrete target frame for external scene readers, after environment-path expansion.""" + return self._cfg.target_frame_prim_path + + @property + def xr_anchor_path(self) -> str: + """Return the exact anchor prim path consumed by external OVXR.""" + return self._anchor_manager.anchor_headset_path + + @property + def xr_render_anchor_path(self) -> str: + """Return the anchor prim path used to frame XR cameras.""" + return self._anchor_manager.render_anchor_path + + @property + def external_anchor_state(self) -> XrAnchorManager | None: + """The dynamic anchor capability consumed by Lab's external scene publisher.""" + return self._anchor_manager if self._anchor_manager.external_reference_path is not None else None + + @property + def required_openxr_extensions(self) -> tuple[str, ...]: + """Return extensions required by the built teleop graphs.""" + return self._session_lifecycle.required_openxr_extensions() + + @property + def session_cleanup_failed(self) -> bool: + """Whether borrowed XR handles could not be safely drained.""" + return self._session_lifecycle.session_cleanup_failed + def reset(self, pause: bool = False) -> None: """Reset the device state. @@ -306,10 +383,25 @@ def last_control_events(self) -> ControlEvents: Returns a :class:`ControlEvents` derived from the teleop control pipeline. When no control channel is configured, returns a - default (no-op) :class:`ControlEvents`. + default (no-op) :class:`ControlEvents`. Repeated property reads return + the same events. RESET is emitted only once per returned input frame, + even if subsequent :meth:`advance` calls reuse that frame. """ return self._session_lifecycle.last_control_events + @property + def hand_tracking_presence(self) -> tuple[bool, bool] | None: + """Left/right hand presence in the latest input frame, or None before hand data is available.""" + lifecycle = self._session_lifecycle + result = lifecycle.last_step_result + if result is None: + return None + left = result.get(lifecycle.HAND_LEFT_KEY) + right = result.get(lifecycle.HAND_RIGHT_KEY) + if left is None or right is None: + return None + return not left.is_none, not right.is_none + def add_callback(self, key: str, func: Callable) -> None: """Add a callback function for teleop commands. @@ -324,8 +416,8 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N If the Isaac Capture session has not been started yet (because the OpenXR handles were not available at ``__enter__`` time), this method will - attempt to start it on each call. Once the user clicks "Start AR" and - the handles become available, the session is created transparently. + attempt to start it on each call. Once the selected Kit or external XR + provider exposes handles, the session is created transparently. Args: target_T_world: Optional 4x4 transform matrix that rebases all @@ -343,10 +435,14 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N (``__dlpack__``), including :class:`numpy.ndarray`, :class:`torch.Tensor`, and ``wp.array``. - When ``None`` and + For Kit sessions, when ``None`` and :attr:`~IsaacTeleopCfg.target_frame_prim_path` is set, the transform is computed automatically by reading the prim's - world matrix from Fabric and inverting it. + world matrix from Fabric and inverting it. External OVXR + sessions do not read Fabric: callers must supply this matrix + to rebase into a target frame. With ``None``, external output + remains in the simulation world frame, using the configured + static anchor or the owner's latest committed dynamic anchor. Returns: A flattened action :class:`torch.Tensor` ready for the Isaac Lab @@ -354,7 +450,8 @@ def advance(self, target_T_world: np.ndarray | torch.Tensor | SupportsDLPack | N (e.g. still waiting for the user to start AR). Raises: - RuntimeError: If called outside of a context manager. + RuntimeError: If called outside of a context manager or an external + dynamic anchor has no committed pose for the current reset epoch. """ if target_T_world is None and self._target_world_matrix is not None: matrix = self._target_world_matrix.Get() @@ -557,6 +654,8 @@ def create_isaac_teleop_device( *, env_prim_path: str | None = None, visualizers: Sequence[BaseVisualizer] = (), + external_oxr_handles_provider: Callable[[], object | None] | None = None, + ovxr_session: bool = False, ) -> IsaacTeleopDevice: """Create an :class:`IsaacTeleopDevice` with required Omniverse extension setup. @@ -596,6 +695,12 @@ def create_isaac_teleop_device( ``isaacteleop`` creates its own OpenXR session through the CloudXR runtime, so teleop I/O works headless without Kit XR rendering. Typically wired to the ``--xr`` CLI flag. + external_oxr_handles_provider: Optional callback returning handles + borrowed from an application-owned OVXR context. That owner supplies + static anchoring or commits dynamic anchor poses before input reads. + ovxr_session: Use application-owned OVXR presentation and the configured + static or dynamic anchor policy even when input comes from an MCAP + reader rather than live XR handles. mcap_record_path: Optional MCAP file path to record the live teleop session into. Debug-grade only. Mutually exclusive with *mcap_replay_path*. @@ -651,6 +756,8 @@ def create_isaac_teleop_device( cloudxr_env_file=cloudxr_env_file, auto_launch_cloudxr=auto_launch_cloudxr, use_kit_xr_bridge=use_kit_xr_bridge, + external_oxr_handles_provider=external_oxr_handles_provider, + ovxr_session=ovxr_session, mcap_record_path=mcap_record_path, mcap_replay_path=mcap_replay_path, enable_debug_visualization=enable_debug_visualization, diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 5fba3cc33da..292754ab3c3 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -244,12 +244,10 @@ class TeleopSessionLifecycle: Only chained when debug visualization is enabled.""" HAND_LEFT_KEY = "hand_left" - """Pipeline output key for the left hand ``TensorGroup`` when the debug - outputs are chained (matches ``HandsSource.LEFT``).""" + """Pipeline output key for left-hand presence when ``HandsSource`` exists.""" HAND_RIGHT_KEY = "hand_right" - """Pipeline output key for the right hand ``TensorGroup`` when the debug - outputs are chained (matches ``HandsSource.RIGHT``).""" + """Pipeline output key for right-hand presence when ``HandsSource`` exists.""" _STEP_FAILURE_RESTART_COOLDOWN_S = 1.0 """Minimum delay [s] before recreating the session after a step failure @@ -270,6 +268,8 @@ def __init__( enable_debug_visualization: bool = False, haptic_cfg: HapticFeedbackCfg | None = None, visualizers: Sequence[BaseVisualizer] = (), + *, + external_oxr_handles_provider: Callable[[], OpenXRSessionHandles | None] | None = None, ): """Initialize the session lifecycle manager. @@ -291,6 +291,10 @@ def __init__( session through the CloudXR runtime, so teleop I/O works headless without Kit XR rendering. Ignored in replay mode (which never touches the XR runtime). + external_oxr_handles_provider: Optional callback returning handles + borrowed from an external OVXR context. Requires + ``use_kit_xr_bridge=False`` and defers session creation while + the callback returns ``None``. mcap_record_path: Optional path to an MCAP file the live teleop session should be recorded into. Mutually exclusive with *mcap_replay_path*. Debug-grade only -- see the Isaac Lab @@ -322,12 +326,17 @@ def __init__( "mcap_record_path and mcap_replay_path are mutually exclusive; " "set at most one to switch the session between LIVE recording and REPLAY playback." ) + if external_oxr_handles_provider is not None and use_kit_xr_bridge: + raise ValueError("An external OpenXR handle provider requires use_kit_xr_bridge=False") + if external_oxr_handles_provider is not None and mcap_replay_path is not None: + raise ValueError("An external OpenXR handle provider is incompatible with replay mode") self._cfg = cfg self._device = torch.device(cfg.sim_device) self._cloudxr_env_file = cloudxr_env_file self._auto_launch_cloudxr = auto_launch_cloudxr self._use_kit_xr_bridge = use_kit_xr_bridge + self._external_oxr_handles_provider = external_oxr_handles_provider self._mcap_record_path = mcap_record_path self._mcap_replay_path = mcap_replay_path self._is_replay = mcap_replay_path is not None @@ -347,6 +356,7 @@ def __init__( # Session state (populated during start) self._session: TeleopSession | None = None + self._session_cleanup_error: BaseException | None = None self._pipeline = None self._teleop_control_pipeline = None self._message_processor: TeleopMessageProcessor | None = None @@ -359,6 +369,8 @@ def __init__( self._last_right_controller = None self._last_left_controller = None self._last_step_result: dict | None = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id: int | None = None self._session_start_deferred_logged = False # Monotonic deadline gating session re-creation after a step failure self._restart_holdoff_until = 0.0 @@ -445,7 +457,12 @@ def __init__( @property def is_active(self) -> bool: """Whether the teleop session is currently running.""" - return self._session is not None + return self._session is not None and not self.session_cleanup_failed + + @property + def session_cleanup_failed(self) -> bool: + """Whether native/session drain failed and further use is prohibited.""" + return self._session_cleanup_error is not None @property def pipeline(self): @@ -475,9 +492,11 @@ def last_left_controller(self): def last_step_result(self) -> dict | None: """Full pipeline output from the most recent :meth:`step`, or ``None``. - Contains at least ``"action"`` and the right-controller entry. When - debug visualization was enabled at session start, :attr:`HAND_LEFT_KEY` - and :attr:`HAND_RIGHT_KEY` are included when a ``HandsSource`` exists. + Contains at least ``"action"`` and the right-controller entry. + :attr:`HAND_LEFT_KEY` and :attr:`HAND_RIGHT_KEY` are included whenever + a ``HandsSource`` exists; a user pipeline may explicitly provide those + outputs after applying a validity policy. Controller-left and visual + updates remain debug-only. """ return self._last_step_result @@ -490,19 +509,11 @@ def has_control_channel(self) -> bool: def last_control_events(self) -> ControlEvents: """Control events from the most recent :meth:`step`. - When a ``teleop_control_pipeline`` is configured, derives - :class:`ControlEvents` from - ``session.last_context.execution_events``. Otherwise returns a - default (no-op) :class:`ControlEvents`. + Reset is an edge-triggered event. In pipelined execution, multiple app + steps can return the same completed retargeting frame, so the event is + latched by returned frame ID and emitted only on its first return. """ - if self._message_processor is None: - return _NO_OP_EVENTS - if self._session is None: - return _NO_OP_EVENTS - ctx = self._session.last_context - if ctx is None: - return _NO_OP_EVENTS - return _execution_events_to_control(ctx.execution_events) + return self._last_control_events def request_reset(self, pause: bool = False) -> None: """Schedule a reset for the next pipeline step. @@ -569,6 +580,8 @@ def start(self) -> None: "Start AR"), session creation is deferred and will be retried on each :meth:`step` call. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error # Measure the workstation before anything expensive starts. Any notice is # queued now and delivered when the client connects. The queue is not # cleared here: send_client_message promises that messages queued before @@ -598,6 +611,8 @@ def start(self) -> None: self._last_right_controller = None self._last_left_controller = None self._last_step_result = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None self._pipeline = self._build_combined_pipeline(user_pipeline) self._pressed_keys.clear() @@ -645,8 +660,12 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: """Shut down the session and clean up resources. Closes the retargeting tuning UI and exits the ``TeleopSession`` - context manager. If the underlying OpenXR session was already torn - down externally (e.g. "Stop AR"), cleanup errors are suppressed. + context manager before stopping CloudXR. Cleanup failures propagate + (or are attached to the caller's original exception). If native drain + fails, latch the error and prohibit reuse or restart. The retained session + reference does not prove native drain; the application must not release + borrowed resources or stop the runtime after that failure. Native owners + classify expected disconnects. Args: exc_type: Exception type (for context manager protocol). @@ -657,20 +676,40 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._pressed_keys.clear() # Close the retargeting tuning UI first + error = exc_val + + def record_failure(cleanup_error: BaseException) -> None: + nonlocal error + if error is None: + error = cleanup_error + else: + error.add_note(f"Isaac Capture cleanup failed: {cleanup_error!r}") + + # A UI failure must not prevent native/session cleanup. if self._retargeting_ui_ctx is not None: - self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) - self._retargeting_ui_ctx = None - self._retargeting_ui = None + try: + self._retargeting_ui_ctx.__exit__(exc_type, exc_val, exc_tb) + except BaseException as cleanup_error: + record_failure(cleanup_error) + else: + self._retargeting_ui_ctx = None + self._retargeting_ui = None if self._session is not None: + if self._session_cleanup_error is not None: + if self._session_cleanup_error is not exc_val: + record_failure(self._session_cleanup_error) + if exc_val is None: + raise error + return try: self._session.__exit__(exc_type, exc_val, exc_tb) - except Exception as e: - # The OpenXR session may have already been torn down externally - # (e.g. user clicked "Stop AR"), so destroying spaces/action - # sets will fail with XR_ERROR_HANDLE_INVALID. This is - # expected and safe to suppress. - logger.debug(f"Suppressed error during Isaac Capture session cleanup: {e}") + except BaseException as cleanup_error: + self._session_cleanup_error = cleanup_error + record_failure(cleanup_error) + if exc_val is None: + raise error + return self._session = None # Always clear pipeline state (session may never have been created if @@ -680,6 +719,8 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._message_processor = None self._last_step_result = None self._last_left_controller = None + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None self._haptic_sink = None self._haptic_tracker = None # Undelivered notices belong to the session that queued them; a later @@ -689,12 +730,14 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: if self._cloudxr_launcher is not None: try: self._cloudxr_launcher.stop() - except RuntimeError: - logger.warning("CloudXR runtime process could not be terminated; handle retained for atexit cleanup") + except RuntimeError as cleanup_error: + record_failure(cleanup_error) else: self._cloudxr_launcher = None logger.info("CloudXR runtime stopped") + if exc_val is None and error is not None: + raise error logger.info("Isaac Capture session ended") # ------------------------------------------------------------------ @@ -728,9 +771,9 @@ def _build_combined_pipeline(self, user_pipeline): # the haptic sink, avoiding a second controller action-set attachment. self._button_controllers = ControllersSource("_button_controllers") pipeline_outputs[self._CONTROLLER_RIGHT_KEY] = self._button_controllers.output(ControllersSource.RIGHT) + self._chain_hand_presence_outputs(user_pipeline, pipeline_outputs) if self._enable_debug_visualization: pipeline_outputs[self._CONTROLLER_LEFT_KEY] = self._button_controllers.output(ControllersSource.LEFT) - self._chain_hand_debug_outputs(user_pipeline, pipeline_outputs) return OutputCombiner(pipeline_outputs) # ------------------------------------------------------------------ @@ -794,15 +837,13 @@ def drain_pressed_keys(self) -> list[int]: return pressed @staticmethod - def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: - """Auto-discover a ``HandsSource`` and chain its raw outputs. + def _chain_hand_presence_outputs(user_pipeline, pipeline_outputs: dict) -> None: + """Chain explicit filtered hand outputs or auto-discover raw source outputs. - Walks the leaf nodes of *user_pipeline*; if a ``HandsSource`` is - found, its raw (XR-anchor-frame) ``hand_left`` / ``hand_right`` - outputs are added to *pipeline_outputs* so they appear in the step - result. The anchor-to-world transform is applied on the Isaac Lab - side by the visualizer, so the markers stay world-frame correct even - when a ``target_T_world`` rebase is active in the pipeline. + A user pipeline may expose its own ``hand_left`` / ``hand_right`` + outputs when validity or another application-neutral policy must be + applied before session safety observes presence. Otherwise walk the + leaf nodes and retain the legacy raw ``HandsSource`` behavior. Args: user_pipeline: The pipeline returned by ``pipeline_builder()``. @@ -811,15 +852,24 @@ def _chain_hand_debug_outputs(user_pipeline, pipeline_outputs: dict) -> None: """ from isaacteleop.retargeting_engine.deviceio_source_nodes import HandsSource + output_types_fn = getattr(user_pipeline, "output_types", None) + output_types = output_types_fn() if callable(output_types_fn) else {} + hand_keys = (TeleopSessionLifecycle.HAND_LEFT_KEY, TeleopSessionLifecycle.HAND_RIGHT_KEY) + if all(key in output_types for key in hand_keys): + for key in hand_keys: + pipeline_outputs[key] = user_pipeline.output(key) + logger.debug("Chained explicit pipeline hand-presence outputs") + return + # HandsSource owns its HandTracker; the tracker API has no reverse # source-node lookup, so use the graph's public leaf discovery API. for leaf in user_pipeline.get_leaf_nodes(): if isinstance(leaf, HandsSource): pipeline_outputs[TeleopSessionLifecycle.HAND_LEFT_KEY] = leaf.output(HandsSource.LEFT) pipeline_outputs[TeleopSessionLifecycle.HAND_RIGHT_KEY] = leaf.output(HandsSource.RIGHT) - logger.debug(f"Auto-discovered HandsSource '{leaf.name}'; chained hand debug outputs") + logger.debug(f"Auto-discovered HandsSource '{leaf.name}'; chained raw hand-presence outputs") return - logger.info("No HandsSource found in the teleop pipeline; hand joint visualization unavailable") + logger.info("No HandsSource found in the teleop pipeline; raw hand presence unavailable") # ------------------------------------------------------------------ # Control pipeline construction @@ -1066,6 +1116,10 @@ def _on_request_required_extensions(self) -> list[str]: logger.info(f"Required extensions: {required_extensions}") return required_extensions + def required_openxr_extensions(self) -> tuple[str, ...]: + """Return extensions required by the fully built teleop graphs.""" + return tuple(self._on_request_required_extensions()) + def _on_xr_enabled_changed(self, item, event_type): import carb.settings @@ -1137,6 +1191,8 @@ def _try_start_session(self) -> bool: ``True`` if the session was successfully started (or was already running), ``False`` if session creation was deferred. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error if self._session is not None: return True @@ -1166,6 +1222,13 @@ def _try_start_session(self) -> bool: ) self._session_start_deferred_logged = True return False + elif self._external_oxr_handles_provider is not None: + oxr_handles = self._external_oxr_handles_provider() + if oxr_handles is None: + if not self._session_start_deferred_logged: + logger.info("External OpenXR handles are not ready; IsaacTeleop session creation deferred") + self._session_start_deferred_logged = True + return False else: # Standalone mode (e.g. no ``--xr``): do not touch Kit's XR bridge. # Passing ``oxr_handles=None`` makes ``TeleopSession`` create and own @@ -1196,8 +1259,9 @@ def _try_start_session(self) -> bool: ) # Create and enter the TeleopSession - self._session = TeleopSession(session_config) - self._session.__enter__() + session = TeleopSession(session_config) + self._enter_session(session) + self._session = session if self._mcap_record_path is not None: logger.info(f"Isaac Capture session started: {self._cfg.app_name} (recording to {self._mcap_record_path})") @@ -1246,12 +1310,25 @@ def _start_replay_session(self) -> bool: sinks=[], ) - self._session = TeleopSession(session_config) - self._session.__enter__() + session = TeleopSession(session_config) + self._enter_session(session) + self._session = session logger.info(f"Isaac Capture replay session started: {self._cfg.app_name} (replaying {self._mcap_replay_path})") return True + def _enter_session(self, session: TeleopSession) -> None: + """Latch Capture's explicit failed-rollback signal before releasing borrowed owners.""" + try: + session.__enter__() + except BaseException as error: + # Capture also rolls back interrupts. Preserve its failed-drain + # signal without converting the original interrupt into an error. + if getattr(session, "cleanup_failed", False) is True: + self._session = session + self._session_cleanup_error = error + raise + # ------------------------------------------------------------------ # Stepping # ------------------------------------------------------------------ @@ -1293,7 +1370,11 @@ def step( Raises: RuntimeError: If called before :meth:`start`. + Exception: If an earlier session cleanup failed, or replay stepping + fails. A failed cleanup prohibits further use and restart. """ + if self._session_cleanup_error is not None: + raise self._session_cleanup_error if self._pipeline is None: raise RuntimeError("TeleopSessionLifecycle.start() must be called before step()") @@ -1320,7 +1401,6 @@ def step( from isaacteleop.retargeting_engine.interface.execution_events import ExecutionEvents, ExecutionState execution_events = ExecutionEvents(reset=True, execution_state=ExecutionState.RUNNING) - self._pending_reset = False # Execute one step of the teleop session. # If the underlying OpenXR session was destroyed externally (e.g. @@ -1337,18 +1417,32 @@ def step( # The async retarget worker dies permanently on any pipeline # exception, so session re-entry is the only recovery either way; # what differs is the cause and the restart pacing. - if not self._is_replay and self._kit_xr_session_is_active(): + if not self._is_replay and ( + self._external_oxr_handles_provider is not None or self._kit_xr_session_is_active() + ): logger.warning( - "Isaac Capture retargeting step failed (pipeline error, XR session still active); " + "IsaacTeleop retargeting step failed (owned/existing XR session); " f"restarting the teleop session in {self._STEP_FAILURE_RESTART_COOLDOWN_S:.0f}s: {e}" ) self._restart_holdoff_until = time.monotonic() + self._STEP_FAILURE_RESTART_COOLDOWN_S else: logger.warning(f"Isaac Capture session step failed (XR session likely torn down): {e}") - self._teardown_dead_session() + try: + self._teardown_dead_session() + except BaseException as cleanup_error: + # Failed drain retains the owner; never restart over live resources. + self._session_cleanup_error = e + e.add_note(f"Teleop session drain also failed: {cleanup_error!r}") + raise e from cleanup_error + if self._is_replay: + # Re-entering replay would silently rewind a failed recording. + raise return None + if execution_events is not None: + self._pending_reset = False self._last_step_result = result + self._update_control_events() # Store the controller TensorGroups for button polling / debug viz self._last_right_controller = result.get(self._CONTROLLER_RIGHT_KEY) @@ -1363,6 +1457,21 @@ def step( return action + def _update_control_events(self) -> None: + """Cache the control snapshot associated with the returned retargeting frame.""" + if self._message_processor is None or self._session is None or self._session.last_context is None: + self._last_control_events = _NO_OP_EVENTS + return + + events = _execution_events_to_control(self._session.last_context.execution_events) + step_info = getattr(self._session, "last_step_info", None) + frame_id = getattr(step_info, "returned_frame_id", None) + if frame_id is not None and frame_id == self._last_control_event_frame_id: + events = ControlEvents(is_active=events.is_active, should_reset=False) + else: + self._last_control_event_frame_id = frame_id + self._last_control_events = events + # ------------------------------------------------------------------ # Dead session teardown # ------------------------------------------------------------------ @@ -1372,20 +1481,24 @@ def _teardown_dead_session(self) -> None: This is called when :meth:`step` detects that the XR session was destroyed externally (e.g. user clicked "Stop AR"). The - ``TeleopSession`` is exited with error suppression (since its XR - resources are already gone), and the internal state is reset so that - the deferred-start logic in :meth:`step` can re-acquire handles if - the user restarts AR. + ``TeleopSession`` classifies expected native disconnects. Failed drain + propagates and latches a terminal error; only successful drain resets + state so deferred start can re-acquire handles when the user restarts AR. """ if self._session is not None: try: self._session.__exit__(None, None, None) - except Exception as e: - logger.debug(f"Suppressed error tearing down dead session: {e}") + except BaseException as error: + # Exit may already be terminal. Never infer successful drain from + # a later no-op __exit__, or resume/restart after this failure. + self._session_cleanup_error = error + raise self._session = None self._session_start_deferred_logged = False self._last_step_result = None - logger.info("Isaac Capture session torn down after external XR shutdown") + self._last_control_events = _NO_OP_EVENTS + self._last_control_event_frame_id = None + logger.info("IsaacTeleop session torn down after external XR shutdown") # ------------------------------------------------------------------ # External input building diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py index 39c4f380e35..328c9aab8cb 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py @@ -9,29 +9,46 @@ import contextlib import logging +import threading import numpy as np from scipy.spatial.transform import Rotation -from .xr_anchor_utils import XrAnchorSynchronizer +from .xr_anchor_utils import XrAnchorSynchronizer, _relative_anchor_transform, _XrAnchorPosePolicy from .xr_cfg import XrCfg # Import Kit components with fallback for sessions without Kit -carb = None -with contextlib.suppress(ModuleNotFoundError): - import carb - XRCore = None XRCoreEventType = None with contextlib.suppress(ModuleNotFoundError): from omni.kit.xr.core import XRCore, XRCoreEventType from isaaclab.sim.utils.prims import create_prim as _create_prim +from isaaclab.sim.utils.stage import get_current_stage +from isaaclab.sim.utils.transforms import convert_world_pose_to_local, standardize_xform_ops from isaaclab.utils.version import has_kit logger = logging.getLogger(__name__) +def _create_or_update_xr_anchor( + prim_path: str, position: tuple[float, float, float], orientation: tuple[float, float, float, float] +) -> None: + """Apply the configured world pose to a stage-scoped XR anchor.""" + if not _xr_anchor_prim_exists(prim_path): + _create_prim( + prim_path, + prim_type="Xform", + position=np.asarray(position, dtype=np.float64), + orientation=np.asarray(orientation, dtype=np.float64), + ) + return + prim = get_current_stage().GetPrimAtPath(prim_path) + translation, rotation = convert_world_pose_to_local(tuple(position), tuple(orientation), prim.GetParent()) + if not standardize_xform_ops(prim, translation, rotation): + raise ValueError(f"XR anchor {prim_path!r} is not transformable") + + def _xr_anchor_prim_exists(prim_path: str) -> bool: """Return True when ``prim_path`` is already a valid prim on the active stage. @@ -39,16 +56,11 @@ def _xr_anchor_prim_exists(prim_path: str) -> bool: :class:`XrAnchorManager` construction in multi-replay batches (the replay agent rebuilds the device per run, but the prim is stage-scoped). Best-effort: returns ``False`` if the stage cannot - be inspected (e.g. unit tests without omni.usd) so the caller falls + be inspected (e.g. before stage creation) so the caller falls through to the create-and-warn path used historically. """ try: - import omni.usd - - context = omni.usd.get_context() - if context is None: - return False - stage = context.get_stage() + stage = get_current_stage() if stage is None: return False return stage.GetPrimAtPath(prim_path).IsValid() @@ -80,19 +92,36 @@ class XrAnchorManager: dtype=np.float64, ) - def __init__(self, xr_cfg: XrCfg): + def __init__(self, xr_cfg: XrCfg, *, ovxr_session: bool = False): """Initialize the XR anchor manager. - Creates the anchor prim, configures carb XR settings, and sets up - the optional anchor synchronizer for dynamic anchoring. + Creates or refreshes the stage-scoped anchor's configured world pose, + configures carb XR settings, and sets up the optional anchor + synchronizer for dynamic anchoring. Reusing the Stage across sessions + does not retain a previous session's anchor configuration. Args: xr_cfg: XR configuration specifying anchor position, rotation, and optional dynamic anchoring prim path. + ovxr_session: An application-owned renderer/Stage supplies XR. + This path must not configure or synchronize Kit XRCore. + + Raises: + ValueError: An existing anchor prim is not transformable. Other anchor + authoring errors also propagate for OVXR presentation, so + rendering cannot continue with a pose that differs from tracking. """ self._xr_cfg = xr_cfg - self._xr_core = XRCore.get_singleton() if XRCore is not None else None + self._xr_core = XRCore.get_singleton() if XRCore is not None and not ovxr_session else None self._xr_pre_sync_update_subscription = None + self._external_lock = threading.RLock() + self._external_policy = ( + _XrAnchorPosePolicy(xr_cfg) if ovxr_session and xr_cfg.anchor_prim_path is not None else None + ) + self._external_epoch = 0 + self._external_world_matrix = None + self._external_error = None + self._external_render_local = None # Resolve the headset anchor path if self._xr_cfg.anchor_prim_path is not None: @@ -103,26 +132,55 @@ def __init__(self, xr_cfg: XrCfg): else: self._xr_anchor_headset_path = "/World/XRAnchor" - # Create the XR anchor prim in USD if it does not already exist. - # The check matters for multi-replay batches (the replay agent - # rebuilds ``IsaacTeleopDevice`` -- and therefore this manager -- - # for each run, but the prim is stage-scoped and survives - # per-run device teardown). XrCfg.anchor_rot is xyzw; create_prim - # orientation expects xyzw. - if not _xr_anchor_prim_exists(self._xr_anchor_headset_path): - x, y, z, w = self._xr_cfg.anchor_rot + render_anchor_overridden = ( + self._xr_cfg.render_anchor_pos is not None or self._xr_cfg.render_anchor_rot is not None + ) + if render_anchor_overridden: + if self._xr_cfg.anchor_prim_path is not None: + anchor_path = self._xr_cfg.anchor_prim_path.rstrip("/") + self._xr_render_anchor_path = f"{anchor_path}/XRRenderAnchor" + else: + self._xr_render_anchor_path = "/World/XRRenderAnchor" + else: + self._xr_render_anchor_path = self._xr_anchor_headset_path + + # Anchors survive per-run device teardown. Reuse their paths while + # applying this run's configuration, rather than retaining a stale pose. + try: + _create_or_update_xr_anchor(self._xr_anchor_headset_path, self._xr_cfg.anchor_pos, self._xr_cfg.anchor_rot) + except Exception as e: + if ovxr_session: + raise + logger.warning(f"Failed to create or update XR anchor prim: {e}") + + if render_anchor_overridden: + render_pos = ( + self._xr_cfg.anchor_pos if self._xr_cfg.render_anchor_pos is None else self._xr_cfg.render_anchor_pos + ) + render_rot = ( + self._xr_cfg.anchor_rot if self._xr_cfg.render_anchor_rot is None else self._xr_cfg.render_anchor_rot + ) try: - pos = np.asarray(self._xr_cfg.anchor_pos, dtype=np.float64) - quat_xyzw = np.asarray([x, y, z, w], dtype=np.float64) - _create_prim(self._xr_anchor_headset_path, prim_type="Xform", position=pos, orientation=quat_xyzw) + _create_or_update_xr_anchor(self._xr_render_anchor_path, render_pos, render_rot) except Exception as e: - logger.warning(f"Failed to create XR anchor prim: {e}") + if ovxr_session: + raise + logger.warning(f"Failed to create or update XR render anchor prim: {e}") + + if self._external_policy is not None and render_anchor_overridden: + self._external_render_local = _relative_anchor_transform( + self._xr_anchor_headset_path, self._xr_render_anchor_path + ) - # Configure carb settings for XR rendering; carb can import while Kit is not running - if carb is not None and has_kit(): - carb.settings.get_settings().set_float("/persistent/xr/render/nearPlane", self._xr_cfg.near_plane) - carb.settings.get_settings().set_string("/persistent/xr/anchorMode", "custom anchor") - carb.settings.get_settings().set_string("/xrstage/customAnchor", self._xr_anchor_headset_path) + # Configure settings only when Kit owns a live XRCore. Kitless external- + # present sessions retain this anchor but have no carb settings interface. + if self._xr_core is not None and has_kit(): + import carb.settings # noqa: PLC0415 + + settings = carb.settings.get_settings() + settings.set_float("/persistent/xr/render/nearPlane", self._xr_cfg.near_plane) + settings.set_string("/persistent/xr/anchorMode", "custom anchor") + settings.set_string("/xrstage/customAnchor", self._xr_render_anchor_path) self._anchor_sync: XrAnchorSynchronizer | None = None if self._xr_core is not None: @@ -131,6 +189,7 @@ def __init__(self, xr_cfg: XrCfg): xr_core=self._xr_core, xr_cfg=self._xr_cfg, xr_anchor_headset_path=self._xr_anchor_headset_path, + xr_render_anchor_path=self._xr_render_anchor_path, ) # Subscribe to pre_sync_update to keep anchor in sync each frame. # Capture the synchronizer in a local to satisfy type narrowing. @@ -157,6 +216,11 @@ def anchor_headset_path(self) -> str: """The USD path of the XR anchor prim.""" return self._xr_anchor_headset_path + @property + def render_anchor_path(self) -> str: + """The USD path consumed by XR rendering.""" + return self._xr_render_anchor_path + def get_world_matrix(self) -> np.ndarray: """Build the combined 4x4 transform from OpenXR local space to Isaac Lab world. @@ -180,19 +244,32 @@ def get_world_matrix(self) -> np.ndarray: core via ``set_world_transform_matrix`` during the most recent ``pre_sync_update``. This works for both dynamic anchoring (``anchor_prim_path`` set) and static anchoring. - * Falls back to raw ``anchor_pos`` / ``anchor_rot`` config values - only when the XR core is unavailable (e.g. unit tests). + * External dynamic sessions read the cached pose sealed by their + frame owner, using application-published physics references. + Unseeded or failed reset epochs raise instead of returning a static pose. + * Static and standalone sessions without Kit retain configured poses. Returns: A (4, 4) float32 numpy array. """ + if self._external_policy is not None: + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + if self._external_world_matrix is None: + raise RuntimeError("Dynamic XR anchor requires a committed post-reset scene pose") + matrix = self._external_world_matrix.copy() + matrix[:3, :3] = matrix[:3, :3] @ self._OXR_TO_USD_ROTATION + return matrix.astype(np.float32) if self._anchor_sync is not None: xform = self._anchor_sync.get_world_transform() if xform is not None: pos, quat_xyzw = xform return self._build_matrix(pos, quat_xyzw) - # Fallback when XR core is unavailable (e.g. unit tests). + # Static and standalone sessions retain their configured-pose fallback. return self._build_matrix(self._xr_cfg.anchor_pos, self._xr_cfg.anchor_rot) def _build_matrix(self, pos, quat_xyzw) -> np.ndarray: @@ -222,14 +299,69 @@ def _build_matrix(self, pos, quat_xyzw) -> np.ndarray: def reset(self) -> None: """Reset the anchor synchronizer state.""" + if self._external_policy is not None: + with self._external_lock: + self._external_epoch += 1 + self._external_policy.reset() + self._external_world_matrix = None if self._anchor_sync is not None: self._anchor_sync.reset() def toggle_anchor_rotation(self) -> None: """Toggle anchor rotation following on the synchronizer.""" + if self._external_policy is not None: + with self._external_lock: + self._external_policy.toggle_anchor_rotation() if self._anchor_sync is not None: self._anchor_sync.toggle_anchor_rotation() + @property + def external_reference_path(self) -> str | None: + """The exact physics reference path, when external dynamic anchoring is active.""" + return self._xr_cfg.anchor_prim_path if self._external_policy is not None else None + + @property + def external_epoch(self) -> tuple[int, bool]: + """Return reset identity and whether the current epoch needs its first publication.""" + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + return self._external_epoch, self._external_world_matrix is None + + @property + def external_anchor_paths(self) -> tuple[str, ...]: + """Native Xforms written together by the OVXR presentation host.""" + return tuple(dict.fromkeys((self._xr_anchor_headset_path, self._xr_render_anchor_path))) + + def commit_external_pose(self, reference, rendering_dt: float, epoch: int, commit) -> bool: + """Advance the shared policy once and expose its input pose only after native seal. + + The host serializes this operation with scene publication. Old committed + references cannot reseed a new reset epoch. A failed write invalidates the + cached input pose and propagates; it never falls back to configured static data. + """ + with self._external_lock: + if self._external_error is not None: + raise RuntimeError( + "Dynamic XR anchor publication failed; recreate the session" + ) from self._external_error + if epoch != self._external_epoch: + return False + try: + matrix = np.asarray(self._external_policy.update(reference, rendering_dt)).T.copy() + matrices = [matrix] + if self._external_render_local is not None: + matrices.append(matrix @ self._external_render_local) + commit(np.stack(matrices)) + except BaseException as exc: + self._external_world_matrix = None + self._external_error = exc + raise + self._external_world_matrix = matrix + return True + def cleanup(self) -> None: """Release event subscriptions.""" self._xr_pre_sync_update_subscription = None diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py index 6e6671a68e8..a9ac5c47932 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py @@ -10,6 +10,7 @@ import contextlib import logging import math +from collections.abc import Callable from typing import Any import numpy as np @@ -23,35 +24,159 @@ with contextlib.suppress(ModuleNotFoundError): import usdrt - from pxr import Gf as pxrGf from usdrt import Rt -class XrAnchorSynchronizer: - """Keeps the XR anchor prim aligned with a reference prim according to XR config.""" +class _XrAnchorPosePolicy: + """Own the shared reference-relative anchor policy, without Kit or Stage I/O.""" - def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): - self._xr_core = xr_core + def __init__(self, xr_cfg): self._xr_cfg = xr_cfg - self._xr_anchor_headset_path = xr_anchor_headset_path + self.reset() + def reset(self): self.__anchor_prim_initial_quat = None self.__anchor_prim_initial_height = None self.__smoothed_anchor_quat = None self.__last_anchor_quat = None self.__anchor_rotation_enabled = True + self.__cached_world_pos = None + self.__cached_world_quat_xyzw = None + + def toggle_anchor_rotation(self): + self.__anchor_rotation_enabled = not self.__anchor_rotation_enabled + logger.info("XR: Toggling anchor rotation: %s", self.__anchor_rotation_enabled) + + def get_world_transform(self): + if self.__cached_world_pos is None: + return None + return self.__cached_world_pos.copy(), self.__cached_world_quat_xyzw.copy() + + def update(self, world_T_reference, dt: float | Callable[[], float], previous_headpose=None): + # Let the selected runtime initialize USD before loading its native math types. + # An eager standalone USD import can prevent Kit RTX startup. + from pxr import Gf as pxrGf + + if world_T_reference is not None: + rt_matrix = pxrGf.Matrix4d(world_T_reference.T.tolist()) + rt_pos = rt_matrix.ExtractTranslation() + if self.__anchor_prim_initial_quat is None: + self.__anchor_prim_initial_quat = rt_matrix.ExtractRotationQuat() + if self._xr_cfg.fixed_anchor_height: + if self.__anchor_prim_initial_height is None: + self.__anchor_prim_initial_height = rt_pos[2] + rt_pos[2] = self.__anchor_prim_initial_height + pxr_anchor_pos = rt_pos + pxrGf.Vec3d(*self._xr_cfg.anchor_pos) + else: + rt_matrix = None + pxr_anchor_pos = pxrGf.Vec3d(*self._xr_cfg.anchor_pos) + x, y, z, w = self._xr_cfg.anchor_rot + pxr_cfg_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) + + pxr_anchor_quat = pxr_cfg_quat + + if rt_matrix is not None: + if self._xr_cfg.anchor_rotation_mode in ( + XrAnchorRotationMode.FOLLOW_PRIM, + XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, + ): + rt_prim_quat = rt_matrix.ExtractRotationQuat() + rt_delta_quat = rt_prim_quat * self.__anchor_prim_initial_quat.GetInverse() + pxr_delta_quat = pxrGf.Quatd(rt_delta_quat.GetReal(), pxrGf.Vec3d(*rt_delta_quat.GetImaginary())) + + # yaw-only about Z (right-handed, Z-up) + wq = pxr_delta_quat.GetReal() + ix, iy, iz = pxr_delta_quat.GetImaginary() + yaw = math.atan2(2.0 * (wq * iz + ix * iy), 1.0 - 2.0 * (iy * iy + iz * iz)) + cy = math.cos(yaw * 0.5) + sy = math.sin(yaw * 0.5) + pxr_delta_yaw_only_quat = pxrGf.Quatd(cy, pxrGf.Vec3d(0.0, 0.0, sy)) + pxr_anchor_quat = pxr_delta_yaw_only_quat * pxr_cfg_quat + + if self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED: + if self.__smoothed_anchor_quat is None: + self.__smoothed_anchor_quat = pxr_anchor_quat + else: + step_dt = dt() if callable(dt) else dt + alpha = 1.0 - math.exp(-step_dt / max(self._xr_cfg.anchor_rotation_smoothing_time, 1e-6)) + alpha = min(1.0, max(0.05, alpha)) + self.__smoothed_anchor_quat = pxrGf.Slerp(alpha, self.__smoothed_anchor_quat, pxr_anchor_quat) + pxr_anchor_quat = self.__smoothed_anchor_quat + + elif self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.CUSTOM: + if self._xr_cfg.anchor_rotation_custom_func is not None: + rt_prim_quat = rt_matrix.ExtractRotationQuat() + anchor_prim_pose = np.array( + [ + rt_pos[0], + rt_pos[1], + rt_pos[2], + rt_prim_quat.GetImaginary()[0], + rt_prim_quat.GetImaginary()[1], + rt_prim_quat.GetImaginary()[2], + rt_prim_quat.GetReal(), + ], + dtype=np.float64, + ) + prev_head = previous_headpose if previous_headpose is not None else np.zeros(7, dtype=np.float64) + np_array_quat = self._xr_cfg.anchor_rotation_custom_func(prev_head, anchor_prim_pose) + x, y, z, w = np_array_quat + pxr_anchor_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) + + pxr_mat = pxrGf.Matrix4d() + pxr_mat.SetTranslateOnly(pxr_anchor_pos) + + if self.__anchor_rotation_enabled: + pxr_final_quat = pxr_anchor_quat + self.__last_anchor_quat = pxr_anchor_quat + else: + if self.__last_anchor_quat is None: + self.__last_anchor_quat = pxr_anchor_quat + pxr_final_quat = self.__last_anchor_quat + self.__smoothed_anchor_quat = self.__last_anchor_quat - # Cached anchor world transform (pos, quat_xyzw) set by sync_headset_to_anchor(). - # Reading back from the prim hierarchy is unreliable when the anchor is a child of a - # physics-driven prim (e.g. the pelvis) because Fabric computes the hierarchy world - # matrix using the physics-updated parent while the local xform was decomposed against - # the USD-level parent, which can diverge. - self.__cached_world_pos: np.ndarray | None = None - self.__cached_world_quat_xyzw: np.ndarray | None = None + pxr_mat.SetRotateOnly(pxr_final_quat) + self.__cached_world_pos = np.array([pxr_anchor_pos[0], pxr_anchor_pos[1], pxr_anchor_pos[2]], dtype=np.float64) + fq_img = pxr_final_quat.GetImaginary() + self.__cached_world_quat_xyzw = np.array( + [fq_img[0], fq_img[1], fq_img[2], pxr_final_quat.GetReal()], dtype=np.float64 + ) + return pxr_mat + + +def _relative_anchor_transform(tracking_path: str, render_path: str) -> np.ndarray: + """Return the authored tracking-to-render offset in column-vector convention.""" + from pxr import UsdGeom + + stage = get_current_stage() + tracking = np.asarray(UsdGeom.Xformable(stage.GetPrimAtPath(tracking_path)).GetLocalTransformation()).T + rendering = np.asarray(UsdGeom.Xformable(stage.GetPrimAtPath(render_path)).GetLocalTransformation()).T + return np.linalg.solve(tracking, rendering) + + +class XrAnchorSynchronizer: + """Adapt the shared anchor policy to Kit Fabric and XRCore pre-sync events.""" + + def __init__( + self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str, *, xr_render_anchor_path: str | None = None + ): + self._xr_core = xr_core + self._xr_cfg = xr_cfg + self._xr_anchor_headset_path = xr_anchor_headset_path + self._policy = _XrAnchorPosePolicy(xr_cfg) stage = get_current_stage() anchor = stage.GetPrimAtPath(xr_anchor_headset_path) - self.__anchor_headset_layer_identifier = anchor.GetPrimStack()[0].layer.identifier + self._layer_identifier = anchor.GetPrimStack()[0].layer.identifier + self._render_anchor_path = xr_render_anchor_path + self._render_anchor_offset = None + if xr_render_anchor_path is not None and xr_render_anchor_path != xr_anchor_headset_path: + from pxr import Gf + + offset = _relative_anchor_transform(xr_anchor_headset_path, xr_render_anchor_path) + self._render_anchor_offset = Gf.Matrix4d(offset.T.tolist()) + render_anchor = stage.GetPrimAtPath(xr_render_anchor_path) + self._render_layer_identifier = render_anchor.GetPrimStack()[0].layer.identifier self._reference_world_matrix = None if xr_cfg.anchor_prim_path is not None: rt_stage = usdrt.Usd.Stage.Attach(get_current_stage_id()) @@ -61,149 +186,32 @@ def __init__(self, xr_core: Any, xr_cfg: Any, xr_anchor_headset_path: str): self._reference_world_matrix = Rt.Xformable(reference).GetFabricHierarchyWorldMatrixAttr() def reset(self): - self.__anchor_prim_initial_quat = None - self.__anchor_prim_initial_height = None - self.__smoothed_anchor_quat = None - self.__last_anchor_quat = None - self.__anchor_rotation_enabled = True - self.__cached_world_pos = None - self.__cached_world_quat_xyzw = None + self._policy.reset() self.sync_headset_to_anchor() def toggle_anchor_rotation(self): - self.__anchor_rotation_enabled = not self.__anchor_rotation_enabled - logger.info(f"XR: Toggling anchor rotation: {self.__anchor_rotation_enabled}") - - def get_world_transform(self) -> tuple[np.ndarray, np.ndarray] | None: - """Return the anchor world transform. - - Returns the cached world transform that was computed by the most recent - call to :meth:`sync_headset_to_anchor`. Using the cached value avoids - a Fabric/USD layer mismatch: when the XR anchor prim is a child of a - physics-driven prim (e.g. the robot pelvis), reading - ``GetFabricHierarchyWorldMatrixAttr`` would compose the Fabric-side - parent transform (updated by physics) with a local xform that was - decomposed against the USD-side parent (which can lag behind), - producing an incorrect world matrix that drifts as the robot moves. - - Returns: - A ``(position, quat_xyzw)`` tuple of numpy float64 arrays, - or ``None`` if :meth:`sync_headset_to_anchor` has not run yet. - """ - if self.__cached_world_pos is not None and self.__cached_world_quat_xyzw is not None: - return self.__cached_world_pos, self.__cached_world_quat_xyzw - return None + self._policy.toggle_anchor_rotation() + + def get_world_transform(self): + return self._policy.get_world_transform() def sync_headset_to_anchor(self): - """Sync XR anchor pose in USD for both dynamic and static anchoring. - - For **dynamic** anchoring (``anchor_prim_path`` is set), the reference - prim's world position is read from Fabric and ``anchor_pos`` is added - as an offset. For **static** anchoring (no prim path), ``anchor_pos`` - is used directly as the world position. - - In both cases the function calls ``set_world_transform_matrix`` on the - XR core so that the rendering anchor and the pipeline's - ``world_T_anchor`` matrix are guaranteed to agree, and caches the - world transform for :meth:`get_world_transform`. - """ + """Read the Fabric reference and update the Kit anchor once per pre-sync.""" try: + reference = None if self._reference_world_matrix is not None: - rt_matrix = self._reference_world_matrix.Get() - rt_pos = rt_matrix.ExtractTranslation() - - if self.__anchor_prim_initial_quat is None: - self.__anchor_prim_initial_quat = rt_matrix.ExtractRotationQuat() - - if self._xr_cfg.fixed_anchor_height: - if self.__anchor_prim_initial_height is None: - self.__anchor_prim_initial_height = rt_pos[2] - rt_pos[2] = self.__anchor_prim_initial_height - - pxr_anchor_pos = pxrGf.Vec3d(*rt_pos) + pxrGf.Vec3d(*self._xr_cfg.anchor_pos) - else: - rt_matrix = None - pxr_anchor_pos = pxrGf.Vec3d(*self._xr_cfg.anchor_pos) - - x, y, z, w = self._xr_cfg.anchor_rot - pxr_cfg_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) - - pxr_anchor_quat = pxr_cfg_quat - - if rt_matrix is not None: - if self._xr_cfg.anchor_rotation_mode in ( - XrAnchorRotationMode.FOLLOW_PRIM, - XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, - ): - rt_prim_quat = rt_matrix.ExtractRotationQuat() - rt_delta_quat = rt_prim_quat * self.__anchor_prim_initial_quat.GetInverse() - pxr_delta_quat = pxrGf.Quatd(rt_delta_quat.GetReal(), pxrGf.Vec3d(*rt_delta_quat.GetImaginary())) - - # yaw-only about Z (right-handed, Z-up) - wq = pxr_delta_quat.GetReal() - ix, iy, iz = pxr_delta_quat.GetImaginary() - yaw = math.atan2(2.0 * (wq * iz + ix * iy), 1.0 - 2.0 * (iy * iy + iz * iz)) - cy = math.cos(yaw * 0.5) - sy = math.sin(yaw * 0.5) - pxr_delta_yaw_only_quat = pxrGf.Quatd(cy, pxrGf.Vec3d(0.0, 0.0, sy)) - pxr_anchor_quat = pxr_delta_yaw_only_quat * pxr_cfg_quat - - if self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED: - if self.__smoothed_anchor_quat is None: - self.__smoothed_anchor_quat = pxr_anchor_quat - else: - dt = SimulationContext.instance().get_rendering_dt() - alpha = 1.0 - math.exp(-dt / max(self._xr_cfg.anchor_rotation_smoothing_time, 1e-6)) - alpha = min(1.0, max(0.05, alpha)) - self.__smoothed_anchor_quat = pxrGf.Slerp( - alpha, self.__smoothed_anchor_quat, pxr_anchor_quat - ) - pxr_anchor_quat = self.__smoothed_anchor_quat - - elif self._xr_cfg.anchor_rotation_mode == XrAnchorRotationMode.CUSTOM: - if self._xr_cfg.anchor_rotation_custom_func is not None: - rt_prim_quat = rt_matrix.ExtractRotationQuat() - anchor_prim_pose = np.array( - [ - rt_pos[0], - rt_pos[1], - rt_pos[2], - rt_prim_quat.GetImaginary()[0], - rt_prim_quat.GetImaginary()[1], - rt_prim_quat.GetImaginary()[2], - rt_prim_quat.GetReal(), - ], - dtype=np.float64, - ) - prev_head = getattr(self, "_previous_headpose", np.zeros(7, dtype=np.float64)) - np_array_quat = self._xr_cfg.anchor_rotation_custom_func(prev_head, anchor_prim_pose) - x, y, z, w = np_array_quat - pxr_anchor_quat = pxrGf.Quatd(w, pxrGf.Vec3d(x, y, z)) - - pxr_mat = pxrGf.Matrix4d() - pxr_mat.SetTranslateOnly(pxr_anchor_pos) - - if self.__anchor_rotation_enabled: - pxr_final_quat = pxr_anchor_quat - self.__last_anchor_quat = pxr_anchor_quat - else: - if self.__last_anchor_quat is None: - self.__last_anchor_quat = pxr_anchor_quat - pxr_final_quat = self.__last_anchor_quat - self.__smoothed_anchor_quat = self.__last_anchor_quat - - pxr_mat.SetRotateOnly(pxr_final_quat) - - self.__cached_world_pos = np.array( - [pxr_anchor_pos[0], pxr_anchor_pos[1], pxr_anchor_pos[2]], dtype=np.float64 - ) - fq_img = pxr_final_quat.GetImaginary() - self.__cached_world_quat_xyzw = np.array( - [fq_img[0], fq_img[1], fq_img[2], pxr_final_quat.GetReal()], dtype=np.float64 - ) - - self._xr_core.set_world_transform_matrix( - self._xr_anchor_headset_path, pxr_mat, self.__anchor_headset_layer_identifier + matrix = self._reference_world_matrix.Get() + reference = np.array([[matrix[i][j] for j in range(4)] for i in range(4)]).T + matrix = self._policy.update( + reference, + lambda: SimulationContext.instance().get_rendering_dt(), + getattr(self, "_previous_headpose", None), ) - except Exception as e: - logger.warning(f"XR: Anchor sync failed: {e}") + self._xr_core.set_world_transform_matrix(self._xr_anchor_headset_path, matrix, self._layer_identifier) + if self._render_anchor_offset is not None: + # Gf uses row vectors: apply the view offset before the filtered world pose. + self._xr_core.set_world_transform_matrix( + self._render_anchor_path, self._render_anchor_offset * matrix, self._render_layer_identifier + ) + except Exception as exc: + logger.warning("XR: Anchor sync failed: %s", exc) diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py index 7ec58e4c1ca..f3dd7fceb16 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py @@ -51,6 +51,33 @@ class XrCfg: This quantity is only effective if :attr:`xr_anchor_pos` is set. """ + render_anchor_pos: tuple[float, float, float] | None = None + """Optional position (in m) used only to frame the XR view. + + When ``None``, rendering and tracked-device poses share :attr:`anchor_pos`, + which preserves the historical behavior. Set this when the operator view + needs a different origin from the world frame used for teleoperation. + With dynamic OVXR anchoring, this configured offset follows the filtered + tracking anchor, preserving its height and rotation policies. + """ + + render_anchor_rot: tuple[float, float, float, float] | None = None + """Optional render-only rotation as a quaternion in ``xyzw`` order. + + When ``None``, the render anchor uses :attr:`anchor_rot`. This field is + effective only when either render-anchor override is configured. + """ + + hidden_visual_prims: list[str] = [] + """Prim-path regex patterns hidden from the kitless OVXR view. + + Tasks can use this to hide robot geometry that encloses the eye cameras. + The OVXR session authors visibility on the live stage before + renderer materialization and fails if a configured pattern matches no + imageable prim. Visibility persists for the application Stage's lifetime + and affects all cameras, including PiP; it is not restored on XR close. + """ + anchor_prim_path: str | None = None """Specifies the prim path to attach the XR anchor to for dynamic positioning. @@ -60,6 +87,10 @@ class XrCfg: :func:`~isaaclab_teleop.create_isaac_teleop_device` expands ``{ENV_REGEX_NS}`` when given an ``env_prim_path``; direct device construction requires a concrete path. + Kit sessions use XRCore to follow the reference. Application-owned OVXR presentation + uses committed physics references for the same rotation and height policies. Input cannot + consume an unseeded or failed publication epoch; no static pose is substituted. + If None, the anchor will use the static :attr:`anchor_pos` and :attr:`anchor_rot` values. """ diff --git a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py index 15513137ab3..16f4dd7deb8 100644 --- a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py +++ b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py @@ -151,6 +151,30 @@ def _make_lifecycle( ) +@pytest.mark.parametrize("entry", ["lifecycle", "device", "factory"]) +def test_positional_recording_options_preserve_public_constructor_contract(entry): + """Existing positional MCAP/debug/haptic arguments must not become XR handles. + + Exercise all three public argument-binding boundaries without opening an XR + session. Native recording/playback correctness has a separate owner. + """ + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device + + cfg = _make_cfg() + if entry == "lifecycle": + lifecycle = TeleopSessionLifecycle(cfg, None, False, False, "capture.mcap", None, True, None) + elif entry == "device": + device = IsaacTeleopDevice(cfg, None, False, False, "capture.mcap", None, True, None) + lifecycle = device._session_lifecycle + else: + device = create_isaac_teleop_device(cfg, "cpu", None, None, False, False, "capture.mcap", None, True, None) + lifecycle = device._session_lifecycle + assert lifecycle._mcap_record_path == "capture.mcap" + assert lifecycle._mcap_replay_path is None + assert lifecycle._enable_debug_visualization is True + assert lifecycle._external_oxr_handles_provider is None + + # ============================================================================ # Shipped .env profile paths # ============================================================================ @@ -209,6 +233,172 @@ def test_session_config_receives_deadline_paced_pipelined_retargeting(self): assert resolved_execution.pacing.safety_margin_s == 0.025 +class TestExternalOpenXRHandles: + """Tests for borrowing OVXR-owned OpenXR handles without the Kit bridge.""" + + def test_provider_handles_are_passed_to_teleop_session(self): + handles = object() + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=lambda: handles, + ) + lifecycle._pipeline = MagicMock() + lifecycle._teleop_control_pipeline = None + session_config_cls = MagicMock(return_value=MagicMock()) + session_cls = MagicMock() + fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"] + + with ( + patch.object(fake_tsm_module, "TeleopSessionConfig", session_config_cls), + patch.object(fake_tsm_module, "TeleopSession", session_cls), + ): + assert lifecycle.try_start_session() is True + + assert session_config_cls.call_args.kwargs["oxr_handles"] is handles + + def test_provider_none_defers_session_creation(self): + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=lambda: None, + ) + lifecycle._pipeline = MagicMock() + lifecycle._teleop_control_pipeline = None + + assert lifecycle.try_start_session() is False + assert lifecycle.is_active is False + + +@pytest.mark.parametrize("replay", [False, True]) +@pytest.mark.parametrize("rollback_fails", [False, True]) +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_failed_session_entry_is_not_published_as_active(tmp_path, monkeypatch, replay, rollback_fails, error_type): + recording = tmp_path / "recording.mcap" + recording.touch() + replay_module = ModuleType("isaacteleop.deviceio_session") + replay_module.McapReplayConfig = MagicMock() + monkeypatch.setitem(sys.modules, replay_module.__name__, replay_module) + lifecycle = TeleopSessionLifecycle( + _make_cfg(), + use_kit_xr_bridge=False, + mcap_replay_path=str(recording) if replay else None, + ) + lifecycle._pipeline = MagicMock() + startup_error = error_type("session entry failed") + session_cls = MagicMock() + session_cls.return_value.cleanup_failed = rollback_fails + session_cls.return_value.__enter__.side_effect = startup_error + fake_tsm_module = sys.modules["isaacteleop.teleop_session_manager"] + with patch.object(fake_tsm_module, "TeleopSession", session_cls): + with pytest.raises(error_type) as caught: + lifecycle.try_start_session() + assert caught.value is startup_error + assert not lifecycle.is_active + if rollback_fails: + assert lifecycle._session is session_cls.return_value + assert lifecycle.session_cleanup_failed + for operation in (lifecycle.try_start_session, lifecycle.step, lifecycle.stop): + with pytest.raises(error_type) as repeated: + operation() + assert repeated.value is startup_error + assert session_cls.call_count == 1 + session_cls.return_value.__exit__.assert_not_called() + else: + assert lifecycle._session is None + session_cls.return_value.__enter__.side_effect = None + assert lifecycle.try_start_session() + assert session_cls.call_count == 2 + assert lifecycle._session is session_cls.return_value + + +def test_device_failed_entry_stops_partial_session(): + """Python does not call __exit__ when __enter__ fails after runtime launch.""" + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + + startup_error = RuntimeError("pipeline construction failed") + cfg = _make_cfg() + cfg.pipeline_builder = MagicMock(side_effect=startup_error) + device = IsaacTeleopDevice(cfg, cloudxr_env_file="test.env", use_kit_xr_bridge=False) + launcher_class = MagicMock() + launcher = launcher_class.return_value + with ( + patch.object(device._session_lifecycle, "_run_system_check"), + patch.object(sys.modules["isaacteleop.cloudxr"], "CloudXRLauncher", launcher_class), + patch.dict(os.environ, {"ISAACLAB_CXR_SKIP_AUTOLAUNCH": "0"}), + pytest.raises(RuntimeError) as caught, + ): + with device: + pytest.fail("entry must fail") + assert caught.value is startup_error + launcher.stop.assert_called_once() + assert device._session_lifecycle._cloudxr_launcher is None + + +@pytest.mark.parametrize("body_error", [False, True]) +def test_device_anchor_cleanup_does_not_skip_session_stop(body_error): + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + + device = IsaacTeleopDevice(_make_cfg(), use_kit_xr_bridge=False) + cleanup_error = RuntimeError("anchor subscription cleanup failed") + primary = ValueError("body failed") if body_error else None + device._anchor_manager.cleanup = MagicMock(side_effect=cleanup_error) + device._session_lifecycle.stop = MagicMock() + try: + if primary is None: + with pytest.raises(RuntimeError) as caught: + device.__exit__(None, None, None) + assert caught.value is cleanup_error + else: + assert device.__exit__(type(primary), primary, None) is False + assert any("anchor subscription" in note for note in primary.__notes__) + device._session_lifecycle.stop.assert_called_once() + finally: + device._anchor_manager.cleanup = MagicMock() + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_ui_cleanup_failure_still_drains_session_and_runtime(error_type): + lifecycle = _make_lifecycle() + cleanup_error = error_type("UI cleanup failed") + lifecycle._retargeting_ui_ctx = MagicMock() + lifecycle._retargeting_ui_ctx.__exit__.side_effect = cleanup_error + session, launcher = MagicMock(), MagicMock() + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + with pytest.raises(error_type) as caught: + lifecycle.stop() + assert caught.value is cleanup_error + session.__exit__.assert_called_once() + launcher.stop.assert_called_once() + assert lifecycle._session is None + assert lifecycle._cloudxr_launcher is None + + +@pytest.mark.parametrize("body_error", [False, True]) +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_session_cleanup_failure_is_visible_and_retains_runtime(body_error, error_type): + lifecycle = _make_lifecycle() + session, launcher = MagicMock(), MagicMock() + cleanup_error = error_type("native drain failed") + primary = ValueError("body failed") if body_error else None + session.__exit__.side_effect = cleanup_error + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + if primary is None: + with pytest.raises(error_type) as caught: + lifecycle.stop() + assert caught.value is cleanup_error + else: + try: + lifecycle.stop(type(primary), primary, None) + except BaseException as error: + pytest.fail(f"Cleanup replaced the original body failure with {error!r}") + assert any("native drain failed" in note for note in primary.__notes__) + assert lifecycle._session is session + assert lifecycle.session_cleanup_failed + assert lifecycle._cloudxr_launcher is launcher + launcher.stop.assert_not_called() + + # ============================================================================ # _ensure_cloudxr_runtime # ============================================================================ @@ -382,7 +572,8 @@ def test_stop_retains_launcher_on_runtime_error(self): lifecycle, mock_launcher = self._make_started_lifecycle() mock_launcher.stop.side_effect = RuntimeError("process not found") - lifecycle.stop() + with pytest.raises(RuntimeError, match="process not found"): + lifecycle.stop() assert lifecycle._cloudxr_launcher is mock_launcher diff --git a/source/isaaclab_teleop/test/test_control_events.py b/source/isaaclab_teleop/test/test_control_events.py index dafe7a2ac18..1eb331bc824 100644 --- a/source/isaaclab_teleop/test/test_control_events.py +++ b/source/isaaclab_teleop/test/test_control_events.py @@ -18,83 +18,17 @@ import dataclasses import json -import sys -from types import ModuleType +from types import SimpleNamespace from unittest.mock import MagicMock import pytest -# --------------------------------------------------------------------------- -# Stub out isaacteleop modules before any isaaclab_teleop imports so the -# tests can run in a plain Python environment without Omniverse. -# --------------------------------------------------------------------------- - -_MODULES_TO_STUB = [ - "isaacteleop", - "isaacteleop.deviceio", - "isaacteleop.deviceio_trackers", - "isaacteleop.retargeting_engine", - "isaacteleop.retargeting_engine.deviceio_source_nodes", - "isaacteleop.retargeting_engine.deviceio_source_nodes.deviceio_tensor_types", - "isaacteleop.retargeting_engine.interface", - "isaacteleop.retargeting_engine.interface.retargeter_core_types", - "isaacteleop.retargeting_engine.interface.tensor_group_type", - "isaacteleop.retargeting_engine_ui", - "isaacteleop.schema", - "isaacteleop.teleop_session_manager", - "isaacteleop.teleop_session_manager.teleop_state_manager_retargeter", - "isaacteleop.teleop_session_manager.teleop_state_manager_types", -] - -_stubs: dict[str, ModuleType | MagicMock] = {} - - -def _install_stubs(): - for name in _MODULES_TO_STUB: - if name not in sys.modules: - _stubs[name] = MagicMock() - sys.modules[name] = _stubs[name] - - from enum import Enum - - class ExecutionState(str, Enum): - UNKNOWN = "unknown" - STOPPED = "stopped" - PAUSED = "paused" - RUNNING = "running" - - @dataclasses.dataclass - class ExecutionEvents: - reset: bool = False - execution_state: ExecutionState = ExecutionState.UNKNOWN - - ee_mod = sys.modules["isaacteleop.retargeting_engine.interface.execution_events"] = ModuleType( - "isaacteleop.retargeting_engine.interface.execution_events" - ) - ee_mod.ExecutionState = ExecutionState # type: ignore[attr-defined] - ee_mod.ExecutionEvents = ExecutionEvents # type: ignore[attr-defined] - - iface = sys.modules["isaacteleop.retargeting_engine.interface"] - iface.ExecutionState = ExecutionState # type: ignore[attr-defined] - iface.ExecutionEvents = ExecutionEvents # type: ignore[attr-defined] - iface.RetargeterIOType = dict # type: ignore[attr-defined] - - class FakeBaseRetargeter: - def __init__(self, name: str) -> None: - self.name = name - - iface.BaseRetargeter = FakeBaseRetargeter # type: ignore[attr-defined] - - tsm_types = sys.modules["isaacteleop.teleop_session_manager.teleop_state_manager_types"] - tsm_types.bool_signal = MagicMock # type: ignore[attr-defined] - - dt_mod = sys.modules["isaacteleop.retargeting_engine.deviceio_source_nodes.deviceio_tensor_types"] - dt_mod.MessageChannelMessagesTrackedGroup = MagicMock # type: ignore[attr-defined] - - -_install_stubs() +# Capture's retargeting API is usable without starting a simulator. Use the real +# optional dependency so this module cannot replace packages seen by other tests. +pytest.importorskip("isaacteleop.retargeting_engine.interface") from isaaclab_teleop.control_events import ControlEvents, poll_control_events # noqa: E402 +from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle # noqa: E402 from isaaclab_teleop.teleop_message_processor import ( # noqa: E402 TeleopMessageProcessor, _classify_command, @@ -535,3 +469,29 @@ def last_control_events(self): result = poll_control_events(FakeDevice()) assert result.is_active is False assert result.should_reset is True + + +class TestPipelinedControlEvents: + def test_reset_edge_is_emitted_once_per_returned_frame(self): + from isaacteleop.retargeting_engine.interface.execution_events import ExecutionEvents, ExecutionState + + lifecycle = TeleopSessionLifecycle.__new__(TeleopSessionLifecycle) + lifecycle._message_processor = object() + lifecycle._last_control_event_frame_id = None + lifecycle._last_control_events = ControlEvents() + lifecycle._session = SimpleNamespace( + last_context=SimpleNamespace( + execution_events=ExecutionEvents(reset=True, execution_state=ExecutionState.RUNNING) + ), + last_step_info=SimpleNamespace(returned_frame_id=7), + ) + + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=True) + + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=False) + + lifecycle._session.last_step_info.returned_frame_id = 8 + lifecycle._update_control_events() + assert lifecycle.last_control_events == ControlEvents(is_active=True, should_reset=True) diff --git a/source/isaaclab_teleop/test/test_hand_debug_viz.py b/source/isaaclab_teleop/test/test_hand_debug_viz.py index c67f6d8da4c..9f056071af7 100644 --- a/source/isaaclab_teleop/test/test_hand_debug_viz.py +++ b/source/isaaclab_teleop/test/test_hand_debug_viz.py @@ -7,7 +7,7 @@ """Tests for hand joint debug visualization plumbing. -Covers :meth:`TeleopSessionLifecycle._chain_hand_debug_outputs` (HandsSource +Covers :meth:`TeleopSessionLifecycle._chain_hand_presence_outputs` (HandsSource auto-discovery), :func:`~isaaclab_teleop.visualizers.hand_joint_visualizer._extract_world_joint_positions`, and :func:`~isaaclab_teleop.visualizers.controller_aim_visualizer._aim_world_pose` @@ -21,7 +21,6 @@ from __future__ import annotations import sys -from types import ModuleType from unittest.mock import MagicMock, patch import numpy as np @@ -64,7 +63,7 @@ "omni.kit.xr.system.openxr", ] -_stubs_installed: dict[str, ModuleType | MagicMock] = {} +_stub_patch: pytest.MonkeyPatch | None = None class _FakeHandsSource: @@ -122,41 +121,37 @@ def __init__(self, name: str) -> None: def _install_stubs(): - """Insert MagicMock modules for all heavy dependencies.""" + """Temporarily replace dependencies, including attributes on already loaded modules.""" + global _stub_patch + assert _stub_patch is None + _stub_patch = pytest.MonkeyPatch() for name in _MODULES_TO_STUB: if name not in sys.modules: - sys.modules[name] = _stubs_installed.setdefault(name, MagicMock()) + _stub_patch.setitem(sys.modules, name, MagicMock()) if "." in name: parent_name, child_name = name.rsplit(".", 1) - setattr(sys.modules[parent_name], child_name, sys.modules[name]) + _stub_patch.setattr(sys.modules[parent_name], child_name, sys.modules[name], raising=False) dsn_mod = sys.modules["isaacteleop.retargeting_engine.deviceio_source_nodes"] - dsn_mod.HandsSource = _FakeHandsSource # type: ignore[attr-defined] - dsn_mod.ControllersSource = _FakeControllersSource # type: ignore[attr-defined] + _stub_patch.setattr(dsn_mod, "HandsSource", _FakeHandsSource, raising=False) + _stub_patch.setattr(dsn_mod, "ControllersSource", _FakeControllersSource, raising=False) iface_mod = sys.modules["isaacteleop.retargeting_engine.interface"] - iface_mod.OutputCombiner = _FakeOutputCombiner # type: ignore[attr-defined] - iface_mod.BaseRetargeter = _FakeBaseRetargeter # type: ignore[attr-defined] - iface_mod.RetargeterIOType = dict # type: ignore[attr-defined] + _stub_patch.setattr(iface_mod, "OutputCombiner", _FakeOutputCombiner, raising=False) + _stub_patch.setattr(iface_mod, "BaseRetargeter", _FakeBaseRetargeter, raising=False) + _stub_patch.setattr(iface_mod, "RetargeterIOType", dict, raising=False) tt_mod = sys.modules["isaacteleop.retargeting_engine.tensor_types"] - tt_mod.HandInputIndex = _FakeHandInputIndex # type: ignore[attr-defined] - tt_mod.ControllerInputIndex = _FakeControllerInputIndex # type: ignore[attr-defined] + _stub_patch.setattr(tt_mod, "HandInputIndex", _FakeHandInputIndex, raising=False) + _stub_patch.setattr(tt_mod, "ControllerInputIndex", _FakeControllerInputIndex, raising=False) def _restore_stubs(): - """Remove stubs installed for this test module from ``sys.modules``.""" - for name in reversed(_MODULES_TO_STUB): - stub = _stubs_installed.get(name) - if stub is None: - continue - if "." in name: - parent_name, child_name = name.rsplit(".", 1) - parent = sys.modules.get(parent_name) - if parent is not None and getattr(parent, child_name, None) is stub: - delattr(parent, child_name) - if sys.modules.get(name) is stub: - del sys.modules[name] + """Restore both inserted modules and overwritten attributes exactly.""" + global _stub_patch + if _stub_patch is not None: + _stub_patch.undo() + _stub_patch = None _install_stubs() @@ -192,8 +187,9 @@ def __init__(self, name: str) -> None: class _FakePipeline: """Stand-in for a user pipeline (leaf nodes + action output).""" - def __init__(self, leaves: list) -> None: + def __init__(self, leaves: list, outputs: tuple[str, ...] = ()) -> None: self._leaves = leaves + self._outputs = outputs def get_leaf_nodes(self) -> list: return self._leaves @@ -202,7 +198,7 @@ def output(self, key: str) -> str: return f"selector:user_pipeline:{key}" def output_types(self) -> dict: - return {"action": None} + return dict.fromkeys(("action", *self._outputs), object()) class _FakeTensorGroup: @@ -218,17 +214,36 @@ def __getitem__(self, idx): # =========================================================================== -# _chain_hand_debug_outputs: HandsSource auto-discovery +# _chain_hand_presence_outputs: HandsSource auto-discovery # =========================================================================== class TestChainHandDebugOutputs: + def test_chains_when_hands_source_present(self): + pipeline = _FakePipeline([_FakeLeaf("controllers"), _FakeHandsSource("hands")]) + mapping: dict = {} + TeleopSessionLifecycle._chain_hand_presence_outputs(pipeline, mapping) + assert mapping[TeleopSessionLifecycle.HAND_LEFT_KEY] == "selector:hands:hand_left" + assert mapping[TeleopSessionLifecycle.HAND_RIGHT_KEY] == "selector:hands:hand_right" + def test_mapping_untouched_without_hands_source(self): pipeline = _FakePipeline([_FakeLeaf("controllers"), _FakeLeaf("head")]) mapping: dict = {"action": "selector:user_pipeline:action"} - TeleopSessionLifecycle._chain_hand_debug_outputs(pipeline, mapping) + TeleopSessionLifecycle._chain_hand_presence_outputs(pipeline, mapping) assert mapping == {"action": "selector:user_pipeline:action"} + def test_explicit_filtered_hand_outputs_take_precedence_over_raw_source(self): + pipeline = _FakePipeline( + [_FakeHandsSource("hands")], + outputs=(TeleopSessionLifecycle.HAND_LEFT_KEY, TeleopSessionLifecycle.HAND_RIGHT_KEY), + ) + mapping: dict = {} + TeleopSessionLifecycle._chain_hand_presence_outputs(pipeline, mapping) + assert mapping == { + TeleopSessionLifecycle.HAND_LEFT_KEY: "selector:user_pipeline:hand_left", + TeleopSessionLifecycle.HAND_RIGHT_KEY: "selector:user_pipeline:hand_right", + } + # =========================================================================== # start(): chaining gated by the enable_debug_visualization lifecycle arg @@ -251,10 +266,10 @@ def test_start_chains_when_enabled(self): assert lifecycle.pipeline.output_mapping[TeleopSessionLifecycle.HAND_RIGHT_KEY] == "selector:hands:hand_right" assert TeleopSessionLifecycle._CONTROLLER_LEFT_KEY in lifecycle.pipeline.output_mapping - def test_start_does_not_chain_when_disabled(self): + def test_start_chains_hand_presence_but_not_left_controller_when_disabled(self): lifecycle = self._start(enable_debug_visualization=False) - assert TeleopSessionLifecycle.HAND_LEFT_KEY not in lifecycle.pipeline.output_mapping - assert TeleopSessionLifecycle.HAND_RIGHT_KEY not in lifecycle.pipeline.output_mapping + assert TeleopSessionLifecycle.HAND_LEFT_KEY in lifecycle.pipeline.output_mapping + assert TeleopSessionLifecycle.HAND_RIGHT_KEY in lifecycle.pipeline.output_mapping assert TeleopSessionLifecycle._CONTROLLER_LEFT_KEY not in lifecycle.pipeline.output_mapping # The right controller is always chained for button polling. assert TeleopSessionLifecycle._CONTROLLER_RIGHT_KEY in lifecycle.pipeline.output_mapping diff --git a/source/isaaclab_teleop/test/test_openxr_device_constructors.py b/source/isaaclab_teleop/test/test_openxr_device_constructors.py index 6f971b8ca64..6e7d59663b1 100644 --- a/source/isaaclab_teleop/test/test_openxr_device_constructors.py +++ b/source/isaaclab_teleop/test/test_openxr_device_constructors.py @@ -7,6 +7,11 @@ from typing import cast import pytest + +from isaaclab.test.utils import launch_test_simulation + +launch_test_simulation() + from isaaclab_teleop.deprecated.openxr import OpenXRDevice, OpenXRDeviceCfg, XrCfg from isaaclab_teleop.deprecated.openxr.retargeters import ( GripperRetargeter, diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index a20d347df44..5f154dd0783 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -10,6 +10,7 @@ import builtins import contextlib import importlib +import math import sys import threading from contextlib import contextmanager @@ -353,6 +354,74 @@ def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monk assert bootstrap.events[-1] == "environment-close" +def test_host_smooths_committed_anchor_on_xr_clock_and_reseeds_reset(xr_module, monkeypatch): + """Use the real pose policy; substitute only frame time, scene I/O, and native sealing.""" + import numpy as np + from isaaclab_teleop import xr_anchor_manager + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode, XrCfg + from scipy.spatial.transform import Rotation + + from pxr import Usd, UsdGeom + + from isaaclab.sim.utils.stage import use_stage + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg( + anchor_prim_path="/World/Robot", + anchor_rotation_mode=XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED, + anchor_rotation_smoothing_time=0.1, + ) + with use_stage(Usd.Stage.CreateInMemory()): + UsdGeom.Xform.Define(xr_anchor_manager.get_current_stage(), "/World/Robot") + manager = xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=True) + + bootstrap = _FakeBootstrap() + reference = np.eye(4) + bootstrap.get_world_transform = lambda _path: reference.copy() + env = SimpleNamespace(sim=SimpleNamespace(get_rendering_dt=lambda: 0.25)) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, env, xr_module.OvLibrariesXrConfig(), (), external_anchor_state=manager + ) + host._scene_publisher = bootstrap.publisher + host._ordinal_ledger = _FakeOrdinalLedger(None) + writes = [] + monkeypatch.setattr( + host, "_write_anchor_matrices", lambda matrices, ordinal: writes.append((matrices.copy(), ordinal)) + ) + frame_times = iter((10.0, 10.01, 10.02, 10.05, 20.0, 20.1, 20.11)) + monkeypatch.setattr(xr_module.time, "monotonic", lambda: next(frame_times)) + + def committed_yaw(): + return Rotation.from_matrix(writes[-1][0][0, :3, :3]).as_euler("xyz")[2] + + seed = host.publish_complete_scene("initial") + assert writes[-1][1] == seed == host._ordinal_ledger.published[-1] + host.prepare_frame_scene() + reference[:3, :3] = Rotation.from_euler("z", 90, degrees=True).as_matrix() + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(0.0) # Unpublished physics must not move the view. + host.publish_complete_scene("moved") + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.01 / 0.1))) + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.04 / 0.1))) + + manager.reset() + write_count = len(writes) + host.prepare_frame_scene() + assert len(writes) == write_count # An old reference cannot seed the new epoch. + reference[:3, :3] = Rotation.from_euler("z", 45, degrees=True).as_matrix() + seed = host.publish_complete_scene("reset") + assert writes[-1][1] == seed == host._ordinal_ledger.published[-1] + assert committed_yaw() == pytest.approx(0.0) + assert np.isfinite(manager.get_world_matrix()).all() # Input is usable before the next XR frame. + host.prepare_frame_scene() + reference[:3, :3] = Rotation.from_euler("z", 135, degrees=True).as_matrix() + host.publish_complete_scene("after-reset") + host.prepare_frame_scene() + assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.01 / 0.1))) + + def test_host_abandons_failed_write_and_failed_publish(xr_module, monkeypatch): host, bootstrap, _env = _make_host(xr_module, monkeypatch) host.attach_on_render_thread(host.offer_render_owner_transfer()) diff --git a/source/isaaclab_teleop/test/test_session_step_failure.py b/source/isaaclab_teleop/test/test_session_step_failure.py index ad33e1e7a5a..4ffa0f2a00e 100644 --- a/source/isaaclab_teleop/test/test_session_step_failure.py +++ b/source/isaaclab_teleop/test/test_session_step_failure.py @@ -133,6 +133,110 @@ def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: class TestStepFailureDiagnosis: + @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) + def test_failed_drain_retains_owner_and_original_step_error(self, error_type): + step_error = ValueError("retarget worker failed") + drain_error = error_type("worker did not join") + lifecycle = _make_failing_lifecycle(step_error) + session = lifecycle._session + session.__exit__.side_effect = drain_error + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + pytest.raises(ValueError, match="retarget worker failed") as raised, + ): + lifecycle.step() + assert raised.value is step_error + assert raised.value.__cause__ is drain_error + assert lifecycle._session is session + assert any("worker did not join" in note for note in step_error.__notes__) + notes = tuple(step_error.__notes__) + lifecycle.stop(type(step_error), step_error, step_error.__traceback__) + assert tuple(step_error.__notes__) == notes + + def test_replay_step_failure_is_not_restarted_from_the_beginning(self): + error = ValueError("invalid recorded input") + lifecycle = _make_failing_lifecycle(error) + lifecycle._is_replay = True + session = lifecycle._session + with ( + patch.object(lifecycle, "_build_external_inputs", return_value=None), + pytest.raises(ValueError, match="invalid recorded input") as raised, + ): + lifecycle.step() + assert raised.value is error + session.__exit__.assert_called_once() + assert lifecycle._session is None + + def test_xr_shutdown_drain_failure_remains_visible_and_owned(self): + lifecycle = _make_failing_lifecycle(RuntimeError("unused")) + session = lifecycle._session + session.__exit__.side_effect = RuntimeError("native drain failed") + with pytest.raises(RuntimeError, match="native drain failed"): + lifecycle._on_pre_shutdown(None) + assert lifecycle._session is session + + @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) + def test_terminal_exit_failure_is_latched_across_callbacks_and_restart(self, error_type): + lifecycle = _make_lifecycle() + lifecycle._pipeline = object() + error = error_type("terminal native cleanup failed") + + class TerminalSession: + def __init__(self): + self.entered = True + self.steps = 0 + self.exits = 0 + + def __exit__(self, *_exc): + self.exits += 1 + if self.entered: + self.entered = False + raise error + + def step(self, **_kwargs): + self.steps += 1 + raise AssertionError("must not step a terminal session") + + session = TerminalSession() + lifecycle._session = session + with pytest.raises(error_type) as raised: + lifecycle._on_pre_shutdown(None) + assert raised.value is error + assert lifecycle.session_cleanup_failed + for operation in (lifecycle.step, lifecycle.step, lifecycle.try_start_session, lifecycle.start, lifecycle.stop): + with pytest.raises(error_type) as raised: + operation() + assert raised.value is error + assert session.exits == 1 + assert session.steps == 0 + assert lifecycle.session_cleanup_failed + assert lifecycle._session is session + + def test_failed_submission_keeps_pending_host_reset_for_reentry(self): + lifecycle = _make_failing_lifecycle(ValueError("persistent pipeline error")) + lifecycle._pending_reset = True + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + assert lifecycle._pending_reset + + def test_ovxr_session_pipeline_failure_also_bounds_restart_rate(self): + lifecycle = _make_failing_lifecycle(ValueError("persistent pipeline error")) + lifecycle._external_oxr_handles_provider = lambda: object() + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + assert lifecycle._restart_holdoff_until > time.monotonic() + with patch.object(lifecycle, "_try_start_session") as try_start: + for _ in range(5): + assert lifecycle.step() is None + try_start.assert_not_called() + def test_pipeline_error_with_active_xr_sets_holdoff(self): lifecycle = _make_failing_lifecycle(ValueError("Found zero norm quaternions in `quat`.")) session = lifecycle._session diff --git a/source/isaaclab_teleop/test/test_target_frame_rebase.py b/source/isaaclab_teleop/test/test_target_frame_rebase.py index f4b523f68a7..068315dc22c 100644 --- a/source/isaaclab_teleop/test/test_target_frame_rebase.py +++ b/source/isaaclab_teleop/test/test_target_frame_rebase.py @@ -178,3 +178,50 @@ def test_advance_selects_target_frame(configured, explicit, expected): assert step_kwargs["target_T_world"] is expected else: np.testing.assert_allclose(step_kwargs["target_T_world"], expected, atol=1e-6) + + +def test_external_replay_reads_resolved_target_frame(monkeypatch): + """A cloned target path reaches the scene reader without mutating the task template.""" + import ast + from pathlib import Path + from types import SimpleNamespace + + from isaaclab_teleop import IsaacTeleopCfg + from isaaclab_teleop.isaac_teleop_device import create_isaac_teleop_device + + from pxr import Usd, UsdGeom + + stage = Usd.Stage.CreateInMemory() + frame = UsdGeom.Xform.Define(stage, "/Lab/Cells/Cell0/Robot/base") + frame.AddTranslateOp().Set(Gf.Vec3d(4, 5, 6)) + cfg = IsaacTeleopCfg(target_frame_prim_path="{ENV_REGEX_NS}/Robot/base", sim_device="cpu") + lifecycle = MagicMock() + monkeypatch.setattr("isaaclab_teleop.isaac_teleop_device.TeleopSessionLifecycle", lambda *a, **k: lifecycle) + monkeypatch.setattr("isaaclab_teleop.isaac_teleop_device.XrAnchorManager", lambda *a, **k: MagicMock()) + device = create_isaac_teleop_device( + cfg, env_prim_path="/Lab/Cells/Cell0", use_kit_xr_bridge=False, ovxr_session=True + ) + device._dispatch_control_callbacks = lambda: None + + def read_target(path): + prim = stage.GetPrimAtPath(path) + assert prim.IsValid(), f"Target path did not resolve to the scene: {path}" + return np.asarray(UsdGeom.XformCache().GetLocalToWorldTransform(prim).GetInverse()).T + + source = ( + Path(__file__).resolve().parents[3] + / "source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py" + ) + node = next( + n + for n in ast.parse(source.read_text()).body + if isinstance(n, ast.FunctionDef) and n.name == "_advance_replay_input" + ) + namespace = {} + exec(compile(ast.Module(body=[node], type_ignores=[]), str(source), "exec"), namespace) + presentation = SimpleNamespace(session=SimpleNamespace(get_target_frame_T_world=read_target)) + namespace["_advance_replay_input"](device, presentation) + expected = np.eye(4) + expected[:3, 3] = [-4, -5, -6] + np.testing.assert_allclose(lifecycle.step.call_args.kwargs["target_T_world"], expected) + assert cfg.target_frame_prim_path == "{ENV_REGEX_NS}/Robot/base" diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py new file mode 100644 index 00000000000..7a03d06b8bc --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -0,0 +1,868 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""In-process teleoperation entrypoint, configuration, and control-loop contracts.""" + +import ast +import os +import re +import signal +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest +import torch + +# Repository root: this file lives at source/isaaclab_teleop/test/.py +ROOT = Path(__file__).resolve().parents[3] + +# XR experience file whose dependencies decide whether ``--xr`` can start offline. +_XR_EXPERIENCE = ROOT / "apps" / "isaaclab.python.xr.openxr.kit" + +# ``.kit`` files repeat section headers, so they cannot be parsed as TOML. +_SECTION_RE = re.compile(r"^\s*\[+(?P[^\[\]]+)\]+\s*$") +_DEPENDENCY_RE = re.compile(r'^\s*"(?P[^"]+)"\s*=') + + +def _kit_dependencies(experience: Path) -> set[str]: + """Collect the extension names declared in an experience file's dependency sections. + + Scoped to ``[dependencies]`` tables so quoted keys elsewhere in the file (settings tables + use them too) are not mistaken for extensions. + """ + dependencies: set[str] = set() + in_dependencies = False + + for line in experience.read_text(encoding="utf-8").splitlines(): + section = _SECTION_RE.match(line) + if section is not None: + in_dependencies = section.group("name").strip() == "dependencies" + continue + if in_dependencies: + dependency = _DEPENDENCY_RE.match(line) + if dependency is not None: + dependencies.add(dependency.group("name")) + + return dependencies + + +def test_xr_experience_declares_only_shipped_extensions(): + """The XR experience depends on XR extensions Isaac Sim ships, not on a registry-only bundle. + + ``omni.kit.xr.bundle.generic`` is not shipped and resolves only from the Kit SDK registry, so + depending on it aborted every ``--xr`` run that could not reach the registry. + + Static on purpose: a machine that reaches the registry caches the bundle on first use and + loads it locally forever after, so a launch test cannot catch this once that cache is warm. + """ + dependencies = _kit_dependencies(_XR_EXPERIENCE) + + assert "omni.kit.xr.system.openxr" in dependencies + assert "omni.kit.xr.ui.window.profile" in dependencies, "omni.kit.xr.profile.ar's Kit 110 successor is missing" + assert not [name for name in dependencies if name.startswith("omni.kit.xr.bundle.")], ( + "XR bundle meta-extensions are not shipped with Isaac Sim and resolve only from the Kit SDK registry" + ) + + +def test_teleop_readme_programmatic_config_returns_pipeline_not_factory_tuple(): + """Execute the documented public config, without entering an XR session.""" + from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource + from isaacteleop.retargeting_engine.interface import OutputCombiner + + readme = ROOT / "source/isaaclab_teleop/docs/README.md" + section = readme.read_text().split("### 4. Programmatic Usage", 1)[1] + example = re.search(r"```python\n(.*?)```", section, re.DOTALL) + assert example is not None + tree = ast.parse(example.group(1)) + config_statements = [] + for statement in tree.body: + if isinstance(statement, ast.With): + break + config_statements.append(statement) + source = ControllersSource(name="documentation_controllers") + pipeline = OutputCombiner({"action": source.output(ControllersSource.RIGHT)}) + retargeters = [] + namespace = {"my_pipeline_builder": lambda: (pipeline, retargeters)} + exec(compile(ast.Module(body=config_statements, type_ignores=[]), str(readme), "exec"), namespace) + cfg = namespace["cfg"] + assert cfg.pipeline_builder() is pipeline + assert cfg.pipeline_builder().output("action") is not None + assert cfg.retargeters_to_tune() is retargeters + + +@pytest.mark.parametrize( + ("kitless", "selection", "expected", "reject"), + [ + (True, None, None, False), + (True, " 1 ", " 1 ", False), + (True, "0", "0", True), + (True, "invalid", "invalid", True), + (False, None, None, False), + ], +) +@pytest.mark.parametrize("with_preparation", [False, True]) +def test_teleop_ovxr_validates_external_stage_without_changing_environment( + monkeypatch, kitless, selection, expected, reject, with_preparation +): + """The task resolver preserves the process setting and rejects incompatible explicit modes before launch.""" + from types import SimpleNamespace + + import isaaclab_teleop + from isaaclab_teleop import ( + IsaacTeleopCfg, + se3_gamepad_teleop_cfg, + se3_keyboard_teleop_cfg, + se3_spacemouse_teleop_cfg, + ) + from isaaclab_teleop.device_selection import select_teleop_cfg + + from isaaclab.devices import DeviceCfg + + from isaaclab_tasks.utils import hydra + + monkeypatch.setattr(hydra, "enumerate_task_presets", lambda _task: None) + name = "ISAAC_LAB_OVRTX_USE_OVSTAGE" + if selection is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, selection) + monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) + preparations = [] + from isaaclab_ov.physics import OvPhysxCfg + + config = SimpleNamespace( + sim=SimpleNamespace(device="cpu", physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1), + terminations=SimpleNamespace(time_out=1), + isaac_teleop=IsaacTeleopCfg( + prepare_environment=(lambda selected: preparations.append(selected)) if with_preparation else None, + ), + ) + scans = [] + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + namespace = { + "__package__": "isaaclab_teleop.entrypoints", + "IsaacTeleopCfg": IsaacTeleopCfg, + "DeviceCfg": DeviceCfg, + "select_teleop_cfg": select_teleop_cfg, + "se3_keyboard_teleop_cfg": se3_keyboard_teleop_cfg, + "se3_spacemouse_teleop_cfg": se3_spacemouse_teleop_cfg, + "se3_gamepad_teleop_cfg": se3_gamepad_teleop_cfg, + "os": os, + "sys": sys, + "ManagerBasedRLEnvCfg": SimpleNamespace, + "resolve_task_config": lambda *_args, **_kwargs: (config, None), + "validate_launch": lambda *_args: (scans.append(os.environ.get(name)) or ((), ())), + "args_cli": SimpleNamespace( + task="Test", + device="cpu", + num_envs=1, + teleop_device=None, + sensitivity=1.0, + using_ovxr=kitless, + ovxr_pip_camera=None, + geometry_publication_device=None, + ovrtx_active_cuda_gpus=None, + disable_external_cameras=False, + physics=None, + ), + } + for key, value in namespace.items(): + monkeypatch.setattr(agent, key, value, raising=False) + if reject: + with pytest.raises(ValueError, match="requires.*OVStage"): + agent._resolve_env_cfg() + assert scans == [] + else: + agent._resolve_env_cfg() + assert scans == [expected] + assert os.environ.get(name) == expected + + assert preparations == ([config] if with_preparation and not reject else []) + + +@pytest.fixture +def run_live_workflow(monkeypatch): + """Run the imported live owner with finite input, renderer, and simulation endpoints.""" + import contextlib + + import isaaclab_teleop + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + def run(env, interface, session, *, conditioner=None, validator=None, step_hz=0.0, refresh=lambda: None): + class Device: + target_frame_prim_path = None + xr_render_anchor_path = None + required_openxr_extensions = () + external_anchor_state = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def advance(self): + return interface.advance() + + def reset(self): + return interface.reset() + + @property + def last_control_events(self): + return interface.control_events() + + camera = SimpleNamespace(bind=lambda _env: contextlib.nullcontext(), refresh=refresh, close=lambda: None) + session.external_oxr_handles = None + session.start_render_worker = lambda *_args, **_kwargs: True + session.close = lambda: None + session.async_worker_owns_environment = False + env.close = lambda: None + env.scene = SimpleNamespace(env_prim_paths=["/World/envs/env_0"]) + env.sim.device = "cpu" + env.sim.visualizers = () + cfg = SimpleNamespace( + sim=object(), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + action_conditioner=conditioner, + action_validator=validator, + ), + ) + monkeypatch.setattr( + agent, + "args_cli", + agent._build_parser().parse_args(["--task", "Test", "--ovxr", "--step_hz", str(step_hz)]), + raising=False, + ) + monkeypatch.setattr(agent, "has_kit", lambda: False) + monkeypatch.setattr(agent.gym, "make", lambda *_args, **_kwargs: SimpleNamespace(unwrapped=env)) + monkeypatch.setattr(agent, "create_teleop_device", lambda *_args, **_kwargs: Device()) + monkeypatch.setattr(agent, "_make_haptic_io", lambda *_args: (lambda: None, lambda: None)) + monkeypatch.setattr(agent, "_install_ovxr_shutdown_handlers", lambda *_args: {}) + monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", lambda *_args, **_kwargs: camera) + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", lambda *_args, **_kwargs: session) + agent.run_teleoperation(cfg, cfg.isaac_teleop, True) + + return run + + +def test_teleop_se3_agent_yields_only_when_ovxr_is_idle(monkeypatch): + """A paused kitless loop must yield without slowing Kit or active physics frames.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + sleeps = [] + monkeypatch.setattr(time, "sleep", sleeps.append) + monkeypatch.setattr(time, "perf_counter", lambda: 10.003) + yield_idle = agent._pace_ovxr_control_loop + + timing = {"iteration_started_s": 10.0, "action_period_s": 0.02} + yield_idle(using_ovxr=False, action_applied=False, **timing) + yield_idle(using_ovxr=True, action_applied=True, **timing) + assert sleeps == [] + + yield_idle(using_ovxr=True, action_applied=False, **timing) + assert sleeps == pytest.approx([0.017]) + yield_idle(using_ovxr=True, action_applied=False, iteration_started_s=9.9, action_period_s=0.02) + assert sleeps == pytest.approx([0.017]) # Slow input work needs no added delay. + + +@pytest.mark.parametrize( + "active,step_hz,expected_polls", + [(False, 0.0, 6), (True, 25.0, 3), (None, 25.0, 3)], +) +def test_kitless_loop_respects_control_rate_and_processes_reset( + monkeypatch, run_live_workflow, active, step_hz, expected_polls +): + """The real caller preserves idle pacing and applies an explicit active limit.""" + from types import SimpleNamespace + + import torch + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + clock = 0.0 + advances = 0 + resets = [] + refreshes = [] + publications = [] + applied = [] + seeded_from = [] + stepped_values = [] + sequence = active is None + anchor_ready = False + + def reset(): + nonlocal anchor_ready + resets.append(clock) + anchor_ready = False + + def request(identity, *, force=False): + nonlocal anchor_ready + publications.append((identity, {"force": force})) + anchor_ready = True + return 1 + + def sleep(duration): + nonlocal clock + clock += duration + + def advance(): + nonlocal advances, clock + assert anchor_ready, "Input cannot read an unseeded post-reset anchor" + advances += 1 + clock += 0.003 # Caller work consumes part of the independent 20ms action budget. + return torch.ones((1, 36)) if active or (sequence and advances != 4) else None + + def step(value): + applied.append(clock) + stepped_values.append(value.clone()) + + def condition(_env, value, state): + previous = state.get("frames", 0) + seeded_from.append(previous) + state["frames"] = previous + 1 + return value + previous + 2 + + sim = SimpleNamespace( + is_running=lambda: clock < (0.199 if sequence else 0.119), + get_physics_step_count=lambda: 0, + get_physics_dt=lambda: 0.005, + render=lambda: None, + ) + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: clock, sleep=sleep)) + run_live_workflow( + SimpleNamespace(sim=sim, reset=reset, step_dt=0.02, num_envs=1, step=step), + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace( + is_active=advances not in (2, 4) if sequence else active, + should_reset=advances == (5 if sequence else 3), + ), + ), + SimpleNamespace( + shutdown_requested=False, + wait_for_shutdown=sleep, + request_render=request, + report_host_timing=lambda *args: None, + ), + conditioner=condition, + validator=lambda value: None, + step_hz=step_hz, + refresh=lambda: refreshes.append(clock), + ) + + assert advances == (5 if sequence else expected_polls) + assert clock == pytest.approx( + 0.20 if sequence else 0.12 + ) # Work time is credited, rather than adding a full-period sleep. + assert len(applied) == (3 if sequence else advances if active else 0) + assert len(resets) == 2 # Initial reset plus the public control event. + assert len(refreshes) == 1 + assert len(publications) == (4 if sequence else advances + 1 if active else 2) + assert publications[-1][1] == {"force": True} + + if sequence: + # Both STOP branches clear an established ramp before the next accepted action. + assert seeded_from == [0, 0, 0] + for value in stepped_values: + torch.testing.assert_close(value, torch.full((1, 36), 3.0)) + elif active: + assert seeded_from == [0, 1, 2] + for index, value in enumerate(stepped_values): + torch.testing.assert_close(value, torch.full((1, 36), 3.0 + index)) + + +def test_paused_loop_retries_the_latest_unaccepted_publication(monkeypatch, run_live_workflow): + """A paused iteration retries the rejected latest state without another physics step.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + advances = 0 + steps = [] + requests = [] + sleeps = [] + session = SimpleNamespace(shutdown_requested=False) + + def advance(): + nonlocal advances + advances += 1 + return torch.ones(1) + + def request(identity, *, force): + requests.append((identity, force)) + if force: + return 1 + if len(requests) == 2: + return None + session.shutdown_requested = True + return 2 + + def sleep(duration): + sleeps.append((duration, len(requests))) + + session.request_render = request + session.wait_for_shutdown = sleep + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: 0.0, sleep=sleep)) + env = SimpleNamespace( + num_envs=1, + step_dt=0.02, + reset=lambda: None, + step=lambda action: steps.append(action), + sim=SimpleNamespace( + is_running=lambda: advances < 4, get_physics_step_count=lambda: len(steps), get_physics_dt=lambda: 0.005 + ), + ) + run_live_workflow( + env, + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace(is_active=advances == 1, should_reset=False), + ), + session, + ) + assert advances == 2 and len(steps) == 1 + assert len(requests) == 3 and requests[0][1] is True + assert requests[1:] == [(1, False), (1, False)] + assert sleeps == [(0.02, 3)] # Pacing follows the retried publication. + + +@pytest.mark.parametrize("selector", [["--physics", "ovphysx"], ["physics=ovphysx"]]) +@pytest.mark.parametrize("ovxr", [False, True]) +def test_teleop_replay_agent_resolves_typed_presets_without_reapplying_physics(monkeypatch, selector, ovxr): + """Public CLI aliases reach task resolution without resetting backend settings.""" + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + + from isaaclab.managers import TerminationTermCfg + from isaaclab.sim import SimulationCfg + + source_path = ROOT / "source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py" + argv = [str(source_path), "--task", "Test", "--replay_file", "recording.mcap", *selector] + if ovxr: + argv.append("--ovxr") + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + monkeypatch.setattr(replay, "args_cli", None, raising=False) + monkeypatch.setattr(replay, "hydra_overrides", None, raising=False) + namespace = vars(replay) + replay._parse_args() + prepared_configs = [] + physics = OvPhysxCfg() + cfg = SimpleNamespace( + sim=SimulationCfg(physics=physics), + scene=SimpleNamespace(num_envs=1), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + prepare_environment=lambda selected: prepared_configs.append(selected) + ), + terminations=SimpleNamespace( + success=TerminationTermCfg(func=lambda env: False), + time_out=TerminationTermCfg(func=lambda env: False), + ), + ) + physics = cfg.sim.physics + resolutions = [] + + def resolve(task, agent, *, overrides): + resolutions.append((task, agent, overrides)) + return cfg, None + + monkeypatch.setattr(replay, "ManagerBasedRLEnvCfg", SimpleNamespace) + monkeypatch.setattr(replay, "resolve_task_config", resolve) + monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) + monkeypatch.setattr(replay, "remove_camera_configs", isaaclab_teleop.remove_camera_configs) + prepared, success = namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) + assert resolutions == [("test-task", "", ["physics=ovphysx"])] + assert prepared is cfg and success is not None + assert cfg.sim.physics is physics + assert namespace["args_cli"].require_kit is False + assert not namespace["args_cli"].start_on_replay + legacy_args = replay._build_parser().parse_args( + ["--task", "Test", "--replay_file", "recording.mcap", "--start_on_replay"] + ) + assert legacy_args.start_on_replay + assert not hasattr(namespace["args_cli"], "physics") + assert prepared_configs == [cfg] + assert "ISAAC_LAB_OVRTX_USE_OVSTAGE" not in os.environ + + +@pytest.mark.parametrize( + "backend,viewer,hard_reset", + [("ovphysx", False, True), ("ovphysx", True, True), ("newton_mjwarp", True, False)], +) +def test_recorded_environment_reset_preserves_backend_policy_and_hand_cursor(monkeypatch, backend, viewer, hard_reset): + """The imported replay loop preserves backend reset order and the input cursor.""" + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + calls = [] + frame = -1 + + class Device: + target_frame_prim_path = None + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def reset(self): + calls.append("device-reset") + + def request_start(self): + calls.append("start") + + def advance(self): + nonlocal frame + frame += 1 + return torch.zeros(1) + + @property + def last_control_events(self): + return SimpleNamespace(is_active=False, should_reset=frame == 0) + + env = SimpleNamespace( + scene=SimpleNamespace(physics_backend=backend, env_prim_paths=["/World/envs/env_0"]), + sim=SimpleNamespace( + device="cpu", is_running=lambda: frame < 1, render=lambda: None, reset=lambda: calls.append("sim-reset") + ), + recorder_manager=SimpleNamespace(reset=lambda: calls.append("recorder-reset")), + reset=lambda: calls.append("env-reset"), + step_dt=0, + ) + presentation = ( + SimpleNamespace( + session=SimpleNamespace(shutdown_requested=False), + start=lambda *_args: None, + publish=lambda *_args, **kwargs: calls.append("publish-reset") if kwargs.get("reset") else None, + ) + if viewer + else None + ) + monkeypatch.setattr( + replay, + "args_cli", + SimpleNamespace( + replay_file="recording.mcap", + start_on_replay=True, + enable_debug_visualization=False, + max_replay_duration_s=1, + ), + raising=False, + ) + monkeypatch.setattr(replay, "create_isaac_teleop_device", lambda *_args, **_kwargs: Device()) + monkeypatch.setattr(replay, "NvmlGpuStatsProvider", lambda: SimpleNamespace(summary=lambda: {})) + replay._run_single_replay(env, SimpleNamespace(), None, 1, 2, presentation) + initial = ["sim-reset"] if hard_reset else [] + initial += ["env-reset", "device-reset", "start"] + recorded = ["sim-reset"] if hard_reset else [] + recorded += ["recorder-reset", "env-reset", "device-reset", "start"] + if viewer: + recorded.append("publish-reset") + assert calls == initial + recorded + assert frame == 1 + + +def test_teleop_replay_reset_policy_skips_hard_reset_only_for_newton(): + """Newton keeps its constructed solver while OVPhysX retains the PR-5507 hard reset.""" + from isaaclab_teleop.entrypoints.teleop_replay_agent import _replay_requires_hard_sim_reset as requires_hard_reset + + assert requires_hard_reset("newton_mjwarp") is False + assert requires_hard_reset("newtonmanager") is False + assert requires_hard_reset("ovphysx") is True + assert requires_hard_reset(None) is True + + +def test_teleop_replay_reset_sequence_preserves_backend_owned_order(): + """The backend policy changes only the optional simulator reset in the reset sequence.""" + from isaaclab_teleop.entrypoints.teleop_replay_agent import _reset_replay_environment as reset_environment + + calls: list[str] = [] + + class _Resetter: + def __init__(self, name: str): + self._name = name + + def reset(self): + calls.append(self._name) + + class _Environment: + sim = _Resetter("sim") + recorder_manager = _Resetter("recorder") + + def reset(self): + calls.append("env") + + env = _Environment() + reset_environment(env, hard_sim_reset=False, reset_recorder=True) + assert calls == ["recorder", "env"] + + calls.clear() + reset_environment(env, hard_sim_reset=True, reset_recorder=True) + assert calls == ["sim", "recorder", "env"] + + calls.clear() + reset_environment(env, hard_sim_reset=False, reset_recorder=False) + assert calls == ["env"] + + +def _load_teleop_agent(monkeypatch, *arguments): + from isaaclab_teleop.entrypoints import teleop_se3_agent as module + + monkeypatch.setattr(sys, "argv", ["teleop_se3_agent.py", *arguments]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + module._parse_args() + return module + + +def test_product_xr_default_resolves_cloudxrjs_and_autolaunch(monkeypatch): + agent = _load_teleop_agent(monkeypatch, "--ovxr") + from isaaclab_teleop import CLOUDXR_JS_ENV + + assert agent.args_cli.cloudxr_env == CLOUDXR_JS_ENV + assert agent.args_cli.auto_launch_cloudxr is True + + +def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch): + """Launch validation must not replace task-specific settings on the selected backend.""" + agent = _load_teleop_agent(monkeypatch) + from isaaclab_newton.physics import NewtonCfg + + from isaaclab.app.sim_launcher import validate_launch + + physics_cfg = NewtonCfg() + physics_cfg.solver_cfg.nconmax = 4096 + physics_cfg.solver_cfg.njmax = 1536 + env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)) + configs, _ = validate_launch(env_cfg, vars(agent.args_cli).copy()) + + assert not hasattr(agent.args_cli, "physics") + assert env_cfg.sim.physics is physics_cfg + assert any(cfg is physics_cfg for cfg in configs) + assert physics_cfg.solver_cfg.nconmax == 4096 + assert physics_cfg.solver_cfg.njmax == 1536 + + +def test_task_action_hook_validates_before_and_after_conditioning(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + calls = [] + + def validate(action): + calls.append(("validate", action)) + return "rejected" if action == 3 else None + + def condition(env, action, state): + calls.append(("condition", env, action, state)) + state["seen"] = True + return action + 1 + + state = {} + action, issue = agent.prepare_teleop_action("env", 1, condition, validate, state) + assert (action, issue, state) == (2, None, {"seen": True}) + assert [call[0] for call in calls] == ["validate", "condition", "validate"] + + calls.clear() + action, issue = agent.prepare_teleop_action("env", 3, condition, validate, state) + assert (action, issue) == (3, "rejected") + assert [call[0] for call in calls] == ["validate"] + + +def test_teleop_agent_installs_late_graceful_shutdown_handlers(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + requests = [] + installed = {} + + class Session: + def request_shutdown(self): + requests.append("shutdown") + + def install(signum, handler): + installed[signum] = handler + return f"previous-{signum}" + + monkeypatch.setattr(agent.signal, "signal", install) + previous = agent._install_ovxr_shutdown_handlers(Session()) + + assert set(installed) == {signal.SIGINT, signal.SIGTERM} + installed[signal.SIGINT](signal.SIGINT, None) + assert requests == ["shutdown"] + + restored = {} + monkeypatch.setattr(agent.signal, "signal", lambda signum, handler: restored.update({signum: handler})) + agent._restore_ovxr_shutdown_handlers(previous) + assert restored == previous + + +@pytest.mark.parametrize( + ("left_active", "right_active", "expected"), + ( + (False, True, "waiting for initial left hand tracking"), + (True, False, "waiting for initial right hand tracking"), + ), +) +def test_kitless_hand_guard_requires_each_hand_once_then_allows_held_dropout( + monkeypatch, left_active, right_active, expected +): + agent = _load_teleop_agent(monkeypatch) + + class OptionalHand: + def __init__(self, active): + self.is_none = not active + + lifecycle = SimpleNamespace( + HAND_LEFT_KEY="hand_left", + HAND_RIGHT_KEY="hand_right", + last_step_result={ + "hand_left": OptionalHand(left_active), + "hand_right": OptionalHand(right_active), + }, + ) + from isaaclab_teleop import IsaacTeleopDevice + + interface = IsaacTeleopDevice.__new__(IsaacTeleopDevice) + interface._session_lifecycle = lifecycle + initialized = set() + + assert agent._required_hand_tracking_issue(interface, initialized) == expected + lifecycle.last_step_result = {"hand_left": OptionalHand(True), "hand_right": OptionalHand(True)} + # Numeric zero is not a sentinel: a present, valid fully-open hand frame is accepted. + assert agent._required_hand_tracking_issue(interface, initialized) is None + assert initialized == {"left", "right"} + + lifecycle.last_step_result = {"hand_left": OptionalHand(False), "hand_right": OptionalHand(False)} + assert agent._required_hand_tracking_issue(interface, initialized) is None + + initialized.clear() + assert agent._required_hand_tracking_issue(interface, initialized) == ( + "waiting for initial left/right hand tracking" + ) + + +def test_kitless_hand_guard_renders_without_step_and_next_valid_frame_resumes(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + + class Sim: + def __init__(self): + self.render_count = 0 + + def render(self): + self.render_count += 1 + + class Env: + def __init__(self): + self.num_envs = 1 + self.sim = Sim() + self.actions = [] + + def step(self, action): + self.actions.append(action.clone()) + + env = Env() + haptics = {"update": 0, "stop": 0} + + def update(): + haptics["update"] += 1 + + def stop(): + haptics["stop"] += 1 + + valid = torch.arange(36, dtype=torch.float32) + + assert agent._apply_teleop_frame(env, valid, None, update, stop) + assert not agent._apply_teleop_frame(env, torch.zeros_like(valid), "missing left hand tracking", update, stop) + assert agent._apply_teleop_frame(env, valid, None, update, stop) + + assert env.sim.render_count == 1 + assert len(env.actions) == 2 + torch.testing.assert_close(env.actions[0], env.actions[1]) + assert haptics == {"update": 2, "stop": 1} + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoint): + """Both actual workflows finalize the selected config before OVXR allocates its context.""" + from contextlib import contextmanager + + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_teleop import ovxr_session + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + from isaaclab.sim import SimulationCfg, SimulationContext + + calls = [] + + def prepare_environment(selected_cfg): + assert selected_cfg is config + calls.append("prepare") + selected_cfg.sim.dt = 0.007 + + config = SimpleNamespace( + sim=SimulationCfg(physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1), + terminations=SimpleNamespace(success=None, time_out=None), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg(prepare_environment=prepare_environment), + ) + + class ContextReached(RuntimeError): + pass + + class ContextAllocation: + def __init__(self, sim_cfg): + assert sim_cfg is config.sim + assert sim_cfg.dt == 0.007 + assert calls == ["prepare", "launch"] + calls.append("context") + raise ContextReached + + @staticmethod + def instance(): + return None + + @contextmanager + def launch(selected_cfg, _args, *, cleanup_context): + assert selected_cfg is config + assert not cleanup_context + calls.append("launch") + yield + + def validate_launch(selected_cfg, _args): + assert selected_cfg is config + return (), () + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + monkeypatch.setattr(module, "ManagerBasedRLEnvCfg", SimpleNamespace) + monkeypatch.setattr(module, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) + monkeypatch.setattr(module, "validate_launch", validate_launch) + monkeypatch.setattr(module, "launch_simulation", launch) + monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) + monkeypatch.setattr(SimulationContext, "instance", lambda: None) + monkeypatch.setattr(ovxr_session, "SimulationContext", ContextAllocation) + monkeypatch.setitem(sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _settings: None)) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + arguments = ["--task", "Test", "--ovxr", "--cloudxr_env", "none"] + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + else: + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + monkeypatch.setattr(module, "remove_camera_configs", isaaclab_teleop.remove_camera_configs) + arguments += ["--replay_file", "recording.mcap"] + module._parse_args(arguments) + + with pytest.raises(ContextReached): + module.main() + assert calls == ["prepare", "launch", "context"] diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py new file mode 100644 index 00000000000..eac106fe1a1 --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -0,0 +1,205 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Execute the real public entry without native startup to prove failure/cleanup contracts.""" + +import contextlib +import signal +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock + +import pytest + + +@pytest.fixture +def entry_namespace(monkeypatch): + """Use the imported entrypoint and restore all globals replaced by each test.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + namespace = vars(agent) + for name in ( + "args_cli", + "gym", + "_rtx_rendering_requested", + "_resolve_cloudxr_env", + "_make_haptic_io", + "has_kit", + "create_teleop_device", + "_install_ovxr_shutdown_handlers", + "_restore_ovxr_shutdown_handlers", + "torch", + ): + monkeypatch.setitem(namespace, name, namespace.get(name)) + monkeypatch.setattr(agent, "_restore_ovxr_shutdown_handlers", Mock()) + return namespace + + +@pytest.mark.parametrize( + "stage,use_capture", [("environment", True), ("device", True), ("haptics", True), ("input", True), ("input", False)] +) +def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, entry_namespace, stage, use_capture): + """Every startup phase and the live loop must retain its failure after cleanup.""" + import isaaclab_teleop + + failure = RuntimeError(f"{stage} failed") + camera = SimpleNamespace(close=Mock(), bind=lambda _env: contextlib.nullcontext()) + env = SimpleNamespace( + sim=SimpleNamespace(device="cpu", has_gui=False, visualizers=(), is_running=lambda: True), + scene=SimpleNamespace(env_prim_paths=["/World/envs/env_0"]), + reset=Mock(), + close=Mock(), + ) + interface = MagicMock() + interface.advance.side_effect = failure + monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", Mock(return_value=camera)) + entry_namespace.update( + { + "args_cli": SimpleNamespace( + task="test-task", + xr=False, + using_ovxr=False, + disable_external_cameras=True, + auto_launch_cloudxr=False, + enable_debug_visualization=False, + cloudxr_env="none", + ), + "gym": SimpleNamespace( + make=Mock( + side_effect=failure if stage == "environment" else None, return_value=SimpleNamespace(unwrapped=env) + ) + ), + "_rtx_rendering_requested": lambda _args: False, + "_resolve_cloudxr_env": lambda *_args: None, + "_make_haptic_io": Mock(side_effect=failure if stage == "haptics" else None, return_value=(Mock(), Mock())), + "has_kit": lambda: False, + "create_teleop_device": Mock(side_effect=failure if stage == "device" else None, return_value=interface), + "_install_ovxr_shutdown_handlers": Mock(return_value={}), + "torch": SimpleNamespace(inference_mode=contextlib.nullcontext), + } + ) + cfg = SimpleNamespace( + isaac_teleop=SimpleNamespace( + teleoperation_active_default=False, + require_initial_bimanual_hand_tracking=False, + action_conditioner=None, + action_validator=None, + ) + ) + if not use_capture: + cfg = SimpleNamespace() # Native inputs must not read Capture-only action policy. + with pytest.raises(RuntimeError) as caught: + entry_namespace["run_teleoperation"](cfg, False, use_capture) + assert caught.value is failure + camera.close.assert_called_once_with() + assert env.close.call_count == int(stage != "environment") + + +def test_input_cleanup_failure_prevents_borrowed_owner_release(entry_namespace): + env, camera, xr = Mock(), Mock(), Mock() + interface = SimpleNamespace(session_cleanup_failed=True) + with pytest.raises(RuntimeError, match="borrowed XR/environment drain is unproved"): + entry_namespace["_close_teleop_resources"](env, camera, xr, {}, interface) + xr.request_shutdown.assert_called_once_with() + xr.close.assert_not_called() + camera.close.assert_not_called() + env.close.assert_not_called() + + +@pytest.mark.parametrize( + ("cleanup_stage", "has_primary_error"), + [("xr", True), ("environment", True), ("camera", False), ("signals", False)], +) +def test_cleanup_preserves_first_error_and_never_frees_undrained_stage( + entry_namespace, cleanup_stage, has_primary_error +): + """Cleanup errors must stay visible without replacing the initial cause or risking UAF.""" + events = [] + cleanup_error = RuntimeError("cleanup failed") + primary_error = RuntimeError("original failure") + + def close(name): + events.append(name) + if name == cleanup_stage: + raise cleanup_error + + xr = SimpleNamespace(close=lambda: close("xr"), async_worker_owns_environment=False) + camera = SimpleNamespace(close=lambda: close("camera")) + env = SimpleNamespace(close=lambda: close("environment")) + entry_namespace["_restore_ovxr_shutdown_handlers"] = lambda _: close("signals") + + def exercise(): + try: + if has_primary_error: + raise primary_error + finally: + entry_namespace["_close_teleop_resources"](env, camera, xr, {}) + + with pytest.raises(RuntimeError) as caught: + exercise() + assert caught.value is (primary_error if has_primary_error else cleanup_error) + assert events == ( + ["xr", "camera", "signals"] if cleanup_stage == "xr" else ["xr", "camera", "environment", "signals"] + ) + if has_primary_error: + assert any("cleanup failed" in note for note in primary_error.__notes__) + + +def test_shutdown_signal_stays_owned_until_xr_drain_finishes(entry_namespace): + """A second Ctrl-C requests shutdown again instead of interrupting native cleanup.""" + events = [] + original = signal.signal(signal.SIGINT, lambda *_args: events.append("shutdown")) + + def restore(handlers): + for signum, handler in handlers.items(): + signal.signal(signum, handler) + + def close_xr(): + signal.raise_signal(signal.SIGINT) + events.append("xr") + + entry_namespace["_restore_ovxr_shutdown_handlers"] = restore + try: + entry_namespace["_close_teleop_resources"]( + SimpleNamespace(close=lambda: events.append("environment")), + SimpleNamespace(close=lambda: events.append("camera")), + SimpleNamespace(close=close_xr, async_worker_owns_environment=False), + {signal.SIGINT: original}, + ) + assert events == ["shutdown", "xr", "camera", "environment"] + assert signal.getsignal(signal.SIGINT) is original + finally: + signal.signal(signal.SIGINT, original) + + +@pytest.mark.parametrize("new_context,drain_failed", [(True, False), (False, False), (True, True)]) +def test_partial_construction_cleans_only_new_unowned_context(monkeypatch, entry_namespace, new_context, drain_failed): + """Failed env construction is cleaned, but preexisting/undrained contexts survive.""" + from isaaclab.app import launch_simulation + from isaaclab.physics import PhysicsCfg + from isaaclab.sim import SimulationContext + + previous = object() + state = {"context": previous} + clear = Mock(side_effect=lambda: state.update(context=None)) + monkeypatch.setattr(SimulationContext, "instance", lambda: state["context"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear) + camera = SimpleNamespace(close=Mock()) + xr = SimpleNamespace( + close=Mock(side_effect=RuntimeError("drain failed") if drain_failed else None), + async_worker_owns_environment=False, + ) + failure = RuntimeError("environment construction failed") + with pytest.raises(RuntimeError) as caught: + with launch_simulation(PhysicsCfg(), cleanup_context=False): + if new_context: + state["context"] = object() + try: + raise failure + finally: + entry_namespace["_close_teleop_resources"](None, camera, xr, {}, previous_context=previous) + assert caught.value is failure + assert clear.call_count == int(new_context and not drain_failed) + if not new_context: + assert state["context"] is previous diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py new file mode 100644 index 00000000000..cbe5d1a7ecd --- /dev/null +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -0,0 +1,346 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Execute the real replay bootstrap without starting a simulator. + +Only device/simulator endpoints are isolated; the replay/reset function bodies +and Kit detection are production code. These tests do not qualify MCAP motion. +""" + +import builtins +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +@pytest.fixture +def replay_presentation_type(): + from isaaclab_teleop.entrypoints.teleop_replay_agent import _ReplayPresentation + + return _ReplayPresentation + + +def test_replay_failed_input_drain_requests_stop_without_releasing_borrowers(replay_presentation_type): + presentation = replay_presentation_type.__new__(replay_presentation_type) + presentation.device = SimpleNamespace(session_cleanup_failed=True) + presentation.session = Mock() + with pytest.raises(RuntimeError, match="Replay input did not drain"): + presentation.close() + presentation.session.request_shutdown.assert_called_once_with() + presentation.session.close.assert_not_called() + + +@pytest.mark.parametrize( + "kit_active,run_index,backend,viewer,hard_reset", + [ + (False, 0, "ovphysx", False, True), + (True, 0, "ovphysx", False, True), + (True, 1, "ovphysx", False, True), + (False, 0, "ovphysx", True, True), + (False, 0, "newton_mjwarp", True, False), + ], +) +def test_replay_bootstrap_waits_only_for_an_active_kit_application( + monkeypatch, kit_active, run_index, backend, viewer, hard_reset +): + """A viewer preserves backend reset policy; kitless replay never waits for Kit assets.""" + calls = [] + + class Device: + def __enter__(self): + calls.append("device-enter") + return self + + def reset(self): + calls.append("device-reset") + + def request_start(self): + calls.append("device-start") + + def __exit__(self, *args): + calls.append("device-exit") + + app = ModuleType("omni.kit.app") + app.get_app = lambda: object() + if kit_active: + monkeypatch.setitem(sys.modules, "omni.kit.app", app) + else: + monkeypatch.delitem(sys.modules, "omni.kit.app", raising=False) + wait_module = ModuleType("isaaclab_physx.renderers.isaac_rtx_renderer_utils") + wait_module.wait_for_stage_load = lambda *args, **kwargs: calls.append("kit-wait") + monkeypatch.setitem(sys.modules, wait_module.__name__, wait_module) + original_import = builtins.__import__ + + def import_with_kit_guard(name, *args, **kwargs): + if name == wait_module.__name__ and not kit_active: + raise AssertionError("Kit-only stage loader imported by a kitless replay") + return original_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", import_with_kit_guard) + + def render(): + calls.append("render") + + env = SimpleNamespace( + scene=SimpleNamespace(physics_backend=backend, env_prim_paths=["/Lab/Cells/Cell0"]), + sim=SimpleNamespace( + device="cpu", render=render, reset=lambda: calls.append("sim-reset"), is_running=lambda: False + ), + reset=lambda: calls.append("env-reset"), + ) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + from isaacteleop.teleop_session_manager import RetargetingExecutionConfig + + cfg = SimpleNamespace(retargeting_execution=RetargetingExecutionConfig(mode="pipelined")) + constructed_modes = [] + + def create_device(selected, **kwargs): + assert kwargs["env_prim_path"] == "/Lab/Cells/Cell0" + constructed_modes.append(selected.retargeting_execution.mode) + assert selected.retargeting_execution.mode == "pipelined" + return Device() + + namespace = { + "args_cli": SimpleNamespace( + replay_file="fixture.mcap", + enable_debug_visualization=False, + max_stage_load_wait_s=5, + replay_start_delay_s=0, + start_on_replay=False, + max_replay_duration_s=1, + ), + "NvmlGpuStatsProvider": lambda: SimpleNamespace(_available=True, summary=lambda: {}), + "create_isaac_teleop_device": create_device, + "_RENDERER_SETTLE_FRAMES": 30, + } + for name, value in namespace.items(): + monkeypatch.setattr(replay, name, value, raising=False) + presentation = SimpleNamespace(start=lambda *_args: calls.append("viewer-start")) if viewer else None + replay._run_single_replay(env, cfg, None, run_index, 2, presentation) + expected = ["device-enter"] + if hard_reset: + expected.append("sim-reset") + expected.extend(["env-reset", "device-reset"]) + if viewer: + expected.append("viewer-start") + if kit_active and run_index == 0: + expected.append("kit-wait") + assert calls == [*expected, "device-exit"] + assert env.sim.render is render + assert constructed_modes == ["pipelined"] + + +@pytest.mark.parametrize( + "stop_at,requested", + [ + ("startup", 2), + ("startup-second-signal", 2), + ("construction-new", 2), + ("construction-previous", 2), + ("after-first", 2), + ("after-first", 1), + ], +) +def test_presented_replay_preserves_startup_ownership_and_batch_completion(monkeypatch, stop_at, requested): + """Failed construction preserves prior owners; a partial batch never reports completion.""" + from isaaclab.sim import SimulationContext + + calls = [] + previous = object() + context_state = {"current": previous} + + def clear_context(): + calls.append("context-clear") + context_state["current"] = None + + monkeypatch.setattr(SimulationContext, "instance", lambda: context_state["current"]) + monkeypatch.setattr(SimulationContext, "clear_instance", clear_context) + handlers = {} + reports = [] + session = SimpleNamespace( + shutdown_requested=False, + async_worker_owns_environment=not stop_at.startswith("construction"), + request_shutdown=lambda: None, + ) + + class Presentation: + def __init__(self, cfg): + self.session = session + self.started = False + + def close(self): + if stop_at == "startup-second-signal": + handlers[2](2, None) + calls.append("presentation-close") + + def install_signal(sig, handler): + handlers[sig] = handler + return + + def make_env(*args, **kwargs): + if stop_at.startswith("startup"): + handlers[1](1, None) + if stop_at.startswith("construction"): + if stop_at == "construction-new": + context_state["current"] = object() + raise RuntimeError("environment construction failed") + return SimpleNamespace(unwrapped=env) + + def replay(**kwargs): + calls.append("replay") + session.shutdown_requested = True + return SimpleNamespace(outcome="success", to_dict=lambda index: {"outcome": "success"}) + + env = SimpleNamespace(sim=SimpleNamespace(is_running=lambda: True)) + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay_agent + + namespace = { + "args_cli": SimpleNamespace( + ovxr=True, + require_kit=False, + task="task", + replay_file="fixture.mcap", + num_replays=requested, + stats_output_file=None, + ), + "_ReplayPresentation": Presentation, + "signal": SimpleNamespace(SIGINT=1, SIGTERM=2, signal=install_signal), + "gym": SimpleNamespace(make=make_env), + "_run_single_replay": replay, + "_extract_env_perf_cfg": lambda cfg: None, + "_print_stdout_summary": reports.append, + } + for name, value in namespace.items(): + monkeypatch.setattr(replay_agent, name, value, raising=False) + if stop_at.startswith("startup"): + with pytest.raises(KeyboardInterrupt): + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + assert calls == ["presentation-close"] + elif stop_at.startswith("construction"): + with pytest.raises(RuntimeError, match="environment construction failed"): + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + assert calls == ["presentation-close"] + (["context-clear"] if stop_at == "construction-new" else []) + assert context_state["current"] is (None if stop_at == "construction-new" else previous) + else: + result = replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + assert result == (0 if requested == 1 else 1) + assert reports[0]["num_replays"] == 1 + assert reports[0]["requested_num_replays"] == requested + assert reports[0]["batch_complete"] is (requested == 1) + assert calls == ["replay", "presentation-close"] + assert handlers == {1: None, 2: None} + + +@pytest.mark.parametrize("start_on_replay", [False, True]) +def test_replay_control_timing_uses_recorded_events_or_explicit_legacy_start(monkeypatch, start_on_replay): + """The real control owners gate input, distinguish RESET from STOP, and retain the replay cursor.""" + import torch + from isaaclab_teleop.control_events import ControlEvents + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg + from isaaclab_teleop.isaac_teleop_device import IsaacTeleopDevice + from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle + from isaaclab_teleop.teleop_message_processor import TeleopMessageProcessor + from isaacteleop.teleop_session_manager import DefaultTeleopStateManager + + processor = TeleopMessageProcessor(name="recorded_control") + state_manager = DefaultTeleopStateManager(name="recorded_state") + lifecycle = TeleopSessionLifecycle.__new__(TeleopSessionLifecycle) + lifecycle._message_processor = processor + lifecycle._last_control_events = ControlEvents() + lifecycle._last_control_event_frame_id = None + lifecycle._haptic_forces = {} + lifecycle._session = SimpleNamespace(last_context=None, last_step_info=None) + messages = {7: b"reset", 12: b"stop"} + if not start_on_replay: + messages.update({3: b"start", 10: b"start"}) + frame = -1 + applied, resets, recorder_resets = [], [], [] + + class Signal(list): + is_none = False + + class RecordedDevice(IsaacTeleopDevice): + def __init__(self): + self._session_lifecycle = lifecycle + self._anchor_manager = SimpleNamespace(reset=lambda: None, cleanup=lambda: None) + + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def advance(self): + nonlocal frame + frame += 1 + payload = messages.get(frame) + tracked = SimpleNamespace(data=[SimpleNamespace(payload=payload)] if payload is not None else []) + outputs = {name: Signal([False]) for name in ("kill", "run_toggle", "reset")} + processor._compute_fn({processor.INPUT_MESSAGES: [tracked]}, outputs, context=None) + events = state_manager._compute_execution_events( + { + state_manager.INPUT_KILL: outputs["kill"], + state_manager.INPUT_RUN_TOGGLE: outputs["run_toggle"], + state_manager.INPUT_RESET: outputs["reset"], + }, + context=None, + ) + lifecycle._session.last_context = SimpleNamespace(execution_events=events) + lifecycle._session.last_step_info = SimpleNamespace(returned_frame_id=frame) + lifecycle._update_control_events() + return torch.zeros(1) + + env = SimpleNamespace( + num_envs=1, + scene=SimpleNamespace(physics_backend="newton_mjwarp", env_prim_paths=["/World/envs/env_0"]), + sim=SimpleNamespace(device="cpu", render=lambda: None, is_running=lambda: frame < 15), + recorder_manager=SimpleNamespace(reset=lambda: recorder_resets.append(frame)), + reset=lambda: resets.append(frame), + ) + + def step(_actions): + applied.append(frame) + env.sim.render() + return None, None, torch.tensor([False]), torch.tensor([False]), {} + + env.step = step + monkeypatch.setattr(replay, "create_isaac_teleop_device", lambda *_args, **_kwargs: RecordedDevice()) + monkeypatch.setattr( + replay, + "NvmlGpuStatsProvider", + lambda: SimpleNamespace(sample=lambda: None, summary=lambda: {}), + ) + monkeypatch.setattr( + replay, + "args_cli", + SimpleNamespace( + replay_file="recording.mcap", + start_on_replay=start_on_replay, + enable_debug_visualization=False, + max_replay_duration_s=10.0, + num_success_steps=1, + ), + raising=False, + ) + conditioning_frames = [] + + def condition(_env, action, state): + previous = state.get("frames", 0) + conditioning_frames.append(previous) + state["frames"] = previous + 1 + return action + + cfg = IsaacTeleopCfg(action_conditioner=condition) + stats = replay._run_single_replay(env, cfg, None, run_index=1, total_runs=2) + assert applied == ([2, 3, 4, 5, 6, 9, 10, 11] if start_on_replay else [5, 6, 10, 11]) + assert resets == [-1, 0, 7] + assert recorder_resets == [0, 7] + assert frame == 12 # STOP terminates; RESET retains the cursor and permits the later START. + assert conditioning_frames == ([0, 1, 2, 3, 4, 0, 1, 2] if start_on_replay else [0, 1, 0, 1]) + assert stats.active_iterations == len(applied) + assert stats.outcome == "incomplete" diff --git a/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py b/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py index 1796331576e..0773650c5dc 100644 --- a/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py +++ b/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py @@ -27,7 +27,6 @@ """ import os -import re import signal import subprocess import sys @@ -36,40 +35,9 @@ import pytest -pytestmark = [pytest.mark.integration, pytest.mark.rendering] - # Repository root: this file lives at source/isaaclab_teleop/test/.py ROOT = Path(__file__).resolve().parents[3] -# XR experience file whose dependencies decide whether ``--xr`` can start offline. -_XR_EXPERIENCE = ROOT / "apps" / "isaaclab.python.xr.openxr.kit" - -# ``.kit`` files repeat section headers, so they cannot be parsed as TOML. -_SECTION_RE = re.compile(r"^\s*\[+(?P[^\[\]]+)\]+\s*$") -_DEPENDENCY_RE = re.compile(r'^\s*"(?P[^"]+)"\s*=') - - -def _kit_dependencies(experience: Path) -> set[str]: - """Collect the extension names declared in an experience file's dependency sections. - - Scoped to ``[dependencies]`` tables so quoted keys elsewhere in the file (settings tables - use them too) are not mistaken for extensions. - """ - dependencies: set[str] = set() - in_dependencies = False - - for line in experience.read_text(encoding="utf-8").splitlines(): - section = _SECTION_RE.match(line) - if section is not None: - in_dependencies = section.group("name").strip() == "dependencies" - continue - if in_dependencies: - dependency = _DEPENDENCY_RE.match(line) - if dependency is not None: - dependencies.add(dependency.group("name")) - - return dependencies - # Lightweight Franka task that configures an Isaac Capture pipeline (controllers), so omitting # ``--teleop_device`` drives the scripts through the Isaac Capture stack rather than the legacy @@ -167,6 +135,8 @@ def _assert_started_cleanly(output: str, markers: list[str], script: str) -> Non ) +@pytest.mark.integration +@pytest.mark.rendering def test_teleop_se3_agent_starts(tmp_path): """teleop_se3_agent.py builds the Isaac Capture device (past the #6656 RTX/camera regression). @@ -185,6 +155,8 @@ def test_teleop_se3_agent_starts(tmp_path): _assert_started_cleanly(output, [_ISAAC_TELEOP_MARKER], "teleop_se3_agent.py") +@pytest.mark.integration +@pytest.mark.rendering def test_record_demos_starts(tmp_path): """record_demos.py builds the Isaac Capture device without AttributeError-ing on enable_cameras. @@ -204,6 +176,8 @@ def test_record_demos_starts(tmp_path): _assert_started_cleanly(output, [_ISAAC_TELEOP_MARKER], "record_demos.py") +@pytest.mark.integration +@pytest.mark.rendering def test_record_demos_starts_when_reward_references_success(tmp_path): """record_demos.py preserves success-term references while disabling automatic resets.""" argv = [ @@ -216,29 +190,14 @@ def test_record_demos_starts_when_reward_references_success(tmp_path): str(tmp_path / "dataset.hdf5"), "physics=isaacsim_physx", "presets=diffik", + *_NO_CLOUDXR, ] output = _launch_until_marker(argv, [_ENV_READY_MARKER], tmp_path / "record_demos_success_reward.log") _assert_started_cleanly(output, [_ENV_READY_MARKER], "record_demos.py with a success reward") -def test_xr_experience_declares_only_shipped_extensions(): - """The XR experience depends on XR extensions Isaac Sim ships, not on a registry-only bundle. - - ``omni.kit.xr.bundle.generic`` is not shipped and resolves only from the Kit SDK registry, so - depending on it aborted every ``--xr`` run that could not reach the registry. - - Static on purpose: a machine that reaches the registry caches the bundle on first use and - loads it locally forever after, so a launch test cannot catch this once that cache is warm. - """ - dependencies = _kit_dependencies(_XR_EXPERIENCE) - - assert "omni.kit.xr.system.openxr" in dependencies - assert "omni.kit.xr.ui.window.profile" in dependencies, "omni.kit.xr.profile.ar's Kit 110 successor is missing" - assert not [name for name in dependencies if name.startswith("omni.kit.xr.bundle.")], ( - "XR bundle meta-extensions are not shipped with Isaac Sim and resolve only from the Kit SDK registry" - ) - - +@pytest.mark.integration +@pytest.mark.rendering def test_record_demos_starts_headless_xr(tmp_path): """record_demos.py starts under ``--xr`` with no ``--viz kit`` (the QA-reported failure). @@ -267,6 +226,8 @@ def test_record_demos_starts_headless_xr(tmp_path): _assert_started_cleanly(output, markers, "record_demos.py --xr") +@pytest.mark.integration +@pytest.mark.rendering def test_teleop_replay_agent_applies_rtx_without_replicator_crash(tmp_path): """teleop_replay_agent.py clears the RTX-apply site (create env) without the replicator crash. diff --git a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py new file mode 100644 index 00000000000..d4d8ccea836 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py @@ -0,0 +1,344 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused tests for kitless XR anchor construction.""" + +from __future__ import annotations + +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import MagicMock + +import numpy as np +import pytest +from isaaclab_teleop import xr_anchor_manager +from isaaclab_teleop.xr_cfg import XrCfg + +from pxr import Usd, UsdGeom + +from isaaclab.sim.utils.stage import use_stage + + +def test_dynamic_external_anchor_commits_pose_before_input_and_preserves_reset(monkeypatch): + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode + from scipy.spatial.transform import Rotation + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg( + anchor_prim_path="/World/Robot", + anchor_pos=(1.0, 2.0, -0.95), + fixed_anchor_height=True, + anchor_rotation_mode=XrAnchorRotationMode.FOLLOW_PRIM, + ) + with use_stage(Usd.Stage.CreateInMemory()): + UsdGeom.Xform.Define(xr_anchor_manager.get_current_stage(), "/World/Robot") + manager = xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=True) + with pytest.raises(RuntimeError, match="committed post-reset"): + manager.get_world_matrix() + reference = np.eye(4) + reference[:3, 3] = (3, 4, 5) + committed = [] + epoch, seed = manager.external_epoch + assert seed + assert manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + np.testing.assert_allclose(committed[-1][0, :3, 3], (4, 6, 4.05)) + reference[:3, 3] = (8, 9, 12) + reference[:3, :3] = Rotation.from_euler("z", 90, degrees=True).as_matrix() + manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + expected = committed[-1][0].copy() + expected[:3, :3] = expected[:3, :3] @ manager._OXR_TO_USD_ROTATION + np.testing.assert_allclose(manager.get_world_matrix(), expected, atol=1e-7) + np.testing.assert_allclose(expected[:3, 3], (9, 11, 4.05)) + before = manager.get_world_matrix() + np.testing.assert_array_equal(manager.get_world_matrix(), before) + manager.toggle_anchor_rotation() + reference[:3, :3] = np.eye(3) + manager.commit_external_pose(reference, 0.02, epoch, lambda m: committed.append(m.copy())) + np.testing.assert_allclose(manager.get_world_matrix()[:3, :3], before[:3, :3], atol=1e-7) + manager.reset() + assert not manager.commit_external_pose(reference, 0.02, epoch, lambda _: pytest.fail("old epoch sealed")) + with pytest.raises(RuntimeError, match="committed post-reset"): + manager.get_world_matrix() + manager.commit_external_pose(reference, 0.02, manager.external_epoch[0], lambda m: committed.append(m.copy())) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], (9, 11, 11.05)) + with pytest.raises(RuntimeError, match="write failed"): + manager.commit_external_pose( + reference, 0.02, manager.external_epoch[0], lambda _: (_ for _ in ()).throw(RuntimeError("write failed")) + ) + with pytest.raises(RuntimeError, match="publication failed"): + manager.get_world_matrix() + + manager.reset() + with pytest.raises(RuntimeError, match="publication failed"): + manager.commit_external_pose(reference, 0.02, epoch, lambda _: pytest.fail("failed pair revived")) + + +def test_standalone_replay_without_kit_xr_preserves_anchor_fallback(monkeypatch): + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) + cfg = XrCfg(anchor_prim_path="/World/Robot", anchor_pos=(1.0, 2.0, 3.0)) + manager = xr_anchor_manager.XrAnchorManager(cfg) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + + +def test_ovxr_session_does_not_borrow_an_available_kit_xr_core(monkeypatch): + core = MagicMock() + monkeypatch.setattr(xr_anchor_manager, "XRCore", core) + with use_stage(Usd.Stage.CreateInMemory()): + manager = xr_anchor_manager.XrAnchorManager(XrCfg(), ovxr_session=True) + assert manager.xr_core is None + core.get_singleton.assert_not_called() + + +@pytest.mark.parametrize("capability", ["none", "live-handles", "recorded-input-viewer"]) +def test_public_device_exposes_dynamic_anchor_only_for_ovxr_session(monkeypatch, capability): + from isaaclab_teleop import isaac_teleop_device + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg + + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(isaac_teleop_device, "CommandHandler", MagicMock()) + lifecycle = MagicMock() + monkeypatch.setattr(isaac_teleop_device, "TeleopSessionLifecycle", lifecycle) + cfg = IsaacTeleopCfg( + pipeline_builder=lambda: None, + xr_cfg=XrCfg(anchor_prim_path="/World/Robot"), + ) + kwargs = dict(use_kit_xr_bridge=False) + if capability == "live-handles": + kwargs["external_oxr_handles_provider"] = lambda: None + elif capability == "recorded-input-viewer": + kwargs.update(ovxr_session=True, mcap_replay_path="recording.mcap") + with use_stage(Usd.Stage.CreateInMemory()): + device = isaac_teleop_device.IsaacTeleopDevice(cfg, **kwargs) + assert device.xr_anchor_path == "/World/Robot/XRAnchor" + assert (device.external_anchor_state is not None) == (capability != "none") + lifecycle.assert_called_once() + + +def test_no_xr_core_does_not_acquire_carb_settings(monkeypatch): + settings = ModuleType("carb.settings") + settings.get_settings = MagicMock(side_effect=AssertionError("kitless anchor touched carb settings")) + carb = ModuleType("carb") + carb.settings = settings + monkeypatch.setitem(sys.modules, "carb", carb) + monkeypatch.setitem(sys.modules, "carb.settings", settings) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) + cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0), anchor_rot=(0.0, 0.0, 0.0, 1.0)) + + manager = xr_anchor_manager.XrAnchorManager(cfg) + + settings.get_settings.assert_not_called() + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + + +def test_render_anchor_does_not_offset_tracked_device_poses(monkeypatch): + created_prims = [] + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: False) + monkeypatch.setattr( + xr_anchor_manager, + "_create_prim", + lambda path, **kwargs: created_prims.append((path, kwargs)), + ) + cfg = XrCfg( + anchor_pos=(0.0, 0.0, 0.0), + render_anchor_pos=(0.0, -2.5, 0.0), + ) + + manager = xr_anchor_manager.XrAnchorManager(cfg) + + assert manager.anchor_headset_path == "/World/XRAnchor" + assert manager.render_anchor_path == "/World/XRRenderAnchor" + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + assert [path for path, _kwargs in created_prims] == ["/World/XRAnchor", "/World/XRRenderAnchor"] + np.testing.assert_allclose(created_prims[1][1]["position"], cfg.render_anchor_pos) + + +def test_replay_reuses_real_stage_scoped_anchors_without_kit(monkeypatch, caplog): + """A second device must not recreate or overwrite the same USD anchors.""" + stage = Usd.Stage.CreateInMemory() + cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0), render_anchor_pos=(0.0, -2.5, 0.0)) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + with use_stage(stage): + first = xr_anchor_manager.XrAnchorManager(cfg) + first.cleanup() + second = xr_anchor_manager.XrAnchorManager(cfg) + second.cleanup() + assert "Failed to create XR" not in caplog.text + for path, position in (("/World/XRAnchor", cfg.anchor_pos), ("/World/XRRenderAnchor", cfg.render_anchor_pos)): + prim = stage.GetPrimAtPath(path) + assert prim.IsValid() + np.testing.assert_allclose( + UsdGeom.Xformable(prim).ComputeLocalToWorldTransform(0).ExtractTranslation(), position + ) + + +def test_replay_refreshes_configured_static_anchors_on_the_same_stage(monkeypatch): + from isaaclab.sim.utils.transforms import resolve_prim_pose + + stage = Usd.Stage.CreateInMemory() + world = UsdGeom.Xform.Define(stage, "/World") + world.AddTranslateOp().Set((2.0, -1.0, 0.5)) + world.AddRotateZOp().Set(90.0) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + first_cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0), render_anchor_pos=(0.0, -2.5, 0.0)) + next_cfg = XrCfg( + anchor_pos=(-1.0, 0.5, 2.0), + anchor_rot=(0.0, 0.0, 1.0, 0.0), + render_anchor_pos=(3.0, -4.0, 0.25), + render_anchor_rot=(0.0, 0.0, 0.0, 1.0), + ) + with use_stage(stage): + first = xr_anchor_manager.XrAnchorManager(first_cfg, ovxr_session=True) + for path, position in ( + (first.anchor_headset_path, first_cfg.anchor_pos), + (first.render_anchor_path, first_cfg.render_anchor_pos), + ): + observed_pos, observed_rot = resolve_prim_pose(stage.GetPrimAtPath(path)) + np.testing.assert_allclose(observed_pos, position, atol=1e-7) + assert abs(np.dot(observed_rot, first_cfg.anchor_rot)) == pytest.approx(1.0) + first.cleanup() + second = xr_anchor_manager.XrAnchorManager(next_cfg, ovxr_session=True) + for path, position, rotation in ( + (second.anchor_headset_path, next_cfg.anchor_pos, next_cfg.anchor_rot), + (second.render_anchor_path, next_cfg.render_anchor_pos, next_cfg.render_anchor_rot), + ): + observed_pos, observed_rot = resolve_prim_pose(stage.GetPrimAtPath(path)) + np.testing.assert_allclose(observed_pos, position, atol=1e-7) + assert abs(np.dot(observed_rot, rotation)) == pytest.approx(1.0) + np.testing.assert_allclose(second.get_world_matrix()[:3, 3], next_cfg.anchor_pos) + second.cleanup() + + +@pytest.mark.parametrize("anchor_path", ["/World/XRAnchor", "/World/XRRenderAnchor"]) +def test_ovxr_session_rejects_an_unusable_stage_anchor(monkeypatch, anchor_path): + """An invalid anchor must not silently misalign tracking and rendering.""" + stage = Usd.Stage.CreateInMemory() + UsdGeom.Scope.Define(stage, anchor_path) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg(render_anchor_pos=(0.0, -2.5, 0.0)) + with use_stage(stage), pytest.raises(ValueError, match="is not transformable"): + xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=True) + + +def test_standalone_replay_keeps_warning_for_an_unusable_anchor(monkeypatch, caplog): + stage = Usd.Stage.CreateInMemory() + UsdGeom.Scope.Define(stage, "/World/XRAnchor") + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0)) + with use_stage(stage): + manager = xr_anchor_manager.XrAnchorManager(cfg) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) + manager.cleanup() + assert "is not transformable" in caplog.text + + +def test_kit_anchor_clock_is_lazy_until_subsequent_smoothed_update(monkeypatch): + from isaaclab_teleop import xr_anchor_utils + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode + + cfg = XrCfg(anchor_prim_path=None) + core = MagicMock() + clock = MagicMock() + clock.instance.side_effect = AssertionError("unused simulation clock") + monkeypatch.setattr(xr_anchor_utils, "SimulationContext", clock) + stage = Usd.Stage.CreateInMemory() + UsdGeom.Xform.Define(stage, "/XRAnchor") + with use_stage(stage): + synchronizer = xr_anchor_utils.XrAnchorSynchronizer(core, cfg, "/XRAnchor") + synchronizer.sync_headset_to_anchor() + core.set_world_transform_matrix.assert_called_once() + clock.instance.assert_not_called() + + cfg.anchor_rotation_mode = XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED + policy = xr_anchor_utils._XrAnchorPosePolicy(cfg) + dt = MagicMock(return_value=0.02) + policy.update(np.eye(4), dt) + dt.assert_not_called() + moved = np.eye(4) + moved[0, 3] = 2.0 + policy.update(moved, dt) + dt.assert_called_once_with() + + +@pytest.mark.parametrize("kit", [False, True]) +@pytest.mark.parametrize("follow,expected_position", [(False, (9.0, 9.0, 4.05)), (True, (11.0, 11.0, 4.05))]) +def test_render_override_preserves_filtered_tracking_pose(monkeypatch, follow, expected_position, kit): + from isaaclab_teleop import xr_anchor_utils + from isaaclab_teleop.xr_cfg import XrAnchorRotationMode + from scipy.spatial.transform import Rotation + + reference = np.eye(4) + reference[:3, 3] = (3.0, 4.0, 5.0) + writes, callbacks = {}, [] + if kit: + core = MagicMock() + core.set_world_transform_matrix.side_effect = lambda path, matrix, _layer: writes.update( + {path: np.asarray(matrix).T.copy()} + ) + core.get_message_bus.return_value.create_subscription_to_pop_by_type.side_effect = ( + lambda _event, callback, **_kwargs: callbacks.append(callback) + ) + monkeypatch.setattr(xr_anchor_manager, "XRCore", SimpleNamespace(get_singleton=lambda: core)) + monkeypatch.setattr(xr_anchor_manager, "XRCoreEventType", SimpleNamespace(pre_sync_update=1)) + monkeypatch.setattr(xr_anchor_manager, "has_kit", lambda: False) + fabric_stage = SimpleNamespace(GetPrimAtPath=lambda _path: SimpleNamespace(IsValid=lambda: True)) + monkeypatch.setattr( + xr_anchor_utils, + "usdrt", + SimpleNamespace(Usd=SimpleNamespace(Stage=SimpleNamespace(Attach=lambda _: fabric_stage))), + raising=False, + ) + monkeypatch.setattr(xr_anchor_utils, "get_current_stage_id", lambda: 1) + monkeypatch.setattr( + xr_anchor_utils, + "Rt", + SimpleNamespace( + Xformable=lambda _: SimpleNamespace( + GetFabricHierarchyWorldMatrixAttr=lambda: SimpleNamespace(Get=lambda: reference.T) + ) + ), + raising=False, + ) + else: + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + cfg = XrCfg( + anchor_prim_path="/World/Robot", + anchor_pos=(1.0, 2.0, -0.95), + render_anchor_pos=(1.0, 0.0, -0.95), + render_anchor_rot=tuple(Rotation.from_euler("z", 45, degrees=True).as_quat()), + fixed_anchor_height=True, + anchor_rotation_mode=XrAnchorRotationMode.FOLLOW_PRIM if follow else XrAnchorRotationMode.FIXED, + ) + with use_stage(Usd.Stage.CreateInMemory()): + UsdGeom.Xform.Define(xr_anchor_manager.get_current_stage(), "/World/Robot") + manager = xr_anchor_manager.XrAnchorManager(cfg, ovxr_session=not kit) + commits = [] + + def update(): + if kit: + assert len(callbacks) == 1 + callbacks[0](None) + return writes[manager.anchor_headset_path], writes[manager.render_anchor_path] + manager.commit_external_pose(reference, 0.02, manager.external_epoch[0], lambda m: commits.append(m.copy())) + return commits[-1] + + tracking, rendering = update() + np.testing.assert_allclose(tracking[:3, 3], (4.0, 6.0, 4.05)) + np.testing.assert_allclose(rendering[:3, 3], (4.0, 4.0, 4.05)) + reference[:3, 3] = (8.0, 9.0, 12.0) + reference[:3, :3] = Rotation.from_euler("xyz", (30, 20, 90), degrees=True).as_matrix() + tracking, rendering = update() + np.testing.assert_allclose(tracking[:3, 3], (9.0, 11.0, 4.05)) + np.testing.assert_allclose(rendering[:3, 3], expected_position) + np.testing.assert_allclose( + rendering[:3, :3], Rotation.from_euler("z", 135 if follow else 45, degrees=True).as_matrix(), atol=1e-7 + ) + np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], tracking[:3, 3]) + manager.reset() + tracking, rendering = update() + np.testing.assert_allclose(tracking[:3, 3], (9.0, 11.0, 11.05)) + np.testing.assert_allclose(rendering[:3, 3], (9.0, 9.0, 11.05)) diff --git a/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst b/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst new file mode 100644 index 00000000000..79924c3b756 --- /dev/null +++ b/source/isaaclab_visualizers/changelog.d/headless-xr.fixed.rst @@ -0,0 +1,2 @@ +* Fixed headless Kit XR sessions remaining idle by continuing scene publication and + application updates without a desktop viewport. diff --git a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py index 66bdbbba80d..11b27512cca 100644 --- a/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py +++ b/source/isaaclab_visualizers/isaaclab_visualizers/kit/kit_visualizer.py @@ -170,6 +170,9 @@ def initialize(self, sim: SimulationContext, *, cameras: list[PerspectiveCameraC def step(self, dt: float) -> None: """Advance visualizer/UI updates for one simulation step. + Headless XR still publishes scene updates and pumps the Kit application. + Ordinary headless image capture renders only when an image is requested. + Args: dt: Simulation time-step in seconds. """ @@ -178,9 +181,9 @@ def step(self, dt: float) -> None: self._app_pumped_this_step = False self._sim_time += dt self._step_counter += 1 - # Headless mode: skip the app update and camera panel refresh; rendering is - # triggered on demand by render_rgb_array() / render_tiled_rgb_array(). - if self._runtime_headless: + # Headless capture renders on demand, but XR needs continuous app updates + # to create its session and submit frames even without a desktop viewport. + if self._runtime_headless and not self._sim.get_setting("/isaaclab/xr/enabled"): return provider = self._sim.get_scene_data_provider() self._fabric.update_transforms(provider) diff --git a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py index 1f393616d38..1ac211a06e6 100644 --- a/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py +++ b/source/isaaclab_visualizers/test/test_kit_visualizer_scene_partitioning.py @@ -59,11 +59,13 @@ def kit_timeline(monkeypatch): @pytest.mark.parametrize("headless", [False, True]) -def test_viewport_pose_publication_is_deferred_for_headless_capture(monkeypatch, kit_app, headless): +@pytest.mark.parametrize("xr", [False, True]) +def test_viewport_pose_publication_is_deferred_only_for_headless_capture(monkeypatch, kit_app, headless, xr): visualizer = KitVisualizer(VisualizerCfg(headless=headless, backend=KitBackendCfg(origin_type="asset"))) visualizer._is_initialized = True visualizer._fabric = MagicMock() visualizer._sim = sim = MagicMock(render_generation=12) + sim.get_setting.side_effect = lambda key: xr if key == "/isaaclab/xr/enabled" else None provider = sim.get_scene_data_provider.return_value monkeypatch.setattr(visualizer, "is_training_paused", lambda: True) tracking = MagicMock() @@ -75,8 +77,8 @@ def test_viewport_pose_publication_is_deferred_for_headless_capture(monkeypatch, visualizer.step(0.1) - assert tracking.call_count == int(not headless) - if headless: + assert tracking.call_count == int(not headless or xr) + if headless and not xr: visualizer._fabric.update_transforms.assert_not_called() visualizer._fabric.update_geometries.assert_not_called() settings.set.assert_not_called() From b18f214180642cf2439a4fe39aaa622a37c7b9fd Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Fri, 30 Oct 2026 18:16:19 -0400 Subject: [PATCH 003/145] Enable G1 and GR1T2 tasks with kitless XR --- .../source/_static/css/environment-browser.js | 8 +- .../pink-hand-joint-order.fixed.rst | 1 + .../mdp/actions/pink_task_space_actions.py | 8 +- .../test/envs/test_pink_hand_joint_order.py | 46 ++ .../changelog.d/kitless-robot-tasks.added.rst | 8 + .../changelog.d/kitless-robot-tasks.minor | 0 .../contrib/locomanip_pick_place/__init__.py | 4 +- .../fixed_base_upper_body_ik_g1_env_cfg.py | 37 +- .../locomanip_pick_place/g1_kitless_newton.py | 242 ++++++ .../locomanipulation_g1_env_cfg.py | 46 +- .../isaaclab_tasks/contrib/packing_table.py | 84 ++ .../contrib/pick_place/__init__.py | 4 +- .../GR1T2_fourier_hand_6dof_kinematics.urdf | 778 ++++++++++++++++++ .../pick_place/gr1t2_kitless_action.py | 181 ++++ .../pick_place/gr1t2_kitless_action_cfg.py | 57 ++ .../pick_place/pickplace_gr1t2_env_cfg.py | 292 ++++++- .../pickplace_gr1t2_waist_enabled_env_cfg.py | 45 +- .../isaaclab_tasks/contrib/steering_wheel.py | 69 ++ .../isaaclab_tasks/contrib/teleop_env.py | 18 + .../contrib/teleop_physics_cfg.py | 57 ++ source/isaaclab_tasks/pyproject.toml | 1 + .../contrib/test_common_packing_collisions.py | 72 ++ .../test/contrib/test_g1_kitless_newton.py | 289 +++++++ .../test/contrib/test_g1_kitless_ovphysx.py | 131 +++ .../test/contrib/test_gr1t2_kitless_action.py | 368 +++++++++ .../core/test_env_cfg_no_forbidden_imports.py | 48 +- .../test/core/test_preset_cli.py | 249 +++++- .../kitless-humanoid-acceptance.skip | 0 source/isaaclab_teleop/docs/kitless_xr.md | 36 +- .../test_ovrtx_transfer_native_harness.py | 177 ++++ .../test/test_teleop_entrypoint_units.py | 44 +- 31 files changed, 3355 insertions(+), 45 deletions(-) create mode 100644 source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst create mode 100644 source/isaaclab/test/envs/test_pink_hand_joint_order.py create mode 100644 source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst create mode 100644 source/isaaclab_tasks/changelog.d/kitless-robot-tasks.minor create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py create mode 100644 source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py create mode 100644 source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py create mode 100644 source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py create mode 100644 source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py create mode 100644 source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py create mode 100644 source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.skip create mode 100644 source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 022715b3db3..87cfbbb378f 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -98,11 +98,11 @@ ["IsaacContrib-Open-Drawer-Franka-IK-Abs", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-Franka-IK-Rel", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-OpenArm", "rl_games,rsl_rl", "", "", "", "environment-previews/openarm_uni_open_drawer.jpg"], - ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], + ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], ["IsaacContrib-PickPlace-G1-InspireFTP-Abs", "", "", "", "", "environment-previews/g1_pick_place.jpg"], - ["IsaacContrib-PickPlace-GR1T2-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/gr_1_pick_place.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], - ["IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "", "", "", "", "environment-previews/gr_1_pick_place_waist.jpg"], - ["IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "", "", "isaacsim_rtx,newton_renderer", "", "environment-previews/g1_pick_place_locomanipulation.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|"]], + ["IsaacContrib-PickPlace-GR1T2-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/gr_1_pick_place.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], + ["IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/gr_1_pick_place_waist.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], + ["IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/g1_pick_place_locomanipulation.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], ["IsaacContrib-Place-Mug-Agibot-Left-Arm-RmpFlow", "", "isaacsim_physx", "", "", "environment-previews/agibot_place_mug.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Place-Toy2Box-Agibot-Right-Arm-RmpFlow", "", "isaacsim_physx", "", "", "environment-previews/agibot_place_toy.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Reach-OpenArm", "rl_games,rsl_rl,skrl", "", "", "", "environment-previews/openarm_uni_reach.jpg"], diff --git a/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst new file mode 100644 index 00000000000..ca76d3f2e2e --- /dev/null +++ b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst @@ -0,0 +1 @@ +* Preserved configured hand joint ordering when applying Pink action targets. diff --git a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py index ebd129bb6f6..454c965562f 100644 --- a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py +++ b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py @@ -72,8 +72,12 @@ def _initialize_joint_info(self) -> None: self._isaaclab_all_joint_ids = list(range(len(self._asset.data.joint_names))) self.cfg.controller.all_joint_names = self._asset.data.joint_names - # Resolve hand joints - self._hand_joint_ids, self._hand_joint_names = self._asset.find_joints(self.cfg.hand_joint_names) + # Hand targets occupy the action tensor in configuration order. Newton's + # articulation order groups these joints differently, so preserve the + # declared order when resolving ids instead of silently permuting targets. + self._hand_joint_ids, self._hand_joint_names = self._asset.find_joints( + self.cfg.hand_joint_names, preserve_order=True + ) # Combine all joint information self._controlled_joint_ids = self._isaaclab_controlled_joint_ids + self._hand_joint_ids diff --git a/source/isaaclab/test/envs/test_pink_hand_joint_order.py b/source/isaaclab/test/envs/test_pink_hand_joint_order.py new file mode 100644 index 00000000000..dfebf344dcd --- /dev/null +++ b/source/isaaclab/test/envs/test_pink_hand_joint_order.py @@ -0,0 +1,46 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Unit coverage for Pink hand-target ordering across articulation backends.""" + +from types import SimpleNamespace + +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction +from isaaclab.utils.string import resolve_matching_names + + +def test_pink_hand_commands_reach_the_configured_joints(): + """A non-self-inverse hand permutation must preserve physical joint targets.""" + + class _Asset: + data = SimpleNamespace(joint_names=["arm", "hand_a", "hand_b", "hand_c"]) + + def __init__(self): + self.joint_targets = torch.zeros((1, 4)) + + def find_joints(self, expressions, preserve_order=False): + return resolve_matching_names(expressions, self.data.joint_names, preserve_order=preserve_order) + + def set_joint_position_target_index(self, target, joint_ids): + self.joint_targets[:, joint_ids] = target + + action = PinkInverseKinematicsAction.__new__(PinkInverseKinematicsAction) + action._asset = _Asset() + action.cfg = SimpleNamespace( + pink_controlled_joint_names=["arm"], + hand_joint_names=["hand_c", "hand_a", "hand_b"], + controller=SimpleNamespace(), + enable_gravity_compensation=False, + ) + + action._initialize_joint_info() + + action._compute_ik_solutions = lambda: torch.tensor([[9.0]]) + action._target_hand_joint_positions = torch.tensor([[30.0, 10.0, 20.0]]) + action.apply_actions() + + torch.testing.assert_close(action._asset.joint_targets, torch.tensor([[9.0, 10.0, 20.0, 30.0]])) diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst new file mode 100644 index 00000000000..1748c52da86 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst @@ -0,0 +1,8 @@ +* Added G1 and GR1T2 kitless teleoperation setup, including portable GR1T2 kinematics, + backend-specific actuators, collision preparation, and bounded action targets. +* Added public OvPhysX and Newton selection while retaining Kit defaults and supporting + standalone backend preparation without optional teleoperation configuration. + Registered construction prepares the selected backend and preserves prepared configuration copies + and serialized spawn callbacks. +* Batched G1 joint-limit preparation and device validation without changing action safety bounds. +* Prepared packing-table mesh collisions for standalone physics. diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.minor b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py index eea4a34bf04..46922bb00ca 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/__init__.py @@ -12,7 +12,7 @@ gym.register( id="IsaacContrib-PickPlace-Locomanipulation-G1-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.locomanipulation_g1_env_cfg:LocomanipulationG1EnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", @@ -22,7 +22,7 @@ gym.register( id="IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.fixed_base_upper_body_ik_g1_env_cfg:FixedBaseUpperBodyIKG1EnvCfg", }, diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index e4e9a1a1e59..caf4a5b349e 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -13,6 +13,7 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR @@ -21,6 +22,11 @@ from isaaclab_tasks.contrib.locomanip_pick_place import mdp as locomanip_mdp from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + kitless_backend_prepared, + prepare_humanoid_teleop_environment, +) from isaaclab_assets.robots.unitree import G1_29DOF_CFG @@ -267,12 +273,12 @@ class FixedBaseUpperBodyIKG1SceneCfg(InteractiveSceneCfg): # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. left_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/left_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/left_hand/left_hand_[^/]*_link", update_period=0.0, history_length=3, ) right_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/right_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/right_hand/right_hand_[^/]*_link", update_period=0.0, history_length=3, ) @@ -374,6 +380,11 @@ class FixedBaseUpperBodyIKG1EnvCfg(ManagerBasedRLEnvCfg): controlled using upper body IK. """ + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + haptic_feedback: object | None = None + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: FixedBaseUpperBodyIKG1SceneCfg = FixedBaseUpperBodyIKG1SceneCfg(num_envs=1, env_spacing=2.5) # MDP settings @@ -388,12 +399,15 @@ class FixedBaseUpperBodyIKG1EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 4 self.episode_length_s = 20.0 # simulation settings self.sim.dt = 1 / 200 # 200Hz self.sim.render_interval = 2 + self.num_rerenders_on_reset = 3 # Set the URDF and mesh paths for the IK controller urdf_omniverse_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 @@ -421,6 +435,7 @@ def __post_init__(self): ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: _build_g1_upper_body_pipeline()[0], # retargeters_to_tune=lambda: _build_g1_upper_body_pipeline()[1], sim_device=self.sim.device, @@ -442,3 +457,21 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + def stabilize_for_kitless_ovphysx(self) -> None: + """Prepare the table collision authoring; let OVPhysX cook the wheel meshes.""" + if kitless_backend_prepared(self, "ovphysx"): + return + from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + + configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) + self._prepared_kitless_backend = "ovphysx" + + def stabilize_for_kitless_newton_mjwarp(self) -> None: + """Configure G1 actions, assets and contacts for kitless Newton.""" + if kitless_backend_prepared(self, "newton_mjwarp"): + return + from .g1_kitless_newton import configure_g1_for_kitless_newton + + configure_g1_for_kitless_newton(self) + self._prepared_kitless_backend = "newton_mjwarp" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py new file mode 100644 index 00000000000..0121037973e --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -0,0 +1,242 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""G1 action and actuator boundaries for kitless Newton teleoperation.""" + +from __future__ import annotations + +from typing import Any + +import numpy as np +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + +POSITION_MAX_STEP_M = 0.10 +"""Maximum Cartesian wrist-target change accepted in one active frame.""" + +ROTATION_MAX_STEP_RAD = 0.60 +"""Maximum wrist-target rotation accepted in one active frame.""" + +JOINT_TARGET_MAX_STEP_RAD = 0.10 +"""Maximum Pink joint-target change accepted in one active frame.""" + +JOINT_LIMIT_MARGIN_RAD = 1.0e-3 +"""Margin retained inside the common Pink/articulation joint limits.""" + +EXPECTED_ACTION_DIM = 28 +"""Two 7-D wrist poses followed by fourteen G1 hand targets.""" + +WRIST_QUATERNION_NORM_BOUNDS = (0.25, 2.0) +"""Reject malformed wrist rotations before the conditioner normalizes them.""" + +HAND_TARGET_MAX_MAGNITUDE_RAD = 6.5 +"""Reject hand joint targets outside the task's accepted angular input range [rad].""" + +ACTUATOR_PROFILE = { + "waist": {"stiffness": 500.0, "damping": 50.0, "armature": 0.10, "joint_effort_limit": 250.0}, + "arms": {"stiffness": 1200.0, "damping": 60.0, "armature": 0.30, "joint_effort_limit": 800.0}, + "hands": {"stiffness": 5.0, "damping": 0.5, "armature": 0.02, "joint_effort_limit": 20.0}, +} +"""Newton implicit-drive settings for G1 joint groups.""" + + +def teleop_action_target_issue(action: torch.Tensor) -> str | None: + """Return why an incoming G1 controller action must not reach Pink/Newton.""" + flat = action.detach().flatten() + if flat.numel() != EXPECTED_ACTION_DIM: + return f"unexpected G1 controller action size {flat.numel()} (expected {EXPECTED_ACTION_DIM})" + wrists = flat[:14].reshape(2, 7) + quaternion_norms = torch.linalg.norm(wrists[:, 3:7], dim=1) + min_norm, max_norm = WRIST_QUATERNION_NORM_BOUNDS + invalid_quaternion = ((quaternion_norms < min_norm) | (quaternion_norms > max_norm)).any() + invalid_hand = (torch.abs(flat[14:]) > HAND_TARGET_MAX_MAGNITUDE_RAD).any() + # CPU flags are already local; accelerator flags share one host transfer. + flags = (torch.isfinite(flat).all(), invalid_quaternion, invalid_hand) + finite, invalid_quaternion, invalid_hand = ( + tuple(bool(flag) for flag in flags) if flat.device.type == "cpu" else torch.stack(flags).cpu().tolist() + ) + if not finite: + return "non-finite action target" + if invalid_quaternion: + return f"invalid wrist quaternion norms {quaternion_norms.detach().cpu().tolist()}" + if invalid_hand: + return "unreasonable hand joint target magnitude" + return None + + +def bound_pink_joint_targets( + current: torch.Tensor, + proposed: torch.Tensor, + lower: torch.Tensor, + upper: torch.Tensor, +) -> torch.Tensor: + """Keep one Pink update finite, inside limits, and close to current state.""" + safe_lower = lower + JOINT_LIMIT_MARGIN_RAD + safe_upper = upper - JOINT_LIMIT_MARGIN_RAD + flags = ( + torch.isfinite(current).all(), + torch.isfinite(proposed).all(), + torch.isfinite(lower).all(), + torch.isfinite(upper).all(), + (safe_lower > safe_upper).any(), + ) + *finite, empty_intersection = ( + tuple(bool(flag) for flag in flags) if current.device.type == "cpu" else torch.stack(flags).cpu().tolist() + ) + if not all(finite): + raise RuntimeError("non-finite Pink joint target boundary") + if empty_intersection: + raise RuntimeError("Pink/articulation joint-limit intersection is empty") + + stepped = torch.clamp(proposed, min=current - JOINT_TARGET_MAX_STEP_RAD, max=current + JOINT_TARGET_MAX_STEP_RAD) + return torch.maximum(torch.minimum(stepped, safe_upper), safe_lower) + + +class G1NewtonBoundedPinkInverseKinematicsAction(PinkInverseKinematicsAction): + """Bound Pink outputs without changing G1's public 28-D action layout.""" + + def __init__(self, cfg, env): + super().__init__(cfg, env) + self._pink_joint_limits = None + + def _compute_ik_solutions(self) -> torch.Tensor: + proposed = super()._compute_ik_solutions() + current = self._asset.data.joint_pos.torch[:, self._isaaclab_controlled_joint_ids] + articulation_limits = self._asset.data.joint_pos_limits.torch[:, self._isaaclab_controlled_joint_ids, :] + + # Pink's controlled model/order are immutable for this action's lifetime. + # Keep articulation limits live: users may update them during a run. + if self._pink_joint_limits is None: + self._pink_joint_limits = tuple( + torch.as_tensor( + np.stack( + [ + np.asarray(getattr(controller.pink_configuration.controlled_model, limit_name))[ + controller.pink_to_isaac_lab_controlled_ordering + ] + for controller in self._ik_controllers + ] + ), + device=self.device, + dtype=proposed.dtype, + ) + for limit_name in ("lowerPositionLimit", "upperPositionLimit") + ) + pink_lower, pink_upper = self._pink_joint_limits + lower = torch.maximum(pink_lower, articulation_limits[..., 0]) + upper = torch.minimum(pink_upper, articulation_limits[..., 1]) + return bound_pink_joint_targets(current, proposed, lower, upper) + + +def _wrist_poses(env, state: dict[str, Any]): + """Return current task-selected wrist poses with body indices cached in state.""" + try: + robot = env.scene["robot"] + if "_wrist_indices" in state: + left_index, right_index = state["_wrist_indices"] + else: + body_names = list(getattr(robot, "body_names", None) or getattr(robot.data, "body_names", [])) + target_links = env.cfg.actions.upper_body_ik.target_eef_link_names + left_index = body_names.index(target_links["left_wrist"]) + right_index = body_names.index(target_links["right_wrist"]) + state["_wrist_indices"] = (left_index, right_index) + pose_w = robot.data.body_link_pose_w + pose_w = (pose_w.torch if hasattr(pose_w, "torch") else pose_w)[0] + except (AttributeError, KeyError, ValueError): + return None + return ( + pose_w[left_index, :3].clone(), + pose_w[left_index, 3:7].clone(), + pose_w[right_index, :3].clone(), + pose_w[right_index, 3:7].clone(), + ) + + +def _step_vector_towards(current: torch.Tensor, target: torch.Tensor, max_step: float) -> torch.Tensor: + delta = target - current + distance = torch.linalg.norm(delta) + alpha = torch.clamp(max_step / torch.clamp(distance, min=1.0e-6), max=1.0) + return current + delta * alpha + + +def _step_quaternion_towards(current: torch.Tensor, target: torch.Tensor, max_step: float) -> torch.Tensor: + current = torch.nn.functional.normalize(current, dim=0) + target = torch.nn.functional.normalize(target, dim=0) + target = torch.where(torch.sum(current * target) < 0.0, -target, target) + dot = torch.clamp(torch.abs(torch.sum(current * target)), 0.0, 1.0) + angle = 2.0 * torch.acos(dot) + alpha = torch.clamp(max_step / torch.clamp(angle, min=1.0e-6), max=1.0) + half_angle = angle * 0.5 + denominator = torch.clamp(torch.sin(half_angle), min=1.0e-6) + interpolated = ( + torch.sin((1.0 - alpha) * half_angle) * current + torch.sin(alpha * half_angle) * target + ) / denominator + # Coincident quaternions need no interpolation and avoid a zero denominator. + return torch.nn.functional.normalize(torch.where(angle < 1.0e-6, target, interpolated), dim=0) + + +def condition_kitless_newton_g1_action(env, action: torch.Tensor, state: dict[str, Any]) -> torch.Tensor: + """Ramp both wrist targets from the current G1 poses before Pink solves IK.""" + flat = action.detach().flatten() + if flat.numel() != EXPECTED_ACTION_DIM: + state.clear() + return action + + conditioned = action.clone() + conditioned_flat = conditioned.flatten() + if "left_pos" not in state: + current_wrist_poses = _wrist_poses(env, state) + if current_wrist_poses is None: + state.clear() + raise RuntimeError("could not resolve the task-selected G1 wrist poses") + left_pos, left_quat, right_pos, right_quat = current_wrist_poses + state["left_pos"] = left_pos.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["left_quat"] = left_quat.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["right_pos"] = right_pos.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + state["right_quat"] = right_quat.to(device=conditioned_flat.device, dtype=conditioned_flat.dtype) + + state["left_pos"] = _step_vector_towards(state["left_pos"], flat[0:3], POSITION_MAX_STEP_M) + state["left_quat"] = _step_quaternion_towards(state["left_quat"], flat[3:7], ROTATION_MAX_STEP_RAD) + state["right_pos"] = _step_vector_towards(state["right_pos"], flat[7:10], POSITION_MAX_STEP_M) + state["right_quat"] = _step_quaternion_towards(state["right_quat"], flat[10:14], ROTATION_MAX_STEP_RAD) + conditioned_flat[0:3] = state["left_pos"] + conditioned_flat[3:7] = state["left_quat"] + conditioned_flat[7:10] = state["right_pos"] + conditioned_flat[10:14] = state["right_quat"] + return conditioned + + +def apply_g1_newton_actuator_profile(robot_cfg) -> None: + """Configure the G1 actuator groups used by teleoperation.""" + groups = robot_cfg.actuators + if not set(ACTUATOR_PROFILE).issubset(groups): + raise RuntimeError(f"Expected G1 waist/arms/hands actuator groups; found {sorted(groups)}") + for name, selected in ACTUATOR_PROFILE.items(): + actuator = groups[name] + for key, value in selected.items(): + setattr(actuator, key, value) + + +def configure_g1_for_kitless_newton(env_cfg) -> None: + """Configure the controller, asset and solver for kitless Newton.""" + from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions + + solver_cfg = getattr(getattr(env_cfg.sim, "physics", None), "solver_cfg", None) + if solver_cfg is None: + raise RuntimeError("G1 kitless Newton stabilization requires an MJWarp solver config") + solver_cfg.nconmax = 4096 + solver_cfg.njmax = 1536 + + apply_g1_newton_actuator_profile(env_cfg.scene.robot) + env_cfg.actions.upper_body_ik.controller.fail_on_joint_limit_violation = False + env_cfg.actions.upper_body_ik.class_type = G1NewtonBoundedPinkInverseKinematicsAction + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.action_conditioner = condition_kitless_newton_g1_action + env_cfg.isaac_teleop.action_validator = teleop_action_target_issue + + configure_packing_table_collisions(env_cfg.scene.packing_table) + condition_newton_steering_wheel_detail_collisions(env_cfg.scene.object) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index 3b8952eadb8..c14e1b9093c 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -13,6 +13,7 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR @@ -23,7 +24,13 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.agile_locomotion_observation_cfg import ( AgileTeacherPolicyObservationsCfg, ) +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + kitless_backend_prepared, + prepare_humanoid_teleop_environment, +) from isaaclab_assets.robots.unitree import G1_29DOF_CFG @@ -296,12 +303,12 @@ class LocomanipulationG1SceneCfg(InteractiveSceneCfg): # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors # on the robot spawn, enabled in the env __post_init__. left_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/left_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/left_hand/left_hand_[^/]*_link", update_period=0.0, history_length=3, ) right_hand_contact = ContactSensorCfg( - prim_path="{ENV_REGEX_NS}/Robot/right_hand_[^/]*_link", + prim_path="{ENV_REGEX_NS}/Robot/right_hand/right_hand_[^/]*_link", update_period=0.0, history_length=3, ) @@ -430,6 +437,12 @@ class LocomanipulationG1EnvCfg(ManagerBasedRLEnvCfg): behaviors. """ + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + haptic_feedback: object | None = None + image_obs_list: list[str] = ["robot_pov_cam"] + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: LocomanipulationG1SceneCfg = LocomanipulationG1SceneCfg(num_envs=1, env_spacing=2.5) # MDP settings @@ -444,6 +457,10 @@ class LocomanipulationG1EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return + # The pretrained Agile policy consumes and produces PhysX-ordered joints. + self.scene.robot.joint_ordering = "physx" # general settings self.decimation = 4 self.episode_length_s = 20.0 @@ -455,8 +472,6 @@ def __post_init__(self): # Set the URDF path for the IK controller. Path resolution (Nucleus → local) happens at runtime. self.actions.upper_body_ik.controller.urdf_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 - self.image_obs_list = ["robot_pov_cam"] - # The scene always contains hand contact sensors, so their target bodies must have the # contact reporter API even when the optional teleop package is unavailable. self.scene.robot.spawn.activate_contact_sensors = True @@ -480,10 +495,10 @@ def __post_init__(self): self.xr.anchor_rotation_mode = XrAnchorRotationMode.FOLLOW_PRIM_SMOOTHED self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=_build_g1_locomanipulation_pipeline, sim_device=self.sim.device, xr_cfg=self.xr, - xr_camera_feed_layout=XrCameraFeedLayoutCfg(placement="head_locked", use_scene_partition=True), xr_camera_feeds=[ XrCameraFeedCfg( camera_name="robot_pov_cam", @@ -493,6 +508,7 @@ def __post_init__(self): max_update_hz=0.0, ) ], + xr_camera_feed_layout=XrCameraFeedLayoutCfg(placement="head_locked", use_scene_partition=True), ) # Drive controller haptics from the per-hand contact sensors. @@ -500,3 +516,23 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + def stabilize_for_kitless_ovphysx(self) -> None: + """Provide the packing-table support shape required by standalone OVPhysX.""" + if kitless_backend_prepared(self, "ovphysx"): + return + configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) + self._prepared_kitless_backend = "ovphysx" + + def stabilize_for_kitless_newton_mjwarp(self) -> None: + """Prepare shared task collision geometry without replacing the locomotion policy.""" + if kitless_backend_prepared(self, "newton_mjwarp"): + return + from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions + + # Match the common locomotion integrator for the floating-base robot. + self.sim.physics.solver_cfg.integrator = "implicitfast" + self.sim.physics.num_substeps = 2 + configure_packing_table_collisions(self.scene.packing_table) + condition_newton_steering_wheel_detail_collisions(self.scene.object) + self._prepared_kitless_backend = "newton_mjwarp" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py new file mode 100644 index 00000000000..c63053c45aa --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/packing_table.py @@ -0,0 +1,84 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared collision preparation for the packing-table USD asset.""" + +from collections.abc import Callable +from dataclasses import MISSING + +from isaaclab.assets import AssetBaseCfg +from isaaclab.sim import UsdFileCfg +from isaaclab.utils import configclass + + +def _spawn_with_mesh_collisions(prim_path, cfg, *args, **kwargs): + from pxr import Usd, UsdGeom, UsdPhysics + + import isaaclab.sim as sim_utils + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + stage = prim.GetStage() + sim_utils.make_uninstanceable(str(prim.GetPath()), stage=stage) + for child in list(Usd.PrimRange(prim)): + if not child.HasAPI(UsdPhysics.MeshCollisionAPI) or child.IsA(UsdGeom.Gprim): + continue + approximation = UsdPhysics.MeshCollisionAPI(child).GetApproximationAttr().Get() + child.RemoveAPI(UsdPhysics.MeshCollisionAPI) + child.RemoveAPI(UsdPhysics.CollisionAPI) + for mesh in Usd.PrimRange(child): + if mesh.IsA(UsdGeom.Mesh): + path = str(mesh.GetPath()) + sim_utils.apply_collision_properties( + path, + [sim_utils.UsdPhysicsCollisionCfg(collision_enabled=True)], + stage=stage, + create_if_missing=True, + ) + sim_utils.apply_mesh_collision_properties( + path, + [sim_utils.UsdPhysicsMeshCollisionCfg(mesh_approximation_name=approximation or "boundingCube")], + stage=stage, + ) + if cfg.physx_offsets: + from isaaclab_physx.sim.schemas import PhysxCollisionCfg + + sim_utils.apply_collision_properties( + str(prim.GetPath()) + "/.*", + [PhysxCollisionCfg(contact_offset=0.001, rest_offset=0.0)], + stage=stage, + ) + return prim + + +@configclass +class PackingTableUsdFileCfg(UsdFileCfg): + """Declared spawn-time collision preparation for the packing-table asset.""" + + original_spawn: Callable = MISSING + physx_offsets: bool = False + func: Callable = _spawn_with_mesh_collisions + + +def configure_packing_table_collisions(table_cfg: AssetBaseCfg, *, physx_offsets: bool = False) -> None: + """Move the asset's non-geometric collision anchor onto its existing meshes. + + The source USD attaches a bounding-cube collision approximation to an Xform. + Both standalone backends need concrete geometry. Preserve the authored mesh + transforms and approximation, then use the common schema writers for each + mesh. The five bin colliders are left intact; no replacement geometry or + backend-specific cooking implementation is needed. + """ + if table_cfg.spawn is None: + return + if not isinstance(table_cfg.spawn, PackingTableUsdFileCfg): + original_spawn = table_cfg.spawn.func + table_cfg.spawn = PackingTableUsdFileCfg(**vars(table_cfg.spawn)) + table_cfg.spawn.original_spawn = original_spawn + elif table_cfg.spawn.func is not _spawn_with_mesh_collisions and not ( + isinstance(table_cfg.spawn.func, str) and table_cfg.spawn.func == f"{__name__}:_spawn_with_mesh_collisions" + ): + table_cfg.spawn.original_spawn = table_cfg.spawn.func + table_cfg.spawn.func = _spawn_with_mesh_collisions + table_cfg.spawn.physx_offsets = physx_offsets diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py index 2e004118995..1d82b7c783c 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/__init__.py @@ -9,7 +9,7 @@ gym.register( id="IsaacContrib-PickPlace-GR1T2-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.pickplace_gr1t2_env_cfg:PickPlaceGR1T2EnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", @@ -39,7 +39,7 @@ gym.register( id="IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", - entry_point="isaaclab.envs:ManagerBasedRLEnv", + entry_point="isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv", kwargs={ "env_cfg_entry_point": f"{__name__}.pickplace_gr1t2_waist_enabled_env_cfg:PickPlaceGR1T2WaistEnabledEnvCfg", "robomimic_bc_cfg_entry_point": f"{agents.__name__}:robomimic/bc_rnn_low_dim.json", diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf new file mode 100644 index 00000000000..f8f8af75677 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf @@ -0,0 +1,778 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py new file mode 100644 index 00000000000..bcb7358ffe2 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py @@ -0,0 +1,181 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""GR1T2 Pink action conditioning for standalone OVPhysX and Newton.""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from typing import TYPE_CHECKING + +import torch + +from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedEnv + + from .gr1t2_kitless_action_cfg import GR1T2KitlessPinkInverseKinematicsActionCfg + + +def bound_ik_joint_targets( + targets: torch.Tensor, + current: torch.Tensor, + max_delta_rad: float, + max_magnitude_rad: float, +) -> torch.Tensor: + """Hold invalid rows and bound finite IK targets relative to measured state.""" + finite_rows = torch.isfinite(targets).all(dim=1, keepdim=True) + safe_targets = torch.where(finite_rows, targets, current) + delta = torch.clamp(safe_targets - current, -max_delta_rad, max_delta_rad) + bounded = torch.clamp(current + delta, -max_magnitude_rad, max_magnitude_rad) + return torch.where(finite_rows, bounded, current) + + +def constrain_hand_joint_targets( + targets: torch.Tensor, + mimic_pairs: tuple[tuple[int, int, float, float], ...], + lower: torch.Tensor, + upper: torch.Tensor, +) -> torch.Tensor: + """Apply software mimic relations and anatomical bounds to hand targets.""" + constrained = targets.clone() + for mimic_index, reference_index, factor, bias in mimic_pairs: + constrained[:, mimic_index] = factor * constrained[:, reference_index] + bias + return torch.maximum(torch.minimum(constrained, upper), lower) + + +class _HandTargetSlewState: + """Per-environment hand target history, independent of measured joint state.""" + + def __init__(self, prototype: torch.Tensor, max_step_rad: float) -> None: + self._previous = torch.zeros_like(prototype) + self._initialized = torch.zeros(prototype.shape[0], dtype=torch.bool, device=prototype.device) + self._max_step_rad = max_step_rad + + def condition(self, targets: torch.Tensor) -> torch.Tensor: + slewed = self._previous + torch.clamp(targets - self._previous, -self._max_step_rad, self._max_step_rad) + result = torch.where(self._initialized[:, None], slewed, targets) + self._previous.copy_(result) + self._initialized.fill_(True) + return result + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + if env_ids is None: + env_ids = slice(None) + self._previous[env_ids] = 0.0 + self._initialized[env_ids] = False + + +class GR1T2KitlessPinkInverseKinematicsAction(PinkInverseKinematicsAction): + """Pink action that owns the GR1T2 kitless target-stability boundary. + + Kit realizes the hand's PhysX mimic constraints and sanitizes the raw asset. + The kitless task removes those hard constraints before physics parsing, so + this term reproduces their target relation in software and bounds hand + targets. OVPhysX widens physical hand limits; Newton retains them. It also + bounds cold-start IK demands against measured state to avoid a first-step + drive impulse. With the feature disabled, behavior is the base Pink action. + """ + + cfg: GR1T2KitlessPinkInverseKinematicsActionCfg + + def __init__(self, cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, env: ManagerBasedEnv) -> None: + super().__init__(cfg, env) + self._kitless_target_conditioning_enabled = cfg.kitless_target_conditioning_enabled + self._mimic_pairs: tuple[tuple[int, int, float, float], ...] = () + self._hand_lower: torch.Tensor | None = None + self._hand_upper: torch.Tensor | None = None + self._hand_target_slew: _HandTargetSlewState | None = None + if not self._kitless_target_conditioning_enabled: + return + + from .pickplace_gr1t2_env_cfg import GR1T2KitlessUsdFileCfg + + spawn_cfg = self._asset.cfg.spawn + if isinstance(spawn_cfg, GR1T2KitlessUsdFileCfg) and spawn_cfg.action_cfg is not None: + # Asset construction copies spawn settings before conditioning the USD. Read + # its result here, rather than coupling independently copied environment configs. + cfg.hand_joint_limits = dict(spawn_cfg.action_cfg.hand_joint_limits) + cfg.hand_mimic_relations = dict(spawn_cfg.action_cfg.hand_mimic_relations) + + for name, value in ( + ("max_ik_joint_delta_rad", cfg.max_ik_joint_delta_rad), + ("max_ik_joint_magnitude_rad", cfg.max_ik_joint_magnitude_rad), + ("max_hand_target_step_rad", cfg.max_hand_target_step_rad), + ): + if not math.isfinite(value) or value <= 0.0: + raise ValueError(f"{name} must be finite and positive.") + if not math.isfinite(cfg.hand_target_margin_rad) or cfg.hand_target_margin_rad < 0.0: + raise ValueError("hand_target_margin_rad must be finite and non-negative.") + + hand_indices = {name: index for index, name in enumerate(self._hand_joint_names)} + missing_limits = [name for name in self._hand_joint_names if name not in cfg.hand_joint_limits] + if missing_limits: + raise ValueError(f"Missing pre-conditioned hand limits for: {missing_limits}") + + pairs = [] + for mimic_name, (reference_name, gearing, offset) in cfg.hand_mimic_relations.items(): + if mimic_name not in hand_indices or reference_name not in hand_indices: + raise ValueError( + f"Mimic relation {mimic_name!r} -> {reference_name!r} is outside the Pink hand joint set." + ) + if not all(math.isfinite(value) for value in (gearing, offset)): + raise ValueError(f"Mimic relation for {mimic_name!r} must be finite.") + # PhysX authors q_mimic + gearing * q_reference + offset = 0. + pairs.append((hand_indices[mimic_name], hand_indices[reference_name], -gearing, -offset)) + self._mimic_pairs = tuple(pairs) + + lower = [] + upper = [] + for name in self._hand_joint_names: + joint_lower, joint_upper = cfg.hand_joint_limits[name] + if not all(math.isfinite(value) for value in (joint_lower, joint_upper)) or joint_lower > joint_upper: + raise ValueError(f"Invalid hand limits for {name!r}: {(joint_lower, joint_upper)}") + lower.append(joint_lower - cfg.hand_target_margin_rad) + upper.append(joint_upper + cfg.hand_target_margin_rad) + self._hand_lower = torch.tensor(lower, dtype=self._raw_actions.dtype, device=self.device) + self._hand_upper = torch.tensor(upper, dtype=self._raw_actions.dtype, device=self.device) + self._hand_target_slew = _HandTargetSlewState( + torch.zeros( + (self.num_envs, len(self._hand_joint_names)), + dtype=self._raw_actions.dtype, + device=self.device, + ), + cfg.max_hand_target_step_rad, + ) + + def process_actions(self, actions: torch.Tensor) -> None: + """Process input without mutating the action-manager input during coupling.""" + super().process_actions(actions) + if self._kitless_target_conditioning_enabled: + assert self._hand_lower is not None and self._hand_upper is not None + assert self._hand_target_slew is not None + constrained = constrain_hand_joint_targets( + self._target_hand_joint_positions, + self._mimic_pairs, + self._hand_lower, + self._hand_upper, + ) + self._target_hand_joint_positions = self._hand_target_slew.condition(constrained) + + def reset(self, env_ids: Sequence[int] | None = None) -> None: + """Clear commanded-target history when the owning session/environment resets.""" + super().reset(env_ids) + if self._hand_target_slew is not None: + self._hand_target_slew.reset(env_ids) + + def _compute_ik_solutions(self) -> torch.Tensor: + targets = super()._compute_ik_solutions() + if not self._kitless_target_conditioning_enabled: + return targets + current = self._asset.data.joint_pos.torch[:, self._isaaclab_controlled_joint_ids] + return bound_ik_joint_targets( + targets, + current, + self.cfg.max_ik_joint_delta_rad, + self.cfg.max_ik_joint_magnitude_rad, + ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py new file mode 100644 index 00000000000..6da9e4abaf6 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pure-data configuration for the GR1T2 standalone action boundary.""" + +from __future__ import annotations + +from dataclasses import field +from typing import TYPE_CHECKING + +from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg +from isaaclab.utils.configclass import configclass + +if TYPE_CHECKING: + from .gr1t2_kitless_action import GR1T2KitlessPinkInverseKinematicsAction + + +@configclass +class GR1T2KitlessPinkInverseKinematicsActionCfg(PinkInverseKinematicsActionCfg): + """Pink action with opt-in, backend-specific target conditioning for GR1T2. + + The feature is disabled by default, preserving the regular Kit/PhysX action + path. The task enables it for kitless OVPhysX or Newton and fills the + hand metadata while conditioning the spawned USD before physics parses it. + """ + + class_type: type[GR1T2KitlessPinkInverseKinematicsAction] | str = ( + "{DIR}.gr1t2_kitless_action:GR1T2KitlessPinkInverseKinematicsAction" + ) + + kitless_target_conditioning_enabled: bool = False + """Enable the shared raw-GR1T2 joint/target policy for standalone OVPhysX and Newton.""" + + max_ik_joint_delta_rad: float = 0.2 + """Maximum IK target offset from measured joint position on each IK application [rad]. + + The bound is re-evaluated when the action is applied. OVPhysX applies actions + on each environment-managed physics substep; Newton can manage decimation + within a physics step. This is not a per-policy-step travel limit. + """ + + max_ik_joint_magnitude_rad: float = 3.5 + """Symmetric absolute bound for commanded IK-joint positions [rad].""" + + max_hand_target_step_rad: float = 0.079 + """Maximum hand-target change per policy step [rad].""" + + hand_target_margin_rad: float = 0.5 + """Extra target travel beyond anatomical limits, used to sustain grip effort [rad].""" + + hand_mimic_relations: dict[str, tuple[str, float, float]] = field(default_factory=dict) + """Mimic joint to ``(reference joint, gearing, offset)`` from the spawned USD.""" + + hand_joint_limits: dict[str, tuple[float, float]] = field(default_factory=dict) + """Original anatomical hand-joint limits recorded before physical limits are widened [rad].""" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index 2a049e692bc..db809f16238 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -5,13 +5,17 @@ import os import tempfile +from collections.abc import Callable +from dataclasses import MISSING + +from isaaclab_physx.sim.schemas import PhysxArticulationCfg, PhysxRigidBodyCfg import isaaclab.envs.mdp as base_mdp import isaaclab.sim as sim_utils +from isaaclab.actuators import ImplicitActuatorCfg from isaaclab.assets import ArticulationCfg, AssetBaseCfg, RigidObjectCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs import ManagerBasedRLEnvCfg -from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg from isaaclab.managers import EventTermCfg as EventTerm from isaaclab.managers import ObservationGroupCfg as ObsGroup from isaaclab.managers import ObservationTermCfg as ObsTerm @@ -19,17 +23,29 @@ from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.scene import InteractiveSceneCfg from isaaclab.sensors import ContactSensorCfg +from isaaclab.sim import SimulationCfg from isaaclab.sim.spawners.from_files.from_files_cfg import GroundPlaneCfg, UsdFileCfg from isaaclab.utils import configclass, replace from isaaclab.utils.assets import ISAAC_NUCLEUS_DIR, ISAACLAB_NUCLEUS_DIR, retrieve_file_path from isaaclab.visualizers import VisualizerCfg from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE +from isaaclab_tasks.contrib.steering_wheel import ( + condition_newton_steering_wheel_detail_collisions, +) +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + kitless_backend_prepared, + prepare_humanoid_teleop_environment, +) from . import mdp +from .gr1t2_kitless_action_cfg import GR1T2KitlessPinkInverseKinematicsActionCfg from isaaclab_assets.robots.fourier import GR1T2_HIGH_PD_CFG # isort: skip + from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions # isort: skip def build_gr1t2_pickplace_pipeline(): @@ -259,7 +275,13 @@ def build_gr1t2_pickplace_pipeline(): } ) - pipeline = OutputCombiner({"action": connected_reorderer.output("output")}) + pipeline = OutputCombiner( + { + "action": connected_reorderer.output("output"), + HandsSource.LEFT: hands.output(HandsSource.LEFT), + HandsSource.RIGHT: hands.output(HandsSource.RIGHT), + } + ) return pipeline, [left_dex, right_dex] @@ -383,7 +405,7 @@ class PickPlaceGR1T2SceneCfg(ObjectTableSceneCfg): class ActionsCfg: """Action specifications for the MDP.""" - upper_body_ik = PinkInverseKinematicsActionCfg( + upper_body_ik = GR1T2KitlessPinkInverseKinematicsActionCfg( pink_controlled_joint_names=[ "left_shoulder_pitch_joint", "left_shoulder_roll_joint", @@ -582,6 +604,12 @@ class EventCfg: class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + haptic_feedback: object | None = None + image_obs_list: list[str] = ["robot_pov_cam"] + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: PickPlaceGR1T2SceneCfg = PickPlaceGR1T2SceneCfg(num_envs=1, env_spacing=2.5) # Basic settings @@ -643,6 +671,9 @@ class PickPlaceGR1T2EnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + # Prepared config copies already contain their backend and teleop settings. + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 6 self.episode_length_s = 20.0 @@ -657,8 +688,6 @@ def __post_init__(self): self.actions.upper_body_ik.controller.usd_path = self.scene.robot.spawn.usd_path self.actions.upper_body_ik.controller.urdf_output_dir = self.temp_urdf_dir - self.image_obs_list = ["robot_pov_cam"] - if ISAACLAB_TELEOP_AVAILABLE: from isaaclab_teleop import GloveHapticFeedbackCfg, IsaacTeleopCfg, XrCameraFeedCfg, XrCfg @@ -668,6 +697,7 @@ def __post_init__(self): anchor_rot=(0.0, 0.0, 0.0, 1.0), ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, @@ -690,3 +720,255 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + def stabilize_for_kitless_ovphysx(self) -> None: + """Configure the GR1T2 asset and action boundaries for kitless OVPhysX.""" + if kitless_backend_prepared(self, "ovphysx"): + condition_kitless_gr1t2_stage(self.scene.robot, self.actions.upper_body_ik) + return + action_cfg = self.actions.upper_body_ik + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + action_cfg.kitless_target_conditioning_enabled = True + # Do not apply retargeter defaults before hand tracking starts. + if self.isaac_teleop is not None: + self.isaac_teleop.require_initial_bimanual_hand_tracking = True + apply_kitless_ovphysx_drive_gains(self.scene.robot) + condition_kitless_ovphysx_robot_spawn_properties(self.scene.robot) + condition_kitless_gr1t2_stage(self.scene.robot, action_cfg) + configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) + self._prepared_kitless_backend = "ovphysx" + + def stabilize_for_kitless_newton_mjwarp(self) -> None: + """Configure GR1T2 actuators and contacts for kitless Newton.""" + prepare_gr1t2_for_kitless_newton_mjwarp(self) + + +def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Share Newton preparation without replacing the configured controlled joints.""" + if kitless_backend_prepared(env_cfg, "newton_mjwarp"): + condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik, widen_hand_limits=False) + return + action_cfg = env_cfg.actions.upper_body_ik + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + action_cfg.kitless_target_conditioning_enabled = True + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True + _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg) + condition_kitless_gr1t2_stage(env_cfg.scene.robot, action_cfg, widen_hand_limits=False) + configure_packing_table_collisions(env_cfg.scene.packing_table) + env_cfg._prepared_kitless_backend = "newton_mjwarp" + + +def _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Apply Newton actuator, allocation-budget, and wheel-collision preparation.""" + from isaaclab_newton.sim.schemas import NewtonArticulationCfg + + # Match the kitless OVPhysX task: internal finger contacts must not force + # compliant thumb drives away from their commands. Object contacts remain. + fragments = env_cfg.scene.robot.spawn.articulation_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, NewtonArticulationCfg): + fragment.self_collision_enabled = False + + groups = env_cfg.scene.robot.actuators + contracts = { + "head": (["head_.*"], 80.0, 8.0, 0.03, 80.0), + "trunk": (["waist_.*"], 500.0, 50.0, 0.10, 250.0), + "legs": ([".*_hip_.*", ".*_knee_.*", ".*_ankle_.*"], 300.0, 30.0, 0.125, 250.0), + "right-arm": (["right_shoulder_.*", "right_elbow_.*", "right_wrist_.*"], 1200.0, 60.0, 0.30, 800.0), + "left-arm": (["left_shoulder_.*", "left_elbow_.*", "left_wrist_.*"], 1200.0, 60.0, 0.30, 800.0), + } + for name, (joint_names, stiffness, damping, armature, effort_limit) in contracts.items(): + actuator = groups.get(name) + if actuator is None: + actuator = ImplicitActuatorCfg(joint_names_expr=joint_names) + groups[name] = actuator + actuator.joint_names_expr = joint_names + actuator.stiffness = stiffness + actuator.damping = damping + actuator.armature = armature + actuator.joint_effort_limit = effort_limit + + for side, name in (("L", "left-hand"), ("R", "right-hand")): + # The action conditions all eleven targets to the asset's mimic + # relation. Driving the whole hand gives Newton the same compliant + # hand targets as the OVPhysX path. + hand = groups[name] + hand.joint_names_expr = [f"{side}_.*"] + hand.stiffness = 5.0 + hand.damping = 0.5 + hand.armature = 0.02 + hand.joint_effort_limit = 20.0 + groups.pop(f"{name}-slaved", None) + + solver_cfg = env_cfg.sim.physics.solver_cfg + solver_cfg.nconmax = 4096 + solver_cfg.njmax = 1536 + condition_newton_steering_wheel_detail_collisions(env_cfg.scene.object) + + +def apply_kitless_ovphysx_drive_gains(robot_cfg: ArticulationCfg) -> None: + """Restore stable actuator coverage around the Kit-tuned arm drives.""" + groups = { + "head": (["head_.*"], 80.0, 8.0, 0.03, 80.0), + "legs": ([".*_hip_.*", ".*_knee_.*", ".*_ankle_.*"], 300.0, 30.0, 0.10, 250.0), + "right-hand": (None, 5.0, 0.5, 0.001, 20.0), + "left-hand": (None, 5.0, 0.5, 0.001, 20.0), + } + for group_name, (joint_expr, stiffness, damping, armature, effort_limit) in groups.items(): + actuator = robot_cfg.actuators.get(group_name) + if actuator is None: + assert joint_expr is not None + actuator = ImplicitActuatorCfg(joint_names_expr=list(joint_expr)) + robot_cfg.actuators[group_name] = actuator + actuator.stiffness = stiffness + actuator.damping = damping + actuator.armature = armature + actuator.joint_effort_limit = effort_limit + + if robot_cfg.spawn is not None and robot_cfg.spawn.articulation_props is not None: + fragments = robot_cfg.spawn.articulation_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, PhysxArticulationCfg): + fragment.enabled_self_collisions = False + + +def condition_kitless_ovphysx_robot_spawn_properties(robot_cfg: ArticulationCfg) -> None: + """Limit robot contact depenetration for kitless OVPhysX.""" + if robot_cfg.spawn is not None and robot_cfg.spawn.rigid_props is not None: + fragments = robot_cfg.spawn.rigid_props + for fragment in fragments if isinstance(fragments, list) else [fragments]: + if isinstance(fragment, PhysxRigidBodyCfg): + fragment.max_depenetration_velocity = 0.1 + + +def _author_kitless_ovphysx_contact_offsets(prim) -> bool: + """Author Kit-parity offsets on a collision prim before OVPhysX parses it.""" + from pxr import Sdf + + schemas = prim.GetMetadata("apiSchemas") + applied = [str(schema) for schema in (schemas.GetAddedOrExplicitItems() if schemas else [])] + if not any(schema.startswith("PhysicsCollisionAPI") for schema in applied): + return False + if not any(schema.startswith("PhysxCollisionAPI") for schema in applied): + list_op = Sdf.TokenListOp() + list_op.explicitItems = [*applied, "PhysxCollisionAPI"] + prim.SetMetadata("apiSchemas", list_op) + prim.CreateAttribute("physxCollision:contactOffset", Sdf.ValueTypeNames.Float).Set(0.001) + prim.CreateAttribute("physxCollision:restOffset", Sdf.ValueTypeNames.Float).Set(0.0) + return True + + +def _spawn_and_condition(prim_path, cfg, *args, **kwargs): + import math + + from pxr import Gf, Sdf, Usd + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + action_cfg = cfg.action_cfg + action_cfg.hand_joint_limits.clear() + action_cfg.hand_mimic_relations.clear() + + # Collision meshes below instanceable ``collisions`` scopes are + # immutable proxies. De-instance those scopes so the parser-visible + # collision prims can receive the OVPhysX contact offsets. + for child in Usd.PrimRange(prim): + if child.GetName() == "collisions" and child.IsInstanceable(): + child.SetInstanceable(False) + + for child in Usd.PrimRange(prim): + _author_kitless_ovphysx_contact_offsets(child) + inertia = child.GetAttribute("physics:diagonalInertia") + if inertia and inertia.HasValue(): + value = inertia.Get() + if min(value) < 1.0e-4: + inertia.Set(Gf.Vec3f(*(max(component, 1.0e-4) for component in value))) + + is_hand_joint = child.GetName()[:2] in ("L_", "R_") and child.GetTypeName() == "PhysicsRevoluteJoint" + schemas = child.GetMetadata("apiSchemas") + applied = list(schemas.GetAddedOrExplicitItems()) if schemas else [] + kept = [schema for schema in applied if not str(schema).startswith("PhysxMimicJointAPI")] + if len(kept) != len(applied): + gearing = child.GetAttribute("physxMimicJoint:rotZ:gearing") + offset = child.GetAttribute("physxMimicJoint:rotZ:offset") + reference = child.GetRelationship("physxMimicJoint:rotZ:referenceJoint") + targets = reference.GetTargets() if reference else [] + if gearing and gearing.HasValue() and targets: + action_cfg.hand_mimic_relations[child.GetName()] = ( + targets[0].name, + float(gearing.Get()), + float(offset.Get()) if offset and offset.HasValue() else 0.0, + ) + list_op = Sdf.TokenListOp() + list_op.explicitItems = kept + child.SetMetadata("apiSchemas", list_op) + for property_name in [name for name in child.GetPropertyNames() if name.startswith("physxMimicJoint:")]: + if not child.RemoveProperty(property_name): + mimic_attr = child.GetAttribute(property_name) + if mimic_attr and mimic_attr.GetTypeName() == Sdf.ValueTypeNames.Float: + mimic_attr.Set(0.0) + + if is_hand_joint: + lower = child.GetAttribute("physics:lowerLimit") + upper = child.GetAttribute("physics:upperLimit") + if lower and upper and lower.HasValue() and upper.HasValue(): + action_cfg.hand_joint_limits[child.GetName()] = ( + math.radians(float(lower.Get())), + math.radians(float(upper.Get())), + ) + if cfg.widen_hand_limits: + lower.Set(-720.0) + upper.Set(720.0) + return prim + + +@configclass +class GR1T2KitlessUsdFileCfg(UsdFileCfg): + """Serializable GR1T2 spawn preparation and asset-owned hand metadata.""" + + original_spawn: Callable = MISSING + action_cfg: GR1T2KitlessPinkInverseKinematicsActionCfg | None = None + widen_hand_limits: bool = True + func: Callable = _spawn_and_condition + + +def condition_kitless_gr1t2_stage( + robot_cfg: ArticulationCfg, + action_cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, + *, + widen_hand_limits: bool = True, +) -> None: + """Condition the spawned GR1T2 hand description before kitless physics parses it. + + The raw asset has near-singular finger inertia, hard PhysX-only mimic + constraints, and hand rest poses exactly on physical limits. Those inputs + are stable after Kit's asset preparation but are ill-conditioned when read + directly by OVPhysX. This spawn boundary floors inertia, removes the hard + constraints that conflict with direct hand drives, and optionally keeps the + physical limits inactive for OVPhysX. Newton retains the anatomical limits + because bounding drive targets alone does not bound simulated positions. The + original mimic relations and anatomical limits are recorded on the typed + action config for target conditioning. The source USD and normal Kit path + remain unchanged. + """ + if robot_cfg.spawn is None: + return + spawn = robot_cfg.spawn + if not isinstance(spawn, GR1T2KitlessUsdFileCfg): + original_spawn = spawn.func + spawn = GR1T2KitlessUsdFileCfg(**vars(spawn)) + spawn.original_spawn = original_spawn + robot_cfg.spawn = spawn + elif spawn.func is not _spawn_and_condition and not ( + isinstance(spawn.func, str) and spawn.func == f"{__name__}:_spawn_and_condition" + ): + # Preserve a caller replacement made after an earlier preparation. + spawn.original_spawn = spawn.func + spawn.func = _spawn_and_condition + # Seed metadata from this environment; Asset construction owns its independent spawn copy. + spawn.action_cfg = action_cfg + spawn.widen_hand_limits = widen_hand_limits diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py index 9e04c5afe25..c05b2fd3d46 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py @@ -3,12 +3,20 @@ # # SPDX-License-Identifier: BSD-3-Clause +import os import tempfile from isaaclab.envs import ManagerBasedRLEnvCfg +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions +from isaaclab_tasks.contrib.teleop_physics_cfg import ( + TeleopPhysicsCfg, + kitless_backend_prepared, + prepare_humanoid_teleop_environment, +) from .pickplace_gr1t2_env_cfg import ( ActionsCfg, @@ -16,7 +24,11 @@ ObjectTableSceneCfg, ObservationsCfg, TerminationsCfg, + apply_kitless_ovphysx_drive_gains, build_gr1t2_pickplace_pipeline, + condition_kitless_gr1t2_stage, + condition_kitless_ovphysx_robot_spawn_properties, + prepare_gr1t2_for_kitless_newton_mjwarp, ) @@ -24,6 +36,11 @@ class PickPlaceGR1T2WaistEnabledEnvCfg(ManagerBasedRLEnvCfg): """Configuration for the GR1T2 environment.""" + class_type: type | str = "isaaclab_tasks.contrib.teleop_env:HumanoidTeleopEnv" + _prepared_kitless_backend: str | None = None + haptic_feedback: object | None = None + sim: SimulationCfg = SimulationCfg(physics=TeleopPhysicsCfg()) + # Scene settings scene: ObjectTableSceneCfg = ObjectTableSceneCfg(num_envs=1, env_spacing=2.5) # Basic settings @@ -43,6 +60,8 @@ class PickPlaceGR1T2WaistEnabledEnvCfg(ManagerBasedRLEnvCfg): def __post_init__(self): """Post initialization.""" + if self._prepared_kitless_backend is not None: + return # general settings self.decimation = 6 self.episode_length_s = 20.0 @@ -53,7 +72,8 @@ def __post_init__(self): # Add waist joint to pink_ik_cfg waist_joint_names = ["waist_yaw_joint", "waist_pitch_joint", "waist_roll_joint"] for joint_name in waist_joint_names: - self.actions.upper_body_ik.pink_controlled_joint_names.append(joint_name) + if joint_name not in self.actions.upper_body_ik.pink_controlled_joint_names: + self.actions.upper_body_ik.pink_controlled_joint_names.append(joint_name) # Defer USD→URDF conversion to controller initialization (requires Isaac Sim at runtime). self.actions.upper_body_ik.controller.usd_path = self.scene.robot.spawn.usd_path @@ -68,6 +88,7 @@ def __post_init__(self): anchor_rot=(0.0, 0.0, 0.0, 1.0), ) self.isaac_teleop = IsaacTeleopCfg( + prepare_environment=prepare_humanoid_teleop_environment, pipeline_builder=lambda: build_gr1t2_pickplace_pipeline()[0], sim_device=self.sim.device, xr_cfg=self.xr, @@ -81,3 +102,25 @@ def __post_init__(self): left_sensor_name="left_hand_contact", right_sensor_name="right_hand_contact", ) + + def stabilize_for_kitless_ovphysx(self) -> None: + """Prepare the existing GR asset while retaining the movable waist.""" + if kitless_backend_prepared(self, "ovphysx"): + condition_kitless_gr1t2_stage(self.scene.robot, self.actions.upper_body_ik) + return + action_cfg = self.actions.upper_body_ik + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + action_cfg.kitless_target_conditioning_enabled = True + if self.isaac_teleop is not None: + self.isaac_teleop.require_initial_bimanual_hand_tracking = True + apply_kitless_ovphysx_drive_gains(self.scene.robot) + condition_kitless_ovphysx_robot_spawn_properties(self.scene.robot) + condition_kitless_gr1t2_stage(self.scene.robot, action_cfg) + configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) + self._prepared_kitless_backend = "ovphysx" + + def stabilize_for_kitless_newton_mjwarp(self) -> None: + """Configure GR1T2 actuators and contacts for kitless Newton.""" + prepare_gr1t2_for_kitless_newton_mjwarp(self) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py new file mode 100644 index 00000000000..cc649548de7 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/steering_wheel.py @@ -0,0 +1,69 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Shared collision preparation for the contributed steering-wheel asset.""" + +from collections.abc import Callable +from dataclasses import MISSING + +import isaaclab.sim as sim_utils +from isaaclab.assets import RigidObjectCfg +from isaaclab.utils import configclass + +# These are the graspable wheel surfaces. Their decomposition preserves the +# rim opening and spoke gaps that a single hull would fill. +_STEERING_WHEEL_DECOMPOSED_MESHES = frozenset( + { + "sm_steeringwheel_a01_wheel_rim_01", + "sm_steeringwheel_a01_wheel_spokes_01", + } +) + + +def _spawn_with_selective_collision(prim_path, cfg, *args, **kwargs): + from pxr import Usd, UsdGeom + + prim = cfg.original_spawn(prim_path, cfg, *args, **kwargs) + for root_path in sim_utils.find_matching_prim_paths(prim_path): + root = prim.GetStage().GetPrimAtPath(root_path) + if not root.IsValid(): + continue + for mesh_prim in Usd.PrimRange(root): + if not mesh_prim.IsA(UsdGeom.Mesh) or mesh_prim.GetName() in _STEERING_WHEEL_DECOMPOSED_MESHES: + continue + approximation = mesh_prim.GetAttribute("physics:approximation") + if approximation and approximation.Get() is not None: + approximation.Set("convexHull") + return prim + + +@configclass +class SteeringWheelUsdFileCfg(sim_utils.UsdFileCfg): + """Declared selective collision preparation for the steering-wheel asset.""" + + original_spawn: Callable = MISSING + func: Callable = _spawn_with_selective_collision + + +def condition_newton_steering_wheel_detail_collisions(object_cfg: RigidObjectCfg) -> None: + """Hull only non-graspable wheel details that MuJoCo cannot compile as solids. + + The authored wheel uses convex decomposition throughout. Tiny hub and + quick-release decomposition pieces can have near-zero volume, which MuJoCo + rejects for solid inertia. The rim and spokes remain decomposed so the + wheel remains graspable; only detail meshes are switched to convex hulls. + """ + if object_cfg.spawn is None: + return + if not isinstance(object_cfg.spawn, SteeringWheelUsdFileCfg): + original_spawn = object_cfg.spawn.func + object_cfg.spawn = SteeringWheelUsdFileCfg(**vars(object_cfg.spawn)) + object_cfg.spawn.original_spawn = original_spawn + elif object_cfg.spawn.func is not _spawn_with_selective_collision and not ( + isinstance(object_cfg.spawn.func, str) + and object_cfg.spawn.func == f"{__name__}:_spawn_with_selective_collision" + ): + object_cfg.spawn.original_spawn = object_cfg.spawn.func + object_cfg.spawn.func = _spawn_with_selective_collision diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py new file mode 100644 index 00000000000..d182b85279c --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_env.py @@ -0,0 +1,18 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Construction boundary for the contributed GR1T2 and G1 teleoperation tasks.""" + +from isaaclab.envs import ManagerBasedRLEnv, ManagerBasedRLEnvCfg + +from .teleop_physics_cfg import prepare_humanoid_teleop_environment + + +class HumanoidTeleopEnv(ManagerBasedRLEnv): + """Prepare the selected task backend before physics and scene construction.""" + + def __init__(self, cfg: ManagerBasedRLEnvCfg, render_mode: str | None = None, **kwargs): + prepare_humanoid_teleop_environment(cfg) + super().__init__(cfg, render_mode, **kwargs) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py new file mode 100644 index 00000000000..e363fa3a3a8 --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py @@ -0,0 +1,57 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Backend selection for humanoid teleoperation tasks with task-owned preparation.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg +from isaaclab_ov.physics import OvPhysxCfg +from isaaclab_physx.physics import PhysxCfg + +from isaaclab.physics import PhysxAutoCfg +from isaaclab.utils import configclass + +from isaaclab_tasks.utils import PresetCfg + +if TYPE_CHECKING: + from isaaclab.envs import ManagerBasedRLEnvCfg + + +def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: + """Prepare humanoid task settings before simulation or environment construction.""" + physics = cfg.sim.physics + if isinstance(physics, OvPhysxCfg): + kitless_backend_prepared(cfg, "ovphysx") + cfg.stabilize_for_kitless_ovphysx() + elif isinstance(physics, NewtonCfg) and isinstance(physics.solver_cfg, MJWarpSolverCfg): + kitless_backend_prepared(cfg, "newton_mjwarp") + cfg.stabilize_for_kitless_newton_mjwarp() + else: + kitless_backend_prepared(cfg, None) + + +@configclass +class TeleopPhysicsCfg(PresetCfg): + """Keep Kit as the default and expose the supported standalone backends.""" + + isaacsim_physx = PhysxCfg() + ovphysx = OvPhysxCfg() + newton_mjwarp = NewtonCfg(solver_cfg=MJWarpSolverCfg()) + physx = PhysxAutoCfg(isaacsim_physx=isaacsim_physx, ovphysx=ovphysx) + default = isaacsim_physx + + +def kitless_backend_prepared(env_cfg, backend: str | None) -> bool: + """Keep completed task preparation stable across repeated callers and config copies.""" + prepared = getattr(env_cfg, "_prepared_kitless_backend", None) + if prepared is not None and prepared != backend: + raise ValueError( + f"Task configuration was already prepared for {prepared}; resolve a fresh task configuration " + "before selecting a different physics backend." + ) + return prepared is not None diff --git a/source/isaaclab_tasks/pyproject.toml b/source/isaaclab_tasks/pyproject.toml index dc860b9bc9b..4e1ca6a07a5 100644 --- a/source/isaaclab_tasks/pyproject.toml +++ b/source/isaaclab_tasks/pyproject.toml @@ -39,3 +39,4 @@ include = ["isaaclab_tasks", "isaaclab_tasks.*"] [tool.setuptools.package-data] "*" = ["*.pyi"] +"isaaclab_tasks.contrib.pick_place" = ["assets/*.urdf"] diff --git a/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py b/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py new file mode 100644 index 00000000000..5410ef913a1 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_common_packing_collisions.py @@ -0,0 +1,72 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Geometry-preserving common collision preparation, independent of a physics engine.""" + +from types import SimpleNamespace + +import pytest + +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions + + +@pytest.mark.parametrize("physx_offsets", [False, True]) +def test_existing_mesh_transforms_and_bin_colliders_are_preserved(physx_offsets): + stage = Usd.Stage.CreateInMemory() + expected = {} + + def spawn(path, cfg): + root = stage.DefinePrim(path, "Xform") + root.SetInstanceable(True) + parent = stage.DefinePrim(path + "/Table", "Xform") + parent.SetInstanceable(True) + UsdGeom.Xformable(parent).AddTranslateOp().Set((0.3, -0.2, 0.4)) + UsdPhysics.CollisionAPI.Apply(parent) + UsdPhysics.MeshCollisionAPI.Apply(parent).CreateApproximationAttr().Set("boundingCube") + parent.CreateAttribute("horde:filterOwner", Sdf.ValueTypeNames.String).Set("table") + for index in range(4): + mesh = UsdGeom.Mesh.Define(stage, path + f"/Table/Mesh_{index}") + mesh.CreatePointsAttr().Set([(0, 0, 0), (1, 0, 0), (0, 1, 0)]) + mesh.CreateFaceVertexCountsAttr().Set([3]) + mesh.CreateFaceVertexIndicesAttr().Set([0, 1, 2]) + UsdGeom.Xformable(mesh).AddTranslateOp().Set((index, index / 2, index / 4)) + stage.DefinePrim(path + "/Bin", "Xform").SetInstanceable(True) + for index in range(5): + cube = UsdGeom.Cube.Define(stage, path + f"/Bin/Cube_{index}") + cube.CreateSizeAttr().Set(0.1 + index) + UsdPhysics.CollisionAPI.Apply(cube.GetPrim()) + cache = UsdGeom.XformCache() + for prim in Usd.PrimRange(root): + if prim.IsA(UsdGeom.Gprim): + expected[str(prim.GetPath())] = cache.GetLocalToWorldTransform(prim) + return root + + cfg = SimpleNamespace(spawn=SimpleNamespace(func=spawn)) + configure_packing_table_collisions(cfg, physx_offsets=physx_offsets) + prepared = cfg.spawn.func + configure_packing_table_collisions(cfg, physx_offsets=physx_offsets) + assert cfg.spawn.func is prepared + root = prepared("/World/PackingTable", cfg.spawn) + shapes = [p for p in Usd.PrimRange(root) if p.HasAPI(UsdPhysics.CollisionAPI)] + assert len(shapes) == 9 + assert {str(p.GetPath()) for p in shapes} == set(expected) + assert sum(p.IsA(UsdGeom.Mesh) for p in shapes) == 4 + assert sum(p.IsA(UsdGeom.Cube) for p in shapes) == 5 + assert not stage.GetPrimAtPath("/World/PackingTable/Table").HasAPI(UsdPhysics.CollisionAPI) + assert stage.GetPrimAtPath("/World/PackingTable/Table").GetAttribute("horde:filterOwner").Get() == "table" + assert not any(prim.IsInstance() or prim.IsInstanceProxy() for prim in Usd.PrimRange(root)) + cache = UsdGeom.XformCache() + for shape in shapes: + assert cache.GetLocalToWorldTransform(shape) == expected[str(shape.GetPath())] + assert UsdPhysics.CollisionAPI(shape).GetCollisionEnabledAttr().Get() + assert UsdGeom.Imageable(shape).GetPurposeAttr().Get() == UsdGeom.Tokens.default_ + if shape.IsA(UsdGeom.Mesh): + assert UsdPhysics.MeshCollisionAPI(shape).GetApproximationAttr().Get() == "boundingCube" + assert list(UsdGeom.Mesh(shape).GetFaceVertexIndicesAttr().Get()) == [0, 1, 2] + if physx_offsets: + assert shape.GetAttribute("physxCollision:contactOffset").Get() == pytest.approx(0.001) + assert shape.GetAttribute("physxCollision:restOffset").Get() == 0.0 diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py new file mode 100644 index 00000000000..4f0c0fdb85f --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py @@ -0,0 +1,289 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sim-free tests for the G1 kitless Newton task boundary.""" + +import math +import re +from types import SimpleNamespace + +import pytest +import torch +from isaaclab_newton.physics import NewtonCfg + +from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, +) +from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( + ACTUATOR_PROFILE, + JOINT_LIMIT_MARGIN_RAD, + JOINT_TARGET_MAX_STEP_RAD, + POSITION_MAX_STEP_M, + ROTATION_MAX_STEP_RAD, + G1NewtonBoundedPinkInverseKinematicsAction, + bound_pink_joint_targets, + condition_kitless_newton_g1_action, + teleop_action_target_issue, +) +from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg +from isaaclab_tasks.contrib.steering_wheel import SteeringWheelUsdFileCfg + +RECORDED_FIRST_ACTION = torch.tensor( + [ + -0.11201994, + 0.43588385, + 0.89166349, + -0.12487421, + -0.11215594, + 0.69946498, + 0.69467705, + 0.14433567, + 0.41181007, + 0.91755670, + 0.00914924, + -0.05320583, + 0.65801126, + 0.75107032, + *([0.0] * 14), + ], + dtype=torch.float32, +) + + +def _quat_angle(a: torch.Tensor, b: torch.Tensor) -> float: + a = torch.nn.functional.normalize(a, dim=0) + b = torch.nn.functional.normalize(b, dim=0) + return float(2.0 * torch.acos(torch.clamp(torch.abs(torch.sum(a * b)), 0.0, 1.0))) + + +def _fake_env(): + left = torch.tensor([0.23491086, 0.21602505, 0.63518274, -0.13474128, 0.79248637, -0.02669795, -0.59422028]) + right = torch.tensor([-0.30430639, 0.52459145, 0.78784978, 0.19782960, 0.52823287, 0.41972736, 0.71109974]) + poses = torch.stack((left, right)).unsqueeze(0) + robot = SimpleNamespace( + body_names=["left_wrist_yaw_link", "right_wrist_yaw_link"], + data=SimpleNamespace(body_link_pose_w=poses), + ) + cfg = SimpleNamespace( + actions=SimpleNamespace( + upper_body_ik=SimpleNamespace( + target_eef_link_names={ + "left_wrist": "left_wrist_yaw_link", + "right_wrist": "right_wrist_yaw_link", + } + ) + ) + ) + return SimpleNamespace(scene={"robot": robot}, cfg=cfg), left, right + + +def test_g1_newton_contract_is_backend_scoped_and_complete() -> None: + default_cfg = FixedBaseUpperBodyIKG1EnvCfg() + newton_cfg = FixedBaseUpperBodyIKG1EnvCfg() + newton_cfg.sim.physics = NewtonCfg() + newton_cfg.stabilize_for_kitless_newton_mjwarp() + + assert newton_cfg.sim.physics.solver_cfg.nconmax == 4096 + assert newton_cfg.sim.physics.solver_cfg.njmax == 1536 + for name, selected in ACTUATOR_PROFILE.items(): + actuator = newton_cfg.scene.robot.actuators[name] + assert {key: getattr(actuator, key) for key in selected} == selected + assert newton_cfg.actions.upper_body_ik.class_type is G1NewtonBoundedPinkInverseKinematicsAction + assert newton_cfg.isaac_teleop.action_conditioner is condition_kitless_newton_g1_action + assert newton_cfg.isaac_teleop.action_validator is teleop_action_target_issue + assert isinstance(newton_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert isinstance(newton_cfg.scene.object.spawn, SteeringWheelUsdFileCfg) + + assert default_cfg.isaac_teleop.action_conditioner is None + assert default_cfg.isaac_teleop.action_validator is None + assert not isinstance(default_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert not isinstance(default_cfg.scene.object.spawn, SteeringWheelUsdFileCfg) + + +@pytest.mark.parametrize("task", ["fixed_base", "locomanipulation"]) +def test_g1_contact_mappings_are_bilateral_and_nested(task) -> None: + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + + cfg = FixedBaseUpperBodyIKG1EnvCfg() if task == "fixed_base" else LocomanipulationG1EnvCfg() + + assert cfg.scene.left_hand_contact.prim_path == "{ENV_REGEX_NS}/Robot/left_hand/left_hand_[^/]*_link" + assert cfg.scene.right_hand_contact.prim_path == "{ENV_REGEX_NS}/Robot/right_hand/right_hand_[^/]*_link" + assert cfg.haptic_feedback.left_sensor_name == "left_hand_contact" + assert cfg.haptic_feedback.right_sensor_name == "right_hand_contact" + + compiled_labels = ( + "/World/envs/env_0/Robot/left_hand/left_hand_index_0_link", + "/World/envs/env_0/Robot/right_hand/right_hand_thumb_2_link", + "/World/envs/env_0/Robot/torso_link", + ) + left_pattern = cfg.scene.left_hand_contact.prim_path.replace("{ENV_REGEX_NS}", r"/World/envs/env_[^/]+") + right_pattern = cfg.scene.right_hand_contact.prim_path.replace("{ENV_REGEX_NS}", r"/World/envs/env_[^/]+") + left = [label for label in compiled_labels if re.fullmatch(left_pattern, label)] + right = [label for label in compiled_labels if re.fullmatch(right_pattern, label)] + assert left == [compiled_labels[0]] + assert right == [compiled_labels[1]] + assert set(left).isdisjoint(right) + + +def test_locomanipulation_preserves_agile_policy_order_and_action_contract() -> None: + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + + cfg = LocomanipulationG1EnvCfg() + controller = cfg.actions.lower_body_joint_pos.copy() + upper_body_class = cfg.actions.upper_body_ik.class_type + cfg.sim.physics = NewtonCfg() + cfg.stabilize_for_kitless_newton_mjwarp() + + assert cfg.scene.robot.joint_ordering == "physx" + assert cfg.actions.lower_body_joint_pos == controller + assert cfg.actions.upper_body_ik.class_type == upper_body_class + assert cfg.isaac_teleop.action_conditioner is None + assert cfg.sim.physics.solver_cfg.integrator == "implicitfast" + assert cfg.sim.physics.num_substeps == 2 + + +def test_recorded_first_action_is_ramped_and_finite() -> None: + env, left, right = _fake_env() + conditioned = condition_kitless_newton_g1_action(env, RECORDED_FIRST_ACTION, {}) + + assert teleop_action_target_issue(conditioned) is None + assert torch.isfinite(conditioned).all() + assert float(torch.linalg.norm(conditioned[:3] - left[:3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert float(torch.linalg.norm(conditioned[7:10] - right[:3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert _quat_angle(conditioned[3:7], left[3:7]) <= ROTATION_MAX_STEP_RAD + 1.0e-5 + assert _quat_angle(conditioned[10:14], right[3:7]) <= ROTATION_MAX_STEP_RAD + 1.0e-5 + assert torch.equal(conditioned[14:], RECORDED_FIRST_ACTION[14:]) + + +@pytest.mark.parametrize( + "angle, axis, sign", [(0.9, (1, 0, 0), 1), (1.2, (0, 1, 1), -1), (0.1, (0, 0, 1), 1), (0, (1, 0, 0), -1)] +) +def test_conditioned_wrist_rotation_obeys_angular_cap(angle, axis, sign): + """The public action conditioner follows the short rotation without exceeding its cap.""" + env, _, _ = _fake_env() + pose = env.scene["robot"].data.body_link_pose_w + pose[0, :, 3:7] = torch.tensor([1.0, 0, 0, 0]) + direction = torch.nn.functional.normalize(torch.tensor(axis, dtype=torch.float32), dim=0) + target = sign * torch.cat((torch.tensor([math.cos(angle / 2)]), direction * math.sin(angle / 2))) + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = action[10:14] = target + conditioned = condition_kitless_newton_g1_action(env, action, {}) + identity = torch.tensor([1.0, 0, 0, 0]) + for result in (conditioned[3:7], conditioned[10:14]): + assert torch.isfinite(result).all() + assert torch.linalg.norm(result).item() == pytest.approx(1, abs=1.0e-6) + assert _quat_angle(identity, result) <= min(angle, ROTATION_MAX_STEP_RAD) + 1.0e-5 + assert _quat_angle(result, target) == pytest.approx(max(0, angle - ROTATION_MAX_STEP_RAD), abs=1.0e-5) + + +def test_invalid_actions_fail_closed() -> None: + nonfinite = RECORDED_FIRST_ACTION.clone() + nonfinite[0] = float("nan") + assert teleop_action_target_issue(nonfinite) == "non-finite action target" + + oversized = RECORDED_FIRST_ACTION.clone() + oversized[14] = 7.0 + assert teleop_action_target_issue(oversized) == "unreasonable hand joint target magnitude" + assert "unexpected G1 controller action size" in teleop_action_target_issue(torch.zeros(27)) + + missing_wrist_env = SimpleNamespace(scene={}, cfg=SimpleNamespace()) + with pytest.raises(RuntimeError, match="could not resolve the task-selected G1 wrist poses"): + condition_kitless_newton_g1_action(missing_wrist_env, RECORDED_FIRST_ACTION, {}) + + +@pytest.mark.parametrize("nonfinite", [False, True]) +def test_invalid_action_errors_preserve_priority(nonfinite) -> None: + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = 0.0 + action[10:14] = torch.tensor([3.0, 0.0, 0.0, 0.0]) + action[14] = 7.0 + if nonfinite: + action[0] = float("nan") + expected = "non-finite action target" if nonfinite else "invalid wrist quaternion norms [0.0, 3.0]" + assert teleop_action_target_issue(action) == expected + + +def test_action_validation_accepts_inclusive_thresholds() -> None: + action = RECORDED_FIRST_ACTION.clone() + action[3:7] = torch.tensor([0.25, 0.0, 0.0, 0.0]) + action[10:14] = torch.tensor([2.0, 0.0, 0.0, 0.0]) + action[14:] = 6.5 + assert teleop_action_target_issue(action) is None + + +def test_pink_targets_are_slew_limited_and_clamped_to_common_limits() -> None: + current = torch.tensor([0.25, 0.0, -0.5]) + proposed = torch.tensor([0.5215, -4.0, 4.0]) + lower = torch.tensor([-0.2617994, -2.0, -1.0]) + upper = torch.tensor([0.2617994, 2.0, 1.0]) + + bounded = bound_pink_joint_targets(current, proposed, lower, upper) + + assert torch.all(bounded >= lower + JOINT_LIMIT_MARGIN_RAD) + assert torch.all(bounded <= upper - JOINT_LIMIT_MARGIN_RAD) + assert float(bounded[0]) == pytest.approx(0.2617994 - JOINT_LIMIT_MARGIN_RAD) + assert float(bounded[1]) == pytest.approx(-JOINT_TARGET_MAX_STEP_RAD) + assert float(bounded[2]) == pytest.approx(-0.5 + JOINT_TARGET_MAX_STEP_RAD) + + +@pytest.mark.parametrize("invalid_input", ["current", "proposed", "lower", "upper"]) +def test_nonfinite_pink_target_fails_closed(invalid_input) -> None: + inputs = { + "current": torch.zeros((2, 1)), + "proposed": torch.zeros((2, 1)), + "lower": -torch.ones((2, 1)), + "upper": torch.ones((2, 1)), + } + inputs[invalid_input][1, 0] = float("nan") + with pytest.raises(RuntimeError, match="non-finite Pink joint target boundary"): + bound_pink_joint_targets(**inputs) + + +@pytest.mark.parametrize("nonfinite", [False, True]) +def test_empty_pink_limit_intersection_fails_closed(nonfinite) -> None: + current = torch.zeros((2, 1)) + if nonfinite: + current[0, 0] = float("nan") + lower = torch.tensor([[-1.0], [0.0]]) + upper = torch.tensor([[1.0], [0.0]]) + message = "non-finite Pink joint target boundary" if nonfinite else "joint-limit intersection is empty" + with pytest.raises(RuntimeError, match=message): + bound_pink_joint_targets(current, torch.zeros_like(current), lower, upper) + + +def test_pink_limit_cache_keeps_live_articulation_limits(monkeypatch): + from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + + proposed = torch.tensor([[0.6, -0.4], [0.8, 0.8]]) + monkeypatch.setattr(PinkInverseKinematicsAction, "_compute_ik_solutions", lambda self: proposed) + action = G1NewtonBoundedPinkInverseKinematicsAction.__new__(G1NewtonBoundedPinkInverseKinematicsAction) + action._env = SimpleNamespace(device="cpu") + limits = torch.tensor([[[-1.0, 1.0]] * 3, [[-0.8, 0.8]] * 3]) + action._asset = SimpleNamespace( + data=SimpleNamespace( + joint_pos=SimpleNamespace(torch=torch.tensor([[0.2, 0.0, 0.0], [0.4, 0.0, -0.2]])), + joint_pos_limits=SimpleNamespace(torch=limits), + ) + ) + action._isaaclab_controlled_joint_ids = [2, 0] + action._ik_controllers = [ + SimpleNamespace( + pink_configuration=SimpleNamespace( + controlled_model=SimpleNamespace( + lowerPositionLimit=lower, + upperPositionLimit=upper, + ) + ), + pink_to_isaac_lab_controlled_ordering=[1, 0], + ) + for lower, upper in [([-0.25, -0.05], [0.5, 0.06]), ([-0.3, -0.4], [0.45, 0.7])] + ] + action._pink_joint_limits = None + torch.testing.assert_close(action._compute_ik_solutions(), torch.tensor([[0.059, 0.1], [-0.1, 0.449]])) + cached = action._pink_joint_limits + limits[0, 2, 1] = 0.03 + limits[1, 0, 1] = 0.42 + torch.testing.assert_close(action._compute_ik_solutions(), torch.tensor([[0.029, 0.1], [-0.1, 0.419]])) + assert action._pink_joint_limits is cached diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py new file mode 100644 index 00000000000..55372fb44e1 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py @@ -0,0 +1,131 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused regressions for the G1 kitless OVPhysX packing-table contract.""" + +import gymnasium as gym +import pytest +import torch +from isaaclab_ov.physics import OvPhysxCfg + +from isaaclab.app import launch_simulation + +import isaaclab_tasks # noqa: F401 +from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, +) +from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg +from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg + +_TASK = "IsaacContrib-PickPlace-Locomanipulation-G1-Abs" + + +def test_fixed_base_g1_ovphysx_conditions_assets_without_changing_other_backends() -> None: + cfg = FixedBaseUpperBodyIKG1EnvCfg() + default_cfg = FixedBaseUpperBodyIKG1EnvCfg() + action_type = cfg.actions.upper_body_ik.class_type + drop_term = cfg.terminations.object_dropping.func + + cfg.stabilize_for_kitless_ovphysx() + table_spawn = cfg.scene.packing_table.spawn.func + wheel_spawn = cfg.scene.object.spawn.func + cfg.stabilize_for_kitless_ovphysx() + + assert isinstance(cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert cfg.scene.object.spawn.func == default_cfg.scene.object.spawn.func + assert cfg.scene.object.spawn.usd_path == default_cfg.scene.object.spawn.usd_path + assert cfg.scene.packing_table.spawn.func is table_spawn + assert cfg.scene.object.spawn.func is wheel_spawn + assert cfg.actions.upper_body_ik.class_type is action_type + assert cfg.terminations.object_dropping.func is drop_term + assert not isinstance(default_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) + assert not isinstance(default_cfg.scene.object.spawn, PackingTableUsdFileCfg) + + +@pytest.mark.isaacsim_ci +@pytest.mark.parametrize( + ("cfg_type", "task"), + [ + (LocomanipulationG1EnvCfg, _TASK), + (FixedBaseUpperBodyIKG1EnvCfg, "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs"), + ], + ids=["locomanipulation", "fixed_base"], +) +def test_g1_wheel_remains_supported_on_ovphysx_cpu(cfg_type, task) -> None: + pytest.importorskip("ovphysx.types", reason="ovphysx wheel not installed") + + env_cfg = cfg_type() + env_cfg.scene.num_envs = 1 + env_cfg.sim.device = "cpu" + env_cfg.sim.physics = OvPhysxCfg() + env_cfg.scene.robot_pov_cam = None + env_cfg.observations.policy.robot_pov_cam = None + if isinstance(env_cfg, FixedBaseUpperBodyIKG1EnvCfg): + env_cfg.scene.left_wrist_cam = None + env_cfg.scene.right_wrist_cam = None + env_cfg.observations.policy.left_wrist_cam = None + env_cfg.observations.policy.right_wrist_cam = None + env_cfg.image_obs_list = [] + env_cfg.stabilize_for_kitless_ovphysx() + + env = None + launcher_args = { + "physics": None, + "device": "cpu", + "visualizer": None, + "visualizer_explicit": True, + } + try: + with launch_simulation(env_cfg, launcher_args): + env = gym.make(task, cfg=env_cfg).unwrapped + phases = ("table", "reset", "bin") if cfg_type is FixedBaseUpperBodyIKG1EnvCfg else ("table",) + for phase in phases: + env.reset() + wheel = env.scene["object"] + if phase == "bin": + from pxr import Usd, UsdGeom + + from isaaclab.sim import get_current_stage + + floor = get_current_stage().GetPrimAtPath( + "/World/envs/env_0/PackingTable/container_h20/container_h20_inst/colliders/Cube" + ) + assert floor.IsValid() + # Include invisible collision geometry when measuring the bin floor. + bounds = ( + UsdGeom.BBoxCache(Usd.TimeCode.Default(), [UsdGeom.Tokens.default_], False, True) + .ComputeWorldBound(floor) + .ComputeAlignedRange() + ) + center = bounds.GetMidpoint() + floor_top = bounds.GetMax()[2] + pose = wheel.data.root_pose_w.torch.clone() + pose[0, :3] = torch.tensor([center[0], center[1], floor_top + 0.10]) + wheel.write_root_pose_to_sim_index(root_pose=pose) + wheel.write_root_velocity_to_sim_index(root_velocity=torch.zeros((1, 6))) + + minimum_z = float("inf") + with torch.inference_mode(): + for _ in range(300): + env.sim.step(render=False) + env.scene.update(env_cfg.sim.dt) + minimum_z = min(minimum_z, float(wheel.data.root_pos_w.torch[0, 2].item())) + assert torch.isfinite(wheel.data.root_pose_w.torch).all() + assert torch.isfinite(wheel.data.root_vel_w.torch).all() + object_dropping = env_cfg.terminations.object_dropping.func( + env, **env_cfg.terminations.object_dropping.params + ) + assert not bool(object_dropping[0].item()), phase + + assert 300 * env_cfg.sim.dt == pytest.approx(1.5) + assert minimum_z > env_cfg.terminations.object_dropping.params["minimum_height"] + if phase == "bin": + final_position = wheel.data.root_pos_w.torch[0] + assert float(final_position[2]) >= floor_top - 0.005 + for axis in (0, 1): + assert bounds.GetMin()[axis] < float(final_position[axis]) < bounds.GetMax()[axis] + finally: + if env is not None: + env.close() diff --git a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py new file mode 100644 index 00000000000..0393148d945 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py @@ -0,0 +1,368 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Sim-free tests for the GR1T2 kitless OVPhysX action boundary.""" + +from copy import deepcopy +from types import SimpleNamespace + +import numpy as np +import pytest +import torch +from isaaclab_newton.physics import NewtonCfg +from isaaclab_ov.physics import OvPhysxCfg + +from pxr import Usd, UsdGeom, UsdPhysics + +from isaaclab_tasks.contrib.pick_place import pickplace_gr1t2_env_cfg as gr1t2_env_cfg +from isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action import ( + GR1T2KitlessPinkInverseKinematicsAction, + _HandTargetSlewState, + bound_ik_joint_targets, + constrain_hand_joint_targets, +) +from isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg import ( + GR1T2KitlessPinkInverseKinematicsActionCfg, +) +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import ( + PickPlaceGR1T2EnvCfg, + condition_kitless_gr1t2_stage, + condition_kitless_ovphysx_robot_spawn_properties, +) +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import PickPlaceGR1T2WaistEnabledEnvCfg +from isaaclab_tasks.contrib.steering_wheel import SteeringWheelUsdFileCfg +from isaaclab_tasks.utils import parse_env_cfg + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +def test_newton_contract_is_backend_scoped_and_complete(config_type): + default_cfg = config_type() + task = ( + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs" + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg + else "IsaacContrib-PickPlace-GR1T2-Abs" + ) + newton_cfg = parse_env_cfg(task, device="cpu", num_envs=1, overrides=["physics=newton_mjwarp"]) + newton_cfg.stabilize_for_kitless_newton_mjwarp() + + expected = { + "head": (80.0, 8.0, 0.03, 80.0), + "trunk": (500.0, 50.0, 0.10, 250.0), + "legs": (300.0, 30.0, 0.125, 250.0), + "right-arm": (1200.0, 60.0, 0.30, 800.0), + "left-arm": (1200.0, 60.0, 0.30, 800.0), + "right-hand": (5.0, 0.5, 0.02, 20.0), + "left-hand": (5.0, 0.5, 0.02, 20.0), + } + assert set(newton_cfg.scene.robot.actuators) == set(expected) + for name, values in expected.items(): + actuator = newton_cfg.scene.robot.actuators[name] + assert (actuator.stiffness, actuator.damping, actuator.armature, actuator.joint_effort_limit) == pytest.approx( + values + ) + assert newton_cfg.scene.robot.actuators["left-hand"].joint_names_expr == ["L_.*"] + assert newton_cfg.scene.robot.actuators["right-hand"].joint_names_expr == ["R_.*"] + assert newton_cfg.sim.physics.solver_cfg.nconmax == 4096 + assert newton_cfg.sim.physics.solver_cfg.njmax == 1536 + assert newton_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is True + assert newton_cfg.actions.upper_body_ik.kitless_target_conditioning_enabled is True + assert default_cfg.actions.upper_body_ik.kitless_target_conditioning_enabled is False + assert isinstance(newton_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) + assert not isinstance(default_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) + assert default_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is False + + assert set(default_cfg.scene.robot.actuators) == { + "trunk", + "right-arm", + "left-arm", + "right-hand", + "left-hand", + } + assert "legs" not in default_cfg.scene.robot.actuators + + +def test_newton_teleop_self_collision_matches_ovphysx_without_changing_asset_default(): + from isaaclab_newton.sim.schemas import NewtonArticulationCfg + from isaaclab_physx.sim.schemas import PhysxArticulationCfg + + default_cfg = PickPlaceGR1T2EnvCfg() + for backend, fragment_type, attribute in ( + ("newton_mjwarp", NewtonArticulationCfg, "self_collision_enabled"), + ("ovphysx", PhysxArticulationCfg, "enabled_self_collisions"), + ): + cfg = PickPlaceGR1T2EnvCfg() + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + getattr(cfg, f"stabilize_for_kitless_{backend}")() + fragment = next(p for p in cfg.scene.robot.spawn.articulation_props if isinstance(p, fragment_type)) + original = next(p for p in default_cfg.scene.robot.spawn.articulation_props if isinstance(p, fragment_type)) + assert getattr(fragment, attribute) is False + assert getattr(original, attribute) is True + + +@pytest.mark.parametrize("backend", ["ovphysx", "newton_mjwarp"]) +@pytest.mark.parametrize("copy_prepared", [False, True, "deepcopy"]) +@pytest.mark.parametrize("replace_spawn", [False, True]) +def test_hand_physical_limits_are_widened_only_for_ovphysx(monkeypatch, backend, copy_prepared, replace_spawn): + stage = Usd.Stage.CreateInMemory() + + def spawn_robot(path, _cfg, **_kwargs): + root = stage.DefinePrim(path, "Xform") + joint = UsdPhysics.RevoluteJoint.Define(stage, path + "/R_thumb_distal_joint") + joint.CreateLowerLimitAttr().Set(-15.0) + joint.CreateUpperLimitAttr().Set(80.0) + return root + + cfg = PickPlaceGR1T2EnvCfg() + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + cfg.scene.robot.spawn.func = spawn_robot + getattr(cfg, f"stabilize_for_kitless_{backend}")() + if replace_spawn: + cfg.scene.robot.spawn.func = spawn_robot + getattr(cfg, f"stabilize_for_kitless_{backend}")() + original_cfg = cfg + if copy_prepared: + cfg = deepcopy(cfg) if copy_prepared == "deepcopy" else cfg.copy() + assert cfg.actions.upper_body_ik is not original_cfg.actions.upper_body_ik + getattr(cfg, f"stabilize_for_kitless_{backend}")() + from isaaclab.assets import Asset, AssetBaseCfg + from isaaclab.sim import use_stage + + cfg.scene.robot.spawn.semantic_tags = [("class", "gr1t2")] + # AssetBase delegates to this exact constructor, which copies the asset config before spawning. + with use_stage(stage): + asset = Asset(AssetBaseCfg(prim_path="/World/Robot", spawn=cfg.scene.robot.spawn)) + assert asset.cfg.spawn is not cfg.scene.robot.spawn + assert asset.cfg.spawn.action_cfg is not cfg.actions.upper_body_ik + asset.cfg.spawn.semantic_tags.append(("instance", "copy")) + assert cfg.scene.robot.spawn.semantic_tags == [("class", "gr1t2")] + assert cfg.scene.robot.spawn.copy().action_cfg is not cfg.actions.upper_body_ik + from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction + + def initialize_solver(self, action_cfg, env): + self.cfg = action_cfg + self._env = env + self._asset = asset + self._hand_joint_names = ["R_thumb_distal_joint"] + self._raw_actions = torch.zeros((1, 8)) + + monkeypatch.setattr(PinkInverseKinematicsAction, "__init__", initialize_solver) + action = GR1T2KitlessPinkInverseKinematicsAction( + cfg.actions.upper_body_ik, SimpleNamespace(device="cpu", num_envs=1) + ) + assert action._hand_lower.shape == action._hand_upper.shape == (1,) + assert cfg.actions.upper_body_ik.hand_joint_limits is not asset.cfg.spawn.action_cfg.hand_joint_limits + joint = UsdPhysics.RevoluteJoint(stage.GetPrimAtPath("/World/Robot/R_thumb_distal_joint")) + expected = (-720.0, 720.0) if backend == "ovphysx" else (-15.0, 80.0) + assert (joint.GetLowerLimitAttr().Get(), joint.GetUpperLimitAttr().Get()) == pytest.approx(expected) + assert cfg.actions.upper_body_ik.hand_joint_limits["R_thumb_distal_joint"] == pytest.approx( + (torch.deg2rad(torch.tensor(-15.0)).item(), torch.deg2rad(torch.tensor(80.0)).item()) + ) + if copy_prepared: + assert not original_cfg.actions.upper_body_ik.hand_joint_limits + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +def test_newton_wheel_wrapper_hulls_only_non_graspable_details(monkeypatch, config_type): + stage = Usd.Stage.CreateInMemory() + + def spawn_wheel(prim_path, _cfg): + root = stage.DefinePrim(prim_path, "Xform") + for name in ( + "sm_steeringwheel_a01_wheel_rim_01", + "sm_steeringwheel_a01_wheel_spokes_01", + "sm_steeringwheel_a01_hub_detail_01", + "sm_steeringwheel_a01_quick_release_detail_01", + ): + mesh = UsdGeom.Mesh.Define(stage, f"{prim_path}/{name}").GetPrim() + UsdPhysics.MeshCollisionAPI.Apply(mesh).CreateApproximationAttr().Set(UsdPhysics.Tokens.convexDecomposition) + return root + + cfg = config_type() + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg: + assert {"waist_yaw_joint", "waist_pitch_joint", "waist_roll_joint"} <= set(controlled_joints) + cfg.sim.physics = NewtonCfg() + spawn = cfg.scene.object.spawn + spawn.func = spawn_wheel + monkeypatch.setattr(gr1t2_env_cfg.sim_utils, "find_matching_prim_paths", lambda _path: ["/World/Wheel"]) + + cfg.stabilize_for_kitless_newton_mjwarp() + spawn = cfg.scene.object.spawn + wrapped_spawn = spawn.func + wrapped_spawn("/World/Wheel", spawn) + + for name in ("sm_steeringwheel_a01_wheel_rim_01", "sm_steeringwheel_a01_wheel_spokes_01"): + assert stage.GetPrimAtPath(f"/World/Wheel/{name}").GetAttribute("physics:approximation").Get() == ( + UsdPhysics.Tokens.convexDecomposition + ) + for name in ("sm_steeringwheel_a01_hub_detail_01", "sm_steeringwheel_a01_quick_release_detail_01"): + assert stage.GetPrimAtPath(f"/World/Wheel/{name}").GetAttribute("physics:approximation").Get() == ( + UsdPhysics.Tokens.convexHull + ) + + gr1t2_env_cfg.condition_newton_steering_wheel_detail_collisions(cfg.scene.object) + assert spawn.func is wrapped_spawn + assert cfg.actions.upper_body_ik.pink_controlled_joint_names == controlled_joints + assert not isinstance(config_type().scene.object.spawn, SteeringWheelUsdFileCfg) + + +@pytest.mark.parametrize("config_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +@pytest.mark.parametrize("backend", ["newton_mjwarp", "ovphysx"]) +def test_kitless_kinematics_model_loads_without_the_kit_exporter(config_type, backend): + from isaaclab.controllers.pink_ik.pink_kinematics_configuration import PinkKinematicsConfiguration + + cfg = config_type() + assert cfg.actions.upper_body_ik.controller.urdf_path is None + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + spawn_usd = cfg.scene.robot.spawn.usd_path + cfg.sim.physics = OvPhysxCfg() if backend == "ovphysx" else NewtonCfg() + getattr(cfg, f"stabilize_for_kitless_{backend}")() + action = cfg.actions.upper_body_ik + assert action.pink_controlled_joint_names == controlled_joints + assert cfg.scene.robot.spawn.usd_path == spawn_usd + assert action.controller.urdf_path is not None, "Kitless IK must not invoke Isaac Sim's USD-to-URDF exporter" + kinematics = PinkKinematicsConfiguration( + controlled_joint_names=action.pink_controlled_joint_names, + urdf_path=action.controller.urdf_path, + ) + assert set(kinematics.controlled_joint_names_pinocchio_order) == set(action.pink_controlled_joint_names) + for frame in ("left_hand_pitch_link", "right_hand_pitch_link"): + assert np.isfinite(kinematics.get_transform_frame_to_world(frame).homogeneous).all() + jacobian = kinematics.get_frame_jacobian(frame) + assert jacobian.shape == (6, len(action.pink_controlled_joint_names)) + assert np.isfinite(jacobian).all() + if config_type is PickPlaceGR1T2WaistEnabledEnvCfg: + waist_yaw = kinematics.controlled_joint_names_pinocchio_order.index("waist_yaw_joint") + # A real movable waist must rotate either hand about its vertical axis. + world_axis = kinematics.get_transform_frame_to_world(frame).rotation @ jacobian[3:, waist_yaw] + np.testing.assert_allclose(world_axis, [0.0, 0.0, 1.0], atol=1e-6) + + +def test_kitless_conditioning_is_opt_in_and_metadata_is_not_shared(): + first = GR1T2KitlessPinkInverseKinematicsActionCfg() + second = GR1T2KitlessPinkInverseKinematicsActionCfg() + + first.hand_joint_limits["joint"] = (-1.0, 1.0) + + assert not first.kitless_target_conditioning_enabled + assert second.hand_joint_limits == {} + + +def test_ik_guard_is_per_environment_and_holds_invalid_rows(): + current = torch.tensor([[0.0, 0.1], [0.25, -0.25], [-0.1, 0.1]]) + targets = torch.tensor([[1.0, -1.0], [float("nan"), 0.0], [-0.15, 0.15]]) + + actual = bound_ik_joint_targets(targets, current, max_delta_rad=0.2, max_magnitude_rad=0.3) + + expected = torch.tensor([[0.2, -0.1], [0.25, -0.25], [-0.15, 0.15]]) + torch.testing.assert_close(actual, expected) + + +def test_hand_constraints_apply_mimic_equation_and_bounds_without_mutating_input(): + targets = torch.tensor([[0.4, 9.0, -0.9], [-0.3, 9.0, 0.8]]) + original = targets.clone() + # q_mimic = 2 * q_reference - 0.1; the first row then reaches its upper bound. + mimic_pairs = ((1, 0, 2.0, -0.1),) + lower = torch.tensor([-0.5, -0.6, -0.7]) + upper = torch.tensor([0.5, 0.6, 0.7]) + + actual = constrain_hand_joint_targets(targets, mimic_pairs, lower, upper) + + torch.testing.assert_close(actual, torch.tensor([[0.4, 0.6, -0.7], [-0.3, -0.6, 0.7]])) + torch.testing.assert_close(targets, original) + + +def test_hand_target_slew_clamps_discontinuity_against_previous_command(): + state = _HandTargetSlewState(torch.zeros((1, 2)), max_step_rad=0.079) + + torch.testing.assert_close(state.condition(torch.tensor([[0.2, -0.2]])), torch.tensor([[0.2, -0.2]])) + torch.testing.assert_close(state.condition(torch.tensor([[0.6, -0.5]])), torch.tensor([[0.279, -0.279]])) + + +def test_hand_target_slew_preserves_smooth_input_without_command_chasing(): + state = _HandTargetSlewState(torch.zeros((1, 2)), max_step_rad=0.08) + state.condition(torch.tensor([[0.2, -0.2]])) + + # The conditioner has no measured-state input: a smooth command passes exactly. + commanded = torch.tensor([[0.25, -0.24]]) + torch.testing.assert_close(state.condition(commanded), commanded) + + +def test_hand_target_slew_reset_clears_previous_command_history(): + state = _HandTargetSlewState(torch.zeros((2, 1)), max_step_rad=0.08) + state.condition(torch.tensor([[0.2], [-0.2]])) + state.condition(torch.tensor([[0.6], [-0.6]])) + + state.reset([0]) + + torch.testing.assert_close(state.condition(torch.tensor([[0.7], [-0.7]])), torch.tensor([[0.7], [-0.36]])) + + +def test_robot_conditioning_authors_collision_offsets_and_deinstances_scopes(): + stage = Usd.Stage.CreateInMemory() + + def spawn_robot(prim_path, _cfg): + robot = stage.DefinePrim(prim_path, "Xform") + collisions = stage.DefinePrim(f"{prim_path}/collisions", "Scope") + collisions.SetInstanceable(True) + collision = UsdGeom.Cube.Define(stage, f"{prim_path}/collision_shape").GetPrim() + UsdPhysics.CollisionAPI.Apply(collision) + return robot + + from isaaclab_physx.sim.schemas import PhysxRigidBodyCfg + + spawn = SimpleNamespace( + func=spawn_robot, + rigid_props=[PhysxRigidBodyCfg(max_depenetration_velocity=1.0)], + ) + robot_cfg = SimpleNamespace(spawn=spawn) + action_cfg = GR1T2KitlessPinkInverseKinematicsActionCfg() + + condition_kitless_ovphysx_robot_spawn_properties(robot_cfg) + condition_kitless_gr1t2_stage(robot_cfg, action_cfg) + spawn = robot_cfg.spawn + spawn.func("/World/Robot", spawn) + + collisions = stage.GetPrimAtPath("/World/Robot/collisions") + collision = stage.GetPrimAtPath("/World/Robot/collision_shape") + schemas = collision.GetMetadata("apiSchemas") + applied = [str(schema) for schema in schemas.GetAddedOrExplicitItems()] + assert not collisions.IsInstanceable() + assert "PhysxCollisionAPI" in applied + assert collision.GetAttribute("physxCollision:contactOffset").Get() == pytest.approx(0.001) + assert collision.GetAttribute("physxCollision:restOffset").Get() == pytest.approx(0.0) + assert spawn.rigid_props[0].max_depenetration_velocity == pytest.approx(0.1) + + +def test_wheel_mesh_collision_geometry_is_not_replaced(): + stage = Usd.Stage.CreateInMemory() + original = {} + + def spawn_wheel(path, cfg): + root = stage.DefinePrim(path, "Xform") + body = stage.DefinePrim(path + "/Body", "Xform") + UsdPhysics.RigidBodyAPI.Apply(body) + for name in ("Rim", "Spokes", "Hub", "QuickRelease"): + mesh = UsdGeom.Mesh.Define(stage, path + "/Body/" + name) + mesh.CreatePointsAttr().Set([(0, 0, 0), (1, 0, 0), (0, 1, 0)]) + mesh.CreateFaceVertexCountsAttr().Set([3]) + mesh.CreateFaceVertexIndicesAttr().Set([0, 1, 2]) + UsdPhysics.CollisionAPI.Apply(mesh.GetPrim()) + UsdPhysics.MeshCollisionAPI.Apply(mesh.GetPrim()).CreateApproximationAttr().Set("convexDecomposition") + original["layer"] = stage.GetRootLayer().ExportToString() + return root + + cfg = PickPlaceGR1T2EnvCfg() + cfg.scene.object.spawn.func = spawn_wheel + cfg.stabilize_for_kitless_ovphysx() + root = cfg.scene.object.spawn.func("/World/Wheel", cfg.scene.object.spawn) + assert stage.GetRootLayer().ExportToString() == original["layer"] + colliders = [p for p in Usd.PrimRange(root) if p.HasAPI(UsdPhysics.CollisionAPI)] + assert len(colliders) == 4 + assert {p.GetName() for p in colliders} == {"Rim", "Spokes", "Hub", "QuickRelease"} + assert all(p.IsA(UsdGeom.Mesh) for p in colliders) + assert all(UsdPhysics.CollisionAPI(p).GetCollisionEnabledAttr().Get() for p in colliders) + assert all(UsdPhysics.MeshCollisionAPI(p).GetApproximationAttr().Get() == "convexDecomposition" for p in colliders) + assert not stage.GetPrimAtPath("/World/Wheel/Body/KitlessPrimitiveCollision") diff --git a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py index 6bf7bff8735..f2f299d2013 100644 --- a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py +++ b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py @@ -194,10 +194,11 @@ def test_config_load_does_not_import_backend_modules(task_name: str, all_cfg_che def test_teleop_configs_resolve_and_validate_without_teleop(): - """Configs with optional Isaac Teleop integrations must resolve and validate without the package.""" + """Optional teleop integrations must not block task validation or standalone physics preparation.""" script = textwrap.dedent("""\ import builtins import importlib.util + from pathlib import Path original_import = builtins.__import__ original_find_spec = importlib.util.find_spec @@ -252,6 +253,51 @@ def hide_teleop(name, *args, **kwargs): assert cfg.scene.left_hand_contact is not None assert cfg.scene.right_hand_contact is not None assert cfg.scene.robot.spawn.activate_contact_sensors + + from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( + G1NewtonBoundedPinkInverseKinematicsAction, + ) + from isaaclab.envs import ManagerBasedRLEnv + + class ConstructionReached(Exception): + pass + + def stop_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + raise ConstructionReached + + ManagerBasedRLEnv.__init__ = stop_before_simulation + + standalone_tasks = ( + ("IsaacContrib-PickPlace-GR1T2-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "newton_mjwarp"), + ) + for task_name, physics in standalone_tasks: + cfg, _ = resolve_task_config(task_name, "", overrides=[f"physics={physics}"]) + assert cfg.isaac_teleop is None + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + try: + gymnasium.make(task_name, cfg=cfg) + except ConstructionReached: + pass + else: + raise AssertionError("Registered constructor did not reach the simulation boundary") + validate(cfg) + assert cfg.xr is None + assert cfg.isaac_teleop is None + action = cfg.actions.upper_body_ik + assert action.pink_controlled_joint_names == controlled_joints + if "GR1T2" in task_name: + assert action.kitless_target_conditioning_enabled + assert Path(action.controller.urdf_path).is_file() + else: + assert action.class_type is G1NewtonBoundedPinkInverseKinematicsAction + if physics == "newton_mjwarp": + assert cfg.sim.physics.solver_cfg.nconmax == 4096 + assert cfg.sim.physics.solver_cfg.njmax == 1536 """) result = subprocess.run( [sys.executable, "-c", script], diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index cd50665f7b7..b9c1a7aa137 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -271,18 +271,31 @@ def load(task, entry): assert phrase in flat, f"Missing phrase: {phrase!r}" +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + "IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", + ], +) @pytest.mark.parametrize("physics", [None, "isaacsim_physx", "ovphysx", "newton_mjwarp"]) -def test_teleop_tasks_resolve_public_physics_selectors(physics): - """Resolve registered tasks through the public flag parser before backend preparation.""" +def test_teleop_tasks_resolve_public_physics_selectors(monkeypatch, task_name, physics): + """Registered construction prepares the selected backend before simulation starts.""" + import gymnasium as gym from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg from isaaclab_ov.physics import OvPhysxCfg from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import instantiate + import isaaclab_tasks # noqa: F401 -- registers the public task IDs from isaaclab_tasks.utils import parse_env_cfg from isaaclab_tasks.utils.hydra import setup_preset_cli - task_name = "IsaacContrib-Stack-Cube-SO101-IK-Abs-v0" arguments = ["--task", task_name] if physics is not None: arguments.extend(["--physics", physics]) @@ -299,3 +312,233 @@ def test_teleop_tasks_resolve_public_physics_selectors(physics): assert cfg.sim.physics.solver_cfg.iterations == 100 assert cfg.sim.physics.solver_cfg.impratio == 10.0 assert cfg.sim.physics.solver_cfg.nconmax == 600 + + class ConstructionReached(Exception): + pass + + from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import GR1T2KitlessUsdFileCfg + + original_action_type = getattr(getattr(cfg.actions, "upper_body_ik", None), "class_type", None) + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + standalone = physics in ("ovphysx", "newton_mjwarp") + if "GR1T2" in task_name: + action = cfg.actions.upper_body_ik + assert action.kitless_target_conditioning_enabled is standalone + if standalone: + assert Path(action.controller.urdf_path).is_file() + assert isinstance(cfg.scene.robot.spawn, GR1T2KitlessUsdFileCfg) + else: + assert action.controller.urdf_path is None + expected_waist = "WaistEnabled" in task_name + assert ("waist_yaw_joint" in action.pink_controlled_joint_names) is expected_waist + if "SO101" not in task_name: + assert isinstance(cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) is standalone + if physics == "newton_mjwarp": + if "Locomanipulation" in task_name: + assert cfg.sim.physics.solver_cfg.integrator == "implicitfast" + assert cfg.sim.physics.num_substeps == 2 + else: + assert cfg.sim.physics.solver_cfg.nconmax == 4096 + assert cfg.sim.physics.solver_cfg.njmax == 1536 + if "FixedBaseUpperBodyIK" in task_name: + from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( + G1NewtonBoundedPinkInverseKinematicsAction, + ) + + assert cfg.actions.upper_body_ik.class_type is G1NewtonBoundedPinkInverseKinematicsAction + elif not standalone: + assert cfg.actions.upper_body_ik.class_type == original_action_type + raise ConstructionReached + + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + with pytest.raises(ConstructionReached): + gym.make(task_name, cfg=cfg) + with pytest.raises(ConstructionReached): + instantiate(cfg) + + +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + ], +) +@pytest.mark.parametrize("physics", ["ovphysx", "newton_mjwarp"]) +def test_humanoid_preparation_preserves_caller_changes_and_rejects_backend_switch(monkeypatch, task_name, physics): + """Teleop preparation, copies and registered construction share one completed preparation.""" + import gymnasium as gym + from isaaclab_physx.physics import PhysxCfg + + from isaaclab.envs import ManagerBasedRLEnv + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) + getattr(cfg, f"stabilize_for_kitless_{physics}")() + cfg.isaac_teleop.require_initial_bimanual_hand_tracking = ( + not cfg.isaac_teleop.require_initial_bimanual_hand_tracking + ) + policy = { + name: getattr(cfg.isaac_teleop, name) + for name in ("require_initial_bimanual_hand_tracking", "action_conditioner", "action_validator") + } + haptics = cfg.haptic_feedback.to_dict() + images = getattr(cfg, "image_obs_list", None) + if images is not None: + images.append("caller_camera") + images = list(images) + cfg.sim.dt = 0.003 + actuator_name = next(iter(cfg.scene.robot.actuators)) + cfg.scene.robot.actuators[actuator_name].stiffness = 123.0 + if physics == "newton_mjwarp": + cfg.sim.physics.solver_cfg.nconmax = 5123 + if "GR1T2" in task_name: + cfg.actions.upper_body_ik.controller.urdf_path = "caller-selected.urdf" + + class ConstructionReached(Exception): + pass + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + assert cfg.actions.upper_body_ik.pink_controlled_joint_names == controlled_joints + assert all(getattr(cfg.isaac_teleop, name) == value for name, value in policy.items()) + assert cfg.haptic_feedback.to_dict() == haptics + assert getattr(cfg, "image_obs_list", None) == images + assert cfg.sim.dt == 0.003 + assert cfg.scene.robot.actuators[actuator_name].stiffness == 123.0 + if physics == "newton_mjwarp": + assert cfg.sim.physics.solver_cfg.nconmax == 5123 + if "GR1T2" in task_name: + assert cfg.actions.upper_body_ik.controller.urdf_path == "caller-selected.urdf" + raise ConstructionReached + + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + for prepared in (cfg, cfg.copy()): + getattr(prepared, f"stabilize_for_kitless_{physics}")() + serialized = prepared.to_dict() + assert serialized["haptic_feedback"] == haptics + if "GR1T2" in task_name: + assert serialized["scene"]["robot"]["spawn"]["func"].endswith(":_spawn_and_condition") + with pytest.raises(ConstructionReached): + gym.make(task_name, cfg=prepared) + prepared.sim.physics = PhysxCfg() + with pytest.raises(ValueError, match="resolve a fresh task configuration"): + gym.make(task_name, cfg=prepared) + + +def _spawn_task_roundtrip_fixture(path, _cfg, **_kwargs): + """Module-level source spawner, preserving the public callable serialization contract.""" + from pxr import UsdGeom, UsdPhysics + + from isaaclab.sim import get_current_stage + + stage = get_current_stage() + root = stage.DefinePrim(path, "Xform") + joint = UsdPhysics.RevoluteJoint.Define(stage, path + "/R_thumb_distal_joint") + joint.CreateLowerLimitAttr().Set(-15.0) + joint.CreateUpperLimitAttr().Set(80.0) + table = stage.DefinePrim(path + "/Table", "Xform") + UsdPhysics.CollisionAPI.Apply(table) + UsdPhysics.MeshCollisionAPI.Apply(table).CreateApproximationAttr().Set("boundingCube") + UsdGeom.Mesh.Define(stage, path + "/Table/Mesh") + for name in ("sm_steeringwheel_a01_wheel_rim_01", "hub_detail"): + mesh = UsdGeom.Mesh.Define(stage, path + "/" + name).GetPrim() + UsdPhysics.MeshCollisionAPI.Apply(mesh).CreateApproximationAttr().Set("convexDecomposition") + return root + + +@pytest.mark.parametrize( + "task_name", + [ + "IsaacContrib-PickPlace-GR1T2-Abs", + "IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", + "IsaacContrib-PickPlace-Locomanipulation-G1-Abs", + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", + ], +) +@pytest.mark.parametrize("physics", ["ovphysx", "newton_mjwarp"]) +@pytest.mark.parametrize("restore_target", ["copy", "fresh_prepared"]) +def test_prepared_task_spawn_callbacks_survive_public_config_roundtrip(monkeypatch, task_name, physics, restore_target): + """Restored prepared configurations reach registered construction with resolvable spawners.""" + import gymnasium as gym + + from pxr import Usd, UsdPhysics + + import isaaclab.sim as sim_utils + from isaaclab.assets import Asset, AssetBaseCfg + from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import to_dict, update_from_dict + from isaaclab.utils.string import string_to_callable + + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + for asset in (cfg.scene.robot, cfg.scene.packing_table, cfg.scene.object): + asset.spawn.func = _spawn_task_roundtrip_fixture + getattr(cfg, f"stabilize_for_kitless_{physics}")() + cfg.sim.dt = 0.003 + serialized = to_dict(cfg) + if "GR1T2" in task_name: + cfg.actions.upper_body_ik.hand_target_margin_rad = 0.123 + if restore_target == "copy": + restored = cfg.copy() + else: + restored = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + getattr(restored, f"stabilize_for_kitless_{physics}")() + update_from_dict(restored, serialized) + if "GR1T2" in task_name: + assert cfg.actions.upper_body_ik.hand_target_margin_rad == 0.123 + + class ConstructionReached(Exception): + pass + + def inspect_before_simulation(self, cfg, *args, **kwargs): + self._is_closed = True + assert cfg.sim.dt == 0.003 + assets = [cfg.scene.packing_table] + if physics == "newton_mjwarp": + assets.append(cfg.scene.object) + if "GR1T2" in task_name: + assets.append(cfg.scene.robot) + stage = Usd.Stage.CreateInMemory() + for index, asset in enumerate(assets): + callback = to_dict(asset.spawn)["func"] + assert callable(string_to_callable(callback)) + path = f"/World/Asset_{index}" + with sim_utils.use_stage(stage): + authored = Asset(AssetBaseCfg(prim_path=path, spawn=asset.spawn)) + root = authored.prim + assert root == stage.GetPrimAtPath(path) + if asset is cfg.scene.packing_table: + assert not stage.GetPrimAtPath(path + "/Table").HasAPI(UsdPhysics.CollisionAPI) + assert stage.GetPrimAtPath(path + "/Table/Mesh").HasAPI(UsdPhysics.CollisionAPI) + elif asset is cfg.scene.object: + assert ( + stage.GetPrimAtPath(path + "/hub_detail").GetAttribute("physics:approximation").Get() + == "convexHull" + ) + assert ( + stage.GetPrimAtPath(path + "/sm_steeringwheel_a01_wheel_rim_01") + .GetAttribute("physics:approximation") + .Get() + == "convexDecomposition" + ) + else: + limits = authored.cfg.spawn.action_cfg.hand_joint_limits["R_thumb_distal_joint"] + assert limits == pytest.approx((-0.2617993878, 1.3962634016)) + if "GR1T2" in task_name: + assert cfg.scene.robot.spawn.action_cfg is cfg.actions.upper_body_ik + raise ConstructionReached + + monkeypatch.setattr(sim_utils, "find_matching_prim_paths", lambda path: [path]) + monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) + with pytest.raises(ConstructionReached): + gym.make(task_name, cfg=restored) diff --git a/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.skip b/source/isaaclab_teleop/changelog.d/kitless-humanoid-acceptance.skip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 665441e43a6..e27492c2b06 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -46,19 +46,43 @@ and restore the caller's arguments on return. The scripts under ```bash uv run --extra ovxr isaaclab teleop run \ - --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --num_envs 1 \ - --device cpu --physics ovphysx --ovxr + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ + --device cuda:0 --physics newton_mjwarp --ovxr ``` -Use `--physics ovphysx --device cpu` for CPU OVPhysX. `--ovxr` selects OVStage -and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported -CloudXR client, then use Play, Pause and Reset as in Kit-based teleoperation. +The same registered-task selectors apply to live input and MCAP replay: + +| Task | Physics selectors | +| --- | --- | +| `IsaacContrib-PickPlace-GR1T2-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-Locomanipulation-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-Stack-Cube-SO101-IK-Abs-v0` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | + +Omitting `--physics` retains the Kit/Isaac Sim default. Use `--physics ovphysx +--device cpu` for standalone CPU physics, or `--physics newton_mjwarp --device +cuda:0` for Newton; these standalone selections support `--ovxr`. Kit's +`isaacsim_physx` selection uses its separate `--xr` workflow and Isaac Sim +installation, not `--ovxr`. The task presets retain backend-specific solver +settings and, for SO101, choose the matching USD Physics variant. The SO101 IK task resets its joints to the configured nominal pose without joint-position noise, giving the clutch a repeatable reset pose. Its configured home transform supplies the clutch calibration. This applies to its Kit and kitless backends. The separate joint-control task retains its randomized reset. +For example, replace the task in this command with any row above: + +```bash +uv run --python 3.12 --extra ovxr --locked isaaclab teleop run \ + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ + --device cpu --physics ovphysx --ovxr --cloudxr_env cloudxrjs +``` + + `--ovxr` selects OVStage +and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported +CloudXR client, then use Play, Pause and Reset as in Kit-based teleoperation. + For desktop IWER, select **VR** and set **Reference Space** to **local-floor** under **Teleoperator adjustment** before connecting. This keeps the emulated head and controllers relative to the floor. Between trials, end the previous XR @@ -90,7 +114,7 @@ environment action interval when no explicit rate is set. This option requires ```bash uv run --extra ovxr python scripts/environments/teleoperation/teleop_replay_agent.py \ - --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --num_envs 1 \ + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ --device cpu --physics ovphysx --ovxr \ --replay_file /absolute/path/to/session.mcap ``` diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py new file mode 100644 index 00000000000..4497488552e --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -0,0 +1,177 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Native scene publication and renderer ownership transfer.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from dataclasses import dataclass +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1", + reason="native OVRTX transfer harness is explicitly opt-in", + ), +] + + +@dataclass +class _CaseResult: + offer_thread: int + claim_thread: int | None = None + renderer_id: int | None = None + stage_id: int | None = None + ordinal: int | None = None + error: BaseException | None = None + + +def _make_env_and_bootstrap(): + import gymnasium as gym + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop import remove_camera_configs + from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap + + import isaaclab_tasks.contrib.pick_place # noqa: F401 -- registers the exact GR1T2 task + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg("IsaacContrib-PickPlace-GR1T2-Abs", device="cpu", num_envs=1) + cfg.sim.physics = OvPhysxCfg() + cfg.stabilize_for_kitless_ovphysx() + cfg = remove_camera_configs(cfg) + bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) + try: + return bootstrap, gym.make("IsaacContrib-PickPlace-GR1T2-Abs", cfg=cfg).unwrapped + except BaseException: + bootstrap.abort() + raise + + +def _run_case(*, transfer: bool) -> _CaseResult: + from isaaclab_teleop.ov_libraries_xr import OvLibrariesXrConfig, _IsaacLabOvxrApplicationInterface + + bootstrap, env = _make_env_and_bootstrap() + result = _CaseResult(offer_thread=threading.get_native_id()) + host = _IsaacLabOvxrApplicationInterface(bootstrap, env, OvLibrariesXrConfig(auto_launch_cloudxr=False), ()) + capability = host.offer_render_owner_transfer() + ready, published = threading.Event(), threading.Event() + + def attach(): + binding = host.attach_on_render_thread(capability) + result.renderer_id, result.stage_id = id(binding.renderer), id(binding.stage) + return binding + + def update_stage(binding): + with host.publication_guard(), host.owner_lane(): + binding.renderer.update_from_stage(result.ordinal) + + def close(): + host.close_publication() + try: + host.close_on_render_thread() + finally: + host.close_environment_on_render_thread() + + if not transfer: + try: + binding = attach() + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-control", 0)) + update_stage(binding) + finally: + close() + return result + + with pytest.raises(RuntimeError, match="transfer is pending"): + _ = bootstrap._lease.ovstage + + def worker(): + try: + result.claim_thread = threading.get_native_id() + binding = attach() + ready.set() + if not published.wait(60): + raise TimeoutError("application did not complete its scene publication") + if result.error is None: + update_stage(binding) + with pytest.raises(RuntimeError, match="capability is invalid"): + bootstrap.claim_render_owner_transfer(capability) + except BaseException as exc: + if result.error is None: + result.error = exc + finally: + ready.set() + try: + close() + except BaseException as exc: + if result.error is None: + result.error = exc + + thread = threading.Thread(target=worker, name="ovrtx-native-transfer-worker") + thread.start() + try: + if not ready.wait(90): + raise TimeoutError("renderer claimant did not finish binding") + if result.error is not None: + raise result.error + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-transfer", 0)) + except BaseException as exc: + result.error = exc + finally: + published.set() + thread.join(timeout=90) + assert not thread.is_alive(), "native transfer worker timed out" + if result.error is not None: + raise result.error + assert result.claim_thread is not None and result.claim_thread != result.offer_thread + with pytest.raises(RuntimeError, match="(owner thread|released)"): + _ = bootstrap._lease.ovstage + return result + + +def _run_native_subprocess(transfer: bool) -> None: + # A native non-daemon thread cannot be bounded by Thread.join alone. + # Keep the native lifetime in a child that the standard subprocess timeout owns. + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), "transfer" if transfer else "same-thread"], + env={**os.environ, "ISAAC_LAB_OVRTX_USE_OVSTAGE": "1"}, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + print(completed.stdout) + assert completed.returncode == 0, completed.stderr + + +def test_gr1t2_same_thread_ovxr_session_native_control(): + """One-thread production scene publication remains viable.""" + _run_native_subprocess(False) + + +def test_gr1t2_cross_thread_preuse_owner_transfer_native(): + """The application publishes while the render thread binds, renders and closes the pair.""" + _run_native_subprocess(True) + + +if __name__ == "__main__": + if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": + raise RuntimeError("native transfer harness requires its explicit opt-in") + result = _run_case(transfer=sys.argv[1] == "transfer") + assert result.renderer_id is not None and result.stage_id is not None + assert result.ordinal is not None + print( + f"offer={result.offer_thread} claim={result.claim_thread} " + f"renderer={result.renderer_id} stage={result.stage_id} ordinal={result.ordinal}" + ) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 7a03d06b8bc..15e419c9708 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -629,24 +629,44 @@ def test_product_xr_default_resolves_cloudxrjs_and_autolaunch(monkeypatch): assert agent.args_cli.auto_launch_cloudxr is True -def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch): - """Launch validation must not replace task-specific settings on the selected backend.""" - agent = _load_teleop_agent(monkeypatch) - from isaaclab_newton.physics import NewtonCfg - +@pytest.mark.parametrize( + "task_name,physics", + [ + ("IsaacContrib-PickPlace-GR1T2-Abs", "isaacsim_physx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-GR1T2-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-Locomanipulation-G1-Abs", "newton_mjwarp"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "ovphysx"), + ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "newton_mjwarp"), + ], +) +def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch, task_name, physics): + """Task preparation must reach launch validation without losing its backend settings.""" from isaaclab.app.sim_launcher import validate_launch - physics_cfg = NewtonCfg() - physics_cfg.solver_cfg.nconmax = 4096 - physics_cfg.solver_cfg.njmax = 1536 - env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)) - configs, _ = validate_launch(env_cfg, vars(agent.args_cli).copy()) + from isaaclab_tasks.utils import parse_env_cfg + + agent = _load_teleop_agent(monkeypatch, "--task", task_name, "--physics", physics) + env_cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + physics_cfg = env_cfg.sim.physics + env_cfg.isaac_teleop.prepare_environment(env_cfg) + if physics == "newton_mjwarp": + if "Locomanipulation-G1" in task_name: + assert physics_cfg.num_substeps == 2 + assert physics_cfg.solver_cfg.integrator == "implicitfast" + else: + assert physics_cfg.solver_cfg.nconmax == 4096 + assert physics_cfg.solver_cfg.njmax == 1536 + else: + from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg + + assert isinstance(env_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) == (physics == "ovphysx") + configs, _ = validate_launch(SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)), vars(agent.args_cli).copy()) assert not hasattr(agent.args_cli, "physics") assert env_cfg.sim.physics is physics_cfg assert any(cfg is physics_cfg for cfg in configs) - assert physics_cfg.solver_cfg.nconmax == 4096 - assert physics_cfg.solver_cfg.njmax == 1536 def test_task_action_hook_validates_before_and_after_conditioning(monkeypatch): From 1f07bb985ded3e947cb19fa808a4b79781a7f668 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Fri, 30 Oct 2026 18:16:20 -0400 Subject: [PATCH 004/145] Add camera panels and the Vulkan desktop mirror --- docs/source/features/isaac_teleop.rst | 40 +- docs/source/how-to/cloudxr_teleoperation.rst | 7 +- .../ovxr-camera-entrypoint-tests.skip | 0 .../test/cli/test_teleop_entrypoints.py | 31 ++ .../leased-camera-products.added.rst | 2 + .../changelog.d/leased-camera-products.minor | 0 .../isaaclab_ov/renderers/ovrtx_renderer.py | 68 +++- .../isaaclab_ov/renderers/render_lease.py | 377 +++++++++++++++++- .../isaaclab_ov/renderers/visual_materials.py | 19 +- source/isaaclab_ov/isaaclab_ov/stage.py | 10 +- .../test/test_ovrtx_renderer_contract.py | 307 +++++++++++++- .../ovxr-robot-camera-observations.added.rst | 1 + ...ovxr-robot-camera-observations.changed.rst | 6 + .../ovxr-robot-camera-observations.major | 0 source/isaaclab_tasks/docs/README.md | 52 +++ .../fixed_base_upper_body_ik_g1_env_cfg.py | 69 +++- .../pick_place/pickplace_gr1t2_env_cfg.py | 16 +- .../pickplace_gr1t2_waist_enabled_env_cfg.py | 2 + .../test/core/test_xr_camera_feed_presets.py | 45 ++- .../changelog.d/ovxr-camera-panels.added.rst | 7 + .../changelog.d/ovxr-camera-panels.minor | 0 source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 17 + source/isaaclab_teleop/docs/README.md | 53 ++- source/isaaclab_teleop/docs/kitless_xr.md | 42 +- .../isaaclab_teleop/docs/kitless_xr_source.md | 9 +- .../isaaclab_teleop/__init__.pyi | 3 +- .../isaaclab_teleop/camera_feed.py | 114 +++++- .../entrypoints/teleop_replay_agent.py | 62 ++- .../entrypoints/teleop_se3_agent.py | 111 +++++- .../isaaclab_teleop/ov_libraries_xr.py | 165 +++++++- .../isaaclab_teleop/ovxr_session.py | 12 +- .../isaaclab_teleop/isaaclab_teleop/xr_cfg.py | 19 +- .../isaaclab_teleop/xr_scene_partition.py | 73 ++++ .../isaaclab_teleop/test/test_entrypoints.py | 27 ++ .../test/test_ov_libraries_xr.py | 338 +++++++++++++++- .../test/test_teleop_entrypoint_units.py | 111 +++++- .../test/test_teleop_failure_exit.py | 6 +- .../test/test_teleop_replay_bootstrap.py | 60 +++ .../test/test_xr_camera_feed.py | 85 ++++ .../test/test_xr_scene_partition.py | 104 +++++ 40 files changed, 2336 insertions(+), 134 deletions(-) create mode 100644 source/isaaclab/changelog.d/ovxr-camera-entrypoint-tests.skip create mode 100644 source/isaaclab_ov/changelog.d/leased-camera-products.added.rst create mode 100644 source/isaaclab_ov/changelog.d/leased-camera-products.minor create mode 100644 source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst create mode 100644 source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst create mode 100644 source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.major create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-camera-panels.minor create mode 100644 source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py create mode 100644 source/isaaclab_teleop/test/test_xr_scene_partition.py diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 5d1fc11eb33..7240c44e311 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1344,6 +1344,34 @@ through this variable -- later runs no longer prompt. The variable applies to ev that launches the runtime, including the process-scoped launcher in ``teleop_replay_agent.py``. +.. _isaac-teleop-ovxr-mcap: + +Replay MCAP with a Kitless OVXR Viewer +-------------------------------------- + +The MCAP replay entrypoint can present recorded input through an independent +OVXR viewer. Follow the :ref:`kitless XR guide ` to install Isaac +Capture with its OVXR extra and select the compatible rendering dependencies. + +.. code-block:: bash + + uv run --extra ovxr python \ + scripts/environments/teleoperation/teleop_replay_agent.py \ + --task IsaacContrib-PickPlace-GR1T2-Abs --physics ovphysx \ + --device cpu --num_envs 1 --ovxr \ + --replay_file /absolute/path/to/session.mcap + +Use ``--physics newton_mjwarp`` for Newton. MCAP supplies tracker and control +input; the connected client supplies the view. A recorded STOP, task success +or failure, or the replay duration limit ends the run. The separate +``isaaclab teleop replay`` command replays HDF5 demonstrations through Kit. + +Add ``--ovxr_pip_camera robot_pov_cam --ovxr_pip_head_locked`` for a camera +panel. OVXR live and MCAP replay open a desktop mirror by default; add +``--headless`` to disable that window while keeping XR and PiP rendering. +See the kitless XR guide for installation, camera selection and display requirements. + + .. _isaac-teleop-xr-anchor: Configure the XR Anchor @@ -1485,7 +1513,7 @@ unchanged. For either G1 task, set ``env.isaac_teleop.xr_camera_feeds=[]`` to di the camera sensor and any configured camera observations. The reference feeds request render-product-local DLSS Ray Reconstruction and ``quality`` execution -mode through :class:`~isaaclab_teleop.XrCameraFeedCfg`. The private PiP adapter authors those two +mode through :class:`~isaaclab_teleop.XrCameraFeedCfg`. The Kit Scene UI presenter authors those two attributes only on the selected camera render product. Ray Reconstruction also requires the process-global responsive-denoising setting, which the teleoperation and recording scripts enable before environment construction when a selected feed requests Ray Reconstruction. @@ -1596,8 +1624,10 @@ Kit Scene UI presentation PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until an enabled feed is requested. If the Scene UI modules cannot be imported, the scripts log a warning and continue without PiP; task-owned cameras and recording observations remain unchanged. This keeps -the camera selection configuration usable when a future kitless entry point no longer provides -Scene UI. Render-product tuning is also best-effort: backends without a compatible render product +the camera selection configuration usable without Scene UI. The +:ref:`kitless XR guide ` uses separate OpenXR quad +layers and does not import Scene UI; those layers are visible in the headset/client, not in the +passive eye-image desktop mirror. Render-product tuning is also best-effort: backends without a compatible render product keep the Camera-buffer path, and schema-authoring failures warn while PiP continues. Other configuration, camera-buffer, and panel-initialization errors still fail during startup. @@ -2031,8 +2061,8 @@ Optimize XR Performance .. note:: - The ``--headless`` CLI flag was removed in Isaac Lab 3.0. Headless is now the **default** - (omit ``--visualizer``), or set ``HEADLESS=1`` in the environment. + For Kit XR, omit ``--visualizer`` to run without the local viewport. + For kitless ``--ovxr``, pass ``--headless`` to disable its default desktop mirror. .. dropdown:: Configure retargeting execution :open: diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index 3e7ceae2f4e..51874952783 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -146,9 +146,10 @@ to capture demonstrations, and ``replay`` to play a dataset back. :ref:`The next .. note:: - ``teleop`` cannot be combined with ``ov`` or ``ovphysx`` in a single ``uv run``: the - bundled Isaac Sim pins ``packaging==26.0`` while those runtimes require ``<24``. Install - the OV runtimes separately when you need them. + For XR teleoperation without Isaac Sim, follow the + :ref:`kitless XR guide `. It explains how to select a public + Isaac Capture wheel and its optional OVXR dependencies. The Kit setup below + uses the published ``teleop`` extra. Complete these steps first: diff --git a/source/isaaclab/changelog.d/ovxr-camera-entrypoint-tests.skip b/source/isaaclab/changelog.d/ovxr-camera-entrypoint-tests.skip new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab/test/cli/test_teleop_entrypoints.py b/source/isaaclab/test/cli/test_teleop_entrypoints.py index 94691d8f266..4011b808a5b 100644 --- a/source/isaaclab/test/cli/test_teleop_entrypoints.py +++ b/source/isaaclab/test/cli/test_teleop_entrypoints.py @@ -59,6 +59,8 @@ def test_teleop_workflow_help_exposes_task_preset_selectors(source_checkout_root assert "presets=NAME[,NAME,...]" in result.stdout if command == "run": assert re.search(r"^\s+--ovxr(?:\s|$)", result.stdout, re.MULTILINE) + assert "--ovxr_pip_camera" in result.stdout + assert "--ovxr_desktop_mirror" in result.stdout def test_ovxr_omitted_launch_options_match_documented_defaults(source_checkout_root, monkeypatch): @@ -88,6 +90,35 @@ def parse(arguments): assert parse(["--ovxr", "--cloudxr_env", "none"]).cloudxr_env is None +def test_ovxr_output_options_reach_existing_runtime_fields(source_checkout_root, monkeypatch): + """Camera options configure panels, the desktop mirror and XR startup.""" + options = [ + "--ovxr", + "--ovxr_session_mode", + "vr", + "--ovxr_desktop_mirror", + "--ovxr_desktop_mirror_fps", + "20", + "--ovxr_pip_camera", + "robot_pov_cam", + "--ovxr_pip_fps", + "10", + "--ovxr_pip_head_locked", + "--ovxr_pip_view_position", + "0", + "0.2", + "-0.5", + ] + script = source_checkout_root / "scripts/environments/teleoperation/teleop_se3_agent.py" + monkeypatch.setattr(sys, "argv", [str(script), *options]) + parsed = importlib.import_module("isaaclab_teleop.entrypoints.teleop_se3_agent")._parse_args() + assert parsed.using_ovxr and parsed.ovxr_session_mode == "vr" + assert parsed.ovxr_desktop_mirror + assert parsed.ovxr_desktop_mirror_fps == 20 + assert parsed.ovxr_pip_camera == ["robot_pov_cam"] and parsed.ovxr_pip_fps == 10 + assert parsed.ovxr_pip_head_locked and parsed.ovxr_pip_view_position == [0, 0.2, -0.5] + + @pytest.mark.parametrize(("command", "script_parts"), TELEOP_WORKFLOWS.items()) def test_teleop_dispatches_to_its_workflow(source_checkout_root: Path, command, script_parts, monkeypatch): """``isaaclab teleop`` runs each workflow's entrypoint in process with the remaining arguments.""" diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst new file mode 100644 index 00000000000..af673174ba2 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst @@ -0,0 +1,2 @@ +* Added leased camera render products and independent camera-panel scheduling to external consumers. +* Registered visual materials before deferred scene population and bound their addresses before publication. diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.minor b/source/isaaclab_ov/changelog.d/leased-camera-products.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 6c34a2cca16..8c6a65a9986 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -378,11 +378,15 @@ def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): @property def visual_material_writer(self): - """Return a writer factory once this renderer has a scene consumer.""" - return self._create_visual_material_writer if self._initialized_scene else None + """Register materials for an initialized or deferred leased scene consumer.""" + return ( + self._create_visual_material_writer + if self._initialized_scene or self._render_lease.stage_deferred + else None + ) def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> OVRTXVisualMaterialWriter: - """Create the detached-scene material writer after scene population.""" + """Retain material buffers; deferred scenes compile addresses after population.""" writer = OVRTXVisualMaterialWriter(self, batches) self._visual_material_writer_ref = weakref.ref(writer) return writer @@ -407,7 +411,9 @@ def _publish_visual_materials(self) -> Iterator[None]: def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" - self._render_lease.prepare_stage(num_envs) + if not self._render_lease.prepare_stage(num_envs): + return + lease = self._render_lease.active logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -437,7 +443,10 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: if self.cfg.temp_usd_dir is not None: _write_file(Path(self.cfg.temp_usd_dir), "ovrtx_renderer_stage.usda", usd) if self._use_ovstage: - self.scene.populate(usd) + if lease is not None: + self._render_lease.populate_stage(usd) + else: + self.scene.populate(usd) else: self.backend.renderer.open_usd_from_string(usd) ovrtx_replicate(self.backend.renderer, self.scene.clone_copies, env_paths, self.scene.clone_positions) @@ -591,24 +600,42 @@ def _setup_geometry_bindings(self) -> None: self._bindings.callback(self._geometry_points_binding.unbind) def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: - """Register a camera's product and bindings in the prepared scene.""" - self._render_lease.reject_ordinary_operation("create ordinary camera render data") + """Register an ordinary camera, or reserve an authorized external camera until handoff.""" + auxiliary = self._render_lease.accept_camera_spec(spec) + if not auxiliary: + self._render_lease.reject_ordinary_operation("create ordinary camera render data") if str(wp.get_device(spec.device)) != self._device: raise ValueError("Cameras sharing an OVRTX renderer must use the simulation device.") + camera = self._render_lease.camera_for_spec(spec) if auxiliary else None + if camera is not None and camera.render_data is not None: + raise RuntimeError("Auxiliary camera render data already exists") camera_paths = list(spec.camera_prim_paths) env_partitions = {Sdf.Path(path): f"env_{i}" for i, path in enumerate(self.scene.clone_env_paths)} source_root = next((root for root in Sdf.Path(camera_paths[0]).GetPrefixes() if root in env_partitions), None) + if source_root is not None and len(camera_paths) < spec.num_instances: + # Native cloning may leave only the source camera in USD. Preserve complete path row order. + suffix = camera_paths[0][len(str(source_root)) :] + camera_paths = [path + suffix for path in self.scene.clone_env_paths] + data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) + if not auxiliary or not self._render_lease.stage_deferred: + self._register_camera(spec, data) + self.scene.next_camera_id += 1 + self._camera_render_data.append(data) + if camera is not None: + camera.render_data = data + return data + + def _register_camera(self, spec: CameraRenderSpec, render_data: OVRTXCameraRenderData) -> None: + """Bind one product in the already populated simulation-owned scene.""" + camera_paths = render_data.camera_paths + env_partitions = {Sdf.Path(path): f"env_{i}" for i, path in enumerate(self.scene.clone_env_paths)} + source_root = next((root for root in Sdf.Path(camera_paths[0]).GetPrefixes() if root in env_partitions), None) camera_partitions = [] if source_root is not None: - # Native cloning may leave only the source camera in USD. Complete paths retain their row order. - if len(camera_paths) < spec.num_instances: - suffix = camera_paths[0][len(str(source_root)) :] - camera_paths = [path + suffix for path in self.scene.clone_env_paths] camera_partitions = [ next(env_partitions[root] for root in Sdf.Path(path).GetPrefixes() if root in env_partitions) for path in camera_paths ] - render_data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) scope, product_path = render_data.render_scope_name, render_data.render_product_path usd = build_render_product_as_string( spec, @@ -699,11 +726,8 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: except Exception: render_data.cleanup() raise - self.scene.next_camera_id += 1 - self._camera_render_data.append(render_data) self.backend.render_products.add(render_data.render_product_path) render_data.resources.callback(self.backend.render_products.discard, render_data.render_product_path) - return render_data def set_outputs(self, render_data: OVRTXCameraRenderData, output_data: dict[str, ProxyArray]) -> None: """Register pre-allocated warp output buffers for rendering. @@ -1134,8 +1158,20 @@ def update_camera( orientations: ProxyArray, intrinsics: ProxyArray, ) -> None: - """Update camera poses using the camera's reusable conversion buffers.""" + """Update an ordinary camera or queue a sample for an external borrower.""" + if self._render_lease.queue_camera_update(render_data, positions, orientations, intrinsics): + return self._render_lease.reject_ordinary_operation("update ordinary cameras") + self._update_camera(render_data, positions, orientations, intrinsics) + + def _update_camera( + self, + render_data: OVRTXCameraRenderData, + positions: ProxyArray, + orientations: ProxyArray, + intrinsics: ProxyArray, + ) -> None: + """Update camera poses using the camera's reusable conversion buffers.""" binding = render_data.camera_xform_binding if binding is None: return diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 4a56beb3bd5..f85d48ca407 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -9,17 +9,28 @@ from __future__ import annotations import contextlib +import math import os +import re import threading -from collections.abc import Hashable, Iterator +from collections.abc import Hashable, Iterator, Sequence +from dataclasses import dataclass from enum import Enum, auto -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast + +import numpy as np +import ovstage +import torch + +from pxr import Gf, Sdf, Usd + +from isaaclab.renderers.camera_render_spec import CameraRenderSpec if TYPE_CHECKING: from ovrtx import Renderer from ovstage import Stage as OvStage - from .ovrtx_renderer import OVRTXRenderer + from .ovrtx_renderer import OVRTXCameraRenderData, OVRTXRenderer _NO_PHYSICS_STEP_ID = object() @@ -31,6 +42,16 @@ class _OvLibrariesRenderLeaseState(Enum): RELEASED = auto() +@dataclass +class _AuxiliaryCamera: + cfg: Any + max_fps: float | None + spec: CameraRenderSpec | None = None + render_data: OVRTXCameraRenderData | None = None + state: tuple[Any, ...] | None = None + product_path: str | None = None + + class OvLibrariesRenderLease: """Exclusive non-owning lease over one exact attached OVRTX/OVStage pair. @@ -51,6 +72,11 @@ def __enter__(self) -> OvLibrariesRenderLease: def __exit__(self, *_exc: object) -> None: self.release() + @property + def auxiliary_render_product_paths(self) -> tuple[str, ...]: + self.validate_live() + return cast("RenderLeaseController", self._controller).product_paths(self) + @property def ovrtx_renderer(self) -> Renderer: """Return the exact retained application renderer.""" @@ -63,6 +89,19 @@ def ovstage(self) -> OvStage: self.validate_live() return cast("OvStage", self._stage) + def configure_auxiliary_camera(self, camera_cfg: Any, *, max_fps: float | None = None) -> None: + """Authorize an exact camera before Stage materialization.""" + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Auxiliary camera configuration must precede Stage materialization") + if camera_cfg is None: + raise ValueError("Auxiliary camera configuration cannot be None") + if any(camera.cfg.prim_path == camera_cfg.prim_path for camera in self._auxiliary_cameras): + raise RuntimeError("This auxiliary camera is already configured") + if max_fps is not None and (not math.isfinite(max_fps) or max_fps <= 0.0): + raise ValueError("Auxiliary camera max_fps must be finite and positive [Hz]") + self._auxiliary_cameras.append(_AuxiliaryCamera(camera_cfg, max_fps)) + def offer_owner_transfer(self) -> object: """Freeze one unbound lease and return its one-shot owner capability.""" owner = cast("RenderLeaseController", self._controller) @@ -78,6 +117,10 @@ def capture_scene_publisher(self) -> OvLibrariesScenePublisher: """Capture the application-thread scene publisher before render ownership transfers.""" return cast("RenderLeaseController", self._controller).capture_scene_publisher(self) + def materialize_deferred_auxiliary_camera_stage(self, stage: Any, num_envs: int) -> bool: + """Complete the selected camera's deferred Stage on the claimed owner.""" + return cast("RenderLeaseController", self._controller).materialize_deferred_stage(self, stage, num_envs) + @contextlib.contextmanager def ovstage_owner_lane(self) -> Iterator[None]: """Guard one non-reentrant operation on the render-owner thread. @@ -87,9 +130,28 @@ def ovstage_owner_lane(self) -> Iterator[None]: """ self.validate_live() owner = cast("RenderLeaseController", self._controller) - with owner.owner_lane("external renderer operation"): + with owner.owner_lane("external renderer operation", lease=self): yield + def configure_external_camera_scene_partitions( + self, environment_root_path: str, camera_paths: Sequence[str] + ) -> str: + """Assign the live environment partition token to external cameras. + + This method must be called inside :meth:`ovstage_owner_lane` after the + external camera prims exist. The environment root's resolved token is + authoritative; empty and token-zero values fail closed. + """ + self.validate_live() + owner = cast("RenderLeaseController", self._controller) + return owner.configure_camera_partitions(self, environment_root_path, camera_paths) + + def validate_external_scene_partitions(self, ordinal: int) -> dict[str, str]: + """Read back and prove environment/camera partition equality.""" + self.validate_live() + owner = cast("RenderLeaseController", self._controller) + return owner.validate_partitions(self, ordinal) + def release(self) -> None: """Release the borrowing intent without destroying either pair member.""" if self._state is _OvLibrariesRenderLeaseState.RELEASED: @@ -121,6 +183,10 @@ def _create_unbound(cls, owner: RenderLeaseController) -> OvLibrariesRenderLease lease._renderer = None lease._stage = None lease._generation = 0 + lease._required_environment_scene_partitions = () + lease._external_camera_scene_partitions = () + lease._external_camera_scene_partitions_pending = False + lease._auxiliary_cameras = [] return lease def _bind(self, renderer: Renderer, stage: OvStage, generation: int) -> None: @@ -187,8 +253,10 @@ def __init__(self, renderer: OVRTXRenderer) -> None: self._publisher: OvLibrariesScenePublisher | None = None self._generation = 0 self._lane_operation: str | None = None + self._lane_lease: OvLibrariesRenderLease | None = None self._last_step_id: Hashable = _NO_PHYSICS_STEP_ID self._poisoned = False + self.stage_deferred = False def acquire(self) -> OvLibrariesRenderLease: """Reserve the exact OVRTX/OVStage pair before stage preparation.""" @@ -273,7 +341,7 @@ def capture_scene_publisher(self, lease: OvLibrariesRenderLease) -> OvLibrariesS return publisher @contextlib.contextmanager - def owner_lane(self, operation: str) -> Iterator[None]: + def owner_lane(self, operation: str, *, lease: OvLibrariesRenderLease | None = None) -> Iterator[None]: """Reject nested operations and access outside the render-owner thread. This lane is not a cross-thread lock. The external consumer must share @@ -285,9 +353,11 @@ def owner_lane(self, operation: str) -> Iterator[None]: f"OVStage owner lane is already active for {self._lane_operation}; nested writers are forbidden" ) self._lane_operation = operation + self._lane_lease = lease try: yield finally: + self._lane_lease = None self._lane_operation = None def validate_active(self, lease: OvLibrariesRenderLease) -> None: @@ -364,7 +434,11 @@ def producer_publication_required(self, lease: OvLibrariesRenderLease, physics_s raise TypeError("physics_step_id must be hashable") from exc writer_ref = self._renderer._visual_material_writer_ref writer = writer_ref() if writer_ref is not None else None - return self._last_step_id != physics_step_id or (writer is not None and writer.has_pending_updates) + return ( + lease._external_camera_scene_partitions_pending + or self._last_step_id != physics_step_id + or (writer is not None and writer.has_pending_updates) + ) def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, physics_step_id: Hashable) -> None: """Write a complete scene batch without advancing its reserved ordinal.""" @@ -372,20 +446,260 @@ def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, phys raise ValueError("OVStage publication ordinal must be nonzero") if not self.producer_publication_required(lease, physics_step_id): raise RuntimeError("OVStage scene publication is not required for this complete state") - self._renderer.scene.ordinal = ordinal + publish_camera_partitions = lease._external_camera_scene_partitions_pending try: - with self._renderer._publish_visual_materials(): - self._renderer._update_transforms() - self._renderer._update_geometries() + self._write_scene_components(ordinal, partition_lease=lease if publish_camera_partitions else None) except Exception as exc: self._poisoned = True raise RuntimeError("Producer scene publication failed; recreate the exact pair") from exc self._last_step_id = physics_step_id + if publish_camera_partitions: + lease._external_camera_scene_partitions_pending = False + + def camera_for_spec(self, spec: CameraRenderSpec) -> _AuxiliaryCamera | None: + lease = self.active + if lease is None: + return None + for camera in lease._auxiliary_cameras: + expected = camera.cfg + if spec.num_instances != 1 or spec.view_count != 1 or len(spec.camera_prim_paths) != 1: + raise RuntimeError("An external camera requires exactly one camera view") + pattern = str(expected.prim_path).replace("{ENV_REGEX_NS}", r"/World/envs/env_[^/]+") + if ( + int(spec.cfg.width) == int(expected.width) + and int(spec.cfg.height) == int(expected.height) + and tuple(spec.cfg.data_types) == tuple(expected.data_types) + and re.fullmatch(f"(?:{pattern})(?:/camera)?", str(spec.camera_prim_paths[0])) is not None + ): + return camera + return None + + def accept_camera_spec(self, spec: CameraRenderSpec) -> bool: + camera = self.camera_for_spec(spec) + if camera is None: + return False + if camera.spec is None: + camera.spec = spec + elif camera.spec != spec: + raise RuntimeError("Auxiliary camera specification changed after authorization") + return True + + def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: + """Materialize the selected camera after render-thread ownership transfer.""" + self.require_owner_thread("materialize the auxiliary camera Stage") + if not self.stage_deferred: + return False + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or not self._transfer_claimed + or not lease._auxiliary_cameras + or any(camera.spec is None or camera.render_data is None for camera in lease._auxiliary_cameras) + ): + raise RuntimeError("Deferred auxiliary camera Stage is not eligible for materialization") + self._renderer.prepare_stage(stage, num_envs) + return True + + def product_paths(self, lease: OvLibrariesRenderLease) -> tuple[str, ...]: + """Return products in application selection order, independently of sensor initialization order.""" + self.validate_live(lease, lease.ovrtx_renderer, lease.ovstage, lease._generation) + paths = tuple(camera.product_path for camera in lease._auxiliary_cameras) + if any(not isinstance(path, str) or not path.startswith("/") for path in paths) or len(set(paths)) != len( + paths + ): + raise RuntimeError("Auxiliary camera did not materialize exactly one RenderProduct per selection") + return paths + + def _write_scene_components(self, ordinal: int, *, partition_lease: OvLibrariesRenderLease | None) -> None: + """Write selected scene components; the caller owns validation and commit.""" + self._renderer.scene.ordinal = ordinal + with self._renderer._publish_visual_materials(): + self._renderer._update_transforms() + self._renderer._update_geometries() + self._update_cameras() + if partition_lease is not None: + self._write_camera_partitions(partition_lease, ordinal) + + def configure_camera_partitions( + self, + lease: OvLibrariesRenderLease, + environment_root_path: str, + camera_paths: Sequence[str], + ) -> str: + """Read the authoritative environment token and arm matching camera writes.""" + self.validate_active(lease) + lease.validate_live() + if self._lane_lease is not lease: + raise RuntimeError("configure_external_camera_scene_partitions() needs the active lease owner lane") + if not environment_root_path or not camera_paths or any(not path for path in camera_paths): + raise ValueError("One environment root and at least one non-empty external camera path are required") + + floor = self._next_ordinal() - 1 + attribute = "primvars:omni:scenePartition" + actual = self._read_partition_tokens([environment_root_path], (attribute,), floor) + token = actual.get((environment_root_path, attribute), "") + if token in {"", "0"}: + raise RuntimeError( + f"Environment root {environment_root_path!r} resolved to invalid scene-partition token {token!r}" + ) - def prepare_stage(self, num_envs: int) -> None: - """Validate leased preparation; ordinary rendering is immediately allowed.""" + lease._required_environment_scene_partitions = ((environment_root_path, token),) + lease._external_camera_scene_partitions = tuple((str(path), token) for path in camera_paths) + lease._external_camera_scene_partitions_pending = True + return token + + def _write_camera_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> None: + """Write the external camera tokens inside the complete scene batch.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot write external camera partitions before Stage materialization") + camera_scene_partitions = lease._external_camera_scene_partitions + if not camera_scene_partitions: + raise RuntimeError("No external camera scene partitions are configured") + camera_paths = [path for path, _token in camera_scene_partitions] + token_ids = np.array( + [self._renderer.scene.paths.intern_token(token) for _path, token in camera_scene_partitions], + dtype=np.uint64, + ) + camera_paths_list = self._renderer.scene.paths.create_path_list_from_strings(camera_paths) + camera_query = None + try: + camera_query = self._renderer.scene.stage.query_from_path_list(camera_paths_list) + if camera_query is None: + raise RuntimeError("External camera scene-partition query is None") + self._renderer.scene.stage.write_attribute( + camera_query, + "omni:scenePartition", + ordinal=ordinal, + tensors=token_ids, + is_array=False, + semantic=ovstage.AttributeSemantic.TOKEN_ID, + ).wait() + finally: + if camera_query is not None: + self._renderer.scene.stage.release_query(camera_query).wait() + self._renderer.scene.paths.destroy_path_list(camera_paths_list) + + def _read_partition_tokens( + self, paths: list[str], attributes: tuple[str, ...], ordinal: int + ) -> dict[tuple[str, str], str]: + """Read resolved partition tokens from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot read scene partitions before Stage materialization") + path_list = self._renderer.scene.paths.create_path_list_from_strings(paths) + attribute_tokens = [self._renderer.scene.paths.intern_token(attribute) for attribute in attributes] + resolved: dict[tuple[str, str], str] = {} + try: + with self._renderer.scene.stage.query_from_path_list(path_list) as query: + query.wait() + with self._renderer.scene.stage.read_attributes( + query, attribute_tokens, ovstage.OrdinalRange.latest(ordinal) + ) as read: + read.wait() + for group in read.groups(): + try: + if group.is_delete or group.is_array or group.tensor_count != 1: + continue + attribute = self._renderer.scene.paths.token_to_string(group.attribute) + if attribute not in attributes: + continue + group_paths = self._renderer.scene.paths.get_path_strings(group.prim_list) + try: + values = np.asarray(group.array(0).copy()).reshape(-1) + except NotImplementedError: + values = torch.from_dlpack(group.dlpack(0)).detach().cpu().numpy().copy().reshape(-1) + logical_count = min(group.prim_count, group.data_count) + for local_index in range(logical_count): + prim_index = group.prim_index(local_index) + data_row_index = group.data_row_index(local_index) + if prim_index >= len(group_paths) or data_row_index >= len(values): + raise RuntimeError("OVStage scene-partition read escaped its group-owned rows") + token_id = int(values[data_row_index]) + token = self._renderer.scene.paths.token_to_string(token_id) if token_id else "" + key = (group_paths[prim_index], attribute) + if key in resolved and resolved[key] != token: + raise RuntimeError(f"OVStage returned conflicting scene-partition rows for {key}") + resolved[key] = token + finally: + self._renderer.scene.stage.release_group(group) + finally: + self._renderer.scene.paths.destroy_path_list(path_list) + return resolved + + def validate_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> dict[str, str]: + """Prove the configured root and external cameras resolve to one valid token.""" + self.validate_active(lease) + lease.validate_live() + if self._lane_lease is not lease: + raise RuntimeError("validate_external_scene_partitions() needs the active lease owner lane") + expected_rows = [ + *( + (path, "primvars:omni:scenePartition", token) + for path, token in lease._required_environment_scene_partitions + ), + *((path, "omni:scenePartition", token) for path, token in lease._external_camera_scene_partitions), + ] + if len(lease._required_environment_scene_partitions) != 1: + raise RuntimeError("Exactly one authoritative environment scene partition is required") + if not lease._external_camera_scene_partitions: + raise RuntimeError("At least one external camera scene partition is required") + paths = list(dict.fromkeys(path for path, _attribute, _token in expected_rows)) + attributes = tuple(dict.fromkeys(attribute for _path, attribute, _token in expected_rows)) + actual = self._read_partition_tokens(paths, attributes, ordinal) + partitions: dict[str, str] = {} + mismatches = [] + for path, attribute, expected_token in expected_rows: + actual_token = actual.get((path, attribute), "") + partitions[path] = actual_token + if actual_token in {"", "0"} or actual_token != expected_token: + mismatches.append(f"{path}.{attribute}: expected {expected_token!r}, got {actual_token!r}") + if mismatches: + raise RuntimeError("External scene-partition proof failed: " + "; ".join(mismatches)) + return partitions + + def _author_camera_schedules(self, usd: str, lease: OvLibrariesRenderLease) -> str: + """Author selected native camera rates on the private export, preserving the application stage.""" + + layer = Sdf.Layer.CreateAnonymous("borrowed-renderer.usda") + if not layer.ImportFromString(usd): + raise RuntimeError("Cannot parse the exported leased Stage") + stage = Usd.Stage.Open(layer) + for camera in lease._auxiliary_cameras: + if camera.max_fps is None: + continue + if camera.spec is None: + raise RuntimeError("Native-rate auxiliary camera was not prepared") + prim = stage.GetPrimAtPath(camera.spec.camera_prim_paths[0]) + if not prim.IsValid(): + raise RuntimeError("Native-rate auxiliary camera is absent from the exported Stage") + prim.CreateAttribute("omni:sensor:frameRate", Sdf.ValueTypeNames.Double2).Set(Gf.Vec2d(camera.max_fps, 1.0)) + prim.CreateAttribute("omni:sensor:Core:partialOutputs", Sdf.ValueTypeNames.Bool).Set(False) + return stage.ExportToString() + + def _materialize_camera_products(self, lease: OvLibrariesRenderLease) -> None: + for camera in lease._auxiliary_cameras: + if camera.spec is None or camera.render_data is None: + raise RuntimeError("Selected auxiliary camera was not prepared") + self._renderer._register_camera(camera.spec, camera.render_data) + camera.product_path = camera.render_data.render_product_path + if lease._auxiliary_cameras: + self._renderer.scene.commit() + + def _update_cameras(self) -> None: + lease = self.active + if lease is None: + return + for camera in lease._auxiliary_cameras: + if camera.state is None or camera.render_data.camera_xform_binding is None: + raise RuntimeError("Auxiliary camera has no live world-pose sample") + self._renderer._update_camera(*camera.state) + + def prepare_stage(self, num_envs: int) -> bool: + """Validate or defer preparation; ordinary rendering is immediately allowed.""" lease = self.active if lease is not None: + if lease._auxiliary_cameras and not self._transfer_claimed: + self.stage_deferred = True + return False self.require_owner_thread("prepare the leased Stage") if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") @@ -393,14 +707,53 @@ def prepare_stage(self, num_envs: int) -> None: raise RuntimeError("The render lease requires exactly one environment") if not self._renderer._use_ovstage or self._renderer.backend._render_lease_owner is not self._renderer: raise RuntimeError("The render lease requires its reserved private OVStage renderer") + return True def bind_stage(self) -> None: """Bind the prepared native pair after renderer initialization succeeds.""" lease = self.active if lease is not None: + self._materialize_camera_products(lease) + self.stage_deferred = False self._generation += 1 lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) + def populate_stage(self, usd: str) -> None: + """Populate the borrowed scene with the authorized camera schedule.""" + lease = self.active + if lease is not None and lease._auxiliary_cameras: + usd = self._author_camera_schedules(usd, lease) + scheduled_paths = ( + [camera.spec.camera_prim_paths[0] for camera in lease._auxiliary_cameras if camera.max_fps is not None] + if lease is not None + else [] + ) + if scheduled_paths: + population = ovstage.population + description = population.Desc( + domains=self._renderer.scene.cfg.population_domains, + selectors=[ + population.Selector( + population.PrimPredicate.has_path(*scheduled_paths), + population.PropertyPredicate.has_name( + "omni:sensor:frameRate", "omni:sensor:Core:partialOutputs" + ), + ) + ], + ) + self._renderer.scene.populate(usd, population_desc=description) + else: + self._renderer.scene.populate(usd) + + def queue_camera_update(self, render_data, positions, orientations, intrinsics) -> bool: + """Retain the latest authorized external-camera sample until publication.""" + if self.active is not None: + for camera in self.active._auxiliary_cameras: + if camera.render_data is render_data: + camera.state = (render_data, positions, orientations, intrinsics) + return True + return False + def reserve_pair(self) -> None: """Reserve a newly acquired private native pair during clone preparation.""" renderer = self._renderer diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 38b0505d6f9..ee1c75580fc 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -62,8 +62,18 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, start = end self._device = str(device) self._event = wp.Event(device=self._device) + self._pending_groups = groups + self._resources = contextlib.ExitStack() + if renderer._initialized_scene: + self._compile_addresses(renderer) + + def _compile_addresses(self, renderer: OVRTXRenderer) -> None: + """Bind addresses only after the leased or ordinary scene has been populated.""" + if not renderer._initialized_scene: + raise RuntimeError("Visual materials require a populated renderer scene before publication.") + addresses = [] with contextlib.ExitStack() as resources: - for channel, attribute_name, shader_paths, rows, dtype, shape in groups: + for channel, attribute_name, shader_paths, rows, dtype, shape in self._pending_groups: if renderer._use_ovstage: address = resources.enter_context(renderer.scene.query(shader_paths)) else: @@ -76,8 +86,10 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, flags=BindingFlag.OPTIMIZE, ) resources.callback(address.unbind) - self._addresses.append((channel, address, attribute_name, rows)) + addresses.append((channel, address, attribute_name, rows)) self._resources = resources.pop_all() + self._addresses = addresses + self._pending_groups = [] @property def has_pending_updates(self) -> bool: @@ -97,6 +109,8 @@ def publish(self) -> None: if not channels: return renderer = self._renderer_ref() + if self._pending_groups: + self._compile_addresses(renderer) operations = [] try: for channel, address, attribute_name, rows in self._addresses: @@ -157,5 +171,6 @@ def close(self) -> None: logger.warning("Error draining in-flight render before material release: %s", error) finally: self._addresses.clear() + self._pending_groups.clear() self._dirty_channels.clear() self._buffers.clear() diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index 230d57d4f8f..3814fd9f922 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -180,7 +180,7 @@ def __init__(self, cfg: OvstageBackendCfg): """Release borrowers before paths and stage, in reverse registration order.""" self.reset() - def populate(self, usda: str) -> None: + def populate(self, usda: str, *, population_desc: ovstage.population.Desc | None = None) -> None: """Import the exported prototypes and apply the clone context's prepared operations. Population commits ordinal 1. Consumers may then author render products and poses @@ -188,12 +188,18 @@ def populate(self, usda: str) -> None: Args: usda: USD scene containing the routed prototypes and their materials. + population_desc: Optional native property selection for this scene population. """ if self.ordinal: return # Ordinal 0 is the empty state; population and cloning form the first committed write. - ovstage.population.open_usd_from_string(self.stage, usda, ordinal=1, domains=self.cfg.population_domains) + if population_desc is None: + ovstage.population.open_usd_from_string(self.stage, usda, ordinal=1, domains=self.cfg.population_domains) + else: + ovstage.population.open_usd_from_string_with_desc( + self.stage, usda, ordinal=1, time=float("nan"), descs=population_desc + ) ovstage_replicate( self.stage, self.paths, self.clone_copies, self.clone_env_paths, self.clone_positions, ordinal=1 ) diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 2e3fa03f601..493cd551095 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1541,36 +1541,149 @@ def owner(): assert renderer.backend is renderer.scene is None -def test_render_lease_complete_publication_deduplicates_and_poison_fails_closed(leased_renderer, monkeypatch): +@pytest.fixture +def complete_scene_publication(leased_renderer, monkeypatch): + """Exercise the application publisher with native component endpoints doubled.""" renderer = leased_renderer lease = renderer.acquire_ov_libraries_render_lease() publisher = lease.capture_scene_publisher() _prepare_leased_renderer(renderer) - renderer.scene.stage.advance_write_floor.reset_mock() - transforms = MagicMock() - monkeypatch.setattr(renderer, "_update_transforms", transforms) - monkeypatch.setattr(renderer, "_update_geometries", lambda: None) - assert publisher.publication_required("step-1") - publisher.write_complete_scene(18, "step-1") - assert not publisher.publication_required("step-1") - transforms.assert_called_once_with() + lease._external_camera_scene_partitions = (("/Camera/left", "env"), ("/Camera/right", "env")) + lease._external_camera_scene_partitions_pending = True + stage = renderer.scene.stage + stage.get_attribute_write_floor.reset_mock() + stage.advance_write_floor.reset_mock() + state = types.SimpleNamespace( + renderer=renderer, + lease=lease, + publisher=publisher, + ordinal=23, + events=[], + failure=None, + cause=ValueError("component endpoint failed"), + initial_id=renderer._render_lease._last_step_id, + ) + + def component(name): + state.events.append((name, renderer.scene.ordinal)) + if state.failure == name: + raise state.cause + + writer = types.SimpleNamespace( + has_pending_updates=False, + publish=lambda: component("material"), + drain=lambda: component("material-drain"), + ) + monkeypatch.setattr(renderer, "_visual_material_writer_ref", lambda: writer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: component("transform")) + monkeypatch.setattr(renderer, "_update_geometries", lambda: component("geometry")) + monkeypatch.setattr(renderer._render_lease, "_update_cameras", lambda: component("auxiliary")) + + def partitions(selected, ordinal): + assert selected is lease + assert selected._external_camera_scene_partitions == (("/Camera/left", "env"), ("/Camera/right", "env")) + assert ordinal == renderer.scene.ordinal + component("partition") + + monkeypatch.setattr(renderer._render_lease, "_write_camera_partitions", partitions) + state.publish = lambda step_id="step": publisher.write_complete_scene(state.ordinal, step_id) + try: + yield state + finally: + lease.release() + + +@pytest.mark.parametrize("pending", [False, True]) +def test_render_lease_complete_scene_components_share_the_supplied_ordinal(complete_scene_publication, pending): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + lease._external_camera_scene_partitions_pending = pending + assert state.publisher.publication_required("step") + assert state.publish() is None + names = ["material", "transform", "geometry", "auxiliary"] + (["partition"] if pending else []) + ["material-drain"] + assert state.events == [(name, state.ordinal) for name in names] + assert renderer._render_lease._last_step_id == "step" + assert not lease._external_camera_scene_partitions_pending + assert renderer.scene.ordinal == state.ordinal + assert not state.publisher.publication_required("step") + with pytest.raises(RuntimeError, match="publication is not required"): + state.publish() + assert state.events == [(name, state.ordinal) for name in names] + renderer.scene.stage.get_attribute_write_floor.assert_not_called() renderer.scene.stage.advance_write_floor.assert_not_called() - transforms.side_effect = ValueError("write failed") - with pytest.raises(RuntimeError, match="scene publication failed"): - publisher.write_complete_scene(19, "step-2") + assert not renderer._render_lease._poisoned + + +@pytest.mark.parametrize("failure", ["material", "transform", "geometry", "auxiliary", "partition", "material-drain"]) +def test_render_lease_complete_scene_failure_preserves_commit_boundary(complete_scene_publication, failure): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + state.failure = failure + with pytest.raises(RuntimeError, match="Producer scene publication failed") as caught: + state.publish() + assert caught.value.__cause__ is state.cause + order = ["material", "transform", "geometry", "auxiliary", "partition", "material-drain"] + expected = order[: order.index(failure) + 1] + if failure != "material-drain": + expected.append("material-drain") + assert state.events == [(name, state.ordinal) for name in expected] + assert renderer._render_lease._last_step_id == state.initial_id + assert lease._external_camera_scene_partitions_pending + assert renderer.scene.ordinal == state.ordinal + assert renderer._render_lease._poisoned with pytest.raises(RuntimeError, match="poisoned"): _ = lease.ovstage - publisher.close() - lease.release() + previous_events = list(state.events) + with pytest.raises(RuntimeError, match="unavailable|poisoned"): + state.publish("next-step") + assert state.events == previous_events + renderer.scene.stage.advance_write_floor.assert_not_called() + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + + +def test_render_lease_complete_scene_public_authority_precedes_components(complete_scene_publication): + import threading + + state = complete_scene_publication + errors = [] + + def foreign_thread(): + try: + state.publish() + except BaseException as exc: + errors.append(exc) + + thread = threading.Thread(target=foreign_thread) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert "thread" in str(errors[0]) + with pytest.raises(TypeError, match="physics_step_id must be hashable"): + state.publish([]) + with pytest.raises(ValueError, match="ordinal must be nonzero"): + state.publisher.write_complete_scene(0, "step") + state.publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + state.publish() + assert not state.events + state.renderer.scene.stage.get_attribute_write_floor.assert_not_called() + state.renderer.scene.stage.advance_write_floor.assert_not_called() + assert not state.renderer._render_lease._poisoned @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA material buffers") -def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch): +@pytest.mark.parametrize("deferred", [False, True]) +def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch, deferred): """Real material composition and the lease publisher must reach native writes, including before physics changes.""" renderer = leased_renderer lease = renderer.acquire_ov_libraries_render_lease() publisher = lease.capture_scene_publisher() context = renderer._sim.render_context + if deferred: + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = "/World/Camera" + lease.configure_auxiliary_camera(cfg) _prepare_leased_renderer(renderer) material = types.SimpleNamespace( channels=("color",), @@ -1584,6 +1697,20 @@ def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monke ) context.register_visual_material(material) context.finalize_consumers([]) + if deferred: + spec = CameraRenderSpec(cfg, "cpu", 1, (cfg.prim_path,), 1) + data = renderer.create_render_data(spec) + monkeypatch.setattr(ovrtx_renderer_module.ovstage.population, "add_usd_reference_from_string", lambda *a: 1) + monkeypatch.setattr(ovrtx_renderer_module.ovstage.population, "apply_usd_changes", lambda *a, **kw: None) + token = lease.offer_owner_transfer() + lease.claim_owner_transfer(token) + assert lease.materialize_deferred_auxiliary_camera_stage(renderer._sim.stage, 1) + renderer.update_camera( + data, + wp.zeros(1, dtype=wp.vec3f, device="cpu"), + wp.array([[0.0, 0.0, 0.0, 1.0]], dtype=wp.quatf, device="cpu"), + None, + ) stage = renderer.scene.stage try: for ordinal, color in ((23, [0.0, 1.0, 0.0]), (24, [0.0, 0.0, 1.0])): @@ -1720,3 +1847,153 @@ def consume_product(spec, render_data, **kwargs): monkeypatch.setattr(ovrtx_renderer_module, "build_render_product_as_string", consume_product) with pytest.raises(AuthoredProduct): renderer.create_render_data(spec) + + +def test_external_camera_native_rate_is_authored_only_on_export(leased_renderer): + from pxr import Usd, UsdGeom + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = "/World/envs/env_0/Camera" + lease.configure_auxiliary_camera(cfg, max_fps=20.0) + lease._auxiliary_cameras[0].spec = types.SimpleNamespace(camera_prim_paths=[cfg.prim_path]) + stage = Usd.Stage.CreateInMemory() + camera = UsdGeom.Camera.Define(stage, cfg.prim_path).GetPrim() + exported = Usd.Stage.CreateInMemory() + assert exported.GetRootLayer().ImportFromString( + renderer._render_lease._author_camera_schedules(stage.ExportToString(), lease) + ) + assert not camera.HasAttribute("omni:sensor:frameRate") + assert not camera.HasAttribute("omni:sensor:Core:partialOutputs") + prim = exported.GetPrimAtPath(cfg.prim_path) + assert tuple(prim.GetAttribute("omni:sensor:frameRate").Get()) == (20.0, 1.0) + assert prim.GetAttribute("omni:sensor:Core:partialOutputs").Get() is False + lease.release() + + +def test_selected_external_camera_order_is_independent_of_sensor_preparation(leased_renderer): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + specs = [] + for name, hz in zip(("Head", "Left", "Right", "Overview"), (20, 10, 10, 5), strict=True): + cfg = types.SimpleNamespace(prim_path=f"/World/{name}", width=640, height=480, data_types=("rgb",)) + lease.configure_auxiliary_camera(cfg, max_fps=hz) + specs.append(types.SimpleNamespace(cfg=cfg, num_instances=1, view_count=1, camera_prim_paths=(cfg.prim_path,))) + for spec in reversed(specs): + assert renderer._render_lease.accept_camera_spec(spec) + assert [camera.spec for camera in lease._auxiliary_cameras] == specs + assert [camera.max_fps for camera in lease._auxiliary_cameras] == [20, 10, 10, 5] + lease.release() + + +def test_external_camera_partition_validation_is_not_stereo_specific(leased_renderer, monkeypatch): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + cameras = ("/Camera/front", "/Camera/side", "/Camera/rear") + tokens = { + ("/World/envs/env_0", "primvars:omni:scenePartition"): "env_0", + **{(path, "omni:scenePartition"): "env_0" for path in cameras}, + } + monkeypatch.setattr(renderer._render_lease, "_read_partition_tokens", lambda *_: tokens) + try: + with lease.ovstage_owner_lane(): + lease.configure_external_camera_scene_partitions("/World/envs/env_0", cameras) + assert lease.validate_external_scene_partitions(1) == { + "/World/envs/env_0": "env_0", + **dict.fromkeys(cameras, "env_0"), + } + tokens[(cameras[-1], "omni:scenePartition")] = "other" + with pytest.raises(RuntimeError, match="scene-partition proof failed"): + lease.validate_external_scene_partitions(1) + finally: + lease.release() + + +def test_external_camera_schedules_reach_native_stage(monkeypatch): + """The renderer's real population path retains only selected camera scheduling properties.""" + import ovrtx + import ovstage + + ovrtx.register_schema_paths() + camera_paths = [f"/World/envs/env_0/{name}" for name in ("Head", "Wrist", "Unselected")] + source = """#usda 1.0 +def Xform "World" { + def Xform "envs" { + def Xform "env_0" { + def Camera "Head" { + double2 omni:sensor:frameRate = (20, 1) + bool omni:sensor:Core:partialOutputs = false + } + def Camera "Wrist" { + double2 omni:sensor:frameRate = (10, 1) + bool omni:sensor:Core:partialOutputs = false + } + def Camera "Unselected" { + double2 omni:sensor:frameRate = (999, 1) + } + } + } +} +""" + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = camera_paths[0] + spec = types.SimpleNamespace(cfg=cfg, num_instances=1, camera_prim_paths=[camera_paths[0]], render_settings={}) + data = OVRTXCameraRenderData(spec, "RenderCamera_0", list(spec.camera_prim_paths)) + from isaaclab_ov.stage import OvstageBackend, OvstageBackendCfg + + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = True + renderer._render_lease.stage_deferred = False + renderer._render_lease.active = None + renderer.backend.renderer = MagicMock() + with contextlib.ExitStack() as resources: + renderer.scene = OvstageBackend(OvstageBackendCfg()) + resources.callback(renderer.scene.close) + stage, paths = renderer.scene.stage, renderer.scene.paths + population = ovstage.population + description = population.Desc( + domains=ovstage.PopulationDomain.RENDERING, + selectors=[ + population.Selector( + population.PrimPredicate.has_path(*camera_paths[:2]), + population.PropertyPredicate.has_name("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs"), + ) + ], + ) + renderer.scene.populate(source, population_desc=description) + try: + renderer._register_camera(spec, data) + renderer.scene.commit() + resolved = {} + plist = paths.create_path_list_from_strings(camera_paths) + try: + with stage.query_from_path_list(plist) as query: + query.wait() + names = ("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs") + with stage.read_attributes( + query, + [paths.intern_token(name) for name in names], + ovstage.OrdinalRange.latest(renderer.scene.ordinal - 1), + ) as read: + read.wait() + for group in read.groups(): + try: + values = np.asarray(group.array(0).copy()).reshape(group.data_count, -1) + group_paths = paths.get_path_strings(group.prim_list) + attribute = paths.token_to_string(group.attribute) + for index in range(min(group.prim_count, group.data_count)): + path = group_paths[group.prim_index(index)] + resolved[path, attribute] = values[group.data_row_index(index)].tolist() + finally: + stage.release_group(group) + finally: + paths.destroy_path_list(plist) + assert resolved.get((camera_paths[0], names[0])) == [20.0, 1.0] + assert resolved.get((camera_paths[1], names[0])) == [10.0, 1.0] + assert resolved.get((camera_paths[0], names[1])) == [False] + assert resolved.get((camera_paths[1], names[1])) == [False] + assert (camera_paths[2], names[0]) not in resolved + finally: + data.resources.close() diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst new file mode 100644 index 00000000000..ecf50db66d4 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst @@ -0,0 +1 @@ +* Added fixed-base G1 wrist cameras and isolated robot head visuals for OVXR views. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst new file mode 100644 index 00000000000..c55f11e4a58 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst @@ -0,0 +1,6 @@ +* **Breaking:** Added head and wrist RGB terms to the default policy observation space of + ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs``, including non-teleoperation consumers. + Existing state-only policies must set ``robot_pov_cam``, ``left_wrist_cam``, and + ``right_wrist_cam`` to ``None`` in both ``env_cfg.scene`` and ``env_cfg.observations.policy``, + and clear ``env_cfg.image_obs_list`` before environment construction. Policies using the + new defaults must accept the three image terms and configure camera rendering. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.major b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.major new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 2ccb195fb5d..699b28823e0 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -58,3 +58,55 @@ gym.register( > **Note:** As a practice, we specify all the environments in a single file to avoid name conflicts between different > tasks or environments. However, this practice is debatable and we are open to suggestions to deal with a large > scaling in the number of tasks or environments. + +## Fixed-base G1 manipulation cameras + +For standalone OVPhysX, the task's backend preparation hook moves the tabletop +collision anchor onto its existing meshes, using the same preparation as GR1T2. +OVPhysX cooks the steering wheel's authored collision meshes. The bin's existing +primitive colliders and the object's drop termination remain enabled. +Registered GR1T2 and G1 tasks prepare the selected backend before scene construction, +including when created through `gym.make()` or `instantiate(cfg)` without the +teleoperation launcher. Newton uses its separate backend preparation. Resolve a +fresh task configuration when changing backends after preparation; copying a prepared +configuration preserves its selected backend and caller overrides. + +`IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` provides three 640×480 RGB +cameras: `robot_pov_cam` at 20 Hz, and `left_wrist_cam` and `right_wrist_cam` at +10 Hz each. The head uses the calibrated G1 head-camera pose; wrist cameras +are mounted on their respective wrist-yaw links and look along wrist-local +X +with +Z up. Their offsets are configurable in the task's scene configuration. + +These cameras and image observations are task defaults, including outside XR. +They add image keys to the policy observation space and require camera rendering. + +To retain the previous state-only observation space for an existing policy or +checkpoint, remove the three sensors, their policy terms, and their image metadata +from the resolved environment configuration before calling `gym.make()` or +`instantiate(cfg)`. This migration does not require the teleoperation package: + +```python +env_cfg.scene.robot_pov_cam = None +env_cfg.scene.left_wrist_cam = None +env_cfg.scene.right_wrist_cam = None +env_cfg.observations.policy.robot_pov_cam = None +env_cfg.observations.policy.left_wrist_cam = None +env_cfg.observations.policy.right_wrist_cam = None +env_cfg.image_obs_list = [] +``` + +To use the new defaults, update the policy's observation schema to consume all +three image terms and configure a supported camera renderer. + +The same names identify policy image observations and the task's XR camera +feeds, with the head panel centered and wrist panels to its left and right. +Sensor `update_period` controls image sampling in simulation time; feed +`max_update_hz` caps uploads. Actual delivery also depends on simulation and +renderer throughput. Camera sampling rates are configuration targets, not +measured headset frame rates. + +The teleoperation launcher's `--disable_external_cameras` option removes these +cameras and their image observations. The kitless XR launcher accepts up to +three camera names with `--ovxr_pip_camera`. Camera feeds are opt-in: without +this option, live OVXR and MCAP replay omit task cameras and PiP layers. +Selected feeds keep their own sampling cadence. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index caf4a5b349e..6a37080c1f9 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -33,6 +33,7 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.pink_controller_cfg import ( # isort: skip G1_UPPER_BODY_IK_ACTION_CFG, ) +from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip def _build_g1_upper_body_pipeline(): @@ -267,7 +268,24 @@ class FixedBaseUpperBodyIKG1SceneCfg(InteractiveSceneCfg): # Unitree G1 Humanoid robot - fixed base configuration robot: ArticulationCfg = replace(G1_29DOF_CFG, prim_path="{ENV_REGEX_NS}/Robot") - robot_pov_cam = g1_robot_pov_camera_cfg() + # Share the calibrated head view with the locomomanipulation G1 task. + robot_pov_cam = g1_robot_pov_camera_cfg().replace(update_period=1.0 / 20.0) + + # Mount above each hand, looking along wrist-local +X with +Z up. + left_wrist_cam = robot_pov_camera_cfg( + parent_prim_path="{ENV_REGEX_NS}/Robot/left_wrist_yaw_link", + offset_pos=(0.06, 0.0, 0.06), + offset_rot=(-0.5, 0.5, -0.5, 0.5), + ).replace( + prim_path="{ENV_REGEX_NS}/Robot/left_wrist_yaw_link/LeftWristCam", + update_period=1.0 / 10.0, + height=480, + width=640, + spawn=sim_utils.PinholeCameraCfg(focal_length=15.0, horizontal_aperture=20.955, clipping_range=(0.01, 5.0)), + ) + right_wrist_cam = left_wrist_cam.replace( + prim_path="{ENV_REGEX_NS}/Robot/right_wrist_yaw_link/RightWristCam", + ) # Per-hand contact sensors over all finger links, used to drive controller # haptics (see HapticFeedbackCfg below). Requires activate_contact_sensors @@ -310,13 +328,11 @@ class ActionsCfg: @configclass class ObservationsCfg: - """Observation specifications for the MDP. - This class is required by the environment configuration but not used in this implementation - """ + """State and camera observations for fixed-base manipulation.""" @configclass class PolicyCfg(ObsGroup): - """Observations for policy group with state values.""" + """Policy observations with state values and head/wrist RGB images.""" actions = ObsTerm(func=manip_mdp.last_action) robot_joint_pos = ObsTerm( @@ -342,6 +358,28 @@ class PolicyCfg(ObsGroup): params={"left_eef_link_name": "left_wrist_yaw_link", "right_eef_link_name": "right_wrist_yaw_link"}, ) + robot_pov_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("robot_pov_cam"), + "normalize": False, + }, + ) + left_wrist_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("left_wrist_cam"), + "normalize": False, + }, + ) + right_wrist_cam = ObsTerm( + func=base_mdp.image_rgb, + params={ + "sensor_cfg": SceneEntityCfg("right_wrist_cam"), + "normalize": False, + }, + ) + def __post_init__(self): self.enable_corruption = False self.concatenate_terms = False @@ -390,6 +428,7 @@ class FixedBaseUpperBodyIKG1EnvCfg(ManagerBasedRLEnvCfg): # MDP settings terminations: TerminationsCfg = TerminationsCfg() observations: ObservationsCfg = ObservationsCfg() + image_obs_list = ["robot_pov_cam", "left_wrist_cam", "right_wrist_cam"] actions: ActionsCfg = ActionsCfg() # Unused managers @@ -432,6 +471,7 @@ def __post_init__(self): self.xr = XrCfg( anchor_pos=(0.0, 0.0, -0.30), anchor_rot=(0.0, 0.0, 0.0, 1.0), + excluded_visual_meshes=["/World/envs/env_0/Robot/torso_link/visuals/head_link/mesh"], ) self.isaac_teleop = IsaacTeleopCfg( @@ -444,11 +484,26 @@ def __post_init__(self): xr_camera_feeds=[ XrCameraFeedCfg( camera_name="robot_pov_cam", + label="Head", enable_dlss_ray_reconstruction=True, dlss_exec_mode="quality", offset_m=(0.0, -0.15), - max_update_hz=0.0, - ) + max_update_hz=20.0, + ), + XrCameraFeedCfg( + camera_name="left_wrist_cam", + label="Left wrist", + panel_width_m=0.30, + offset_m=(-0.43, -0.15), + max_update_hz=10.0, + ), + XrCameraFeedCfg( + camera_name="right_wrist_cam", + label="Right wrist", + panel_width_m=0.30, + offset_m=(0.43, -0.15), + max_update_hz=10.0, + ), ], ) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index db809f16238..08726ebe838 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -689,12 +689,21 @@ def __post_init__(self): self.actions.upper_body_ik.controller.urdf_output_dir = self.temp_urdf_dir if ISAACLAB_TELEOP_AVAILABLE: - from isaaclab_teleop import GloveHapticFeedbackCfg, IsaacTeleopCfg, XrCameraFeedCfg, XrCfg + from isaaclab_teleop import ( + GloveHapticFeedbackCfg, + IsaacTeleopCfg, + XrCameraFeedCfg, + XrCameraFeedLayoutCfg, + XrCfg, + ) # Isaac Capture-based teleoperation pipeline. self.xr = XrCfg( anchor_pos=(0.0, 0.0, 0.0), anchor_rot=(0.0, 0.0, 0.0, 1.0), + # The shared anchor places the default eye inside the robot head. + # Exclude visual meshes from the eye partition, not source visibility. + excluded_visual_meshes=["/World/envs/env_0/Robot/head_.*_link/visuals/.*"], ) self.isaac_teleop = IsaacTeleopCfg( prepare_environment=prepare_humanoid_teleop_environment, @@ -710,6 +719,11 @@ def __post_init__(self): max_update_hz=0.0, ) ], + xr_camera_feed_layout=XrCameraFeedLayoutCfg( + placement="world", + world_position_m=(0.0, 0.8, 1.6), + world_orientation_xyzw=(0.70710678, 0.0, 0.0, 0.70710678), + ), ) # Per-finger haptic glove feedback: vibrate each finger of the operator's diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py index c05b2fd3d46..f00a63801ee 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py @@ -86,6 +86,8 @@ def __post_init__(self): self.xr = XrCfg( anchor_pos=(0.0, 0.0, 0.0), anchor_rot=(0.0, 0.0, 0.0, 1.0), + # Exclude head visuals from the operator camera, not the source scene. + excluded_visual_meshes=["/World/envs/env_0/Robot/head_.*_link/visuals/.*"], ) self.isaac_teleop = IsaacTeleopCfg( prepare_environment=prepare_humanoid_teleop_environment, diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py index 67630cca26c..7282af0dc51 100644 --- a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -3,15 +3,20 @@ # # SPDX-License-Identifier: BSD-3-Clause +import re + +import numpy as np import pytest from isaaclab_ov.renderers import OVRTXRendererCfg from isaaclab_physx.renderers import IsaacRtxRendererCfg +from isaaclab_teleop import xr_anchor_manager from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( FixedBaseUpperBodyIKG1EnvCfg, ) from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg +from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import PickPlaceGR1T2WaistEnabledEnvCfg from isaaclab_tasks.utils.hydra import resolve_presets @@ -48,18 +53,28 @@ def test_g1_xr_camera_uses_calibration_and_head_locked_panel(env_cfg_type): assert feed_cfg.distance_m == 0.8 assert feed_cfg.enable_dlss_ray_reconstruction is True assert feed_cfg.dlss_exec_mode == "quality" - assert feed_cfg.max_update_hz == 0.0 + assert feed_cfg.max_update_hz == (20.0 if env_cfg_type is FixedBaseUpperBodyIKG1EnvCfg else 0.0) assert camera_cfg.renderer_cfg.default == IsaacRtxRendererCfg() assert camera_cfg.renderer_cfg.isaacsim_rtx == IsaacRtxRendererCfg() def test_fixed_base_g1_pip_preserves_policy_observations_and_fixed_root(): - """PiP adds a sensor without changing the fixed-base policy observation schema.""" + """Retain the existing three-camera observation contract and fixed-base root.""" cfg = FixedBaseUpperBodyIKG1EnvCfg() + assert cfg.image_obs_list == ["robot_pov_cam", "left_wrist_cam", "right_wrist_cam"] assert hasattr(cfg.scene, "robot_pov_cam") - assert not hasattr(cfg.observations.policy, "robot_pov_cam") + for name in ("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"): + assert hasattr(cfg.observations.policy, name) assert cfg.scene.robot.spawn.fix_root_link is True + assert [feed.camera_name for feed in cfg.isaac_teleop.xr_camera_feeds] == [ + "robot_pov_cam", + "left_wrist_cam", + "right_wrist_cam", + ] + assert [feed.max_update_hz for feed in cfg.isaac_teleop.xr_camera_feeds] == [20.0, 10.0, 10.0] + assert cfg.xr.hidden_visual_prims == [] + assert cfg.xr.excluded_visual_meshes def test_locomanipulation_g1_retains_recorded_camera(): @@ -71,6 +86,30 @@ def test_locomanipulation_g1_retains_recorded_camera(): assert cfg.image_obs_list == ["robot_pov_cam"] +@pytest.mark.parametrize("env_cfg_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) +def test_gr1t2_xr_anchor_preserves_operator_view_and_head_partition(monkeypatch, env_cfg_type): + """The operator faces the table; head exclusion remains camera-selective.""" + cfg = env_cfg_type() + assert cfg.xr.anchor_pos == (0.0, 0.0, 0.0) + assert cfg.xr.render_anchor_pos is None + assert cfg.xr.render_anchor_rot is None + assert not cfg.xr.hidden_visual_prims + assert any( + re.fullmatch(pattern, "/World/envs/env_0/Robot/head_yaw_link/visuals/head_yaw_link/mesh") + for pattern in cfg.xr.excluded_visual_meshes + ) + monkeypatch.setattr(xr_anchor_manager, "XRCore", None) + monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) + anchor = xr_anchor_manager.XrAnchorManager(cfg.xr) + assert anchor.render_anchor_path == anchor.anchor_headset_path + transform = anchor.get_world_matrix() + eye = transform @ np.array([0.0, 1.6, 0.0, 1.0]) + forward = transform @ np.array([0.0, 0.0, -1.0, 0.0]) + np.testing.assert_allclose(eye, (0.0, 0.0, 1.6, 1.0)) + np.testing.assert_allclose(forward, (0.0, 1.0, 0.0, 0.0)) + assert np.dot(np.asarray(cfg.scene.packing_table.init_state.pos) - eye[:3], forward[:3]) > 0.0 + + def test_xr_camera_reference_renderer_resolves_for_supported_backends(): """Reference cameras retain Isaac RTX defaults and OVRTX compatibility.""" camera_renderer_cfg = PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst new file mode 100644 index 00000000000..ae5bcc606fd --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -0,0 +1,7 @@ +* Added optional OVXR camera panels and a desktop mirror as application-configured + Isaac Capture frame consumers using completed render products, with OVUI's default + Vulkan backend for the mirror. +* Enabled the mirror by default in OVXR live and MCAP replay entrypoints; ``--headless`` + disables the local window while preserving XR and camera-panel rendering. +* Supported task-configured automatic layouts for head-locked panels + in live teleoperation and replay. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.minor b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 4b10b2b20a8..03a92a876c8 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -57,3 +57,20 @@ ordinary renderer operations and does not start an XR session. While a consumer holds the lease, ordinary camera rendering, scene writes and renderer destruction are rejected to prevent concurrent access to the borrowed resources. The same generic borrowing interface is available to applications outside teleoperation. + +## Optional presentation + +After resolving the selected camera render products, the Lab adapter supplies +Capture with configured `CameraPanelConsumer` and `OwnedDesktopMirror` instances +in `OvxrSessionBinding.frame_consumers`. Each panel configuration pairs its product +path with its placement and image bound. The mirror receives its camera choices +separately, so it can run without headset panels. + +Capture runs both consumers through the same frame lifecycle. It shares CPU +snapshots within a frame and keeps native layers alive through submission; Lab +continues to own sensor production and scene publication. These interfaces are +application-neutral and do not require Lab-specific scene types. + +The mirror uses OVUI's default Vulkan backend. Neither Lab nor Capture modifies +`OMNIUI_BACKEND`; the mirror rejects an active non-Vulkan backend. Neither optional +consumer is created when disabled. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index d64137cfd11..f5af33f7df2 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -19,8 +19,9 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. modes for controlling how the anchor orientation tracks the reference prim. - **Retargeting tuning UI** -- optional ImGui window for real-time adjustment of retargeter parameters when `retargeters_to_tune` is provided. -- **XR camera feedback** -- selected scene cameras can be shown through Kit Scene UI using a - feed-owned CUDA view of their existing RTX render product, with camera-buffer fallback. +- **XR camera feedback** -- selected scene cameras use Kit Scene UI or the optional + [OVXR path](kitless_xr.md)'s OpenXR quad layers. The latter also + supports a passive desktop eye-image mirror; separate quad layers are not in that image. ## Architecture @@ -36,6 +37,14 @@ renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; | `TeleopSessionLifecycle` | Pipeline building, OpenXR handle acquisition, session create/destroy, action-tensor extraction | | `CommandHandler` | Callback registration and XR message-bus command dispatch | +For kitless XR, `isaaclab_teleop` owns session preparation, tracking anchors, +camera-panel selection and presentation policy. It borrows the application renderer +through `isaaclab_ov`'s `OvLibrariesRenderLease` interface. The renderer retains +ownership of native resources, thread synchronization and GPU scene publication. +Robot-specific action conditioning and physics preparation remain in the task +package. Both live teleoperation and replay use the shared camera-feed configuration +helpers in `isaaclab_teleop.camera_feed`. + ## Usage ### 1. Configure Your Environment @@ -109,8 +118,11 @@ uv run python scripts/environments/teleoperation/teleop_se3_agent.py \ --task My-Isaac-Capture-Env-v0 --xr ``` +For kitless teleoperation, installation, MCAP replay, camera panels and the +optional desktop mirror, see [Kitless XR teleoperation](kitless_xr.md). + See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for -reference tasks, camera selection, layout, placement, renderer, disable, and kitless behavior. +reference tasks, camera selection, layout, placement, renderer, disable, and OVXR behavior. ### 4. Programmatic Usage @@ -151,7 +163,10 @@ rendering without blocking. | `sim_device` | `str` | `"cuda:0"` | Torch device for output action tensors | | `retargeting_execution` | `RetargetingExecutionConfig \| None` | `None` (resolved at session start to `mode="pipelined", pacing=DeadlinePacingConfig(safety_margin_s=0.025)`) | Isaac Capture retargeting execution settings; deferred so the config imports without `isaacteleop` | | `teleoperation_active_default` | `bool` | `False` | Whether teleoperation is active on session start | -| `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the Isaac Capture session | +| `require_initial_bimanual_hand_tracking` | `bool` | `False` | Wait for initial hand presence before stepping | +| `action_conditioner` | `Callable \| None` | `None` | Task-owned stateful pre-step action conditioning | +| `action_validator` | `Callable \| None` | `None` | Task-owned fail-closed action validation | +| `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the IsaacTeleop session | The 25 ms `DeadlinePacingConfig` safety margin staggers Isaac Capture's Python work behind Isaac Lab's step Python, giving native work such as rendering time to overlap instead of having both Python stacks @@ -163,13 +178,34 @@ contend for the GIL at the start of the step. |---|---|---|---| | `anchor_pos` | `tuple[float, float, float]` | `(0, 0, 0)` | XR anchor position in world frame | | `anchor_rot` | `tuple[float, float, float, float]` | `(0, 0, 0, 1)` | XR anchor rotation (quaternion xyzw) | -| `anchor_prim_path` | `str \| None` | `None` | Prim to attach anchor to for dynamic positioning | +| `render_anchor_pos` | `tuple[float, float, float] \| None` | `None` | Optional render-only anchor position | +| `render_anchor_rot` | `tuple[float, float, float, float] \| None` | `None` | Optional render-only anchor rotation | +| `hidden_visual_prims` | `list[str]` | `[]` | Prim-path regexes hidden before OVXR session materialization | +| `excluded_visual_meshes` | `list[str]` | `[]` | Visual-mesh regexes assigned to a separate render-export partition, without changing source USD visibility | +| `anchor_prim_path` | `str \| None` | `None` | Reference prim for dynamic positioning; OVXR reads committed physics poses and shares the same anchor policy | | `anchor_rotation_mode` | `XrAnchorRotationMode` | `FIXED` | How anchor rotation tracks the reference prim | | `anchor_rotation_smoothing_time` | `float` | `1.0` | Slerp time constant (seconds) for `FOLLOW_PRIM_SMOOTHED` mode | | `anchor_rotation_custom_func` | `Callable` | identity | Custom rotation function for `CUSTOM` mode | | `near_plane` | `float` | `0.15` | Near clipping plane distance for the XR device | | `fixed_anchor_height` | `bool` | `True` | Fix anchor height to initial value of the reference prim | +For single-environment OVXR, prefer `excluded_visual_meshes` for a robot +head surrounding the eyes. Match only visual mesh paths, not head links or +collision geometry. Environment cameras select `env_0`; excluded meshes use +`xr_excluded`, with `showAllPartitionsByDefault=false`. This also excludes the +head from PiP cameras selecting `env_0`; other camera views require an explicit +partition policy. The source Stage's visibility and instancing are restored after +render export. GR1T2 and fixed-base G1 use this policy instead of global hiding. + +## Capture and replay boundaries + +MCAP records DeviceIO tracker/control input, not historical simulation scenes. +Environment/control reset does not rewind MCAP input. HDF5 +demonstration scripts use the Kit/Isaac Sim application path; these OVXR +commands do not qualify HDF5 recording/replay. PiP and mirror consume latest +completed images rather than a history of simulation scenes; they are not lossless +recorders or rolling-shutter playback. + ## Utilities - **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy @@ -187,5 +223,8 @@ startup and client connection. For OVXR, use the separate ## Dependencies - **`isaaclab`** -- core Isaac Lab framework -- **`isaacteleop`** -- Isaac Capture retargeting engine, device I/O, and session management -- **`isaacsim`** -- Isaac Sim runtime (provides the Kit XR bridge for OpenXR handle acquisition) +- **`isaaclab_ov`** -- required by the optional OVXR session adapter; the standalone `isaaclab_teleop` package does not install this backend automatically +- **`isaacteleop`** -- IsaacTeleop retargeting engine, device I/O, and session management +- **`isaacsim`** -- required for the Kit XR bridge, not for the OVXR session path +- **OVXR / OVRTX / OVStage** -- matched native runtime and stock peers for OVXR +- **OVUI** -- optional desktop-mirror extra; headset-only operation does not import it diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index e27492c2b06..216e6b2c972 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -93,8 +93,10 @@ this client origin before changing the renderer. `--ovxr` and Kit's `--xr` select separate XR runtimes and are mutually exclusive. An explicit `ISAAC_LAB_OVRTX_USE_OVSTAGE=0` is incompatible with OVXR. -No local window opens by default. Headset rendering still uses the GPU and works -with `DISPLAY` unset. The teleoperation CLI does not need a `--headless` flag. +OVXR live teleoperation and MCAP replay open a desktop mirror by default. Add +`--headless` to disable the local window, including on hosts without a desktop +display. Headset rendering still uses the GPU and works with `DISPLAY` unset +when `--headless` is selected. PiP remains available in headless mode. Renderer startup has a bounded attachment timeout; it does not bound waiting for a client to establish its OpenXR session. Installing Capture and resolving Python packages do not warm application-specific RTX shaders or Newton/Warp @@ -131,3 +133,39 @@ between steps. Application render-call timing statistics are unavailable when OVXR owns rendering independently. HDF5 demonstration recording and replay continue to use the Kit/Isaac Sim path. + +## Camera panels and desktop mirror + +Select up to three task cameras and a head-locked layout: + +```bash +uv run --extra ovxr isaaclab teleop run --ovxr \ + --task IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs \ + --num_envs 1 --device cpu --physics ovphysx \ + --ovxr_pip_camera robot_pov_cam left_wrist_cam right_wrist_cam \ + --ovxr_pip_head_locked +``` + +`--ovxr_pip_fps` limits panel updates. `--ovxr_pip_view_position X Y Z` overrides +one panel's position; multiple panels use the task layout. Panel cameras must be +enabled in the task's `xr_camera_feeds`. World and viewer-start layouts are not +supported by the OVXR presenter. The session snapshots its camera selection at +preparation; create a new session to change it. Panels use the same renderer as +the headset and do not require a desktop display. + +GR1T2 and G1 locomanipulation provide `robot_pov_cam`; fixed-base G1 also provides +`left_wrist_cam` and `right_wrist_cam`. SO101 does not declare a PiP camera in +its scene. Use the desktop mirror without `--ovxr_pip_camera` for that task. + +The default desktop mirror shows completed eye or camera images locally. +`--headless` disables it and takes precedence over `--ovxr_desktop_mirror`. +The explicit mirror flag is also accepted, but is unnecessary for the default. +`--ovxr_desktop_mirror_fps` limits its update rate. The mirror requires the +`desktop-mirror` extra, Python 3.12 or newer, and a usable display. On Linux, +use a GPU-backed desktop session for the mirror. OVUI's default Vulkan backend +requires no `OMNIUI_BACKEND` setting. Lab and Capture preserve the application's +backend choice; an already initialized OVUI backend cannot be reconfigured. +The mirror requires Vulkan; remove conflicting backend settings before OVUI +initialization. The mirror opens after the first submitted +XR image; closing it requests application shutdown. Eye images exclude the +separate OpenXR camera panels; select a camera product to inspect that feed. diff --git a/source/isaaclab_teleop/docs/kitless_xr_source.md b/source/isaaclab_teleop/docs/kitless_xr_source.md index 9d95e824dbd..b1fc336fbf9 100644 --- a/source/isaaclab_teleop/docs/kitless_xr_source.md +++ b/source/isaaclab_teleop/docs/kitless_xr_source.md @@ -1,7 +1,7 @@ # Installing kitless XR Use Python 3.12 on Linux x86-64. The Isaac Capture wheel includes the -OVXR session integration. +OVXR session integration, native camera-panel helper and desktop mirror modules. Complete the [CloudXR, client, network and license prerequisites](kitless_xr.md#prerequisites) before starting a session. @@ -11,7 +11,7 @@ extra. Lab already configures NVIDIA's public Python index and allows prerelease ```bash uv add --optional ovxr --frozen \ - "isaaccapture[ovxr,retargeters,ui,cloudxr]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" + "isaaccapture[ovxr,retargeters,ui,cloudxr,desktop-mirror]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" uv add --optional ovxr --frozen \ "isaaclab-dev[ov] ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ "isaaclab-teleop ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ @@ -39,7 +39,10 @@ The `teleop` extra selects Capture packages for Kit and non-Kit input workflows. Use the separate `ovxr` selection for the XR commands here. Isaac Sim and Viser require older WebSocket versions than this Capture release, so these selections cannot be combined. The `all` extra -includes Viser. +includes Viser. The wheel requirement includes the optional `desktop-mirror` extra; +omit that extra only when running with `--headless`. OVXR live and MCAP replay +open the mirror by default. The mirror requires Python 3.12 or newer and a +usable GPU-backed desktop display. Resolve the selected environment and check its dependencies: diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index ce01c7f56cf..850f0e8405a 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,6 +15,7 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", + "OvLibrariesXrCameraConfig", "OvLibrariesXrConfig", "OvLibrariesXrSession", "SupportsControlEvents", @@ -68,7 +69,7 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device -from .ov_libraries_xr import OvLibrariesXrConfig, OvLibrariesXrSession +from .ov_libraries_xr import OvLibrariesXrCameraConfig, OvLibrariesXrConfig, OvLibrariesXrSession from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index 624ce3f42b8..4252a07b7af 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -12,6 +12,7 @@ import math import time from collections import Counter +from collections.abc import Sequence from contextlib import suppress from copy import deepcopy from dataclasses import dataclass @@ -25,13 +26,87 @@ from .isaac_teleop_cfg import XrCameraFeedCfg, XrCameraFeedLayoutCfg if TYPE_CHECKING: - from isaaclab.sensors import Camera + from types import TracebackType + + from isaaclab_ov.renderers import OVRTXRendererCfg + + from isaaclab.sensors import Camera, CameraCfg + + from .ov_libraries_xr import OvLibrariesXrCameraConfig logger = logging.getLogger(__name__) _DLSS_EXEC_MODES = frozenset({"performance", "balanced", "quality", "auto", "rtxaa", "manual"}) +def select_ovxr_camera_feeds( + env_cfg: Any, names: Sequence[str], renderer_cfg: OVRTXRendererCfg +) -> list[tuple[str, CameraCfg, XrCameraFeedCfg]]: + """Validate selected PiP cameras and assign their shared renderer before launch. + + Selection order is retained. Each name must identify a scene camera and exactly + one enabled feed. Validation completes before any camera configuration changes. + """ + from isaaclab.sensors import CameraCfg + + if len(names) > 3 or len(names) != len(set(names)): + raise ValueError("Select up to three unique OVXR PiP cameras") + selected = [] + for name in names: + camera = getattr(env_cfg.scene, name, None) + if not isinstance(camera, CameraCfg): + raise ValueError(f"OVXR PiP camera {name!r} is not a configured CameraCfg") + feeds = [feed for feed in env_cfg.isaac_teleop.xr_camera_feeds if feed.enabled and feed.camera_name == name] + if len(feeds) != 1: + raise ValueError(f"OVXR PiP camera {name!r} must have exactly one enabled XrCameraFeedCfg") + selected.append((name, camera, feeds[0])) + for _name, camera, _feed in selected: + camera.renderer_cfg = renderer_cfg + return selected + + +def resolve_ovxr_camera_configs( + selected: Sequence[tuple[str, CameraCfg, XrCameraFeedCfg]], + layout: XrCameraFeedLayoutCfg, + *, + max_fps: float, + position_override: tuple[float, float, float] | None = None, +) -> tuple[OvLibrariesXrCameraConfig, ...]: + """Resolve selected feeds into OVXR panels without creating native resources. + + A feed's positive update cap limits the caller's rate; zero leaves that rate + unchanged. A position override applies only to a single selected panel. + """ + from .ov_libraries_xr import OvLibrariesXrCameraConfig + + _validate_layout_cfg(layout) + if selected and layout.placement != "head_locked": + raise ValueError("Kitless XR camera quads require head_locked placement") + if position_override is not None and len(selected) > 1: + raise ValueError("A PiP position override requires one camera; multiple panels use task layout") + for _name, _camera, feed in selected: + _validate_feed_cfg(feed, layout.placement) + feeds = _layout_feed_cfgs( + [feed for _name, _camera, feed in selected], + [(camera.width, camera.height) for _name, camera, _feed in selected], + layout, + ) + cameras = [] + for (name, camera, _feed), feed in zip(selected, feeds, strict=True): + cameras.append( + OvLibrariesXrCameraConfig( + name=name, + camera_cfg=camera, + max_fps=min(max_fps, feed.max_update_hz) if feed.max_update_hz > 0.0 else max_fps, + panel_width_m=feed.panel_width_m, + position=(float(feed.offset_m[0]), float(feed.offset_m[1]), -float(feed.distance_m)) + if position_override is None + else tuple(position_override), + ) + ) + return tuple(cameras) + + @lru_cache(maxsize=1) def _camera_type() -> type[Camera]: """Load the concrete camera type only when feeds are presented.""" @@ -64,7 +139,7 @@ def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list if duplicates: raise ValueError(f"XR camera feeds must have unique camera names. Duplicates: {duplicates}.") - scene = getattr(env_cfg, "scene", None) + scene = env_cfg.scene if scene is None: raise ValueError("XR camera feeds require an environment configuration with a scene.") for cfg in enabled_cfgs: @@ -159,7 +234,7 @@ def __init__( presenter: Any | None, *, requires_responsive_denoising: bool, - ): + ) -> None: self._cfgs = cfgs self._layout_cfg = layout_cfg self._presenter = presenter @@ -345,10 +420,24 @@ def __enter__(self) -> XrCameraFeedSession: raise RuntimeError("Call bind(env) before entering an XR camera feed session.") return self - def __exit__(self, exc_type, exc_value, traceback) -> None: + def __exit__( + self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None + ) -> None: self.close() +def _validate_feed_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: + """Validate original feed values before layout or backend conversion.""" + if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: + raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") + if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): + raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") + if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): + raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") + if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: + raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") + + def _panel_size_m(cfg: XrCameraFeedCfg, image_size: tuple[int, int]) -> tuple[float, float]: width, height = image_size if width <= 0 or height <= 0: @@ -474,7 +563,7 @@ def __init__( cfgs: list[XrCameraFeedCfg], layout_cfg: XrCameraFeedLayoutCfg | None, presenter: Any, - ): + ) -> None: self._env = env self._feeds: list[_ActiveFeed] = [] self._presenter = presenter @@ -484,7 +573,7 @@ def __init__( try: bound_feeds = [] for cfg in cfgs: - self._validate_cfg(cfg, self._layout_cfg.placement) + _validate_feed_cfg(cfg, self._layout_cfg.placement) camera, image = self._bind_image(cfg) bound_feeds.append((camera, image)) image_sizes = [(int(image.shape[1]), int(image.shape[0])) for _, image in bound_feeds] @@ -516,19 +605,8 @@ def __init__( def _on_frame(self, _event: Any) -> None: self.update() - @staticmethod - def _validate_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: - if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: - raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") - if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): - raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") - if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): - raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") - if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: - raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") - def _bind_image(self, cfg: XrCameraFeedCfg) -> tuple[Camera, torch.Tensor]: - sensors = getattr(getattr(self._env, "scene", None), "sensors", {}) + sensors = self._env.scene.sensors if cfg.camera_name not in sensors: raise ValueError( f"XR camera feed {cfg.camera_name!r} is not present in the interactive scene. " diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 8d3b82289fe..5972c894602 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -14,9 +14,10 @@ OVXR viewer: ``--ovxr`` presents replay through an independently owned OVXR session while - tracker and control input remain sourced from MCAP. See - ``source/isaaclab_teleop/docs/kitless_xr.md`` for installation and launch - instructions. + tracker and control input remain sourced from MCAP. Optional camera quads + use ``--ovxr_pip_camera NAME --ovxr_pip_head_locked``. The desktop mirror is enabled + by default; ``--headless`` disables it. See ``source/isaaclab_teleop/docs/kitless_xr.md`` + for installation and launch instructions. Inputs: ``--replay_file`` is an MCAP capture produced by Isaac Capture's @@ -316,6 +317,14 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument( "--ovxr", action="store_true", help="Present recorded motion through kitless OVXR; input stays MCAP." ) + parser.add_argument("--ovxr_pip_camera", nargs="+", default=None, help="Task camera names to show as OVXR panels.") + parser.add_argument("--ovxr_pip_head_locked", action="store_true", help="Use the supported head-locked PiP layout.") + parser.add_argument( + "--ovxr_desktop_mirror", + action="store_true", + help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", + ) + parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") parser.add_argument("--ovxr_session_mode", choices=("ar", "vr"), default="ar") parser.add_argument( "--replay_file", @@ -443,6 +452,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _build_parser() args_cli, hydra_overrides = setup_preset_cli(parser) + if args_cli.ovxr: + args_cli.ovxr_desktop_mirror = not args_cli.headless sys.argv = [sys.argv[0]] + hydra_overrides # Enable external camera rendering by default so the replay mimics the production teleop env (perf @@ -1055,7 +1066,10 @@ def _maybe_launch_cloudxr(cloudxr_env_path: str | None, auto_launch: bool): def _prepare_env_cfg( - task: str, num_envs: int, device: str, overrides: Sequence[str] = () + task: str, + num_envs: int, + device: str, + overrides: Sequence[str] = (), ) -> tuple[ManagerBasedRLEnvCfg, object | None]: """Build and tweak an env config suitable for non-interactive replay. @@ -1114,12 +1128,26 @@ def _prepare_env_cfg( ) if hasattr(env_cfg.terminations, "time_out"): env_cfg.terminations.time_out = None + # Keep camera configs when external cameras are enabled (defaulted on) so the replay + # renders them for production parity; otherwise strip them for a lighter headless replay. if args_cli.ovxr: if args_cli.num_envs != 1 or args_cli.xr: raise ValueError("--ovxr requires one environment and no Kit XR or local visualizer") if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") - env_cfg = remove_camera_configs(env_cfg) + from isaaclab_ov.renderers import OVRTXRendererCfg + + from ..camera_feed import select_ovxr_camera_feeds + + names = args_cli.ovxr_pip_camera or [] + if names and args_cli.disable_external_cameras: + raise ValueError("OVXR PiP conflicts with --disable_external_cameras") + if names and not args_cli.ovxr_pip_head_locked: + raise ValueError("OVXR replay PiP requires --ovxr_pip_head_locked") + select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) + env_cfg = remove_camera_configs(env_cfg, keep=tuple(names)) + elif args_cli.ovxr_pip_camera or args_cli.ovxr_desktop_mirror or args_cli.ovxr_pip_head_locked: + raise ValueError("OVXR presentation options require --ovxr") elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) if env_cfg.sim.physics is None: @@ -1213,7 +1241,23 @@ def __init__(self, env_cfg): from isaaclab_ov.renderers import OVRTXRendererCfg from .. import OvLibrariesXrConfig, OvLibrariesXrSession - + from ..camera_feed import resolve_ovxr_camera_configs + from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg + + selected = [ + ( + name, + getattr(env_cfg.scene, name), + next( + feed for feed in env_cfg.isaac_teleop.xr_camera_feeds if feed.enabled and feed.camera_name == name + ), + ) + for name in args_cli.ovxr_pip_camera or [] + ] + layout = (env_cfg.isaac_teleop.xr_camera_feed_layout or XrCameraFeedLayoutCfg()).replace( + placement="head_locked" + ) + cameras = resolve_ovxr_camera_configs(selected, layout, max_fps=30.0) self.session = OvLibrariesXrSession.prepare( env_cfg.sim, OvLibrariesXrConfig( @@ -1221,8 +1265,10 @@ def __init__(self, env_cfg): session_mode=args_cli.ovxr_session_mode, cloudxr_env_file=args_cli.cloudxr_env, auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, + auxiliary_cameras=cameras, ), - OVRTXRendererCfg(), + cameras[0].camera_cfg.renderer_cfg if cameras else OVRTXRendererCfg(), ) self.xr_cfg = env_cfg.isaac_teleop.xr_cfg self.device = None @@ -1237,6 +1283,7 @@ def start(self, env, device): env, tracking_anchor_path=device.xr_render_anchor_path, hidden_visual_prims=self.xr_cfg.hidden_visual_prims, + excluded_visual_meshes=self.xr_cfg.excluded_visual_meshes, external_anchor_state=device.external_anchor_state, timeout_s=args_cli.max_stage_load_wait_s, ): @@ -1255,6 +1302,7 @@ def publish(self, env, *, changed=False, reset=False): if self.pending is not None and self.session.request_render(self.pending, force=self.force) is not None: self.pending = None self.force = False + self.session.report_host_timing(env.sim.get_physics_step_count(), env.sim.get_physics_dt()) def close(self): if self.device is not None and self.device.session_cleanup_failed: diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 20594dfbbed..8ef95d731a7 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -135,6 +135,44 @@ def _build_parser() -> argparse.ArgumentParser: default="ar", help="OVXR session mode. Defaults to 'ar'; pass 'vr' for an opaque background.", ) + parser.add_argument( + "--ovxr_desktop_mirror", + action="store_true", + default=False, + help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", + ) + parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") + parser.add_argument( + "--ovxr_desktop_mirror_fps", + type=float, + default=30.0, + help="Maximum desktop-mirror update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_camera", + type=str, + nargs="+", + default=None, + help="Show up to three task-configured cameras as application-owned XR panels.", + ) + parser.add_argument( + "--ovxr_pip_fps", + type=float, + default=30.0, + help="Maximum OVXR PiP update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_head_locked", + action="store_true", + help="Place the application camera quad in the OpenXR VIEW space.", + ) + parser.add_argument( + "--ovxr_pip_view_position", + type=float, + nargs=3, + metavar=("X", "Y", "Z"), + help="Optional VIEW-space camera panel center in meters; requires --ovxr_pip_head_locked.", + ) add_launcher_args(parser) return parser @@ -153,6 +191,22 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--step_hz must be finite and non-negative; 0 disables active-control pacing") if args_cli.step_hz > 0.0 and not args_cli.using_ovxr: parser.error("--step_hz requires --ovxr") + if args_cli.ovxr_desktop_mirror and not args_cli.using_ovxr: + parser.error("--ovxr_desktop_mirror requires --ovxr") + args_cli.ovxr_desktop_mirror = args_cli.using_ovxr and not args_cli.headless + if args_cli.ovxr_pip_camera is not None and not args_cli.using_ovxr: + parser.error("--ovxr_pip_camera requires --ovxr") + if args_cli.ovxr_pip_camera is not None and args_cli.disable_external_cameras: + parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") + if not math.isfinite(args_cli.ovxr_pip_fps) or args_cli.ovxr_pip_fps <= 0.0: + parser.error("--ovxr_pip_fps must be finite and positive") + if args_cli.ovxr_pip_view_position is not None: + if not args_cli.ovxr_pip_head_locked or args_cli.ovxr_pip_camera is None: + parser.error("--ovxr_pip_view_position requires a head-locked PiP camera") + if not all(math.isfinite(value) for value in args_cli.ovxr_pip_view_position): + parser.error("--ovxr_pip_view_position must be finite") + if not math.isfinite(args_cli.ovxr_desktop_mirror_fps) or args_cli.ovxr_desktop_mirror_fps <= 0: + parser.error("--ovxr_desktop_mirror_fps must be finite and positive") # Call an external callback if requested. if args_cli.external_callback: @@ -256,7 +310,9 @@ def _ovxr_publication_identity( return ("reset", reset_generation, physics_step_id) -def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool]: +def _resolve_env_cfg() -> tuple[ + ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, list[tuple[str, object, object]] +]: """Resolve task configuration and teleop ownership before runtime launch.""" overrides = list(sys.argv[1:]) env_cfg, _ = resolve_task_config(args_cli.task, "", overrides=overrides) @@ -311,10 +367,27 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg stage_mode = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") if stage_mode.strip() != "1": raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") + + pip_selection = [] + keep_cameras: tuple[str, ...] = () + if args_cli.ovxr_pip_camera is not None: + from isaaclab_ov.renderers import OVRTXRendererCfg + + from ..camera_feed import select_ovxr_camera_feeds + + names = args_cli.ovxr_pip_camera + if len(names) > 1 and args_cli.ovxr_pip_view_position is not None: + raise ValueError("A PiP position override requires one camera; multiple panels use task layout") + pip_selection = select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) + keep_cameras = tuple(names) + + # The leased OVRTX renderer is the only render path in OVXR. Strip + # every task camera before the launcher scans the config. if args_cli.using_ovxr or args_cli.disable_external_cameras: from .. import remove_camera_configs - env_cfg = remove_camera_configs(env_cfg) + env_cfg = remove_camera_configs(env_cfg, keep=keep_cameras) + if env_cfg.sim.physics is None: from isaaclab.physics import PhysicsCfg @@ -327,7 +400,7 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg raise ValueError("--ovxr requires no Kit runtime or local visualizer") if use_isaac_teleop and teleop_cfg.prepare_environment is not None: teleop_cfg.prepare_environment(env_cfg) - return (env_cfg, teleop_cfg, use_isaac_teleop) + return env_cfg, teleop_cfg, use_isaac_teleop, pip_selection def _install_ovxr_shutdown_handlers(ov_libraries_xr_session) -> dict: @@ -397,7 +470,7 @@ def report_cleanup_error(error: Exception) -> None: except Exception as error: camera_drained = False report_cleanup_error(error) - if env is not None and xr_drained: + if env is not None and xr_drained and camera_drained: if ov_libraries_xr_session is None or not ov_libraries_xr_session.async_worker_owns_environment: try: env.close() @@ -421,7 +494,7 @@ def report_cleanup_error(error: Exception) -> None: report_cleanup_error(error) if cleanup_failure is not None: raise cleanup_failure - if env is not None and xr_drained: + if env is not None and xr_drained and camera_drained: print("Environment closed") @@ -429,6 +502,7 @@ def run_teleoperation( # noqa: C901 env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, use_isaac_teleop: bool, + pip_selection: list[tuple[str, object, object]], ) -> None: """Run teleoperation through the selected Kit or OVXR runtime.""" @@ -471,7 +545,18 @@ def run_teleoperation( # noqa: C901 from isaaclab_ov.renderers import OVRTXRendererCfg from .. import OvLibrariesXrConfig, OvLibrariesXrSession - + from ..camera_feed import resolve_ovxr_camera_configs + from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg + + layout = env_cfg.isaac_teleop.xr_camera_feed_layout + if args_cli.ovxr_pip_head_locked: + layout = (layout or XrCameraFeedLayoutCfg()).replace(placement="head_locked") + selected_cameras = resolve_ovxr_camera_configs( + pip_selection, + layout, + max_fps=args_cli.ovxr_pip_fps, + position_override=args_cli.ovxr_pip_view_position, + ) ov_libraries_xr_session = OvLibrariesXrSession.prepare( env_cfg.sim, OvLibrariesXrConfig( @@ -479,8 +564,11 @@ def run_teleoperation( # noqa: C901 session_mode=args_cli.ovxr_session_mode, cloudxr_env_file=args_cli.cloudxr_env, auto_launch_cloudxr=args_cli.auto_launch_cloudxr, + desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, + desktop_mirror_max_fps=args_cli.ovxr_desktop_mirror_fps, + auxiliary_cameras=selected_cameras, ), - renderer_cfg=OVRTXRendererCfg(), + renderer_cfg=selected_cameras[0].camera_cfg.renderer_cfg if selected_cameras else OVRTXRendererCfg(), ) try: @@ -716,6 +804,10 @@ def run_loop(): pending_publication_identity = None pending_publication_force = False + if ov_libraries_xr_session is not None: + ov_libraries_xr_session.report_host_timing( + env.sim.get_physics_step_count(), env.sim.get_physics_dt() + ) _pace_ovxr_control_loop( ov_libraries_xr_session is not None, action_applied, @@ -746,6 +838,7 @@ def run_loop(): tracking_anchor_path=teleop_interface.xr_render_anchor_path, required_openxr_extensions=teleop_interface.required_openxr_extensions, hidden_visual_prims=env_cfg.isaac_teleop.xr_cfg.hidden_visual_prims, + excluded_visual_meshes=env_cfg.isaac_teleop.xr_cfg.excluded_visual_meshes, external_anchor_state=teleop_interface.external_anchor_state, timeout_s=_RENDERER_ATTACH_TIMEOUT_S, ) @@ -770,9 +863,9 @@ def run_loop(): def main() -> None: """Resolve the task and run teleoperation with its selected runtime.""" - resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop = _resolve_env_cfg() + resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, pip_selection = _resolve_env_cfg() with launch_simulation(resolved_env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): - run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop) + run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, pip_selection) def run_teleop_se3_agent_cli(argv: list[str] | None = None) -> int: diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index bae4e0a0c4a..c1bf4c4e3fe 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -13,8 +13,9 @@ import re import threading import time +from collections import deque from collections.abc import Hashable, Iterable -from dataclasses import dataclass, replace +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any, Literal from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger @@ -27,6 +28,7 @@ from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding from isaacteleop.oxr import OpenXRSessionHandles + from isaaclab.sensors import CameraCfg from isaaclab.sim import SimulationCfg from .xr_anchor_manager import XrAnchorManager @@ -46,6 +48,28 @@ "/Render/Vars/XRDepth", ) _DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" +_ENVIRONMENT_ROOT_PATH = "/World/envs/env_0" + + +class _CounterRate: + """Measure a cumulative application counter over a recent wall-clock window.""" + + def __init__(self, window_s: float = 1.0) -> None: + self._window_ns = int(window_s * 1_000_000_000) + self._samples: deque[tuple[int, int]] = deque() + + def sample(self, count: int, now_ns: int | None = None) -> float: + now_ns = time.perf_counter_ns() if now_ns is None else now_ns + if self._samples and count < self._samples[-1][1]: + self._samples.clear() + self._samples.append((now_ns, count)) + while len(self._samples) > 2 and self._samples[1][0] <= now_ns - self._window_ns: + self._samples.popleft() + if len(self._samples) < 2: + return 0.0 + elapsed_s = (self._samples[-1][0] - self._samples[0][0]) / 1_000_000_000 + count_delta = self._samples[-1][1] - self._samples[0][1] + return 0.0 if elapsed_s <= 0.0 else max(0, count_delta) / elapsed_s def _apply_hidden_visual_prims(env: Any, patterns: Iterable[str]) -> tuple[str, ...]: @@ -220,6 +244,42 @@ def destroy(self) -> None: self._closed = True +@dataclass(frozen=True) +class OvLibrariesXrCameraConfig: + """One application-selected sensor and its camera-panel policy.""" + + name: str + """Scene sensor name used for camera selection and panel labeling.""" + camera_cfg: CameraCfg = field(repr=False) + """Selected sensor configuration, assigned to the session's shared renderer.""" + max_fps: float = 30.0 + """Finite positive image update cap [Hz].""" + panel_width_m: float = 0.48 + """Physical panel width [m]; image aspect ratio determines its height.""" + position: tuple[float, float, float] = (0.0, -0.15, -0.8) + """Panel center in OpenXR head space [m], with forward along negative Z.""" + orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + """Panel orientation in head space as an (x, y, z, w) quaternion.""" + space: Literal["head_locked"] = "head_locked" + """Placement space; only head-locked panels are supported.""" + + def __post_init__(self) -> None: + if not self.name or self.camera_cfg is None: + raise ValueError("camera name and configuration are required") + from isaaccapture.ovxr_session_manager import CameraPanelConfig + + if not math.isfinite(self.max_fps) or self.max_fps <= 0: + raise ValueError("camera max_fps must be finite and positive") + # Validate panel policy before native preparation; bind the resolved path later. + CameraPanelConfig( + render_product_path=f"/{self.name}", + panel_width_m=self.panel_width_m, + position=self.position, + orientation_xyzw=self.orientation_xyzw, + space=self.space, + ) + + @dataclass class OvLibrariesXrConfig: """Isaac Lab configuration for an IsaacTeleop-owned OVXR session.""" @@ -250,14 +310,25 @@ class OvLibrariesXrConfig: """USD tracking-anchor path; an empty path uses the stage origin.""" host_wait_render_output: bool = True """Whether OVXR waits for render output before returning from frame rendering.""" + desktop_mirror_enabled: bool = False + """Whether to display the owner-managed desktop mirror.""" + desktop_mirror_max_fps: float = 30.0 + """Finite positive desktop mirror refresh cap [Hz].""" + auxiliary_cameras: tuple[OvLibrariesXrCameraConfig, ...] = () + """Up to three uniquely named camera panels; an empty selection disables panels.""" extra_openxr_extensions: tuple[str, ...] = () """Additional OpenXR extension names requested when creating the session.""" def __post_init__(self) -> None: + cameras = self.auxiliary_cameras + if len(cameras) > 3 or len({camera.name for camera in cameras}) != len(cameras): + raise ValueError("Select at most three uniquely named auxiliary cameras") if self.session_mode not in ("ar", "vr"): raise ValueError("session_mode must be 'ar' or 'vr'") if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: raise ValueError("meters_per_unit must be finite and positive") + if not math.isfinite(self.desktop_mirror_max_fps) or self.desktop_mirror_max_fps <= 0.0: + raise ValueError("desktop_mirror_max_fps must be finite and positive") if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( not math.isfinite(self.depth_near_z) or not math.isfinite(self.depth_far_z) @@ -278,18 +349,22 @@ def __init__( env: Any, cfg: OvLibrariesXrConfig, hidden_visual_prims: Iterable[str], + excluded_visual_meshes: Iterable[str] = (), external_anchor_state: XrAnchorManager | None = None, ) -> None: self._bootstrap = bootstrap self._env = env self._cfg = cfg self._hidden_visual_prims = tuple(hidden_visual_prims) + self._excluded_visual_meshes = tuple(excluded_visual_meshes) self._publication_lock = threading.Lock() self._scene_publisher = None self._lease = None self._ordinal_ledger = None self._last_published_ordinal: int | None = None self._publication_closed = False + self._auxiliary_cameras = cfg.auxiliary_cameras + self.desktop_mirror = None self._external_anchor = external_anchor_state self._anchor_snapshot: tuple[int, np.ndarray] | None = None self._last_anchor_frame_time: float | None = None @@ -306,10 +381,23 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding self._bootstrap.claim_render_owner_transfer(capability) + unwrapped = getattr(self._env, "unwrapped", self._env) _apply_hidden_visual_prims(self._env, self._hidden_visual_prims) _declare_ovxr_runtime_prims(self._env, self._cfg) - self._lease = self._bootstrap.bind_environment(self._env) + from .xr_scene_partition import partition_visual_meshes + + stage = unwrapped.scene.stage + with partition_visual_meshes(stage, self._excluded_visual_meshes): + self._lease = self._bootstrap.bind_environment(self._env) + auxiliary_paths = () + if self._auxiliary_cameras: + sensors = unwrapped.scene.sensors + for camera in self._auxiliary_cameras: + if sensors.get(camera.name) is None: + raise RuntimeError(f"Configured auxiliary camera {camera.name!r} is not present in the environment") + auxiliary_paths = self._lease.auxiliary_render_product_paths with self._lease.ovstage_owner_lane(): + self._lease.configure_external_camera_scene_partitions(_ENVIRONMENT_ROOT_PATH, self._cfg.camera_paths) self._ordinal_ledger = _StageOrdinalLedger(self._lease.ovstage) if self._external_anchor is not None: import ovstage @@ -330,10 +418,37 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: self._anchor_retirement_failed = True setup_error.add_note(f"Anchor query cleanup also failed: {cleanup_error!r}") raise + from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig, CameraPanelConsumer + from isaaccapture.ovxr_session_manager.desktop_mirror import DesktopMirrorConfig + from isaaccapture.ovxr_session_manager.owned_mirror import MirrorProduct, OwnedDesktopMirror + + panels = tuple( + CameraPanelConfig( + render_product_path=path, + label=camera.name, + max_image_bytes=int(camera.camera_cfg.width) * int(camera.camera_cfg.height) * 4, + panel_width_m=camera.panel_width_m, + position=camera.position, + orientation_xyzw=camera.orientation_xyzw, + space=camera.space, + ) + for path, camera in zip(auxiliary_paths, self._auxiliary_cameras, strict=True) + ) + consumers = [CameraPanelConsumer(panels)] if panels else [] + if self._cfg.desktop_mirror_enabled: + self.desktop_mirror = OwnedDesktopMirror( + DesktopMirrorConfig( + max_fps=self._cfg.desktop_mirror_max_fps, + initial_status_text="Waiting for headset", + ), + camera_products=tuple(MirrorProduct(panel.render_product_path, panel.label) for panel in panels), + ) + consumers.append(self.desktop_mirror) return OvxrSessionBinding( renderer=self._lease.ovrtx_renderer, stage=self._lease.ovstage, ordinal_ledger=self._ordinal_ledger, + frame_consumers=tuple(consumers), prepare_frame_scene=self.prepare_frame_scene if self._external_anchor is not None else None, ) @@ -357,6 +472,12 @@ def publish_complete_scene(self, publication_id: Hashable) -> int: if self._last_published_ordinal is None: raise RuntimeError("OVStage publication deduplicated before a sealed ordinal exists") return self._last_published_ordinal + if self._auxiliary_cameras: + unwrapped = getattr(self._env, "unwrapped", self._env) + for camera in self._auxiliary_cameras: + # OVXR presentation bypasses Camera's image-update path. Refresh only poses + # after simulation/reset kinematics, before publishing the matching scene. + unwrapped.scene.sensors[camera.name]._update_poses() snapshot = None seed = False if self._external_anchor is not None: @@ -471,7 +592,7 @@ def validate_first_published_scene(self, ordinal: int) -> None: if self._lease is None: raise RuntimeError("OVRTX/OVStage lease is not attached") with self._lease.ovstage_owner_lane(): - self._lease.validate_live() + self._lease.validate_external_scene_partitions(ordinal) def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: return self._bootstrap.get_target_frame_T_world(prim_path) @@ -520,8 +641,10 @@ def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrConfig) -> self._bootstrap = bootstrap self._cfg = cfg self._owner = None + self._desktop_mirror = None self._host = None self._closed = False + self._physics_rate = _CounterRate() @classmethod def prepare( @@ -536,7 +659,11 @@ def prepare( use one configured session per process. No supported readback exists here to restore a previous renderer's settings on close. """ - cfg = replace(cfg) + cameras = tuple(cfg.auxiliary_cameras) + for camera in cameras: + if camera.space != "head_locked": + raise ValueError("Kitless XR camera quads currently require head_locked placement") + cfg = replace(cfg, auxiliary_cameras=cameras) validate_ovxr_session_preparation() import ovxr @@ -544,6 +671,8 @@ def prepare( ovxr.apply_renderer_settings(cfg.renderer_settings.strip()) bootstrap = OvxrSessionBootstrap(sim_cfg, renderer_cfg) try: + for camera in cameras: + bootstrap.configure_auxiliary_camera(camera.camera_cfg, max_fps=camera.max_fps) return cls(bootstrap, cfg) except BaseException as preparation_error: try: @@ -560,6 +689,7 @@ def start_render_worker( tracking_anchor_path: str | None = None, required_openxr_extensions: Iterable[str] = (), hidden_visual_prims: Iterable[str] = (), + excluded_visual_meshes: Iterable[str] = (), max_blank_frames: int = 1, timeout_s: float = 30.0, external_anchor_state: XrAnchorManager | None = None, @@ -571,6 +701,7 @@ def start_render_worker( tracking_anchor_path: Override the configured USD tracking-anchor path. required_openxr_extensions: Extensions required by the input pipeline. hidden_visual_prims: Patterns hidden persistently from every scene view. + excluded_visual_meshes: Mesh patterns excluded from the XR eye partition. max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. timeout_s: Maximum worker startup wait [s]. external_anchor_state: Dynamic anchor policy seeded from committed scene publications. @@ -589,7 +720,7 @@ def start_render_worker( anchor_path = self._cfg.tracking_anchor_path if tracking_anchor_path is None else tracking_anchor_path host = _IsaacLabOvxrApplicationInterface( - self._bootstrap, env, self._cfg, hidden_visual_prims, external_anchor_state + self._bootstrap, env, self._cfg, hidden_visual_prims, excluded_visual_meshes, external_anchor_state ) self._host = host self._owner = OvxrSession( @@ -614,7 +745,10 @@ def start_render_worker( required_openxr_extensions=required_openxr_extensions, max_blank_frames=max_blank_frames, ) - return self._owner.start(timeout_s=timeout_s) + try: + return self._owner.start(timeout_s=timeout_s) + finally: + self._desktop_mirror = host.desktop_mirror def external_oxr_handles(self) -> OpenXRSessionHandles | None: """Return IsaacTeleop's typed external handles once its owner is ready.""" @@ -631,6 +765,25 @@ def request_render(self, physics_step_id: Hashable, *, force: bool = False) -> i self._require_started() return self._owner.request_render(physics_step_id, force=force) + def report_host_timing(self, physics_step_count: int, physics_dt_s: float | None = None) -> None: + """Publish physics counter and optional simulation timestep [s] to the mirror.""" + self._require_started() + if self._desktop_mirror is None: + return + self._desktop_mirror.record_host_frame() + physics_hz = self._physics_rate.sample(physics_step_count) + if physics_dt_s is not None and math.isfinite(physics_dt_s) and physics_dt_s > 0.0: + target_hz = 1.0 / physics_dt_s + physics_text = f"Physics {physics_hz:5.1f}/{target_hz:.0f} Hz ({physics_hz / target_hz:.2f}× RT)" + else: + physics_text = f"Physics {physics_hz:5.1f} Hz" + mirror_status = self._desktop_mirror.status + if mirror_status is not None: + self._desktop_mirror.set_status_text( + f"Main {mirror_status.main_thread_fps:5.1f} FPS " + f"{physics_text} Render {mirror_status.render_thread_fps:5.1f} FPS" + ) + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: """Read world-to-target transform [4, 4], float32, with translation in scene units.""" self._require_started() diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index e3d9a060f52..8de9b46bef0 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -20,6 +20,7 @@ import numpy as np from isaaclab_ov.renderers.render_lease import OvLibrariesScenePublisher + from isaaclab.sensors import CameraCfg from isaaclab.sim import SimulationCfg @@ -78,6 +79,13 @@ def capture_scene_publisher(self) -> OvLibrariesScenePublisher: raise RuntimeError("Scene publisher is available only before OVXR Stage binding") return self._lease.capture_scene_publisher() + def configure_auxiliary_camera(self, camera_cfg: CameraCfg, *, max_fps: float | None = None) -> None: + """Authorize a camera with an update cap [Hz]; None retains its configured cadence.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Auxiliary camera must be configured before OVXR Stage binding") + self._lease.configure_auxiliary_camera(camera_cfg, max_fps=max_fps) + @property def simulation_context(self) -> SimulationContext: """Return the exact pre-created simulation context.""" @@ -100,8 +108,10 @@ def bind_environment(self, env: Any) -> OvLibrariesRenderLease: num_envs = unwrapped.num_envs if not isinstance(num_envs, int) or num_envs <= 0: raise RuntimeError(f"Environment has invalid num_envs: {num_envs!r}") + assert self._sim is not None and self._lease is not None - self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) + if not self._lease.materialize_deferred_auxiliary_camera_stage(scene.stage, num_envs): + self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) self._lease.validate_live() self._bound = True return self._lease diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py index f3dd7fceb16..a1c58d78443 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py @@ -78,6 +78,15 @@ class XrCfg: and affects all cameras, including PiP; it is not restored on XR close. """ + excluded_visual_meshes: list[str] = [] + """Mesh-path regexes excluded from kitless environment cameras by scene partition. + + Unlike ``hidden_visual_prims``, this leaves USD visibility and the source + scene unchanged. Only matching visual meshes in the single-environment + render export enter the ``xr_excluded`` partition. Camera selectors remain + independent; collisions and link/camera transforms must not be matched. + """ + anchor_prim_path: str | None = None """Specifies the prim path to attach the XR anchor to for dynamic positioning. @@ -142,7 +151,7 @@ class XrCfg: from typing import Any -def remove_camera_configs(env_cfg: Any) -> Any: +def remove_camera_configs(env_cfg: Any, *, keep: tuple[str, ...] = ()) -> Any: """Removes cameras from environments when using XR devices. Having additional cameras cause operation performance issues. This function scans the environment @@ -151,6 +160,9 @@ def remove_camera_configs(env_cfg: Any) -> Any: Args: env_cfg: The environment configuration to modify. + keep: Camera attribute names to retain for the OVXR renderer. + Observation terms are removed for retained cameras as well, so the + application loop never requests a second camera render. Returns: The modified environment configuration with cameras removed. @@ -167,8 +179,9 @@ def remove_camera_configs(env_cfg: Any) -> Any: for attr_name in dir(env_cfg.scene): attr = getattr(env_cfg.scene, attr_name) if isinstance(attr, CameraCfg): - delattr(env_cfg.scene, attr_name) - logger.info(f"Removed camera config: {attr_name}") + if attr_name not in keep: + delattr(env_cfg.scene, attr_name) + logger.info(f"Removed camera config: {attr_name}") # Remove any ObsTerms for the camera if hasattr(env_cfg.observations, "policy"): diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py new file mode 100644 index 00000000000..79eff4aba1d --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py @@ -0,0 +1,73 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Camera-selective visual exclusions for the single-environment XR export.""" + +import re +from contextlib import contextmanager + + +@contextmanager +def partition_visual_meshes(stage, patterns): + """Exclude matching visual meshes from environment cameras during export. + + A temporary session sublayer is flattened into the borrowed render Stage. + The application's source layers, visibility, physics and cameras are not + changed. Patterns match mesh paths, including instance proxies, not links. + """ + if not patterns: + yield () + return + + from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + envs = stage.GetPrimAtPath("/World/envs") + if not envs or [str(p.GetPath()) for p in envs.GetChildren()] != ["/World/envs/env_0"]: + raise ValueError("XR visual partitions require exactly env_0") + expressions = [re.compile(pattern) for pattern in patterns] + matches = [[] for _ in expressions] + for prim in Usd.PrimRange(envs, Usd.TraverseInstanceProxies()): + if not prim.IsA(UsdGeom.Mesh): + continue + for index, expression in enumerate(expressions): + if expression.fullmatch(str(prim.GetPath())): + if prim.HasAPI(UsdPhysics.CollisionAPI) or "collisions" in prim.GetPath().pathString.split("/"): + raise ValueError(f"XR visual exclusion matched collision geometry: {prim.GetPath()}") + matches[index].append(str(prim.GetPath())) + if any(not paths for paths in matches): + raise ValueError("XR visual exclusion pattern matched no visual mesh") + paths = tuple(dict.fromkeys(path for group in matches for path in group)) + layer = Sdf.Layer.CreateAnonymous("xr-visual-partitions.usda") + session = stage.GetSessionLayer() + session.subLayerPaths.insert(0, layer.identifier) + try: + with Usd.EditContext(stage, layer): + for path in paths: + while stage.GetPrimAtPath(path).IsInstanceProxy(): + parent = stage.GetPrimAtPath(path).GetParent() + while parent and not parent.IsInstance(): + parent = parent.GetParent() + if not parent: + raise RuntimeError(f"Cannot author XR visual partition: {path}") + parent.SetInstanceable(False) + if parent.IsInstance(): + raise RuntimeError(f"Stronger source opinion prevents XR deinstancing: {parent.GetPath()}") + partition = UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).CreatePrimvar( + "omni:scenePartition", Sdf.ValueTypeNames.Token, UsdGeom.Tokens.constant + ) + partition.Set("xr_excluded") + if partition.Get() != "xr_excluded": + raise RuntimeError(f"Stronger source opinion prevents XR visual partition: {path}") + settings_path = stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" + settings = stage.DefinePrim(settings_path, "RenderSettings") + show_all = settings.CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ) + show_all.Set(False) + if show_all.Get() is not False: + raise RuntimeError("Stronger source opinion prevents XR camera-selective partitions") + yield paths + finally: + session.subLayerPaths.remove(layer.identifier) diff --git a/source/isaaclab_teleop/test/test_entrypoints.py b/source/isaaclab_teleop/test/test_entrypoints.py index 5f814345e6c..2dd220b02e6 100644 --- a/source/isaaclab_teleop/test/test_entrypoints.py +++ b/source/isaaclab_teleop/test/test_entrypoints.py @@ -8,6 +8,7 @@ import importlib import importlib.metadata import sys +from contextlib import contextmanager from unittest import mock import pytest @@ -15,6 +16,32 @@ pytestmark = pytest.mark.unit +def test_live_entrypoint_preserves_resolved_camera_selection(monkeypatch): + """The public workflow carries its resolved PiP selection into the selected runtime.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as workflow + + config, teleop_config, camera = object(), object(), object() + selection = [("robot_pov_cam", camera, object())] + resolved = (config, teleop_config, True, selection) + monkeypatch.setattr(workflow, "_resolve_env_cfg", lambda: resolved) + calls = [] + + @contextmanager + def launch(env_config, args, *, cleanup_context): + assert env_config is config and args.using_ovxr + assert not cleanup_context + calls.append("launch") + yield + calls.append("close") + + monkeypatch.setattr(workflow, "launch_simulation", launch) + monkeypatch.setattr(workflow, "run_teleoperation", lambda *args: calls.append(args)) + monkeypatch.setattr(sys, "argv", ["caller"]) + assert workflow.run_teleop_se3_agent_cli(["--ovxr", "--ovxr_pip_camera", "robot_pov_cam"]) == 0 + assert calls == ["launch", resolved, "close"] + assert sys.argv == ["caller"] + + @pytest.mark.parametrize( ("module_name", "cli_name"), [ diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 5f154dd0783..b92ac03cb3f 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -24,15 +24,21 @@ class _FakeLease: def __init__(self): self.ovrtx_renderer = object() self.ovstage = object() + self.partition_calls = [] + self.validated_ordinals = [] self.owner_lane_entries = 0 + self.auxiliary_render_product_paths = () @contextmanager def ovstage_owner_lane(self): self.owner_lane_entries += 1 yield - def validate_live(self): - pass + def configure_external_camera_scene_partitions(self, environment_root_path, camera_paths): + self.partition_calls.append((environment_root_path, camera_paths)) + + def validate_external_scene_partitions(self, ordinal): + self.validated_ordinals.append(ordinal) class _FakePublisher: @@ -88,6 +94,8 @@ def __init__(self): self.publisher = _FakePublisher() self.events = [] self.closed = False + self.aborted = False + self.auxiliary_camera_cfgs = [] def capture_scene_publisher(self): self.events.append("capture") @@ -104,12 +112,18 @@ def bind_environment(self, env): self.events.append(("bind", env)) return self.lease + def configure_auxiliary_camera(self, camera_cfg, *, max_fps=30.0): + self.auxiliary_camera_cfgs.append(camera_cfg) + def get_target_frame_T_world(self, path): return ("target", path) def close(self): self.closed = True + def abort(self): + self.aborted = True + @pytest.fixture def real_lease_type(): @@ -341,6 +355,7 @@ def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monk assert ordinal == 11 assert bootstrap.publisher.calls[0][:2] == (11, "step-1") host.validate_first_published_scene(ordinal) + assert bootstrap.lease.validated_ordinals == [11] with host.owner_lane(): pass assert bootstrap.lease.owner_lane_entries == 3 @@ -422,6 +437,108 @@ def committed_yaw(): assert committed_yaw() == pytest.approx(math.pi / 2 * (1.0 - math.exp(-0.01 / 0.1))) +def test_host_lends_the_selected_auxiliary_product_without_rendering_it(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + bootstrap.lease.auxiliary_render_product_paths = ("/IsaacLabAuxiliaryRender/RenderProduct_robot_pov_cam",) + camera = object() + env = SimpleNamespace( + unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={"robot_pov_cam": camera})) + ) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_apply_hidden_visual_prims", lambda *_args: None) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, + env, + xr_module.OvLibrariesXrConfig( + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ), + ), + (), + ) + + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + + assert binding.frame_consumers[0].render_product_paths == bootstrap.lease.auxiliary_render_product_paths + assert not hasattr(binding, "auxiliary_frame_source") + + +def test_host_rejects_a_missing_selected_auxiliary_camera(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={}))) + monkeypatch.setattr(xr_module, "_apply_hidden_visual_prims", lambda *_args: None) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, + env, + xr_module.OvLibrariesXrConfig( + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ), + ), + (), + ) + + with pytest.raises(RuntimeError, match="not present"): + host.attach_on_render_thread(host.offer_render_owner_transfer()) + + +def test_selected_camera_poses_follow_motion_and_reset_before_publication(xr_module, monkeypatch): + host, bootstrap, env = _make_host(xr_module, monkeypatch) + names = ("head", "left_wrist", "right_wrist") + events = [] + + class PoseOnlyCamera: + live = 0 + cached = -1 + fail = False + + def _update_poses(self): + if self.fail: + raise RuntimeError("pose unavailable") + self.cached = self.live + events.append(self) + + @property + def data(self): + pytest.fail("Publication must not fetch/render camera images") + + cameras = {name: PoseOnlyCamera() for name in (*names, "unselected")} + env.scene = SimpleNamespace(stage=None, sensors=cameras) + host._auxiliary_cameras = tuple( + xr_module.OvLibrariesXrCameraConfig(name, SimpleNamespace(width=720, height=450)) for name in names + ) + bootstrap.lease.auxiliary_render_product_paths = tuple(f"/Render/{name}" for name in names) + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + published = [] + bootstrap.publisher.write_complete_scene = lambda *_args: published.append( + tuple(cameras[name].cached for name in names) + ) + + for identity, positions in ((1, (0, 2, 3)), (("reset", 1, 1), (0, -2, -3)), (2, (0, 4, 5))): + for name, value in zip(names, positions, strict=True): + cameras[name].live = value + with host.publication_guard(): + host.publish_complete_scene(identity) + assert published[-1] == positions + assert events == [cameras[name] for name in names] * 3 + assert cameras["unselected"].cached == -1 + + bootstrap.publisher.required = False + with host.publication_guard(): + host.publish_complete_scene(2) + assert len(events) == 9 # Reused scene/cadence gaps do not resample poses. + + bootstrap.publisher.required = True + cameras["left_wrist"].fail = True + reserved = list(binding.ordinal_ledger.reserved) + with host.publication_guard(), pytest.raises(RuntimeError, match="pose unavailable"): + host.publish_complete_scene(3) + assert binding.ordinal_ledger.reserved == reserved + assert len(published) == 3 + + def test_host_abandons_failed_write_and_failed_publish(xr_module, monkeypatch): host, bootstrap, _env = _make_host(xr_module, monkeypatch) host.attach_on_render_thread(host.offer_render_owner_transfer()) @@ -580,7 +697,8 @@ def fail_abandon(_ordinal): assert any("abandon failed" in note for note in caught.value.__notes__) -def test_session_delegates_lifecycle_to_isaacteleop_owner(xr_module, monkeypatch): +@pytest.mark.parametrize("camera_enabled", [False, True]) +def test_session_delegates_lifecycle_to_isaacteleop_owner(xr_module, monkeypatch, camera_enabled): calls = [] class FakeOwner: @@ -589,9 +707,11 @@ class FakeOwner: def __init__(self, host, config, **kwargs): calls.append(("init", host, config, kwargs)) + self.host = host def start(self, timeout_s): calls.append(("start", timeout_s)) + self.binding = self.host.attach_on_render_thread(self.host.offer_render_owner_transfer()) return True def external_oxr_handles(self): @@ -611,28 +731,75 @@ def get_target_frame_T_world(self, path): def close(self): calls.append(("close",)) + class FakeMirror: + status = SimpleNamespace(main_thread_fps=30.0, render_thread_fps=40.0) + + def __init__(self, config, *, camera_products=()): + self.config = config + + def record_host_frame(self): + calls.append(("timing",)) + + def set_status_text(self, text): + calls.append(("status", text)) + + mirror_module = importlib.import_module("isaaccapture.ovxr_session_manager.owned_mirror") + monkeypatch.setattr(mirror_module, "OwnedDesktopMirror", FakeMirror) external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") monkeypatch.setattr(external, "OvxrSession", FakeOwner) bootstrap = _FakeBootstrap() assert xr_module.OvLibrariesXrConfig().extra_openxr_extensions == () session = xr_module.OvLibrariesXrSession( - bootstrap, xr_module.OvLibrariesXrConfig(extra_openxr_extensions=("XR_EXT_debug_utils",)) + bootstrap, + xr_module.OvLibrariesXrConfig( + extra_openxr_extensions=("XR_EXT_debug_utils",), + desktop_mirror_enabled=True, + desktop_mirror_max_fps=20.0, + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + ) + if camera_enabled + else (), + ), ) - env = SimpleNamespace() + env = SimpleNamespace(scene=SimpleNamespace(stage=None, sensors={"robot_pov_cam": object()})) + bootstrap.lease.auxiliary_render_product_paths = ("/Render/robot_pov_cam",) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_apply_hidden_visual_prims", lambda *_args: None) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda *_args: None) + assert session.start_render_worker( - env, required_openxr_extensions=("XR_EXT_hand_tracking",), max_blank_frames=0, timeout_s=4.0 + env, + required_openxr_extensions=("XR_EXT_hand_tracking",), + max_blank_frames=0 if camera_enabled else 1, + timeout_s=4.0, ) + assert calls[0][0] == "init" assert calls[0][2].session_mode == "ar" assert calls[0][3]["required_openxr_extensions"] == ("XR_EXT_hand_tracking",) - assert calls[0][3]["max_blank_frames"] == 0 + assert calls[0][3]["max_blank_frames"] == (0 if camera_enabled else 1) assert calls[0][2].extra_openxr_extensions == ("XR_EXT_debug_utils",) assert calls[0][2].host_wait_render_output is True + consumers = session._owner.binding.frame_consumers + consumer = consumers[-1] + assert consumer.config.max_fps == 20.0 + assert consumer.config.initial_status_text == "Waiting for headset" + if camera_enabled: + assert consumers[0].panels[0].label == "robot_pov_cam" + else: + assert len(consumers) == 1 assert calls[1] == ("start", 4.0) assert session.external_oxr_handles() == "handles" assert session.request_render("step") == 17 assert session.request_render("reset", force=True) == 17 assert calls[-2:] == [("request", "step", False), ("request", "reset", True)] + monkeypatch.setattr(session._physics_rate, "sample", lambda _count: 114.7) + session.report_host_timing(42, 1.0 / 120.0) + assert calls[-2:] == [ + ("timing",), + ("status", "Main 30.0 FPS Physics 114.7/120 Hz (0.96× RT) Render 40.0 FPS"), + ] assert session.get_target_frame_T_world("/target") == ("target", "/target") assert session.async_worker_owns_environment assert not session.shutdown_requested @@ -719,6 +886,32 @@ def test_bootstrap_construction_failure_restores_terminal_mode(bootstrap_context assert any("context cleanup failed" in note for note in error.__notes__) +def test_desktop_mirror_rate_must_be_positive_and_finite(xr_module): + with pytest.raises(ValueError, match="desktop_mirror_max_fps"): + xr_module.OvLibrariesXrConfig(desktop_mirror_max_fps=0.0) + with pytest.raises(ValueError, match="desktop_mirror_max_fps"): + xr_module.OvLibrariesXrConfig(desktop_mirror_max_fps=float("nan")) + + +def test_auxiliary_camera_config_is_preconfigured(xr_module, monkeypatch): + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraConfig("", object()) + with pytest.raises(ValueError, match="name and configuration"): + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", None) + with pytest.raises(ValueError, match="max_fps"): + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", object(), max_fps=0.0) + + bootstrap = _FakeBootstrap() + camera_cfg = SimpleNamespace(width=720, height=450) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_args: bootstrap) + camera = xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", camera_cfg) + session = xr_module.OvLibrariesXrSession.prepare( + object(), xr_module.OvLibrariesXrConfig(auxiliary_cameras=(camera,)) + ) + assert isinstance(session, xr_module.OvLibrariesXrSession) + assert bootstrap.auxiliary_camera_cfgs == [camera_cfg] + + def test_caller_renderer_settings_precede_renderer_creation(xr_module, monkeypatch): settings = " --/rtx/rendermode=MinimalRendering " events = [] @@ -736,6 +929,49 @@ def create_bootstrap(*_args): xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(renderer_settings=settings)) +def test_prepare_rejects_existing_context_before_global_settings(xr_module, monkeypatch): + from isaaclab.sim import SimulationContext + + monkeypatch.setattr(SimulationContext, "instance", lambda: object()) + monkeypatch.setitem( + sys.modules, + "ovxr", + SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed existing renderer settings")), + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated second bootstrap")) + with pytest.raises(RuntimeError, match="must create the SimulationContext"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig()) + + +@pytest.mark.parametrize("terminal_mode", [False, None, 1]) +def test_prepare_rejects_extension_mode_before_global_settings(xr_module, monkeypatch, terminal_mode): + monkeypatch.setattr(builtins, "ISAAC_LAUNCHED_FROM_TERMINAL", terminal_mode, raising=False) + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(RuntimeError, match="extension mode"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig()) + + +def test_counter_rate_uses_application_counter_and_resets_cleanly(xr_module): + rate = xr_module._CounterRate() + assert rate.sample(10, now_ns=0) == 0.0 + assert rate.sample(15, now_ns=500_000_000) == pytest.approx(10.0) + assert rate.sample(20, now_ns=1_000_000_000) == pytest.approx(10.0) + assert rate.sample(0, now_ns=1_100_000_000) == 0.0 + + +def test_disabled_desktop_mirror_does_not_collect_or_format_status(xr_module): + session = xr_module.OvLibrariesXrSession( + _FakeBootstrap(), + xr_module.OvLibrariesXrConfig(desktop_mirror_enabled=False), + ) + session._owner = SimpleNamespace(report_host_timing=lambda: pytest.fail("unexpected mirror work")) + + session.report_host_timing(42, 1.0 / 120.0) + + def test_config_rejects_unknown_session_mode(xr_module): with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): xr_module.OvLibrariesXrConfig(session_mode="automatic") @@ -769,8 +1005,96 @@ def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, assert bootstrap.events == [] +def test_three_auxiliary_camera_configs_prepare_the_same_bootstrap(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + cameras = tuple( + xr_module.OvLibrariesXrCameraConfig( + name=name, camera_cfg=SimpleNamespace(width=640, height=480), max_fps=hz, space="head_locked" + ) + for name, hz in zip(("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"), (20, 10, 10), strict=True) + ) + session = xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(auxiliary_cameras=cameras)) + assert bootstrap.auxiliary_camera_cfgs == [camera.camera_cfg for camera in cameras] + assert session is not None + extra = xr_module.OvLibrariesXrCameraConfig("overview", SimpleNamespace(width=640, height=480)) + with pytest.raises(ValueError, match="at most three"): + xr_module.OvLibrariesXrConfig(auxiliary_cameras=(*cameras, extra)) + with pytest.raises(ValueError, match="exactly two"): + xr_module.OvLibrariesXrConfig(camera_paths=("/Camera/mono",)) + + +def test_prepare_snapshots_camera_selection_without_mutating_caller(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + camera_cfg = SimpleNamespace(width=640, height=480) + camera = xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", camera_cfg) + cfg = xr_module.OvLibrariesXrConfig(auxiliary_cameras=(camera,)) + session = xr_module.OvLibrariesXrSession.prepare(object(), cfg) + cfg.auxiliary_cameras = () + assert session._cfg.auxiliary_cameras == (camera,) + assert session._cfg.auxiliary_cameras[0].camera_cfg is bootstrap.auxiliary_camera_cfgs[0] + + +@pytest.mark.parametrize("fail_during", ["camera", "session"]) +@pytest.mark.parametrize("abort_fails", [False, True]) +def test_prepare_aborts_partial_bootstrap_and_preserves_first_error(xr_module, monkeypatch, fail_during, abort_fails): + bootstrap = _FakeBootstrap() + first_error = RuntimeError("preparation failed") + original_cause = ValueError("invalid camera input") + first_error.add_note("Original preparation diagnostic") + abort_error = RuntimeError("abort failed") + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + + def fail(*_args, **_kwargs): + raise first_error from original_cause + + def abort(): + bootstrap.aborted = True + if abort_fails: + raise abort_error + + monkeypatch.setattr(bootstrap, "abort", abort) + if fail_during == "camera": + monkeypatch.setattr(bootstrap, "configure_auxiliary_camera", fail) + else: + monkeypatch.setattr(xr_module.OvLibrariesXrSession, "__init__", fail) + + cfg = xr_module.OvLibrariesXrConfig( + auxiliary_cameras=( + xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=640, height=480)), + ), + ) + with pytest.raises(RuntimeError) as caught: + xr_module.OvLibrariesXrSession.prepare(object(), cfg) + + assert bootstrap.aborted + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert caught.value.__notes__ == ["Original preparation diagnostic"] + ( + [f"Cleanup after failed XR preparation also failed: {abort_error!r}"] if abort_fails else [] + ) + + +@pytest.mark.parametrize("space", ["viewer_start", "world"]) +def test_prepare_rejects_unsupported_quad_placement_before_native_preparation(xr_module, monkeypatch, space): + monkeypatch.setitem( + sys.modules, + "ovxr", + SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("renderer settings changed before validation")), + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated before validation")) + with pytest.raises(ValueError, match="head_locked"): + camera = xr_module.OvLibrariesXrCameraConfig( + name="robot_pov_cam", camera_cfg=SimpleNamespace(width=640, height=480), space=space + ) + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(auxiliary_cameras=(camera,))) + + def test_public_xr_exports_resolve_to_implemented_types(xr_module): import isaaclab_teleop assert isaaclab_teleop.OvLibrariesXrConfig is xr_module.OvLibrariesXrConfig + assert isaaclab_teleop.OvLibrariesXrCameraConfig is xr_module.OvLibrariesXrCameraConfig assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 15e419c9708..6d711a38d0d 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -246,7 +246,7 @@ def last_control_events(self): monkeypatch.setattr(agent, "_install_ovxr_shutdown_handlers", lambda *_args: {}) monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", lambda *_args, **_kwargs: camera) monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", lambda *_args, **_kwargs: session) - agent.run_teleoperation(cfg, cfg.isaac_teleop, True) + agent.run_teleoperation(cfg, cfg.isaac_teleop, True, []) return run @@ -385,6 +385,7 @@ def test_paused_loop_retries_the_latest_unaccepted_publication(monkeypatch, run_ steps = [] requests = [] sleeps = [] + reports = [] session = SimpleNamespace(shutdown_requested=False) def advance(): @@ -405,6 +406,7 @@ def sleep(duration): sleeps.append((duration, len(requests))) session.request_render = request + session.report_host_timing = lambda step, dt: reports.append((step, dt, len(requests))) session.wait_for_shutdown = sleep monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: 0.0, sleep=sleep)) env = SimpleNamespace( @@ -428,6 +430,7 @@ def sleep(duration): assert advances == 2 and len(steps) == 1 assert len(requests) == 3 and requests[0][1] is True assert requests[1:] == [(1, False), (1, False)] + assert reports == [(1, 0.005, 2), (1, 0.005, 3)] assert sleeps == [(0.02, 3)] # Pacing follows the retried publication. @@ -804,6 +807,112 @@ def stop(): assert haptics == {"update": 2, "stop": 1} +def test_desktop_mirror_cli_is_explicit_and_bounded(monkeypatch): + agent = _load_teleop_agent( + monkeypatch, + "--ovxr", + "--ovxr_desktop_mirror", + "--ovxr_desktop_mirror_fps", + "20", + ) + + assert agent.args_cli.ovxr_desktop_mirror + assert agent.args_cli.ovxr_desktop_mirror_fps == 20.0 + + +def test_auxiliary_camera_pip_cli_is_explicit_and_bounded(monkeypatch): + agent = _load_teleop_agent( + monkeypatch, + "--ovxr", + "--ovxr_pip_camera", + "robot_pov_cam", + "--ovxr_pip_fps", + "20", + ) + + assert agent.args_cli.ovxr_pip_camera == ["robot_pov_cam"] + assert agent.args_cli.ovxr_pip_fps == 20.0 + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize( + "flags, expected", + [ + ([], True), + (["--headless"], False), + (["--ovxr_desktop_mirror"], True), + (["--headless", "--ovxr_desktop_mirror"], False), + (["--ovxr_desktop_mirror", "--headless"], False), + ], +) +def test_ovxr_mirror_cli_reaches_session_construction(monkeypatch, entrypoint, flags, expected): + """The real entrypoint parser controls the config passed to XR session preparation.""" + import isaaclab_teleop + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + argv = ["--task", "Test", "--ovxr", *flags] + if entrypoint == "replay": + argv += ["--replay_file", "recording.mcap"] + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + module._parse_args(argv) + assert module.args_cli.headless is ("--headless" in flags) + cfg = SimpleNamespace(sim=object(), isaac_teleop=isaaclab_teleop.IsaacTeleopCfg()) + received = [] + + class PreparationReached(Exception): + pass + + def prepare(_sim, xr_cfg, *_args, **_kwargs): + assert isinstance(xr_cfg, isaaclab_teleop.OvLibrariesXrConfig) + received.append(xr_cfg.desktop_mirror_enabled) + raise PreparationReached + + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + with pytest.raises(PreparationReached): + module.run_teleoperation(cfg, cfg.isaac_teleop, True, []) + else: + with pytest.raises(PreparationReached): + module._ReplayPresentation(cfg) + assert received == [expected] + assert isaaclab_teleop.OvLibrariesXrConfig().desktop_mirror_enabled is False + + +@pytest.mark.parametrize( + "entrypoint, flags", + [ + ("live", []), + ("live", ["--headless"]), + ("live", ["--xr"]), + ("live", ["--xr", "--headless"]), + ("replay", []), + ("replay", ["--headless"]), + ], +) +def test_non_ovxr_cli_keeps_mirror_disabled(monkeypatch, entrypoint, flags): + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + argv = ["--task", "Test", *flags] + if entrypoint == "replay": + argv += ["--replay_file", "recording.mcap"] + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + args = module._parse_args(argv) + assert args.ovxr_desktop_mirror is False + assert args.headless is ("--headless" in flags) + + @pytest.mark.parametrize("entrypoint", ["live", "replay"]) def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoint): """Both actual workflows finalize the selected config before OVXR allocates its context.""" diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py index eac106fe1a1..45a60cf45ac 100644 --- a/source/isaaclab_teleop/test/test_teleop_failure_exit.py +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -90,7 +90,7 @@ def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, en if not use_capture: cfg = SimpleNamespace() # Native inputs must not read Capture-only action policy. with pytest.raises(RuntimeError) as caught: - entry_namespace["run_teleoperation"](cfg, False, use_capture) + entry_namespace["run_teleoperation"](cfg, False, use_capture, []) assert caught.value is failure camera.close.assert_called_once_with() assert env.close.call_count == int(stage != "environment") @@ -109,7 +109,7 @@ def test_input_cleanup_failure_prevents_borrowed_owner_release(entry_namespace): @pytest.mark.parametrize( ("cleanup_stage", "has_primary_error"), - [("xr", True), ("environment", True), ("camera", False), ("signals", False)], + [("xr", True), ("environment", True), ("camera", False), ("camera", True), ("signals", False)], ) def test_cleanup_preserves_first_error_and_never_frees_undrained_stage( entry_namespace, cleanup_stage, has_primary_error @@ -140,7 +140,7 @@ def exercise(): exercise() assert caught.value is (primary_error if has_primary_error else cleanup_error) assert events == ( - ["xr", "camera", "signals"] if cleanup_stage == "xr" else ["xr", "camera", "environment", "signals"] + ["xr", "camera", "signals"] if cleanup_stage in ("xr", "camera") else ["xr", "camera", "environment", "signals"] ) if has_primary_error: assert any("cleanup failed" in note for note in primary_error.__notes__) diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index cbe5d1a7ecd..fc3b5959c11 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -34,6 +34,66 @@ def test_replay_failed_input_drain_requests_stop_without_releasing_borrowers(rep presentation.session.close.assert_not_called() +@pytest.mark.parametrize("layout_mode", ["horizontal", "manual", None]) +def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_presentation_type, layout_mode): + """The actual viewer constructor changes placement without discarding task packing.""" + import isaaclab_teleop + from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg + + from isaaclab.sensors import CameraCfg + + names = ["left", "right"] + config = SimpleNamespace( + scene=SimpleNamespace(**{name: CameraCfg(prim_path=f"/World/{name}", width=640, height=480) for name in names}), + sim=object(), + isaac_teleop=IsaacTeleopCfg( + xr_camera_feeds=[XrCameraFeedCfg(camera_name=name, panel_width_m=0.4) for name in names], + xr_camera_feed_layout=( + XrCameraFeedLayoutCfg( + mode=layout_mode, + placement="viewer_start", + center_offset_m=(0.1, -0.2), + panel_gap_m=0.1, + distance_m=1.2, + ) + if layout_mode is not None + else None + ), + ), + ) + original_layout = config.isaac_teleop.xr_camera_feed_layout + original_layout_fields = original_layout.to_dict() if original_layout is not None else None + original_feeds = [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] + monkeypatch.setitem( + replay_presentation_type.__init__.__globals__, + "args_cli", + SimpleNamespace( + ovxr_pip_camera=names, + task="Test", + ovxr_session_mode="ar", + cloudxr_env="none", + auto_launch_cloudxr=False, + ovxr_desktop_mirror=False, + ), + ) + prepare = Mock() + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + presentation = replay_presentation_type(config) + assert presentation.session is prepare.return_value + session_config = prepare.call_args.args[1] + expected = ( + [(-0.15, -0.2, -1.2), (0.35, -0.2, -1.2)] + if layout_mode == "horizontal" + else [(0.0, 0.0, -0.8), (0.0, 0.0, -0.8)] + ) + for camera, position in zip(session_config.auxiliary_cameras, expected, strict=True): + assert camera.position == pytest.approx(position) + assert config.isaac_teleop.xr_camera_feed_layout is original_layout + if original_layout is not None: + assert original_layout.to_dict() == original_layout_fields + assert [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] == original_feeds + + @pytest.mark.parametrize( "kit_active,run_index,backend,viewer,hard_reset", [ diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index 67aa9e45bb2..d46d13820f6 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -33,6 +33,75 @@ def test_layout_preserves_positional_constructor(): assert camera_feed._PanelDescriptor(**descriptor).use_scene_partition is False +def test_ovxr_feed_selection_preserves_order_shared_renderer_and_rate_limits(): + from isaaclab_ov.renderers import OVRTXRendererCfg + + feeds = [ + XrCameraFeedCfg(camera_name="robot_pov_cam", max_update_hz=12.0), + XrCameraFeedCfg(camera_name="wrist_cam", max_update_hz=0.0, offset_m=(0.1, -0.2), distance_m=0.9), + ] + cfg = _teleop_env_cfg(feeds, camera=_camera_cfg()) + cfg.scene.wrist_cam = _camera_cfg() + original = cfg.scene.robot_pov_cam.renderer_cfg + renderer = OVRTXRendererCfg() + with pytest.raises(ValueError, match="missing.*CameraCfg"): + camera_feed.select_ovxr_camera_feeds(cfg, ["robot_pov_cam", "missing"], renderer) + assert cfg.scene.robot_pov_cam.renderer_cfg is original + + selected = camera_feed.select_ovxr_camera_feeds(cfg, ["wrist_cam", "robot_pov_cam"], renderer) + cameras = camera_feed.resolve_ovxr_camera_configs( + selected, XrCameraFeedLayoutCfg(placement="head_locked"), max_fps=20.0 + ) + assert [camera.name for camera in cameras] == ["wrist_cam", "robot_pov_cam"] + assert [camera.max_fps for camera in cameras] == [20.0, 12.0] + assert all(camera.camera_cfg.renderer_cfg is renderer for camera in cameras) + assert all(camera.space == "head_locked" for camera in cameras) + assert cameras[0].position == (0.1, -0.2, -0.9) + with pytest.raises(ValueError, match="position override requires one camera"): + camera_feed.resolve_ovxr_camera_configs( + selected, + XrCameraFeedLayoutCfg(placement="head_locked"), + max_fps=20.0, + position_override=(0.1, 0.2, -0.8), + ) + overridden = camera_feed.resolve_ovxr_camera_configs( + selected[:1], + XrCameraFeedLayoutCfg(placement="head_locked"), + max_fps=20.0, + position_override=(0.1, 0.2, -0.8), + ) + assert overridden[0].position == (0.1, 0.2, -0.8) + + +@pytest.mark.parametrize("placement", ["viewer_start", "world"]) +def test_ovxr_rejects_unsupported_panel_placement(placement): + selected = [("robot_pov_cam", _camera_cfg(), XrCameraFeedCfg(camera_name="robot_pov_cam"))] + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(0.0, 0.8, 1.6)) + with pytest.raises(ValueError, match="require head_locked"): + camera_feed.resolve_ovxr_camera_configs(selected, layout, max_fps=20.0) + + +@pytest.mark.parametrize( + "field,value,mode", + [ + ("max_update_hz", -1.0, "manual"), + ("max_update_hz", float("nan"), "manual"), + ("max_update_hz", float("inf"), "manual"), + ("distance_m", 0.0, "manual"), + ("distance_m", -0.5, "manual"), + ("offset_m", (0.0, 0.0, 1.0), "manual"), + ("offset_m", (float("nan"), 0.0), "horizontal"), + ], +) +def test_ovxr_rejects_invalid_feed_before_layout_conversion(field, value, mode): + feed = XrCameraFeedCfg(camera_name="robot_pov_cam", **{field: value}) + selected = [(feed.camera_name, _camera_cfg(), feed)] + with pytest.raises(ValueError, match=field): + camera_feed.resolve_ovxr_camera_configs( + selected, XrCameraFeedLayoutCfg(mode=mode, placement="head_locked"), max_fps=20.0 + ) + + @pytest.fixture def isolated_settings(monkeypatch): values = {ISAAC_RTX_SHOW_ALL_PARTITIONS_BY_DEFAULT_SETTING: True} @@ -729,3 +798,19 @@ def test_isaac_teleop_default_has_no_camera_feeds(): cfg = IsaacTeleopCfg(pipeline_builder=lambda: None) assert cfg.xr_camera_feeds == [] + + +def test_ovxr_automatic_panels_use_shared_layout_without_mutating_feeds(): + feeds = [XrCameraFeedCfg(camera_name=name, panel_width_m=0.4) for name in ("left", "right")] + selected = [(feed.camera_name, _camera_cfg(), feed) for feed in feeds] + layout = XrCameraFeedLayoutCfg( + mode="horizontal", + placement="head_locked", + center_offset_m=(0.1, -0.2), + panel_gap_m=0.1, + distance_m=1.2, + ) + panels = camera_feed.resolve_ovxr_camera_configs(selected, layout, max_fps=20.0) + assert panels[0].position == pytest.approx((-0.15, -0.2, -1.2)) + assert panels[1].position == pytest.approx((0.35, -0.2, -1.2)) + assert feeds[0].offset_m == XrCameraFeedCfg().offset_m diff --git a/source/isaaclab_teleop/test/test_xr_scene_partition.py b/source/isaaclab_teleop/test/test_xr_scene_partition.py new file mode 100644 index 00000000000..e8a21f9e912 --- /dev/null +++ b/source/isaaclab_teleop/test/test_xr_scene_partition.py @@ -0,0 +1,104 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +import pytest +from isaaclab_ov.stage import export_stage_to_string +from isaaclab_teleop.xr_scene_partition import partition_visual_meshes + +from pxr import Sdf, Usd, UsdGeom, UsdPhysics + + +def _scene(): + stage = Usd.Stage.CreateInMemory() + env = UsdGeom.Xform.Define(stage, "/World/envs/env_0").GetPrim() + env.CreateAttribute("primvars:omni:scenePartition", Sdf.ValueTypeNames.Token).Set("env_0") + UsdGeom.Mesh.Define(stage, "/Asset/head/mesh") + UsdGeom.Mesh.Define(stage, "/Asset/body/mesh") + visual = UsdGeom.Xform.Define(stage, "/World/envs/env_0/Robot/visuals").GetPrim() + visual.GetReferences().AddInternalReference("/Asset") + visual.SetInstanceable(True) + collision = UsdGeom.Mesh.Define(stage, "/World/envs/env_0/Robot/collisions/head").GetPrim() + UsdPhysics.CollisionAPI.Apply(collision) + UsdGeom.Camera.Define(stage, "/World/envs/env_0/Robot/Camera") + return stage + + +@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) +def test_partition_export_preserves_source_visibility_physics_and_camera(settings_path): + stage = _scene() + if settings_path is not None: + stage.DefinePrim(settings_path, "RenderSettings") + stage.SetMetadata("renderSettingsPrimPath", settings_path) + original = stage.GetRootLayer().ExportToString() + session = stage.GetSessionLayer().ExportToString() + head = "/World/envs/env_0/Robot/visuals/head/mesh" + with partition_visual_meshes(stage, [head]): + prim = stage.GetPrimAtPath(head) + assert UsdGeom.PrimvarsAPI(prim).FindPrimvarWithInheritance("omni:scenePartition").Get() == "xr_excluded" + assert UsdGeom.Imageable(prim).ComputeVisibility() == "inherited" + for path in ("/World/envs/env_0/Robot/visuals/body/mesh", "/World/envs/env_0/Robot/collisions/head"): + assert ( + UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).FindPrimvarWithInheritance("omni:scenePartition").Get() + == "env_0" + ) + assert stage.GetPrimAtPath("/World/envs/env_0/Robot/Camera").IsA(UsdGeom.Camera) + layer = Sdf.Layer.CreateAnonymous() + layer.ImportFromString( + export_stage_to_string(stage, ("/World/envs/env_0",), source_paths=("/World/envs/env_0",)) + ) + exported = Usd.Stage.Open(layer) + settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) + assert settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault").Get() is False + assert ( + UsdGeom.PrimvarsAPI(exported.GetPrimAtPath(head)).FindPrimvarWithInheritance("omni:scenePartition").Get() + == "xr_excluded" + ) + assert stage.GetRootLayer().ExportToString() == original + assert stage.GetSessionLayer().ExportToString() == session + assert stage.GetPrimAtPath(head).IsInstanceProxy() + + +@pytest.mark.parametrize("pattern", ["/missing", "/World/envs/env_0/Robot/collisions/head"]) +def test_reject_missing_or_collision_mesh_before_edit(pattern): + stage = _scene() + original = stage.GetSessionLayer().ExportToString() + with pytest.raises(ValueError): + with partition_visual_meshes(stage, [pattern]): + pass + assert stage.GetSessionLayer().ExportToString() == original + + +def test_export_failure_removes_temporary_layer(): + stage = _scene() + before = stage.GetSessionLayer().ExportToString() + with pytest.raises(RuntimeError): + with partition_visual_meshes(stage, ["/World/envs/env_0/Robot/visuals/head/mesh"]): + raise RuntimeError("export failed") + assert stage.GetSessionLayer().ExportToString() == before + + +@pytest.mark.parametrize("conflict", ["instance", "partition", "show_all"]) +def test_stronger_session_opinion_fails_closed_and_is_preserved(conflict): + stage = _scene() + head = "/World/envs/env_0/Robot/visuals/head/mesh" + with Usd.EditContext(stage, stage.GetSessionLayer()): + visual = stage.GetPrimAtPath("/World/envs/env_0/Robot/visuals") + if conflict == "instance": + visual.SetInstanceable(True) + elif conflict == "partition": + visual.SetInstanceable(False) + UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(head)).CreatePrimvar( + "omni:scenePartition", Sdf.ValueTypeNames.Token, UsdGeom.Tokens.constant + ).Set("env_0") + else: + settings = stage.DefinePrim("/Render/IsaacLabSettings", "RenderSettings") + settings.CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ).Set(True) + before = stage.GetSessionLayer().ExportToString() + with pytest.raises(RuntimeError, match="Stronger source opinion"): + with partition_visual_meshes(stage, [head]): + pytest.fail("A stronger source opinion must not silently defeat the exclusion") + assert stage.GetSessionLayer().ExportToString() == before From 614c319ff493be687a9c589090c5f7c6a72d8853 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Fri, 30 Oct 2026 21:46:31 -0400 Subject: [PATCH 005/145] Limit changes to the kitless feature scope --- .../isaaclab_ov/renderers/ovrtx_renderer.py | 36 +++++++++---------- source/isaaclab_ov/test/test_ovrtx_usd.py | 6 ++-- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 6c34a2cca16..64fc8a6833e 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -76,14 +76,8 @@ from isaaclab.sim import PreviewSurfaceCfg, SimulationContext, use_stage from isaaclab.utils.warp import ProxyArray -from ..cloner import ovrtx_replicate -from ..stage import ( - export_stage_to_string, - points_tensor_from_warp, - xform_tensor_from_numpy, - xform_tensor_from_warp, -) -from .ovrtx_annotator_utils import ( +from isaaclab_ov.cloner import ovrtx_replicate +from isaaclab_ov.renderers.ovrtx_annotator_utils import ( IdentifierMap, build_instance_id_to_labels_and_semantics, build_semantic_id_to_labels, @@ -91,10 +85,10 @@ decode_stable_id_map, decode_stable_id_semantic_id_map, ) -from .ovrtx_compat import OVRTX_USES_NAMED_SEGMENTATION_MAPS -from .ovrtx_compat import OVRTX_VERSION as _OVRTX_VERSION -from .ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg -from .ovrtx_renderer_kernels import ( +from isaaclab_ov.renderers.ovrtx_compat import OVRTX_USES_NAMED_SEGMENTATION_MAPS +from isaaclab_ov.renderers.ovrtx_compat import OVRTX_VERSION as _OVRTX_VERSION +from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg +from isaaclab_ov.renderers.ovrtx_renderer_kernels import ( accumulate_mesh_normals, create_camera_transforms_kernel, create_marker_transforms_kernel, @@ -102,15 +96,21 @@ generate_random_colors_from_ids_kernel, normalize_mesh_normals, ) -from .ovrtx_shader_cache import redirect_shader_cache -from .ovrtx_usd import ( +from isaaclab_ov.renderers.ovrtx_shader_cache import redirect_shader_cache +from isaaclab_ov.renderers.ovrtx_usd import ( _RTX_MINIMAL_MODES, build_render_product_as_string, render_var_prim_names_by_source, ) -from .publication_buffers import _AsyncWriteBuffers -from .render_lease import OvLibrariesRenderLease, RenderLeaseController -from .visual_materials import OVRTXVisualMaterialWriter +from isaaclab_ov.renderers.publication_buffers import _AsyncWriteBuffers +from isaaclab_ov.renderers.render_lease import OvLibrariesRenderLease, RenderLeaseController +from isaaclab_ov.renderers.visual_materials import OVRTXVisualMaterialWriter +from isaaclab_ov.stage import ( + export_stage_to_string, + points_tensor_from_warp, + xform_tensor_from_numpy, + xform_tensor_from_warp, +) if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline @@ -118,7 +118,7 @@ from isaaclab.renderers.base_renderer import VisualMaterialBatch - from ..stage import OvstageBackend + from isaaclab_ov.stage import OvstageBackend from isaaclab.renderers.camera_render_spec import CameraRenderSpec diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 51c6abd7d96..54ddb03a01a 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -435,11 +435,11 @@ def test_export_stage_homogeneous_keeps_only_env0_prototype(): def test_export_stage_without_keep_env_roots_trims_non_source_env_roots(): - """Whole-environment cloning trims the non-source environment roots themselves. + """The ovstage clone path also trims the non-source env roots themselves. ``ovstage.Stage.clone`` requires every target path to not already exist, so the exported stage - must not retain environment roots that the clone will recreate. Asset-level clone plans - retain typed parent roots on both renderer paths. + must not retain env roots that the clone will recreate. This is the only difference from the + legacy ``renderer.clone_usd`` path, which keeps the roots as placeholders. """ num_envs = 4 stage = _make_multi_env_stage(num_envs) From 804868f8f77ff0aee132453ac7baa21eb96ebb13 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:08:54 -0400 Subject: [PATCH 006/145] Defer partition-visibility export change to its consumer Restore the unconditional showAllPartitionsByDefault=True export at this layer. Preserving an authored False only matters once the XR scene partition (#8790) authors it; here it only changed ordinary OVRTX exports. The change and its changelog entry move to #8790. Also document the RuntimeError that OvstageBackend.close/reset raise while a render lease is active. Review: #8787-6, #8787-8 --- .../changelog.d/renderer-lease.changed.rst | 3 --- source/isaaclab_ov/isaaclab_ov/stage.py | 20 ++++++++++++------- source/isaaclab_ov/test/test_ovrtx_usd.py | 12 +++-------- 3 files changed, 16 insertions(+), 19 deletions(-) delete mode 100644 source/isaaclab_ov/changelog.d/renderer-lease.changed.rst diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst b/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst deleted file mode 100644 index c12b3b953e6..00000000000 --- a/source/isaaclab_ov/changelog.d/renderer-lease.changed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Preserved authored ``showAllPartitionsByDefault`` values in render-stage exports, - including exports without an external lease. Scenes that relied on export forcing - all partitions visible should explicitly author ``showAllPartitionsByDefault=True``. diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index 230d57d4f8f..36a438b4720 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -225,14 +225,22 @@ def commit(self) -> int: return ordinal def close(self) -> None: - """Release borrowers, then the path dictionary and stage.""" + """Release borrowers, then the path dictionary and stage. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ if self._render_lease_owner is not None: raise RuntimeError("Cannot destroy or reset a stage with an active render lease") self.resources.close() self.stage = self.paths = None def reset(self) -> None: - """Recreate the native stage after all consumers detach, retaining the prepared clone operations.""" + """Recreate the native stage after all consumers detach, retaining the prepared clone operations. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ self.close() self.ordinal = self.next_camera_id = 0 with contextlib.ExitStack() as resources: @@ -273,15 +281,13 @@ def export_stage_to_string( export_session.subLayerPaths = [stage.GetSessionLayer().identifier] export_stage = Usd.Stage.Open(stage.GetRootLayer(), export_session) export_stage.SetEditTarget(export_session) - # Preserve authored partition visibility; default to every partition when absent. + # Perspective cameras see every partition; sensor cameras retain their explicit partition. settings_path = export_stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" settings = UsdRender.Settings.Define(export_stage, settings_path) settings.GetPrim().AddAppliedSchema("OmniRtxSettingsGlobalCommonAPI_1") - show_all = settings.GetPrim().CreateAttribute( + settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ) - if not show_all.HasAuthoredValueOpinion(): - show_all.Set(True) + ).Set(True) export_stage.SetMetadata("renderSettingsPrimPath", settings_path) prim_paths = [] if len(clone_env_paths) > 1: diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 54ddb03a01a..522a4d5d479 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -528,20 +528,14 @@ def test_export_stage_heterogeneous_keeps_multiple_sources(): assert 'def Camera "Camera"' not in exported -@pytest.mark.parametrize( - "settings_path, authored_show_all", - [(None, None), ("/Render/AuthoredSettings", False), ("/Render/AuthoredSettings", True)], -) -def test_export_stage_preserves_authored_scene(settings_path, authored_show_all): +@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) +def test_export_stage_preserves_authored_scene(settings_path): """Renderer settings and clone trimming are confined to the exported scene.""" num_envs = 4 stage = _make_multi_env_stage(num_envs) if settings_path is not None: settings = UsdRender.Settings.Define(stage, settings_path) settings.CreateResolutionAttr(Gf.Vec2i(1280, 720)) - settings.GetPrim().CreateAttribute( - "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ).Set(authored_show_all) stage.SetMetadata("renderSettingsPrimPath", settings_path) before = stage.ExportToString() layer = Sdf.Layer.CreateAnonymous() @@ -554,7 +548,7 @@ def test_export_stage_preserves_authored_scene(settings_path, authored_show_all) assert exported.HasAuthoredMetadata("renderSettingsPrimPath") settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) show_all = settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault") - assert show_all and show_all.Get() is (True if authored_show_all is None else authored_show_all) + assert show_all and show_all.Get() is True if settings_path is not None: assert settings.GetPath() == settings_path assert settings.GetAttribute("resolution").Get() == (1280, 720) From 7b3e5b3fc1286d4dc8dc0ab74433af5f298ddbc4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:08:54 -0400 Subject: [PATCH 007/145] Keep poisoned render leases retirable A failed producer write poisoned the whole lease, so Capture's OvxrFramePump.close could not enter the owner lane to destroy its OVXR context, and the lease, runtime and environment were never retired. Poison now gates producer publication only; the owner lane validates identity, thread and activity. After a poisoned lease is released the renderer keeps rejecting ordinary use (except close), because its ordinal was not rebased. The leased renderer also tolerates the rest of the environment: - Stage preparation requested on the application thread (camera sensor initialization) is deferred until the claimed render owner materializes it, when the borrower opts in. - Eager scene updates (lazy_sensor_update=False) return without writing; the lease's scene publisher owns those writes. - Visual-material writers register while the lease is pending and compile their addresses on first publication, so materials finalized by env reset reach the XR scene. Add the lease to the API reference and document Raises on the guarded backend operations. Review: #8787-2, #8787-3, #8787-4, #8787-7, #8787-8 --- .../api/lab_ov/isaaclab_ov.renderers.rst | 15 +++ .../changelog.d/renderer-lease.added.rst | 6 +- .../isaaclab_ov/renderers/ovrtx_renderer.py | 51 ++++++-- .../isaaclab_ov/renderers/render_lease.py | 120 ++++++++++++++++-- .../isaaclab_ov/renderers/visual_materials.py | 19 ++- .../test/test_ovrtx_renderer_contract.py | 109 +++++++++++++++- 6 files changed, 296 insertions(+), 24 deletions(-) diff --git a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst index 4909893f624..05cc82ff841 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst @@ -16,9 +16,11 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. OVRTXRenderer OVRTXRendererCfg OVRTXBackendCfg + OvLibrariesRenderLease .. autoclass:: OVRTXRenderer :show-inheritance: + :members: acquire_ov_libraries_render_lease .. autoclass:: OVRTXRendererCfg :show-inheritance: @@ -26,6 +28,19 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. .. autoclass:: OVRTXBackendCfg :show-inheritance: +Renderer leases +--------------- + +An external consumer, such as the OVXR teleoperation adapter, can borrow one exact OVRTX +renderer and its private OVStage with :meth:`OVRTXRenderer.acquire_ov_libraries_render_lease` +before clone preparation. While the lease is active, ordinary camera rendering, scene writes and +renderer/stage destruction are rejected; eager scene updates are left to the lease's scene +publisher. A failed leased publication poisons publication only; the borrower can still retire +the pair, after which the renderer must be recreated. + +.. autoclass:: OvLibrariesRenderLease + :members: + Simulation-owned stages ----------------------- diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst index 886b2872dc0..2060408dcb7 100644 --- a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst +++ b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst @@ -1,4 +1,6 @@ -* Added application-owned OVRTX/OVStage borrowing for external consumers, with - private-stage selection, capability-checked render-thread ownership transfer, scene publication, and ordered cleanup. +* Added :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` and + :meth:`~isaaclab_ov.renderers.OVRTXRenderer.acquire_ov_libraries_render_lease` for application-owned + OVRTX/OVStage borrowing by external consumers, with private-stage selection, capability-checked + render-thread ownership transfer, deferred Stage materialization, scene publication, and ordered cleanup. * Supported CPU scene transforms and camera buffers alongside GPU rendering. * Published dirty visual materials with leased scene updates, including material-only changes. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 64fc8a6833e..40323737203 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -248,7 +248,11 @@ def __init__(self, cfg: OVRTXBackendCfg): self.renderer = Renderer(native_cfg) def detach(self) -> None: - """Release the borrowed stage before it is rebuilt or destroyed.""" + """Release the borrowed stage before it is rebuilt or destroyed. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ if self._render_lease_owner is not None: raise RuntimeError("Cannot detach a backend with an active render lease") if self.attached: @@ -256,7 +260,11 @@ def detach(self) -> None: self.attached = False def close(self) -> None: - """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward.""" + """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ if self._render_lease_owner is not None: raise RuntimeError("Cannot destroy a backend with an active render lease") if self.renderer is not None: @@ -378,8 +386,17 @@ def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): @property def visual_material_writer(self): - """Return a writer factory once this renderer has a scene consumer.""" - return self._create_visual_material_writer if self._initialized_scene else None + """Return a writer factory once this renderer has a scene consumer. + + A leased pair registers its writer before the borrower binds the Stage, because + environment reset finalizes material consumers first; addresses compile on the + first publication after population. + """ + return ( + self._create_visual_material_writer + if self._initialized_scene or self._render_lease.active is not None + else None + ) def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> OVRTXVisualMaterialWriter: """Create the detached-scene material writer after scene population.""" @@ -407,7 +424,8 @@ def _publish_visual_materials(self) -> Iterator[None]: def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" - self._render_lease.prepare_stage(num_envs) + if not self._render_lease.prepare_stage(num_envs): + return logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -1030,7 +1048,13 @@ def _process_render_products( ) def update_transforms(self) -> None: - """Publish ordinary scene data when the native pair is not leased.""" + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes, so eager scene + updates (for example ``lazy_sensor_update=False``) return without writing. + """ + if self._render_lease.owns_scene_updates(): + return self._render_lease.reject_ordinary_operation("update ordinary transforms") self._update_transforms() @@ -1071,7 +1095,12 @@ def _update_transforms(self) -> None: self._transforms_timestamp = timestamp def update_geometries(self) -> None: - """Publish ordinary scene data when the native pair is not leased.""" + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes. + """ + if self._render_lease.owns_scene_updates(): + return self._render_lease.reject_ordinary_operation("update ordinary geometries") self._update_geometries() @@ -1275,8 +1304,12 @@ def _remove_usd_reference(self, reference: int) -> None: ovstage.population.apply_usd_changes(self.scene.stage, ordinal=self.scene.ordinal) def close(self) -> None: - """Complete borrowed-buffer reads and release bindings; the registry owns the native engine.""" - self._render_lease.reject_ordinary_operation("close the renderer") + """Complete borrowed-buffer reads and release bindings; the registry owns the native engine. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + self._render_lease.reject_ordinary_operation("close the renderer", retiring=True) try: with contextlib.ExitStack() as resources: resources.callback(self._bindings.close) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 4a56beb3bd5..62bb53ce179 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -13,7 +13,7 @@ import threading from collections.abc import Hashable, Iterator from enum import Enum, auto -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, Any, cast if TYPE_CHECKING: from ovrtx import Renderer @@ -69,11 +69,51 @@ def offer_owner_transfer(self) -> object: return owner.offer_owner_transfer(self) def claim_owner_transfer(self, capability: object) -> None: + """Make the calling thread the render owner using the offered capability. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If the capability is stale, already claimed, or the lease is no longer unbound. + """ cast("RenderLeaseController", self._controller).claim_owner_transfer(self, capability) def cancel_owner_transfer(self, capability: object) -> None: + """Withdraw an unclaimed offer on the original owner thread. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If called off the original owner thread or with a stale capability. + """ cast("RenderLeaseController", self._controller).cancel_owner_transfer(self, capability) + def defer_stage_until_owner_transfer(self) -> None: + """Defer Stage materialization requested before the owner transfer is claimed. + + Camera initialization calls ``prepare_stage`` for every registered renderer on the + application thread. With deferral, that call only records the request; the claimed + render owner completes it with :meth:`materialize_deferred_stage`. + """ + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Stage deferral must be selected before Stage materialization") + cast("RenderLeaseController", self._controller).defer_until_transfer = True + + def materialize_deferred_stage(self, stage: Any, num_envs: int) -> bool: + """Complete a deferred Stage preparation on the claimed render owner. + + Args: + stage: The environment USD stage. + num_envs: Number of environments; the lease supports exactly one. + + Returns: + Whether a deferred preparation was completed. ``False`` means nothing was deferred. + """ + return cast("RenderLeaseController", self._controller).materialize_deferred_stage(self, stage, num_envs) + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: """Capture the application-thread scene publisher before render ownership transfers.""" return cast("RenderLeaseController", self._controller).capture_scene_publisher(self) @@ -83,9 +123,11 @@ def ovstage_owner_lane(self) -> Iterator[None]: """Guard one non-reentrant operation on the render-owner thread. The application's publication guard must separately exclude concurrent - producer writes while this operation uses the borrowed stage. + producer writes while this operation uses the borrowed stage. A failed + producer write poisons publication only: the lane stays available so the + borrower can destroy its consumers and retire the pair. """ - self.validate_live() + self._validate_live(allow_poisoned=True) owner = cast("RenderLeaseController", self._controller) with owner.owner_lane("external renderer operation"): yield @@ -101,7 +143,10 @@ def release(self) -> None: self._state = _OvLibrariesRenderLeaseState.RELEASED def validate_live(self) -> None: - """Verify that this lease still refers to its live renderer and stage.""" + """Verify that this lease still refers to its live, unpoisoned renderer and stage.""" + self._validate_live(allow_poisoned=False) + + def _validate_live(self, *, allow_poisoned: bool) -> None: self._validate_active() if self._state is not _OvLibrariesRenderLeaseState.BOUND: raise RuntimeError("OvLibrariesRenderLease is not bound to a prepared Stage") @@ -111,6 +156,7 @@ def validate_live(self) -> None: cast("Renderer", self._renderer), cast("OvStage", self._stage), self._generation, + allow_poisoned=allow_poisoned, ) @classmethod @@ -149,6 +195,15 @@ def __init__(self, owner: RenderLeaseController, lease: OvLibrariesRenderLease) self._closed = False def publication_required(self, physics_step_id: Hashable) -> bool: + """Return whether a new physics step or dirty material requires a scene write. + + Args: + physics_step_id: Hashable identity of the complete physics state to publish. + + Raises: + RuntimeError: If the lease is not bound, the publisher is inactive, or a failed + write poisoned the pair. + """ self._require_application_thread() return self._controller.producer_publication_required(self._lease, physics_step_id) @@ -189,6 +244,8 @@ def __init__(self, renderer: OVRTXRenderer) -> None: self._lane_operation: str | None = None self._last_step_id: Hashable = _NO_PHYSICS_STEP_ID self._poisoned = False + self.defer_until_transfer = False + self.stage_deferred = False def acquire(self) -> OvLibrariesRenderLease: """Reserve the exact OVRTX/OVStage pair before stage preparation.""" @@ -303,10 +360,15 @@ def validate_live( ovrtx_renderer: Renderer, ovstage_instance: OvStage, generation: int, + *, + allow_poisoned: bool = False, ) -> None: - """Validate retained Python-object identity for the live attached pair.""" + """Validate retained Python-object identity for the live attached pair. + + ``allow_poisoned`` admits retirement work after a failed publication. + """ self.validate_active(lease) - if self._poisoned: + if self._poisoned and not allow_poisoned: raise RuntimeError("OvLibrariesRenderLease owner pair is poisoned") if generation != self._generation: raise RuntimeError("OvLibrariesRenderLease has a stale Stage generation") @@ -328,18 +390,33 @@ def release(self, lease: OvLibrariesRenderLease) -> None: self._publisher.close() self._publisher = None self._transfer_claimed = False + self.defer_until_transfer = False + self.stage_deferred = False self.active = None if self._renderer.backend is not None: self._renderer.backend._render_lease_owner = None if self._renderer.scene is not None: self._renderer.scene._render_lease_owner = None - def reject_ordinary_operation(self, operation: str) -> None: - """Reject ordinary OVRTX operations while an external lease is active.""" + def reject_ordinary_operation(self, operation: str, *, retiring: bool = False) -> None: + """Reject ordinary OVRTX operations while an external lease is active. + + A pair poisoned by a failed leased write keeps rejecting ordinary use after + release, because its ordinal is not rebased; only ``retiring`` operations + (closing the renderer) are admitted so the pair can be recreated. + """ if self.active is not None or ( self._renderer.backend is not None and self._renderer.backend._render_lease_owner is not None ): raise RuntimeError(f"Cannot {operation} while an OvLibrariesRenderLease is active") + if self._poisoned and not retiring: + raise RuntimeError( + f"Cannot {operation}: a failed leased publication poisoned this OVRTX/OVStage pair; recreate it" + ) + + def owns_scene_updates(self) -> bool: + """Whether an active lease's scene publisher owns transform and geometry writes.""" + return self.active is not None def _next_ordinal(self) -> int: """Return ``floor + 1`` from the exact renderer-owned Stage.""" @@ -382,10 +459,19 @@ def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, phys raise RuntimeError("Producer scene publication failed; recreate the exact pair") from exc self._last_step_id = physics_step_id - def prepare_stage(self, num_envs: int) -> None: - """Validate leased preparation; ordinary rendering is immediately allowed.""" + def prepare_stage(self, num_envs: int) -> bool: + """Validate or defer leased preparation; ordinary rendering is immediately allowed. + + Returns: + Whether the renderer should materialize the Stage now. + """ lease = self.active if lease is not None: + if self.defer_until_transfer and not self._transfer_claimed: + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + self.stage_deferred = True + return False self.require_owner_thread("prepare the leased Stage") if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") @@ -393,11 +479,25 @@ def prepare_stage(self, num_envs: int) -> None: raise RuntimeError("The render lease requires exactly one environment") if not self._renderer._use_ovstage or self._renderer.backend._render_lease_owner is not self._renderer: raise RuntimeError("The render lease requires its reserved private OVStage renderer") + return True + + def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: + """Materialize a deferred Stage after render-thread ownership transfer.""" + self.require_owner_thread("materialize a deferred Stage") + if not self.stage_deferred: + return False + if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Deferred Stage is not eligible for materialization") + if not self._transfer_claimed: + raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") + self._renderer.prepare_stage(stage, num_envs) + return True def bind_stage(self) -> None: """Bind the prepared native pair after renderer initialization succeeds.""" lease = self.active if lease is not None: + self.stage_deferred = False self._generation += 1 lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 38b0505d6f9..ee1c75580fc 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -62,8 +62,18 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, start = end self._device = str(device) self._event = wp.Event(device=self._device) + self._pending_groups = groups + self._resources = contextlib.ExitStack() + if renderer._initialized_scene: + self._compile_addresses(renderer) + + def _compile_addresses(self, renderer: OVRTXRenderer) -> None: + """Bind addresses only after the leased or ordinary scene has been populated.""" + if not renderer._initialized_scene: + raise RuntimeError("Visual materials require a populated renderer scene before publication.") + addresses = [] with contextlib.ExitStack() as resources: - for channel, attribute_name, shader_paths, rows, dtype, shape in groups: + for channel, attribute_name, shader_paths, rows, dtype, shape in self._pending_groups: if renderer._use_ovstage: address = resources.enter_context(renderer.scene.query(shader_paths)) else: @@ -76,8 +86,10 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, flags=BindingFlag.OPTIMIZE, ) resources.callback(address.unbind) - self._addresses.append((channel, address, attribute_name, rows)) + addresses.append((channel, address, attribute_name, rows)) self._resources = resources.pop_all() + self._addresses = addresses + self._pending_groups = [] @property def has_pending_updates(self) -> bool: @@ -97,6 +109,8 @@ def publish(self) -> None: if not channels: return renderer = self._renderer_ref() + if self._pending_groups: + self._compile_addresses(renderer) operations = [] try: for channel, address, attribute_name, rows in self._addresses: @@ -157,5 +171,6 @@ def close(self) -> None: logger.warning("Error draining in-flight render before material release: %s", error) finally: self._addresses.clear() + self._pending_groups.clear() self._dirty_channels.clear() self._buffers.clear() diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 2e3fa03f601..b2474e0e963 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1564,6 +1564,109 @@ def test_render_lease_complete_publication_deduplicates_and_poison_fails_closed( lease.release() +def test_render_lease_poison_blocks_publication_but_not_retirement(leased_renderer, monkeypatch): + """A failed producer write must not prevent the borrower from retiring the pair (Capture's close order).""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", MagicMock(side_effect=ValueError("write failed"))) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + with pytest.raises(RuntimeError, match="scene publication failed"): + publisher.write_complete_scene(19, "step-1") + with pytest.raises(RuntimeError, match="unavailable"): + publisher.publication_required("step-2") + with pytest.raises(RuntimeError, match="poisoned"): + lease.validate_live() + # OvxrFramePump.close: close publication, destroy the OVXR context inside the owner lane, release the lease. + publisher.close() + with lease.ovstage_owner_lane(): + pass + ordinal = renderer.scene.ordinal + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene.ordinal == ordinal # The abandoned ordinal is not rebased onto a sealed floor. + with pytest.raises(RuntimeError, match="poisoned this OVRTX/OVStage pair; recreate it"): + renderer.render_batch((object(),)) + with pytest.raises(RuntimeError, match="recreate it"): + renderer.update_transforms() + renderer.close() + + +def test_render_lease_release_rebases_ordinal_and_restores_ordinary_use(leased_renderer): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + floor = renderer.scene.stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 41 + lease.release() + assert renderer.scene.ordinal == 42 + renderer.update_transforms() + renderer.update_geometries() + + +@pytest.mark.parametrize( + ("variable", "value"), [("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0"), ("ISAAC_LAB_SHARE_OVSTAGE", "1")] +) +def test_render_lease_rejects_explicit_stage_opt_out(leased_renderer, monkeypatch, variable, value): + monkeypatch.setenv(variable, value) + with pytest.raises(RuntimeError, match="OVSTAGE=1|private rendering OVStage"): + leased_renderer.acquire_ov_libraries_render_lease() + + +def test_render_lease_defers_application_thread_stage_until_owner_transfer(leased_renderer): + """Camera initialization prepares every renderer on the application thread before XR transfers ownership.""" + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + lease.defer_stage_until_owner_transfer() + _prepare_leased_renderer(renderer, bind=False) + context = renderer._sim.render_context + context.ensure_prepare_stage(renderer._sim.stage, 1) + assert not renderer._initialized_scene + assert renderer._render_lease.stage_deferred + # The deferred call must leave the pair eligible for the XR worker handoff. + publisher = lease.capture_scene_publisher() + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + assert not lease.materialize_deferred_stage(renderer._sim.stage, 1) + lease.validate_live() + publisher.close() + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._initialized_scene + + +def test_render_lease_eager_scene_updates_defer_to_the_publisher(leased_renderer, monkeypatch): + """``lazy_sensor_update=False`` refreshes every renderer; the leased renderer's publisher owns those writes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + transforms, geometries = MagicMock(), MagicMock() + monkeypatch.setattr(renderer, "_update_transforms", transforms) + monkeypatch.setattr(renderer, "_update_geometries", geometries) + renderer._sim.render_context.update_scene_state(1) + transforms.assert_not_called() + geometries.assert_not_called() + lease.release() + renderer._sim.render_context.update_scene_state(2) + transforms.assert_called_once_with() + geometries.assert_called_once_with() + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA material buffers") def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch): """Real material composition and the lease publisher must reach native writes, including before physics changes.""" @@ -1571,7 +1674,9 @@ def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monke lease = renderer.acquire_ov_libraries_render_lease() publisher = lease.capture_scene_publisher() context = renderer._sim.render_context - _prepare_leased_renderer(renderer) + # Real lifecycle: clone preparation, then environment reset finalizes material consumers, + # then the XR borrower binds the Stage. + _prepare_leased_renderer(renderer, bind=False) material = types.SimpleNamespace( channels=("color",), is_per_env=False, @@ -1584,6 +1689,8 @@ def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monke ) context.register_visual_material(material) context.finalize_consumers([]) + assert renderer._visual_material_writer_ref is not None + renderer.prepare_stage(renderer._sim.stage, 1) stage = renderer.scene.stage try: for ordinal, color in ((23, [0.0, 1.0, 0.0]), (24, [0.0, 0.0, 1.0])): From 2342d8f26ce63e27e0af434a852a1d9538703d13 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:08:54 -0400 Subject: [PATCH 008/145] Split OVXR config and type the anchor contract - Move OvLibrariesXrConfig to the runtime-free isaaclab_teleop.ovxr_cfg so launchers can build it without Capture's OVXR stack, and raise a clear error naming the install guide when the pinned Capture release lacks ovxr_session_manager. Skip the adapter tests in that case. - Ship the kitless install guide with this layer. - Replace the XrAnchorManager hint with the OvLibrariesXrSession.ExternalAnchorState protocol and reject other objects on the caller before the worker claims the lease. - Validate the environment's sim device and dt against the prepared context, and opt the bootstrap into deferred Stage materialization. - Wait on the SceneData stream instead of the whole device when reading anchor reference transforms. - Keep the stub-bound adapter out of sys.modules after the fixture. - Add tests for poisoned retirement in Capture's close order, anchor seeding, epoch changes, re-adoption and protocol rejection. Review: #8787-1, #8787-5, #8787-8, #8787-9, #8787-10 --- .../source/api/lab_teleop/isaaclab_teleop.rst | 11 + .../changelog.d/kitless-ovxr.added.rst | 8 +- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 35 +++- .../isaaclab_teleop/docs/kitless_xr_source.md | 63 ++++++ .../isaaclab_teleop/__init__.pyi | 3 +- .../isaaclab_teleop/ov_libraries_xr.py | 147 +++++++------ .../isaaclab_teleop/ovxr_cfg.py | 70 +++++++ .../isaaclab_teleop/ovxr_session.py | 59 +++++- .../test/test_ov_libraries_xr.py | 198 +++++++++++++++++- 9 files changed, 507 insertions(+), 87 deletions(-) create mode 100644 source/isaaclab_teleop/docs/kitless_xr_source.md create mode 100644 source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 26ceeec6f5b..149ef166652 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -22,6 +22,8 @@ isaaclab_teleop XrCfg XrAnchorRotationMode XrAnchorSynchronizer + OvLibrariesXrConfig + OvLibrariesXrSession .. rubric:: Functions @@ -61,6 +63,15 @@ XR Camera Feedback .. autoclass:: XrCameraFeedSession :members: +Kitless OVXR Session +-------------------- + +.. autoclass:: OvLibrariesXrConfig + :members: + +.. autoclass:: OvLibrariesXrSession + :members: + Device ------ diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst index 646b66b93b0..f6bd105ca1c 100644 --- a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst +++ b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst @@ -1,4 +1,6 @@ -* Added an OVXR application adapter and session bootstrap that borrow the simulation-owned - OVRTX renderer and OVStage, publish scene updates, and coordinate render-thread ownership - and shutdown with Isaac Capture. +* Added :class:`~isaaclab_teleop.OvLibrariesXrSession`, an OVXR application adapter and session + bootstrap that borrow the simulation-owned OVRTX renderer and OVStage, publish scene updates, + and coordinate render-thread ownership and shutdown with Isaac Capture. Publication failures close admission, and failed lease retirement retains borrowed resources. +* Added :class:`~isaaclab_teleop.OvLibrariesXrConfig` in :mod:`isaaclab_teleop.ovxr_cfg`, which + imports without the native OVXR packages. diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 4b10b2b20a8..3078be76127 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -33,13 +33,40 @@ shutdown and retain the borrowed resources until their retirement is proved. The lease currently requires one environment and a private rendering OVStage. Ownership transfer is single-use. Failed retirement must preserve resources for the owning thread to finish cleanup; a failed drain is not successful -shutdown. Importing the adapter requires the compatible Capture OVXR packages. +shutdown. Importing `OvLibrariesXrSession` requires a Capture release with OVXR +session support; see [Installing kitless XR](kitless_xr_source.md). The +`OvLibrariesXrConfig` dataclass lives in `isaaclab_teleop.ovxr_cfg` and imports +without those packages. A failed scene write or seal stops further publication. Recreate the session after retiring the failed pair; retrying the same generation cannot return an -older scene as if the failed update succeeded. During shutdown, publication -storage drains before lease guards are removed. The adapter retains the -environment if lease retirement cannot establish completion. +older scene as if the failed update succeeded. The failure poisons producer +publication only: the render-owner lane stays available so Capture can destroy +its OVXR context and the adapter can release the lease and close the +environment. After a poisoned lease is released, the renderer keeps rejecting +ordinary use (except closing it) because its write ordinal was not rebased. +During shutdown, publication storage drains before lease guards are removed. +The adapter retains the environment if lease retirement cannot establish completion. + +## Environment requirements + +- `OvLibrariesXrSession.prepare()` creates the `SimulationContext` from the + `sim_cfg` it receives, and the environment reuses that context. Pass the + environment's own `env_cfg.sim`; `start_render_worker()` rejects an + environment whose simulation device or time step differs. +- Only environments that accept a pre-created simulation context are + supported. Manager-based environments do; `DirectRLEnv` always creates its own + context and fails with "Simulation context already exists". +- Camera sensors on other renderers are supported: their initialization + prepares every registered renderer on the application thread, and the leased + renderer defers its Stage until the XR worker claims ownership. A camera that + resolves to the leased renderer itself (for example an OVRTX camera with the + default `OVRTXRendererCfg()`) is rejected while the lease is active. +- Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its + scene publisher writes the same physics state at the reserved ordinal. +- Dynamic anchoring takes any object implementing + `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects + other objects before the worker claims the lease. ## Backend implementation diff --git a/source/isaaclab_teleop/docs/kitless_xr_source.md b/source/isaaclab_teleop/docs/kitless_xr_source.md new file mode 100644 index 00000000000..389c1119ecd --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr_source.md @@ -0,0 +1,63 @@ +# Installing kitless XR + +Use Python 3.12 on Linux x86-64. The Isaac Capture wheel includes the +OVXR session integration. + +Complete the CloudXR, client, network and license prerequisites before +starting a session. + +From the Isaac Lab root, select the Capture version in an `ovxr` project +extra. Lab already configures NVIDIA's public Python index and allows prereleases: + +```bash +uv add --optional ovxr --frozen \ + "isaaccapture[ovxr,retargeters,ui,cloudxr]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +uv add --optional ovxr --frozen \ + "isaaclab-dev[ov] ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-teleop ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-mimic ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "ipywidgets>=8.1.5 ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +``` + +The `ov` extra supplies the compatible OVRTX, OVStage, OVPhysX and PhysX USD +schemas. Keep its provider requirements and the workspace package metadata unchanged. + +Add these mutually exclusive selections to `tool.uv.conflicts`, preserving existing entries: + +```toml +[tool.uv] +conflicts = [ + [{ extra = "ovxr" }, { extra = "teleop" }], + [{ extra = "ovxr" }, { extra = "isaacsim" }], + [{ extra = "ovxr" }, { extra = "viser" }], + [{ extra = "ovxr" }, { extra = "all" }], +] +``` + +The `teleop` extra selects Capture packages for Kit and non-Kit input +workflows. Use the separate `ovxr` selection for the XR commands here. +Isaac Sim and Viser require older WebSocket versions than this Capture +release, so these selections cannot be combined. The `all` extra +includes Viser. + +Resolve the selected environment and check its dependencies: + +```bash +uv lock --refresh-package isaaccapture --upgrade-package isaaccapture \ + --upgrade-package numpy==2.3.5 --upgrade-package packaging==23.2 +uv sync --python 3.12 --locked --extra ovxr +uv run --extra ovxr python -c 'import importlib.metadata as m; print(m.version("isaaccapture"), m.version("ovxr"))' +uv pip check +``` + +The lock selects compatible NumPy 2.3.5 and packaging 23.2 versions without +changing project-wide overrides. OVPhysX declares `packaging<24`; the explicit +selection above satisfies its metadata in this non-Kit selection. Stop applications before changing selections. +The Kit and OVXR provider selections are different; preserve the source metadata +and lockfile when changing between them, and check the resulting dependency graph. +Repeat this selection if a later lock upgrade replaces it. Isaac Capture's +`ovxr` extra selects OVXR and its compatible OVRTX/OVStage dependencies; see its +OVXR guide for application version overrides. + +Continue with the [OVXR application adapter](OVXR_ADAPTER.md). diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index ce01c7f56cf..1f0da9f8be1 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -68,7 +68,8 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device -from .ov_libraries_xr import OvLibrariesXrConfig, OvLibrariesXrSession +from .ov_libraries_xr import OvLibrariesXrSession +from .ovxr_cfg import OvLibrariesXrConfig from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index bae4e0a0c4a..6f1dd4aa54d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -9,17 +9,27 @@ import contextlib import logging -import math import re import threading import time -from collections.abc import Hashable, Iterable -from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Any, Literal - -from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger +from collections.abc import Callable, Hashable, Iterable +from dataclasses import replace +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +try: + from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger +except ModuleNotFoundError as exc: + if not (exc.name or "").startswith("isaaccapture"): + raise + raise ModuleNotFoundError( + "OvLibrariesXrSession requires an Isaac Capture release with OVXR session support" + " (isaaccapture[ovxr]>=1.7.93a1). The 'teleop' extra selects the Kit release; install the" + " kitless selection described in source/isaaclab_teleop/docs/kitless_xr_source.md.", + name=exc.name, + ) from exc from isaaclab_ov.renderers import OVRTXRendererCfg +from .ovxr_cfg import OvLibrariesXrConfig from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation if TYPE_CHECKING: @@ -29,14 +39,10 @@ from isaaclab.sim import SimulationCfg - from .xr_anchor_manager import XrAnchorManager +__all__ = ["OvLibrariesXrConfig", "OvLibrariesXrSession"] logger = logging.getLogger(__name__) -_DEFAULT_OVXR_CAMERA_PATHS = ( - "/OVXRGeneratedCameras/OVXR_LeftEyeCamera", - "/OVXRGeneratedCameras/OVXR_RightEyeCamera", -) _DEFAULT_OVXR_RENDER_PRODUCT_PATHS = ( "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", @@ -220,55 +226,6 @@ def destroy(self) -> None: self._closed = True -@dataclass -class OvLibrariesXrConfig: - """Isaac Lab configuration for an IsaacTeleop-owned OVXR session.""" - - application_name: str = "isaaclab-kitless-teleop" - """Application identifier passed to the OpenXR runtime.""" - session_mode: Literal["ar", "vr"] = "ar" - """OpenXR presentation mode: augmented or virtual reality.""" - renderer_settings: str = "--/app/hydra/aperture/conform=stretch" - """Process-global OVXR renderer settings applied before renderer construction.""" - cloudxr_env_file: str | None = None - """CloudXR environment-file path; None leaves runtime launch to the caller.""" - auto_launch_cloudxr: bool = True - """Whether the session owner launches the CloudXR runtime automatically.""" - render_products: tuple[str, ...] = () - """Stereo RenderProduct paths; an empty selection uses OVXR's default paths.""" - camera_paths: tuple[str, ...] = _DEFAULT_OVXR_CAMERA_PATHS - """Exactly two non-empty USD camera paths, ordered left eye then right eye.""" - create_render_products: bool = True - """Whether to declare and configure the selected stereo rendering prims.""" - meters_per_unit: float = 1.0 - """Finite positive scene scale [m per USD unit].""" - depth_near_z: float = 0.0 - """Near depth distance [m]; both depth distances zero select provider defaults.""" - depth_far_z: float = 0.0 - """Far depth distance [m]; explicit distances must satisfy 0 < near < far.""" - tracking_anchor_path: str = "" - """USD tracking-anchor path; an empty path uses the stage origin.""" - host_wait_render_output: bool = True - """Whether OVXR waits for render output before returning from frame rendering.""" - extra_openxr_extensions: tuple[str, ...] = () - """Additional OpenXR extension names requested when creating the session.""" - - def __post_init__(self) -> None: - if self.session_mode not in ("ar", "vr"): - raise ValueError("session_mode must be 'ar' or 'vr'") - if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: - raise ValueError("meters_per_unit must be finite and positive") - if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( - not math.isfinite(self.depth_near_z) - or not math.isfinite(self.depth_far_z) - or self.depth_near_z <= 0.0 - or self.depth_far_z <= self.depth_near_z - ): - raise ValueError("depth_near_z and depth_far_z must both be zero or finite with 0 < near < far") - if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): - raise ValueError("Kitless stereo requires exactly two non-empty OVXR camera paths") - - class _IsaacLabOvxrApplicationInterface(OvxrApplicationInterface): """Supply only the renderer/Stage/coordinator and simulation publication.""" @@ -278,7 +235,7 @@ def __init__( env: Any, cfg: OvLibrariesXrConfig, hidden_visual_prims: Iterable[str], - external_anchor_state: XrAnchorManager | None = None, + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, ) -> None: self._bootstrap = bootstrap self._env = env @@ -454,7 +411,7 @@ def commit(matrices: np.ndarray) -> None: if self._external_anchor.commit_external_pose(reference, dt, epoch, commit): self._last_anchor_frame_time = now - def set_external_anchor_state(self, state: XrAnchorManager | None) -> None: + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: """Replace a replay device's policy while preserving native query ownership.""" if self._external_anchor is not None: # A new replay manager must not revive a failed borrowed native pair. @@ -516,6 +473,53 @@ def abort(self) -> None: class OvLibrariesXrSession: """Thin Isaac Lab adapter around IsaacTeleop's OVXR session.""" + @runtime_checkable + class ExternalAnchorState(Protocol): + """Dynamic XR-anchor policy sealed together with complete scene publications. + + :class:`~isaaclab_teleop.xr_anchor_manager.XrAnchorManager` implements this protocol + when it is constructed for an OVXR session. + """ + + @property + def external_anchor_paths(self) -> tuple[str, ...]: + """Anchor Xform paths written together at each committed pose.""" + ... + + @property + def external_reference_path(self) -> str | None: + """Physics prim whose world transform drives the anchor.""" + ... + + @property + def external_epoch(self) -> tuple[int, bool]: + """Reset identity and whether that epoch still needs its seeding publication. + + Raises: + RuntimeError: If a previous anchor publication failed. + """ + ... + + def commit_external_pose( + self, + reference: np.ndarray, + rendering_dt: float, + epoch: int, + commit: Callable[[np.ndarray], None], + ) -> bool: + """Advance the policy from ``reference`` and seal its matrices through ``commit``. + + Args: + reference: World transform of :attr:`external_reference_path` [4, 4]. + rendering_dt: Elapsed XR-frame time since the last committed pose [s]. + epoch: Reset identity read from :attr:`external_epoch`. + commit: Callable that writes and seals the anchor matrices [N, 4, 4]. + + Returns: + ``False`` when ``epoch`` is stale and nothing was committed. + """ + ... + def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrConfig) -> None: self._bootstrap = bootstrap self._cfg = cfg @@ -562,7 +566,7 @@ def start_render_worker( hidden_visual_prims: Iterable[str] = (), max_blank_frames: int = 1, timeout_s: float = 30.0, - external_anchor_state: XrAnchorManager | None = None, + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, ) -> bool: """Give IsaacTeleop the application pair and report whether XR started. @@ -574,12 +578,18 @@ def start_render_worker( max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. timeout_s: Maximum worker startup wait [s]. external_anchor_state: Dynamic anchor policy seeded from committed scene publications. + + Raises: + TypeError: If ``external_anchor_state`` does not implement :class:`ExternalAnchorState`. """ self._require_open() if self._owner is not None: raise RuntimeError("OVXR async render worker is already started") if isinstance(max_blank_frames, bool) or not isinstance(max_blank_frames, int) or max_blank_frames < 0: raise ValueError("max_blank_frames must be a non-negative integer") + _require_external_anchor_state(external_anchor_state) + # Reject an inconsistent environment before the worker claims the lease. + self._bootstrap.validate_environment(env) from isaaccapture.ovxr_session_manager.ovxr_session import ( OvxrSession, OvxrSessionConfig, @@ -661,9 +671,10 @@ def request_shutdown(self) -> None: if self._owner is not None: self._owner.request_shutdown() - def set_external_anchor_state(self, state: XrAnchorManager | None) -> None: + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: """Adopt the current replay device's anchor policy between runs.""" self._require_started() + _require_external_anchor_state(state) with self._host.publication_guard(): self._host.set_external_anchor_state(state) @@ -687,3 +698,11 @@ def __enter__(self) -> OvLibrariesXrSession: def __exit__(self, *_exc: object) -> None: self.close() + + +def _require_external_anchor_state(state: object) -> None: + """Reject anchor policies that lack the external-publication protocol on the caller's thread.""" + if state is not None and not isinstance(state, OvLibrariesXrSession.ExternalAnchorState): + raise TypeError( + f"external_anchor_state must implement OvLibrariesXrSession.ExternalAnchorState; got {type(state).__name__}" + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py new file mode 100644 index 00000000000..676f399b7e8 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -0,0 +1,70 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for an Isaac Capture-owned OVXR session. + +This module has no runtime dependencies, so launchers can build and validate the +configuration without the native OVXR, OVRTX or OVStage packages. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from typing import Literal + +_DEFAULT_OVXR_CAMERA_PATHS = ( + "/OVXRGeneratedCameras/OVXR_LeftEyeCamera", + "/OVXRGeneratedCameras/OVXR_RightEyeCamera", +) + + +@dataclass +class OvLibrariesXrConfig: + """Isaac Lab configuration for an IsaacTeleop-owned OVXR session.""" + + application_name: str = "isaaclab-kitless-teleop" + """Application identifier passed to the OpenXR runtime.""" + session_mode: Literal["ar", "vr"] = "ar" + """OpenXR presentation mode: augmented or virtual reality.""" + renderer_settings: str = "--/app/hydra/aperture/conform=stretch" + """Process-global OVXR renderer settings applied before renderer construction.""" + cloudxr_env_file: str | None = None + """CloudXR environment-file path; None leaves runtime launch to the caller.""" + auto_launch_cloudxr: bool = True + """Whether the session owner launches the CloudXR runtime automatically.""" + render_products: tuple[str, ...] = () + """Stereo RenderProduct paths; an empty selection uses OVXR's default paths.""" + camera_paths: tuple[str, ...] = _DEFAULT_OVXR_CAMERA_PATHS + """Exactly two non-empty USD camera paths, ordered left eye then right eye.""" + create_render_products: bool = True + """Whether to declare and configure the selected stereo rendering prims.""" + meters_per_unit: float = 1.0 + """Finite positive scene scale [m per USD unit].""" + depth_near_z: float = 0.0 + """Near depth distance [m]; both depth distances zero select provider defaults.""" + depth_far_z: float = 0.0 + """Far depth distance [m]; explicit distances must satisfy 0 < near < far.""" + tracking_anchor_path: str = "" + """USD tracking-anchor path; an empty path uses the stage origin.""" + host_wait_render_output: bool = True + """Whether OVXR waits for render output before returning from frame rendering.""" + extra_openxr_extensions: tuple[str, ...] = () + """Additional OpenXR extension names requested when creating the session.""" + + def __post_init__(self) -> None: + if self.session_mode not in ("ar", "vr"): + raise ValueError("session_mode must be 'ar' or 'vr'") + if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: + raise ValueError("meters_per_unit must be finite and positive") + if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( + not math.isfinite(self.depth_near_z) + or not math.isfinite(self.depth_far_z) + or self.depth_near_z <= 0.0 + or self.depth_far_z <= self.depth_near_z + ): + raise ValueError("depth_near_z and depth_far_z must both be zero or finite with 0 < near < far") + if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): + raise ValueError("Kitless stereo requires exactly two non-empty OVXR camera paths") diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index e3d9a060f52..ef1c7a54848 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -32,7 +32,14 @@ def validate_ovxr_session_preparation() -> None: class OvxrSessionBootstrap: - """Reserve an OVRTX/OVStage lease before a real environment is constructed.""" + """Reserve an OVRTX/OVStage lease before a real environment is constructed. + + The bootstrap creates the :class:`~isaaclab.sim.SimulationContext` from the ``sim_cfg`` + given here, and the environment reuses it. Only environments that accept a pre-created + context are supported (manager-based environments; :class:`~isaaclab.envs.DirectRLEnv` + always creates its own). :meth:`validate_environment` rejects an environment whose + simulation device or time step disagrees with the prepared context. + """ def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None = None) -> None: validate_ovxr_session_preparation() @@ -57,6 +64,10 @@ def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None if not isinstance(renderer, OVRTXRenderer): raise TypeError(f"OVRTXRendererCfg resolved to unexpected renderer {type(renderer).__name__}") lease = renderer.acquire_ov_libraries_render_lease() + # Camera sensors prepare every registered renderer on the application thread + # during environment construction. Only the claimed render owner materializes + # this pair, so that call is recorded and completed in bind_environment(). + lease.defer_stage_until_owner_transfer() except BaseException as preparation_error: sim = SimulationContext.instance() try: @@ -85,23 +96,50 @@ def simulation_context(self) -> SimulationContext: assert self._sim is not None return self._sim + def validate_environment(self, env: Any) -> None: + """Check that a constructed environment uses the prepared simulation context. + + Args: + env: Constructed environment, possibly wrapped. + + Raises: + RuntimeError: If the environment did not reuse the prepared context, has no + completed stage or a non-positive environment count, or its simulation + configuration selects a different device or physics time step [s]. + """ + self._require_open() + unwrapped = getattr(env, "unwrapped", env) + if unwrapped.sim is not self._sim: + raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") + if unwrapped.scene.stage is None: + raise RuntimeError("Environment has no completed InteractiveScene Stage") + num_envs = unwrapped.num_envs + if not isinstance(num_envs, int) or num_envs <= 0: + raise RuntimeError(f"Environment has invalid num_envs: {num_envs!r}") + env_sim_cfg = getattr(getattr(unwrapped, "cfg", None), "sim", None) + prepared_cfg = self._sim.cfg + if env_sim_cfg is not None and env_sim_cfg is not prepared_cfg: + for field in ("device", "dt"): + if getattr(env_sim_cfg, field, None) != getattr(prepared_cfg, field, None): + raise RuntimeError( + f"Environment sim.{field}={getattr(env_sim_cfg, field, None)!r} differs from the OVXR" + f" prepared simulation context ({getattr(prepared_cfg, field, None)!r}); prepare the" + " session with the environment's sim configuration" + ) + def bind_environment(self, env: Any) -> OvLibrariesRenderLease: """Prepare and bind the completed environment Stage to the reserved pair.""" self._require_open() if self._bound: raise RuntimeError("OVXR bootstrap is already bound") try: + self.validate_environment(env) unwrapped = getattr(env, "unwrapped", env) - if unwrapped.sim is not self._sim: - raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") scene = unwrapped.scene - if scene.stage is None: - raise RuntimeError("Environment has no completed InteractiveScene Stage") num_envs = unwrapped.num_envs - if not isinstance(num_envs, int) or num_envs <= 0: - raise RuntimeError(f"Environment has invalid num_envs: {num_envs!r}") assert self._sim is not None and self._lease is not None - self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) + if not self._lease.materialize_deferred_stage(scene.stage, num_envs): + self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) self._lease.validate_live() self._bound = True return self._lease @@ -158,7 +196,10 @@ def get_world_transform(self, prim_path: str) -> np.ndarray: if output.matrices is None: raise RuntimeError(f"SceneData returned no matrix buffer for target frame {prim_path!r}") - wp.synchronize_device(str(output.matrices.device)) + # SceneData converts on the device's current stream. Wait only for that stream so a + # publication does not serialize with unrelated work (such as in-flight XR rendering). + if output.matrices.device.is_cuda: + wp.synchronize_stream(wp.get_stream(output.matrices.device)) world_T_target = np.asarray(output.matrices.numpy()[0], dtype=np.float64) if world_T_target.shape != (4, 4) or not np.all(np.isfinite(world_T_target)): raise RuntimeError(f"Target-frame transform {prim_path!r} is missing or non-finite") diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index a20d347df44..52013f92047 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -18,6 +18,10 @@ import pytest +# The adapter needs a Capture release with OVXR session support. The Kit ``teleop`` extra +# selects an older release without it; skip rather than error in that environment. +pytest.importorskip("isaaccapture.ovxr_session_manager") + class _FakeLease: def __init__(self): @@ -103,6 +107,9 @@ def bind_environment(self, env): self.events.append(("bind", env)) return self.lease + def validate_environment(self, env): + pass + def get_target_frame_T_world(self, path): return ("target", path) @@ -119,7 +126,13 @@ def real_lease_type(): @pytest.fixture def xr_module(monkeypatch, real_lease_type): - """Import the adapter with native renderer dependencies stubbed.""" + """Import the adapter with native renderer dependencies stubbed. + + The stub-bound adapter is removed from ``sys.modules`` and its package on teardown so + later tests import the real module. + """ + import isaaclab_teleop + isaaclab_ov = ModuleType("isaaclab_ov") isaaclab_ov.__path__ = [] renderers = ModuleType("isaaclab_ov.renderers") @@ -133,10 +146,20 @@ def xr_module(monkeypatch, real_lease_type): ovxr_session = importlib.util.module_from_spec(spec) monkeypatch.setitem(sys.modules, spec.name, ovxr_session) spec.loader.exec_module(ovxr_session) - sys.modules.pop("isaaclab_teleop.ov_libraries_xr", None) - module = importlib.import_module("isaaclab_teleop.ov_libraries_xr") - monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) - return module + name = "isaaclab_teleop.ov_libraries_xr" + saved_module = sys.modules.pop(name, None) + saved_attribute = isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + try: + module = importlib.import_module(name) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + yield module + finally: + sys.modules.pop(name, None) + isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + if saved_module is not None: + sys.modules[name] = saved_module + if saved_attribute is not None: + isaaclab_teleop.ov_libraries_xr = saved_attribute def _fake_ovxr(): @@ -604,7 +627,9 @@ def clear_instance(cls): def get_or_create_backend(self, _cfg): if self.failure is not None: raise self.failure - lease = SimpleNamespace(release=lambda: events.append("lease-release")) + lease = SimpleNamespace( + release=lambda: events.append("lease-release"), defer_stage_until_owner_transfer=lambda: None + ) return SimpleNamespace(acquire_ov_libraries_render_lease=lambda: lease) monkeypatch.setattr(module, "SimulationContext", Context) @@ -705,3 +730,164 @@ def test_public_xr_exports_resolve_to_implemented_types(xr_module): assert isaaclab_teleop.OvLibrariesXrConfig is xr_module.OvLibrariesXrConfig assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession + + +def test_poisoned_publication_still_retires_in_capture_order(xr_module, monkeypatch, real_lease_type): + """After a failed scene write, Capture's OvxrFramePump.close must still retire the pair and the env.""" + host, bootstrap, env = _make_host(xr_module, monkeypatch) + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + native = object() + + def failed_transforms(): + raise ValueError("geometry callback failed") + + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _update_transforms=failed_transforms, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + bootstrap.publisher = lease.capture_scene_publisher() + # Stand in for clone preparation and Stage binding of the reserved pair. + renderer.backend = SimpleNamespace(renderer=native, attached=True, _render_lease_owner=renderer) + renderer.scene = SimpleNamespace(stage=stage, ordinal=10, _render_lease_owner=renderer) + renderer._initialized_scene = True + controller._generation = 1 + lease._bind(native, stage, 1) + bootstrap.lease = lease + + def close_bootstrap(): + lease.release() + bootstrap.closed = True + + bootstrap.close = close_bootstrap + host.attach_on_render_thread(host.offer_render_owner_transfer()) + with host.publication_guard(), pytest.raises(RuntimeError, match="recreate the exact pair"): + host.publish_complete_scene("step-1") + + # OvxrFramePump.close(): close publication, destroy the context in the owner lane, then retire. + with host.publication_guard(): + host.close_publication() + with host.publication_guard(): + with host.owner_lane(): + pass + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.closed + assert controller.active is None + assert bootstrap.events[-1] == "environment-close" + + +class _FakeAnchorState: + """Minimal :class:`OvLibrariesXrSession.ExternalAnchorState` with scripted epochs.""" + + external_anchor_paths = ("/World/XrAnchor", "/World/XrRenderAnchor") + external_reference_path = "/World/Robot" + + def __init__(self): + self.epoch = (0, True) + self.commits = [] + self.accept = True + + @property + def external_epoch(self): + return self.epoch + + def commit_external_pose(self, reference, rendering_dt, epoch, commit): + if not self.accept or epoch != self.epoch[0]: + return False + import numpy as np + + self.commits.append((reference.copy(), rendering_dt, epoch)) + commit(np.stack([reference, reference])) + self.epoch = (self.epoch[0], False) + return True + + +def _make_anchor_host(xr_module, monkeypatch, anchor): + import numpy as np + + bootstrap = _FakeBootstrap() + bootstrap.get_world_transform = lambda _path: np.eye(4) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrConfig(), (), external_anchor_state=anchor + ) + host._scene_publisher = bootstrap.publisher + host._ordinal_ledger = _FakeOrdinalLedger(None) + writes = [] + monkeypatch.setattr(host, "_write_anchor_matrices", lambda matrices, ordinal: writes.append(ordinal)) + return host, writes + + +def test_anchor_seed_is_sealed_with_scene_and_frames_use_the_xr_clock(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + times = iter((5.0, 5.25)) + monkeypatch.setattr(xr_module.time, "monotonic", lambda: next(times)) + + seed = host.publish_complete_scene("initial") + assert writes == [seed] == host._ordinal_ledger.published + assert anchor.commits[-1][1] == 0.0 # The seed has no elapsed interval. + host.prepare_frame_scene() + host.prepare_frame_scene() + assert [commit[1] for commit in anchor.commits] == [0.0, 0.0, 0.25] + assert writes == host._ordinal_ledger.published + + +def test_anchor_epoch_change_during_seed_fails_closed(xr_module, monkeypatch): + anchor = _FakeAnchorState() + anchor.accept = False + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + with pytest.raises(RuntimeError, match="anchor reset changed"): + host.publish_complete_scene("initial") + assert writes == [] + assert host._ordinal_ledger.abandoned == [11] + with pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("initial") + + +def test_anchor_state_readoption_requires_the_same_paths(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, _writes = _make_anchor_host(xr_module, monkeypatch, anchor) + host.publish_complete_scene("initial") + replacement = _FakeAnchorState() + host.set_external_anchor_state(replacement) + assert host._anchor_snapshot is None and host._last_anchor_frame_time is None + moved = _FakeAnchorState() + moved.external_anchor_paths = ("/World/Other",) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(moved) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(None) + + +def test_start_rejects_anchor_state_without_the_protocol_before_owner_creation(xr_module, monkeypatch): + """A policy lacking the external-anchor API must fail on the caller, before the worker claims the lease.""" + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + with pytest.raises(TypeError, match="ExternalAnchorState"): + session.start_render_worker(SimpleNamespace(), external_anchor_state=SimpleNamespace(reset=lambda: None)) + assert bootstrap.events == [] + assert isinstance(_FakeAnchorState(), xr_module.OvLibrariesXrSession.ExternalAnchorState) + + +def test_config_imports_without_native_ovxr(monkeypatch): + """Launchers build the configuration without importing Capture's OVXR session stack.""" + monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager", None) + from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig + + assert OvLibrariesXrConfig(session_mode="vr").session_mode == "vr" From da73ee780ec226b936f9ccedba9ee7f788e3a501 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:13:28 -0400 Subject: [PATCH 009/145] Keep unset task physics on the simulation default The live and replay entrypoints replaced sim.physics=None with the abstract PhysicsCfg(), whose class_type is MISSING, so SimulationContext failed in gym.make for every task without a physics selection (all GR1T2/G1 Capture tasks at this layer and any downstream task), and Kit was no longer requested. Leave None so SimulationContext applies its PhysxCfg default. Under --ovxr, reject such a task before launch with the kitless selectors to use. Also: - Read the launcher-recorded kit_visualizer in the RTX/DLSS gate; setup_preset_cli consumes --viz, so args.visualizer never exists. - Do not render on the control thread when OVXR rejects a frame. - Drop a redundant action_state.clear() on the paused path. Review: #8788-1, #8788-2, #8788-7, #8788-8 --- .../entrypoints/_ovxr_launch.py | 29 +++++++ .../entrypoints/teleop_replay_agent.py | 7 +- .../entrypoints/teleop_se3_agent.py | 41 +++++---- .../test/test_teleop_entrypoint_units.py | 87 +++++++++++++++++++ 4 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py new file mode 100644 index 00000000000..0efc91498e3 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -0,0 +1,29 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Launch checks shared by the live and replay OVXR entrypoints.""" + +from __future__ import annotations + +from typing import Any + + +def require_kitless_physics(env_cfg: Any) -> None: + """Reject a task whose physics selection needs the Kit runtime before an OVXR launch. + + A task that leaves ``sim.physics`` unset selects Isaac Sim PhysX, which needs Kit. + The kitless OVXR session cannot run it, so fail before launch with the selector to use. + + Args: + env_cfg: Resolved environment configuration. + + Raises: + ValueError: If ``env_cfg.sim.physics`` is ``None``. + """ + if env_cfg.sim.physics is None: + raise ValueError( + "--ovxr requires a kitless physics preset, but the task leaves sim.physics unset (Isaac Sim" + " PhysX through Kit). Select one, for example physics=ovphysx or physics=newton_mjwarp." + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 9973e5445f4..bbe3207011b 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -295,6 +295,7 @@ poll_control_events, ) from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action +from ._ovxr_launch import require_kitless_physics logger = logging.getLogger(__name__) @@ -1122,12 +1123,10 @@ def _prepare_env_cfg( if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") env_cfg = remove_camera_configs(env_cfg) + require_kitless_physics(env_cfg) elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) - if env_cfg.sim.physics is None: - from isaaclab.physics import PhysicsCfg - - env_cfg.sim.physics = PhysicsCfg() + # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) if args_cli.ovxr: from isaaclab.visualizers import VisualizerBackendCfg diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 20594dfbbed..15199bcbdcf 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -54,6 +54,7 @@ ) from ..action_policy import prepare_teleop_action from ..device_selection import create_teleop_device, select_teleop_cfg +from ._ovxr_launch import require_kitless_physics logger = logging.getLogger(__name__) @@ -205,16 +206,18 @@ def _rtx_rendering_requested(args: argparse.Namespace) -> bool: """Return whether the CLI selects a renderer that actually drives RTX rendering. The RTX/DLSS global settings are only meaningful when something renders through RTX. - That happens when the Kit visualizer is enabled (``--viz kit``), when external cameras - are rendered (on by default; see ``--disable_external_cameras``), or in XR mode (``--xr``). - A pure-headless session with none of these renders nothing. - - This reads the resolved namespace intent rather than any Kit/carb runtime state so the - check keeps working as these scripts grow support for other rendering backends. + That happens when the Kit visualizer is enabled (``--viz kit`` or a task-declared Kit + visualizer), when external cameras are rendered (on by default; see + ``--disable_external_cameras``), or in XR mode (``--xr``). A pure-headless session with + none of these renders nothing. + + This reads the resolved launcher namespace rather than any Kit/carb runtime state. + :func:`~isaaclab.app.launch_simulation` records ``kit_visualizer`` on it; ``--viz`` itself + is consumed by the preset CLI and is not kept on the namespace. """ - visualizers = getattr(args, "visualizer", None) or [] external_cameras = not getattr(args, "disable_external_cameras", False) - return external_cameras or ("kit" in visualizers) or bool(getattr(args, "xr", False)) + kit_visualizer = bool(getattr(args, "kit_visualizer", False)) + return external_cameras or kit_visualizer or bool(getattr(args, "xr", False)) def _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop: bool): @@ -236,10 +239,14 @@ def _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop: bool): return driver.update, driver.stop -def _apply_teleop_frame(env, action, issue: str | None, haptic_update, haptic_stop) -> bool: - """Step one valid action, or render-only while an unsafe tracker frame is held.""" +def _apply_teleop_frame(env, action, issue: str | None, haptic_update, haptic_stop, *, render: bool = True) -> bool: + """Step one valid action, or hold the scene while an unsafe tracker frame is rejected. + + ``render`` is false under OVXR, which owns rendering and republishes the unchanged scene. + """ if issue is not None: - env.sim.render() + if render: + env.sim.render() haptic_stop() return False env.step(action.repeat(env.num_envs, 1)) @@ -315,10 +322,9 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg from .. import remove_camera_configs env_cfg = remove_camera_configs(env_cfg) - if env_cfg.sim.physics is None: - from isaaclab.physics import PhysicsCfg - - env_cfg.sim.physics = PhysicsCfg() + if args_cli.using_ovxr: + require_kitless_physics(env_cfg) + # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) if args_cli.using_ovxr: from isaaclab.visualizers import VisualizerBackendCfg @@ -668,7 +674,9 @@ def run_loop(): action, issue = prepare_teleop_action( env, action, action_conditioner, action_validator, action_state ) - action_applied = _apply_teleop_frame(env, action, issue, haptic_update, haptic_stop) + action_applied = _apply_teleop_frame( + env, action, issue, haptic_update, haptic_stop, render=ov_libraries_xr_session is None + ) if issue is not None: skipped_tracking_frames += 1 if skipped_tracking_frames <= 5 or skipped_tracking_frames % 200 == 0: @@ -678,7 +686,6 @@ def run_loop(): skipped_tracking_frames, ) else: - action_state.clear() if ov_libraries_xr_session is None: env.sim.render() # not stepping: zero haptics so a paused grip stops buzzing diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 7a03d06b8bc..d310b6d86da 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -866,3 +866,90 @@ def validate_launch(selected_cfg, _args): with pytest.raises(ContextReached): module.main() assert calls == ["prepare", "launch", "context"] + + +_UNSET_PHYSICS_TASK = "IsaacContrib-PickPlace-GR1T2-Abs" + + +def _unset_task_physics(monkeypatch, module): + """Resolve the real task configuration, then leave ``sim.physics`` at its default ``None``.""" + real_resolve = module.resolve_task_config + + def resolve(*args, **kwargs): + env_cfg, agent_cfg = real_resolve(*args, **kwargs) + env_cfg.sim.physics = None + return env_cfg, agent_cfg + + monkeypatch.setattr(module, "resolve_task_config", resolve) + + +def test_live_unset_task_physics_keeps_simulation_default(monkeypatch): + """``physics=None`` must reach SimulationContext's PhysxCfg default and launch Kit for it.""" + from isaaclab.app.sim_launcher import validate_launch + + agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, "--disable_external_cameras") + _unset_task_physics(monkeypatch, agent) + env_cfg, _teleop_cfg, _use_isaac_teleop = agent._resolve_env_cfg() + assert env_cfg.sim.physics is None + _configs, kit_sources = validate_launch(env_cfg, vars(agent.args_cli).copy()) + assert kit_sources + + +def test_live_ovxr_rejects_unset_task_physics_before_launch(monkeypatch): + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, "--ovxr") + _unset_task_physics(monkeypatch, agent) + with pytest.raises(ValueError, match="kitless physics preset"): + agent._resolve_env_cfg() + + +@pytest.mark.parametrize("ovxr", [False, True]) +def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monkeypatch, ovxr): + from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + + argv = ["teleop_replay_agent.py", "--task", _UNSET_PHYSICS_TASK, "--replay_file", "recording.mcap"] + if ovxr: + argv.append("--ovxr") + monkeypatch.setattr(sys, "argv", argv) + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + monkeypatch.setattr(replay, "args_cli", None, raising=False) + monkeypatch.setattr(replay, "hydra_overrides", None, raising=False) + replay._parse_args() + _unset_task_physics(monkeypatch, replay) + if ovxr: + with pytest.raises(ValueError, match="kitless physics preset"): + replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) + return + env_cfg, _success = replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) + assert env_cfg.sim.physics is None + assert replay.args_cli.require_kit + + +@pytest.mark.parametrize( + "arguments,expected", + [ + (("--viz", "kit", "--disable_external_cameras"), True), + (("--disable_external_cameras",), False), + ((), True), + ], +) +def test_rtx_gate_reads_the_launcher_kit_visualizer(monkeypatch, arguments, expected): + """``setup_preset_cli`` consumes ``--viz``; the gate must read what the launcher resolved from it.""" + from isaaclab.app.sim_launcher import validate_launch + + agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, *arguments) + assert not hasattr(agent.args_cli, "visualizer") + env_cfg, _teleop_cfg, _use_isaac_teleop = agent._resolve_env_cfg() + # launch_simulation validates against the live namespace, which records kit_visualizer. + validate_launch(env_cfg, vars(agent.args_cli)) + assert agent._rtx_rendering_requested(agent.args_cli) is expected + + +def test_rejected_ovxr_frame_does_not_render_on_the_control_thread(monkeypatch): + agent = _load_teleop_agent(monkeypatch) + env = SimpleNamespace(sim=SimpleNamespace(render=lambda: pytest.fail("rendered under OVXR"))) + stops = [] + assert not agent._apply_teleop_frame( + env, torch.zeros(1, 2), "missing left hand tracking", lambda: None, lambda: stops.append(True), render=False + ) + assert stops == [True] From 8fe1bce80a6eb071de7e25cb28fe6d8ea84e41d1 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:16:48 -0400 Subject: [PATCH 010/145] Apply stack grasp contact tuning on SO101 OvPhysX The Isaac Sim PhysX stack preset sets bounce_threshold_velocity, friction_correlation_distance and solve_articulation_contact_last so object contacts stall the position-driven gripper instead of tunneling. OvPhysxCfg had no equivalent, so the kitless SO101 selection ran without them. OvPhysX consumes the same physxScene attributes from the PhysicsScene prim (see its PhysxSceneAPI population docs); add optional OvPhysxCfg fields that author them, and select the stack values for SO101. Unset fields keep the stage value. Review: #8788-4, #8788-8 --- .../ovphysx-contact-tuning.added.rst | 3 +++ .../changelog.d/ovphysx-contact-tuning.minor | 0 .../isaaclab_ov/physics/ovphysx_manager.py | 11 ++++++-- .../physics/ovphysx_manager_cfg.py | 22 ++++++++++++++++ .../physics/test_ovphysx_manager_lifecycle.py | 25 +++++++++++++++++++ .../config/so101/stack_ik_abs_env_cfg.py | 7 +++++- .../test/core/test_preset_cli.py | 12 ++++----- 7 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst create mode 100644 source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.minor diff --git a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst new file mode 100644 index 00000000000..0400b82d7de --- /dev/null +++ b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst @@ -0,0 +1,3 @@ +* Added ``bounce_threshold_velocity``, ``friction_correlation_distance`` and + ``solve_articulation_contact_last`` to :class:`~isaaclab_ov.physics.OvPhysxCfg`. Unset values keep + the stage's scene settings. diff --git a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.minor b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py index 73cd1f00ec4..67ce8451293 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager.py @@ -811,8 +811,8 @@ def _configure_physx_scene_prim(self, scene_prim, cfg, device: str) -> None: """Apply the OVPhysX scene configuration to the simulation's physics scene prim. The schema, dynamics device and broadphase (see :func:`_set_scene_dynamics_device`), - scene-query-support, and solver-determinism/accuracy attributes are applied regardless of - device. The GPU buffer-capacity attributes are applied only to GPU scenes. + scene-query-support, solver-determinism/accuracy and selected contact attributes are applied + regardless of device. The GPU buffer-capacity attributes are applied only to GPU scenes. Args: scene_prim: The /World/PhysicsScene prim to configure. @@ -838,6 +838,13 @@ def _configure_physx_scene_prim(self, scene_prim, cfg, device: str) -> None: scene_prim.CreateAttribute("physxScene:enableExternalForcesEveryIteration", Sdf.ValueTypeNames.Bool).Set( cfg.enable_external_forces_every_iteration ) + for attr, value_type, value in ( + ("bounceThreshold", Sdf.ValueTypeNames.Float, cfg.bounce_threshold_velocity), + ("frictionCorrelationDistance", Sdf.ValueTypeNames.Float, cfg.friction_correlation_distance), + ("solveArticulationContactLast", Sdf.ValueTypeNames.Bool, cfg.solve_articulation_contact_last), + ): + if value is not None: + scene_prim.CreateAttribute(f"physxScene:{attr}", value_type).Set(value) if device == "gpu" and cfg is not None: for attr, val in [ diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py index c9c4a0d3274..02b56a2f5a4 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py @@ -89,6 +89,28 @@ class OvPhysxCfg(PhysicsCfg): setting can affect simulated velocities and contact forces. """ + bounce_threshold_velocity: float | None = None + """Relative contact velocity below which contacts do not bounce [m/s]. + + ``None`` leaves the stage value (OvPhysX's default is effectively 0, so every contact bounces). + Authored as ``physxScene:bounceThreshold``. + """ + + friction_correlation_distance: float | None = None + """Distance below which contact points merge into one friction anchor [m]. + + ``None`` leaves the stage value (OvPhysX's default is 0.025 / metersPerUnit). + Authored as ``physxScene:frictionCorrelationDistance``. + """ + + solve_articulation_contact_last: bool | None = None + """Whether the solver resolves articulation contacts after other constraints. + + Enable it so object contacts can stall position-driven grippers before they tunnel through a + grasp. ``None`` leaves the stage value (off by default). Authored as + ``physxScene:solveArticulationContactLast``. + """ + gpu_max_rigid_contact_count: int = 2**23 """Size of the GPU rigid-body contact buffer.""" diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py index dcc61931efa..797058724b4 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py @@ -555,6 +555,31 @@ def test_scene_settings(monkeypatch, manager, device, rate, override, expected): assert prim.GetAttribute("physxScene:enableSceneQuerySupport").Get() is sim_cfg.enable_scene_query_support +@pytest.mark.parametrize("device", ["cpu", "gpu"]) +def test_scene_contact_settings_are_authored_only_when_selected(monkeypatch, manager, device): + """Optional contact tuning reaches the scene prim; unset fields keep the stage value.""" + from isaaclab_ov.physics.ovphysx_manager_cfg import OvPhysxCfg + + from pxr import Usd + + monkeypatch.setattr( + manager, "_sim", SimpleNamespace(cfg=SimpleNamespace(dt=1.0 / 60.0, enable_scene_query_support=False)) + ) + stage = Usd.Stage.CreateInMemory() + default_prim = stage.DefinePrim("/World/DefaultScene", "PhysicsScene") + manager._configure_physx_scene_prim(default_prim, OvPhysxCfg(), device) + for attr in ("bounceThreshold", "frictionCorrelationDistance", "solveArticulationContactLast"): + assert not default_prim.GetAttribute(f"physxScene:{attr}").HasAuthoredValue() + prim = stage.DefinePrim("/World/PhysicsScene", "PhysicsScene") + cfg = OvPhysxCfg( + bounce_threshold_velocity=0.01, friction_correlation_distance=0.00625, solve_articulation_contact_last=True + ) + manager._configure_physx_scene_prim(prim, cfg, device) + assert prim.GetAttribute("physxScene:bounceThreshold").Get() == pytest.approx(0.01) + assert prim.GetAttribute("physxScene:frictionCorrelationDistance").Get() == pytest.approx(0.00625) + assert prim.GetAttribute("physxScene:solveArticulationContactLast").Get() is True + + @pytest.mark.parametrize(("device", "gpu_dynamics", "broadphase"), [("cpu", False, "MBP"), ("gpu", True, "GPU")]) def test_scenes_author_device_dynamics_and_broadphase(monkeypatch, manager, device, gpu_dynamics, broadphase): """Every physics scene selects the simulation device, since CPU and GPU scenes share one process.""" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py index 4381e241ce8..0302b4cde78 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/stack/config/so101/stack_ik_abs_env_cfg.py @@ -197,7 +197,12 @@ def __post_init__(self): isaacsim_physx=self.sim.physics.isaacsim_physx, physx=self.sim.physics.physx, newton_mjwarp=self.sim.physics.newton_mjwarp, - ovphysx=OvPhysxCfg(), + # Match the stack family's Isaac Sim PhysX grasp tuning on the kitless OvPhysX backend. + ovphysx=OvPhysxCfg( + bounce_threshold_velocity=self.sim.physics.isaacsim_physx.bounce_threshold_velocity, + friction_correlation_distance=self.sim.physics.isaacsim_physx.friction_correlation_distance, + solve_articulation_contact_last=self.sim.physics.isaacsim_physx.solve_articulation_contact_last, + ), ) # Switch to a stiffer PD controller so the arm tracks IK targets well. Replace the robot diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index cd50665f7b7..28ce5dd527b 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -291,11 +291,11 @@ def test_teleop_tasks_resolve_public_physics_selectors(physics): expected_type = {None: PhysxCfg, "isaacsim_physx": PhysxCfg, "ovphysx": OvPhysxCfg, "newton_mjwarp": NewtonCfg} assert isinstance(cfg.sim.physics, expected_type[physics]) assert cfg.scene.num_envs == 1 and cfg.sim.device == "cpu" - if "SO101" in task_name: - assert cfg.scene.robot.spawn.variants["Physics"] == ("physics" if physics == "newton_mjwarp" else "physx") + assert cfg.scene.robot.spawn.variants["Physics"] == ("physics" if physics == "newton_mjwarp" else "physx") if physics == "newton_mjwarp": assert isinstance(cfg.sim.physics.solver_cfg, MJWarpSolverCfg) - if "SO101" in task_name: - assert cfg.sim.physics.solver_cfg.iterations == 100 - assert cfg.sim.physics.solver_cfg.impratio == 10.0 - assert cfg.sim.physics.solver_cfg.nconmax == 600 + if physics == "ovphysx": + # The kitless OvPhysX selection keeps the stack family's PhysX grasp tuning. + assert cfg.sim.physics.bounce_threshold_velocity == pytest.approx(0.01) + assert cfg.sim.physics.friction_correlation_distance == pytest.approx(0.00625) + assert cfg.sim.physics.solve_articulation_contact_last is True From f6b3c796d7df5219ac081febff53371cb7108288 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:16:48 -0400 Subject: [PATCH 011/145] Complete 8788 changelog and test conventions - Record launch_simulation(cleanup_context=...) as a minor addition. - List the new IsaacTeleopCfg/XrCfg hooks and IsaacTeleopDevice API. - Record the non-zero exit, --disable_external_cameras and replay backend selection changes with migration guidance. - File the SO101 reset-distribution change under changed. - Correct the xr_anchor_path docstring: OVXR consumes the render anchor. - Skip the SO101 pipeline test when Capture's retargeting internals are absent, and drop its no-op aliases. Review: #8788-5, #8788-6, #8788-8 --- .../kitless-ovxr-lifecycle.added.rst | 2 ++ .../kitless-ovxr-lifecycle.fixed.rst | 1 - .../changelog.d/kitless-ovxr-lifecycle.minor | 0 .../changelog.d/kitless-teleop.changed.rst | 3 +++ .../changelog.d/kitless-teleop.fixed.rst | 4 ++-- .../test/contrib/test_so101_teleop_pipeline.py | 18 ++++++++---------- .../changelog.d/ovxr-live-replay.added.rst | 8 ++++++++ .../changelog.d/ovxr-live-replay.changed.rst | 8 ++++++++ .../isaaclab_teleop/isaac_teleop_device.py | 6 +++++- 9 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst create mode 100644 source/isaaclab/changelog.d/kitless-ovxr-lifecycle.minor create mode 100644 source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst new file mode 100644 index 00000000000..3b4e265d5cf --- /dev/null +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.added.rst @@ -0,0 +1,2 @@ +* Added the ``cleanup_context`` keyword to :func:`~isaaclab.app.launch_simulation` so an + asynchronous XR owner can drain before the simulation context is released. diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst index 4274dd1920f..27950c93cdb 100644 --- a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst @@ -1,2 +1 @@ -* Allowed asynchronous XR owners to drain before releasing the simulation context. * Supported empty CPU Warp arrays in Torch conversion. diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.minor b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.minor new file mode 100644 index 00000000000..e69de29bb2d diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst new file mode 100644 index 00000000000..0ca6527bba7 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst @@ -0,0 +1,3 @@ +* Changed the SO101 IK-Abs stack task to reset the arm to its nominal joint pose without joint + noise and to use a full-pose clutch home. Policies or datasets that relied on randomized + initial arm joints should restore ``events.randomize_joint_state.params["std"]``. diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst index 404ba0c707a..4ad6c82a7c1 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.fixed.rst @@ -1,2 +1,2 @@ -* Fixed standalone OvPhysX selection for SO101 teleoperation while retaining the Kit default and Newton settings. -* Updated SO101 clutch construction and nominal gripper reset poses for the supported Isaac Capture API. +* Fixed standalone OvPhysX selection for SO101 teleoperation while retaining the Kit default and + Newton settings. The OvPhysX selection applies the stack family's grasp contact tuning. diff --git a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py index f6088a4f2aa..e7f42bfeec1 100644 --- a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py +++ b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py @@ -9,7 +9,8 @@ import pytest from scipy.spatial.transform import Rotation -pytest.importorskip("isaacteleop") +# The test drives Capture's retargeting-engine internals; skip releases that do not ship them. +pytest.importorskip("isaacteleop.retargeting_engine.interface.retargeter_core_types") from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource from isaacteleop.retargeting_engine.interface import ( @@ -75,21 +76,18 @@ def check(action, position, rotation, gripper): home_rotation = Rotation.from_matrix(_BASE_T_GRIPPER_HOME[:3, :3]) origin = Rotation.from_euler("xyz", [0.2, -0.4, 0.3]) position = np.array([0.3, -0.1, 0.5]) - reset_rotation = home_rotation - engaged_rotation = home_rotation - check(step(position, origin, running=False, reset=True), home_position, reset_rotation, 0.0) - check(step(position, origin), home_position, engaged_rotation, 0.0) + check(step(position, origin, running=False, reset=True), home_position, home_rotation, 0.0) + check(step(position, origin), home_position, home_rotation, 0.0) delta = Rotation.from_rotvec([0.0, 0.0, 0.3]) displacement = np.array([0.02, -0.03, 0.01]) moved_position = home_position + displacement - moved_rotation = delta * engaged_rotation + moved_rotation = delta * home_rotation check(step(position + displacement, delta * origin, trigger=1.0), moved_position, moved_rotation, 1.0) other = Rotation.from_euler("xyz", [-0.5, 0.2, 0.8]) check(step(-position, other, running=False), moved_position, moved_rotation, 0.0) - resumed_rotation = moved_rotation - check(step(-position, other), moved_position, resumed_rotation, 0.0) + check(step(-position, other), moved_position, moved_rotation, 0.0) # Re-engaging composes from the held full pose. delta2 = Rotation.from_rotvec([0.2, 0.0, 0.0]) - check(step(-position + displacement, delta2 * other), moved_position + displacement, delta2 * resumed_rotation, 0.0) - check(step(position, origin, running=False, reset=True), home_position, reset_rotation, 0.0) + check(step(-position + displacement, delta2 * other), moved_position + displacement, delta2 * moved_rotation, 0.0) + check(step(position, origin, running=False, reset=True), home_position, home_rotation, 0.0) check(step(-position, other), home_position, home_rotation, 0.0) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst index 075c72fe464..9b8f9553348 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -1,3 +1,11 @@ * Added kitless OVXR teleoperation and MCAP replay with independently paced rendering, backend-derived anchors, and ordered shutdown. * Preserved recorded replay controls by default, with explicit ``--start_on_replay`` for recordings without start events. +* Added task hooks to :class:`~isaaclab_teleop.IsaacTeleopCfg`: ``prepare_environment``, + ``action_conditioner``, ``action_validator`` and ``require_initial_bimanual_hand_tracking``. +* Added ``render_anchor_pos``, ``render_anchor_rot`` and ``hidden_visual_prims`` to + :class:`~isaaclab_teleop.XrCfg` for a render-only anchor offset and globally hidden prims. +* Added external OVXR inputs to :class:`~isaaclab_teleop.IsaacTeleopDevice`, together with the + ``target_frame_prim_path``, ``xr_anchor_path``, ``xr_render_anchor_path``, + ``external_anchor_state``, ``required_openxr_extensions``, ``session_cleanup_failed`` and + ``hand_tracking_presence`` properties. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst index 6dac77682f0..046da91163e 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -5,3 +5,11 @@ to the caller instead of silently suppressing them. Handle the reported failure; resources whose asynchronous users have not retired must remain alive until cleanup succeeds. +* **Breaking:** Live teleoperation now exits non-zero when environment creation, device + creation or a teleoperation step fails, instead of logging the error and exiting 0. + Automation that inspected the log for failures should check the exit code. +* Changed ``--disable_external_cameras`` to remove camera sensors in every live teleoperation + mode, not only with ``--xr``. +* Changed replay to launch Kit only when the resolved task configuration requires it (for + example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit + physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index 113f8a3808f..e6d0841d4ea 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -315,7 +315,11 @@ def target_frame_prim_path(self) -> str | None: @property def xr_anchor_path(self) -> str: - """Return the exact anchor prim path consumed by external OVXR.""" + """Return the tracking-anchor prim path that maps OpenXR poses into the scene. + + External OVXR frames its cameras from :attr:`xr_render_anchor_path`; the two paths differ + only when :attr:`~isaaclab_teleop.XrCfg.render_anchor_pos` or ``render_anchor_rot`` is set. + """ return self._anchor_manager.anchor_headset_path @property From 855ac4ec91c05a45de9f5396c9b3bc1762c1acbe Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:27:22 -0400 Subject: [PATCH 012/145] Re-seed G1 wrist ramp after auto-resets The kitless Newton G1 conditioner seeded its wrist ramp only once and kept ramping from its own previous target, so after a success or drop termination auto-reset inside env.step Pink received a full-distance jump from the reset pose. Re-seed from the measured wrists whenever the episode counter goes backwards. Locomanipulation G1 on Newton also gets at least fixed-base G1's MJWarp contact/constraint budget (nconmax 4096, njmax 1536): it has the same upper body, hands, table and wheel plus foot contacts, and the MJWarp default njmax is 300. Review: #8789-M2, #8789-M4 --- .../locomanip_pick_place/g1_kitless_newton.py | 22 +++++++++++++- .../locomanipulation_g1_env_cfg.py | 10 +++++-- .../test/contrib/test_g1_kitless_newton.py | 29 +++++++++++++++++-- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py index 0121037973e..6b146a2ed80 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -178,13 +178,33 @@ def _step_quaternion_towards(current: torch.Tensor, target: torch.Tensor, max_st return torch.nn.functional.normalize(torch.where(angle < 1.0e-6, target, interpolated), dim=0) +_WRIST_RAMP_KEYS = ("left_pos", "left_quat", "right_pos", "right_quat") + + +def _episode_step(env) -> int | None: + """Return environment 0's episode step, or ``None`` when the environment has no counter.""" + buffer = getattr(getattr(env, "unwrapped", env), "episode_length_buf", None) + return None if buffer is None else int(buffer[0]) + + def condition_kitless_newton_g1_action(env, action: torch.Tensor, state: dict[str, Any]) -> torch.Tensor: - """Ramp both wrist targets from the current G1 poses before Pink solves IK.""" + """Ramp both wrist targets from the current G1 poses before Pink solves IK. + + The ramp re-seeds from the measured wrist poses whenever the episode restarts, including + termination auto-resets inside ``env.step`` that the teleoperation loop does not observe. + """ flat = action.detach().flatten() if flat.numel() != EXPECTED_ACTION_DIM: state.clear() return action + episode_step = _episode_step(env) + previous_step = state.get("_episode_step") + if episode_step is not None and previous_step is not None and episode_step < previous_step: + for key in _WRIST_RAMP_KEYS: + state.pop(key, None) + state["_episode_step"] = episode_step + conditioned = action.clone() conditioned_flat = conditioned.flatten() if "left_pos" not in state: diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index c14e1b9093c..e0528727008 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -28,7 +28,7 @@ from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp from isaaclab_tasks.contrib.teleop_physics_cfg import ( TeleopPhysicsCfg, - kitless_backend_prepared, + check_kitless_backend_preparation, prepare_humanoid_teleop_environment, ) @@ -519,20 +519,24 @@ def __post_init__(self): def stabilize_for_kitless_ovphysx(self) -> None: """Provide the packing-table support shape required by standalone OVPhysX.""" - if kitless_backend_prepared(self, "ovphysx"): + if check_kitless_backend_preparation(self, "ovphysx"): return configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) self._prepared_kitless_backend = "ovphysx" def stabilize_for_kitless_newton_mjwarp(self) -> None: """Prepare shared task collision geometry without replacing the locomotion policy.""" - if kitless_backend_prepared(self, "newton_mjwarp"): + if check_kitless_backend_preparation(self, "newton_mjwarp"): return from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions # Match the common locomotion integrator for the floating-base robot. self.sim.physics.solver_cfg.integrator = "implicitfast" self.sim.physics.num_substeps = 2 + # The upper body, hands, packing table and wheel match fixed-base G1, plus foot-ground + # contacts; keep at least its contact/constraint budget (MJWarp defaults: njmax=300). + self.sim.physics.solver_cfg.nconmax = max(self.sim.physics.solver_cfg.nconmax or 0, 4096) + self.sim.physics.solver_cfg.njmax = max(self.sim.physics.solver_cfg.njmax or 0, 1536) configure_packing_table_collisions(self.scene.packing_table) condition_newton_steering_wheel_detail_collisions(self.scene.object) self._prepared_kitless_backend = "newton_mjwarp" diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py index 4f0c0fdb85f..dcf320127f0 100644 --- a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py @@ -142,6 +142,9 @@ def test_locomanipulation_preserves_agile_policy_order_and_action_contract() -> assert cfg.isaac_teleop.action_conditioner is None assert cfg.sim.physics.solver_cfg.integrator == "implicitfast" assert cfg.sim.physics.num_substeps == 2 + # Same robot and props as fixed-base G1, plus foot contacts: keep at least its MJWarp budget. + assert cfg.sim.physics.solver_cfg.nconmax >= 4096 + assert cfg.sim.physics.solver_cfg.njmax >= 1536 def test_recorded_first_action_is_ramped_and_finite() -> None: @@ -157,6 +160,26 @@ def test_recorded_first_action_is_ramped_and_finite() -> None: assert torch.equal(conditioned[14:], RECORDED_FIRST_ACTION[14:]) +def test_wrist_ramp_reseeds_after_termination_auto_reset() -> None: + """A success/drop auto-reset inside env.step moves the wrists; the ramp must restart from there.""" + env, left, right = _fake_env() + env.episode_length_buf = torch.tensor([0]) + state = {} + target = RECORDED_FIRST_ACTION.clone() + for step in range(1, 60): + conditioned = condition_kitless_newton_g1_action(env, target, state) + env.episode_length_buf[0] = step + # The ramp has converged near the operator's hands, far from the reset pose. + assert float(torch.linalg.norm(conditioned[:3] - left[:3])) > 2 * POSITION_MAX_STEP_M + # Auto-reset: the robot returns to a different pose and the episode counter restarts. + poses = env.scene["robot"].data.body_link_pose_w + poses[0, :, :3] += torch.tensor([0.0, 0.0, -0.2]) + env.episode_length_buf[0] = 0 + conditioned = condition_kitless_newton_g1_action(env, target, state) + assert float(torch.linalg.norm(conditioned[:3] - poses[0, 0, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert float(torch.linalg.norm(conditioned[7:10] - poses[0, 1, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + + @pytest.mark.parametrize( "angle, axis, sign", [(0.9, (1, 0, 0), 1), (1.2, (0, 1, 1), -1), (0.1, (0, 0, 1), 1), (0, (1, 0, 0), -1)] ) @@ -164,17 +187,19 @@ def test_conditioned_wrist_rotation_obeys_angular_cap(angle, axis, sign): """The public action conditioner follows the short rotation without exceeding its cap.""" env, _, _ = _fake_env() pose = env.scene["robot"].data.body_link_pose_w + # Any fixed unit quaternion works as the start: slerp is convention-independent. This is the + # (x, y, z, w) = (1, 0, 0, 0) half-turn about x, not identity. pose[0, :, 3:7] = torch.tensor([1.0, 0, 0, 0]) direction = torch.nn.functional.normalize(torch.tensor(axis, dtype=torch.float32), dim=0) target = sign * torch.cat((torch.tensor([math.cos(angle / 2)]), direction * math.sin(angle / 2))) action = RECORDED_FIRST_ACTION.clone() action[3:7] = action[10:14] = target conditioned = condition_kitless_newton_g1_action(env, action, {}) - identity = torch.tensor([1.0, 0, 0, 0]) + start = torch.tensor([1.0, 0, 0, 0]) for result in (conditioned[3:7], conditioned[10:14]): assert torch.isfinite(result).all() assert torch.linalg.norm(result).item() == pytest.approx(1, abs=1.0e-6) - assert _quat_angle(identity, result) <= min(angle, ROTATION_MAX_STEP_RAD) + 1.0e-5 + assert _quat_angle(start, result) <= min(angle, ROTATION_MAX_STEP_RAD) + 1.0e-5 assert _quat_angle(result, target) == pytest.approx(max(0, angle - ROTATION_MAX_STEP_RAD), abs=1.0e-5) From ed9888e40594b22bb9833bfe3cb4ba7561dcada0 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:27:22 -0400 Subject: [PATCH 013/145] Share GR1T2 kitless OvPhysX preparation - Extract prepare_gr1t2_for_kitless_ovphysx, used by both GR1T2 configurations, mirroring the Newton helper. - Rename kitless_backend_prepared to check_kitless_backend_preparation, type it, document Raises, and keep the check in one place. - Drop the dead '*-slaved' actuator pop and the no-op hand joint re-assignment, and the explicit GR1T2 HandsSource outputs that the session lifecycle already discovers. - Document that GR1T2 spawn conditioning runs on both kitless backends and floors every inertia component below 1e-4 kg m^2. Review: #8789-L1, #8789-L2, #8789-L3, #8789-L5 --- .../fixed_base_upper_body_ik_g1_env_cfg.py | 6 +- .../pick_place/pickplace_gr1t2_env_cfg.py | 112 +++++++++++------- .../pickplace_gr1t2_waist_enabled_env_cfg.py | 23 +--- .../contrib/teleop_physics_cfg.py | 35 +++++- 4 files changed, 106 insertions(+), 70 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index caf4a5b349e..93ce7b32468 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -24,7 +24,7 @@ from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg from isaaclab_tasks.contrib.teleop_physics_cfg import ( TeleopPhysicsCfg, - kitless_backend_prepared, + check_kitless_backend_preparation, prepare_humanoid_teleop_environment, ) @@ -460,7 +460,7 @@ def __post_init__(self): def stabilize_for_kitless_ovphysx(self) -> None: """Prepare the table collision authoring; let OVPhysX cook the wheel meshes.""" - if kitless_backend_prepared(self, "ovphysx"): + if check_kitless_backend_preparation(self, "ovphysx"): return from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions @@ -469,7 +469,7 @@ def stabilize_for_kitless_ovphysx(self) -> None: def stabilize_for_kitless_newton_mjwarp(self) -> None: """Configure G1 actions, assets and contacts for kitless Newton.""" - if kitless_backend_prepared(self, "newton_mjwarp"): + if check_kitless_backend_preparation(self, "newton_mjwarp"): return from .g1_kitless_newton import configure_g1_for_kitless_newton diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index db809f16238..f517a3473fd 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -35,7 +35,7 @@ ) from isaaclab_tasks.contrib.teleop_physics_cfg import ( TeleopPhysicsCfg, - kitless_backend_prepared, + check_kitless_backend_preparation, prepare_humanoid_teleop_environment, ) @@ -275,13 +275,8 @@ def build_gr1t2_pickplace_pipeline(): } ) - pipeline = OutputCombiner( - { - "action": connected_reorderer.output("output"), - HandsSource.LEFT: hands.output(HandsSource.LEFT), - HandsSource.RIGHT: hands.output(HandsSource.RIGHT), - } - ) + # The session lifecycle discovers the raw HandsSource outputs for hand-presence safety. + pipeline = OutputCombiner({"action": connected_reorderer.output("output")}) return pipeline, [left_dex, right_dex] @@ -723,31 +718,59 @@ def __post_init__(self): def stabilize_for_kitless_ovphysx(self) -> None: """Configure the GR1T2 asset and action boundaries for kitless OVPhysX.""" - if kitless_backend_prepared(self, "ovphysx"): - condition_kitless_gr1t2_stage(self.scene.robot, self.actions.upper_body_ik) - return - action_cfg = self.actions.upper_body_ik - action_cfg.controller.urdf_path = os.path.join( - os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" - ) - action_cfg.kitless_target_conditioning_enabled = True - # Do not apply retargeter defaults before hand tracking starts. - if self.isaac_teleop is not None: - self.isaac_teleop.require_initial_bimanual_hand_tracking = True - apply_kitless_ovphysx_drive_gains(self.scene.robot) - condition_kitless_ovphysx_robot_spawn_properties(self.scene.robot) - condition_kitless_gr1t2_stage(self.scene.robot, action_cfg) - configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) - self._prepared_kitless_backend = "ovphysx" + prepare_gr1t2_for_kitless_ovphysx(self) def stabilize_for_kitless_newton_mjwarp(self) -> None: """Configure GR1T2 actuators and contacts for kitless Newton.""" prepare_gr1t2_for_kitless_newton_mjwarp(self) +def prepare_gr1t2_for_kitless_ovphysx(env_cfg: ManagerBasedRLEnvCfg) -> None: + """Configure a GR1T2 task's asset and action boundaries for kitless OVPhysX. + + Selects the vendored kinematics URDF for Pink, enables target conditioning, requires initial + bimanual hand tracking, restores actuator coverage, conditions the robot spawn, and authors + PhysX contact offsets on the packing table. The configured controlled joints (with or without + the waist) are kept. A prepared configuration only re-attaches its spawn conditioning. + + Args: + env_cfg: GR1T2 pick-place task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "ovphysx"): + condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik) + return + action_cfg = env_cfg.actions.upper_body_ik + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + action_cfg.kitless_target_conditioning_enabled = True + # Do not apply retargeter defaults before hand tracking starts. + if env_cfg.isaac_teleop is not None: + env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True + apply_kitless_ovphysx_drive_gains(env_cfg.scene.robot) + condition_kitless_ovphysx_robot_spawn_properties(env_cfg.scene.robot) + condition_kitless_gr1t2_stage(env_cfg.scene.robot, action_cfg) + configure_packing_table_collisions(env_cfg.scene.packing_table, physx_offsets=True) + env_cfg._prepared_kitless_backend = "ovphysx" + + def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: - """Share Newton preparation without replacing the configured controlled joints.""" - if kitless_backend_prepared(env_cfg, "newton_mjwarp"): + """Configure a GR1T2 task's actuators, contacts and action boundaries for kitless Newton. + + Shares the URDF and target conditioning with :func:`prepare_gr1t2_for_kitless_ovphysx`, + applies Newton actuator gains and MJWarp contact budgets, and keeps the configured + controlled joints. Hand joint limits are not widened on Newton. + + Args: + env_cfg: GR1T2 pick-place task configuration. + + Raises: + ValueError: If ``env_cfg`` was already prepared for a different backend. + """ + if check_kitless_backend_preparation(env_cfg, "newton_mjwarp"): condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik, widen_hand_limits=False) return action_cfg = env_cfg.actions.upper_body_ik @@ -793,17 +816,14 @@ def _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> actuator.armature = armature actuator.joint_effort_limit = effort_limit - for side, name in (("L", "left-hand"), ("R", "right-hand")): - # The action conditions all eleven targets to the asset's mimic - # relation. Driving the whole hand gives Newton the same compliant - # hand targets as the OVPhysX path. + for name in ("left-hand", "right-hand"): + # The action conditions all eleven targets to the asset's mimic relation. Give Newton the + # same compliant hand drives as the OVPhysX path. hand = groups[name] - hand.joint_names_expr = [f"{side}_.*"] hand.stiffness = 5.0 hand.damping = 0.5 hand.armature = 0.02 hand.joint_effort_limit = 20.0 - groups.pop(f"{name}-slaved", None) solver_cfg = env_cfg.sim.physics.solver_cfg solver_cfg.nconmax = 4096 @@ -846,8 +866,12 @@ def condition_kitless_ovphysx_robot_spawn_properties(robot_cfg: ArticulationCfg) fragment.max_depenetration_velocity = 0.1 -def _author_kitless_ovphysx_contact_offsets(prim) -> bool: - """Author Kit-parity offsets on a collision prim before OVPhysX parses it.""" +def _author_kitless_contact_offsets(prim) -> bool: + """Author Kit-parity PhysX contact offsets on a collision prim before kitless parsing. + + Both kitless backends receive the same spawned stage: OVPhysX consumes these + ``physxCollision`` offsets, and Newton parses the same collision prims. + """ from pxr import Sdf schemas = prim.GetMetadata("apiSchemas") @@ -881,7 +905,7 @@ def _spawn_and_condition(prim_path, cfg, *args, **kwargs): child.SetInstanceable(False) for child in Usd.PrimRange(prim): - _author_kitless_ovphysx_contact_offsets(child) + _author_kitless_contact_offsets(child) inertia = child.GetAttribute("physics:diagonalInertia") if inertia and inertia.HasValue(): value = inertia.Get() @@ -947,11 +971,19 @@ def condition_kitless_gr1t2_stage( The raw asset has near-singular finger inertia, hard PhysX-only mimic constraints, and hand rest poses exactly on physical limits. Those inputs are stable after Kit's asset preparation but are ill-conditioned when read - directly by OVPhysX. This spawn boundary floors inertia, removes the hard - constraints that conflict with direct hand drives, and optionally keeps the - physical limits inactive for OVPhysX. Newton retains the anatomical limits - because bounding drive targets alone does not bound simulated positions. The - original mimic relations and anatomical limits are recorded on the typed + directly by a kitless backend. This spawn boundary runs for both OVPhysX and + Newton. It: + + - de-instances ``collisions`` scopes and authors Kit-parity PhysX contact + offsets (contact 1 mm, rest 0) on every collision prim; + - floors every diagonal inertia component below 1e-4 [kg·m²] on every body, + which in this asset affects the finger links; + - removes the hard mimic constraints that conflict with direct hand drives; + - optionally widens the hand joint limits (``widen_hand_limits``, used for + OVPhysX). Newton retains the anatomical limits because bounding drive + targets alone does not bound simulated positions. + + The original mimic relations and anatomical limits are recorded on the typed action config for target conditioning. The source USD and normal Kit path remain unchanged. """ diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py index c05b2fd3d46..cb152477b62 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -import os import tempfile from isaaclab.envs import ManagerBasedRLEnvCfg @@ -11,10 +10,8 @@ from isaaclab.utils import configclass from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE -from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions from isaaclab_tasks.contrib.teleop_physics_cfg import ( TeleopPhysicsCfg, - kitless_backend_prepared, prepare_humanoid_teleop_environment, ) @@ -24,11 +21,9 @@ ObjectTableSceneCfg, ObservationsCfg, TerminationsCfg, - apply_kitless_ovphysx_drive_gains, build_gr1t2_pickplace_pipeline, - condition_kitless_gr1t2_stage, - condition_kitless_ovphysx_robot_spawn_properties, prepare_gr1t2_for_kitless_newton_mjwarp, + prepare_gr1t2_for_kitless_ovphysx, ) @@ -105,21 +100,7 @@ def __post_init__(self): def stabilize_for_kitless_ovphysx(self) -> None: """Prepare the existing GR asset while retaining the movable waist.""" - if kitless_backend_prepared(self, "ovphysx"): - condition_kitless_gr1t2_stage(self.scene.robot, self.actions.upper_body_ik) - return - action_cfg = self.actions.upper_body_ik - action_cfg.controller.urdf_path = os.path.join( - os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" - ) - action_cfg.kitless_target_conditioning_enabled = True - if self.isaac_teleop is not None: - self.isaac_teleop.require_initial_bimanual_hand_tracking = True - apply_kitless_ovphysx_drive_gains(self.scene.robot) - condition_kitless_ovphysx_robot_spawn_properties(self.scene.robot) - condition_kitless_gr1t2_stage(self.scene.robot, action_cfg) - configure_packing_table_collisions(self.scene.packing_table, physx_offsets=True) - self._prepared_kitless_backend = "ovphysx" + prepare_gr1t2_for_kitless_ovphysx(self) def stabilize_for_kitless_newton_mjwarp(self) -> None: """Configure GR1T2 actuators and contacts for kitless Newton.""" diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py index e363fa3a3a8..933e76586c7 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py @@ -23,16 +23,25 @@ def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: - """Prepare humanoid task settings before simulation or environment construction.""" + """Prepare humanoid task settings before simulation or environment construction. + + Dispatches to the configuration's ``stabilize_for_kitless_ovphysx`` or + ``stabilize_for_kitless_newton_mjwarp`` for the selected standalone backend. Kit PhysX + needs no preparation. Repeated calls on a prepared configuration are no-ops. + + Args: + cfg: Resolved humanoid teleoperation task configuration. + + Raises: + ValueError: If ``cfg`` was already prepared for a different physics backend. + """ physics = cfg.sim.physics if isinstance(physics, OvPhysxCfg): - kitless_backend_prepared(cfg, "ovphysx") cfg.stabilize_for_kitless_ovphysx() elif isinstance(physics, NewtonCfg) and isinstance(physics.solver_cfg, MJWarpSolverCfg): - kitless_backend_prepared(cfg, "newton_mjwarp") cfg.stabilize_for_kitless_newton_mjwarp() else: - kitless_backend_prepared(cfg, None) + check_kitless_backend_preparation(cfg, None) @configclass @@ -46,8 +55,22 @@ class TeleopPhysicsCfg(PresetCfg): default = isaacsim_physx -def kitless_backend_prepared(env_cfg, backend: str | None) -> bool: - """Keep completed task preparation stable across repeated callers and config copies.""" +def check_kitless_backend_preparation(env_cfg: ManagerBasedRLEnvCfg, backend: str | None) -> bool: + """Check a task's kitless preparation against the selected backend. + + Preparation is recorded on the configuration, so it stays stable across repeated callers and + configuration copies. + + Args: + env_cfg: Task configuration that may carry ``_prepared_kitless_backend``. + backend: Selected backend name (``"ovphysx"``, ``"newton_mjwarp"``) or ``None`` for Kit PhysX. + + Returns: + Whether ``env_cfg`` is already prepared for ``backend``. + + Raises: + ValueError: If ``env_cfg`` was prepared for a different backend. + """ prepared = getattr(env_cfg, "_prepared_kitless_backend", None) if prepared is not None and prepared != backend: raise ValueError( From 472daaea0ed965f98a4d034a6a61cda4a884a3bd Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:27:23 -0400 Subject: [PATCH 014/145] Check teleop task paths against the spawned assets Add asset-backed tests (skipped when the asset server is unreachable): - the G1 hand contact patterns resolve to rigid hand links on the Isaac 6.2 g1.usd, where the links are nested under left_hand and right_hand and the previous flat pattern matched nothing; - the vendored GR1T2 kinematics URDF keeps the spawned USD's 54 movable joints and limits. Record the URDF's source asset and the regeneration check in its header, and add the Kit-visible G1 contact-sensor fix to the changelog. Review: #8789-M1, #8789-M3 --- .../changelog.d/kitless-robot-tasks.fixed.rst | 2 + .../GR1T2_fourier_hand_6dof_kinematics.urdf | 7 ++ .../test/contrib/test_teleop_task_assets.py | 97 +++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst create mode 100644 source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst new file mode 100644 index 00000000000..2a5564f291c --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst @@ -0,0 +1,2 @@ +* Fixed the G1 hand contact sensors to match the hand links nested under ``left_hand``/``right_hand`` + in the Isaac 6.2 ``g1.usd``. The previous flat pattern matched no prims, including on Kit PhysX. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf index f8f8af75677..9269c43f21c 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf @@ -4,6 +4,13 @@ Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/is All rights reserved. SPDX-License-Identifier: BSD-3-Clause + +Kinematics-only model of the Isaac 6.2 asset +Robots/FourierIntelligence/GR-1/GR1T2_fourier_hand_6dof/GR1T2_fourier_hand_6dof.usd, the robot +spawned by GR1T2_CFG. Kit regenerates the equivalent URDF from that USD at runtime; kitless Pink +uses this copy. Regenerate it whenever the asset changes: +source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py checks its joints and limits against +the spawned USD. --> diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py new file mode 100644 index 00000000000..64c63976fe9 --- /dev/null +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -0,0 +1,97 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Check hard-coded teleoperation task paths against the robot USD assets the tasks spawn. + +These tests read the published assets without a simulator. They skip when the asset server is +unreachable. +""" + +from __future__ import annotations + +import math +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +import pytest + +from pxr import Usd, UsdPhysics + +from isaaclab.utils.assets import retrieve_file_path + +_ENV_ROBOT = "{ENV_REGEX_NS}/Robot" + + +def _open_asset(usd_path: str) -> Usd.Stage: + try: + local_path = retrieve_file_path(usd_path) + except Exception as exc: # The asset server is optional for local test runs. + pytest.skip(f"Robot asset is unavailable: {exc}") + stage = Usd.Stage.Open(local_path) + assert stage is not None, local_path + return stage + + +def _robot_paths(stage: Usd.Stage, pattern: str) -> list[Usd.Prim]: + """Return prims of the asset's default prim (the spawned ``Robot``) that fully match ``pattern``.""" + root = stage.GetDefaultPrim() + assert pattern.startswith(_ENV_ROBOT), pattern + expression = re.compile(re.escape(str(root.GetPath())) + pattern[len(_ENV_ROBOT) :]) + return [ + prim for prim in Usd.PrimRange(root, Usd.TraverseInstanceProxies()) if expression.fullmatch(str(prim.GetPath())) + ] + + +@pytest.mark.parametrize("task", ["fixed_base", "locomanipulation"]) +def test_g1_hand_contact_sensors_resolve_to_rigid_hand_links(task): + """The Isaac 6.2 G1 nests its hand links under ``left_hand``/``right_hand``; flat paths match nothing.""" + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg + + cfg = FixedBaseUpperBodyIKG1EnvCfg() if task == "fixed_base" else LocomanipulationG1EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + matched = {} + for side in ("left", "right"): + prims = _robot_paths(stage, getattr(cfg.scene, f"{side}_hand_contact").prim_path) + assert prims, side + assert all(prim.HasAPI(UsdPhysics.RigidBodyAPI) for prim in prims) + matched[side] = {str(prim.GetPath()) for prim in prims} + assert matched["left"].isdisjoint(matched["right"]) + + +def test_gr1t2_kitless_kinematics_urdf_matches_the_spawned_asset(): + """The vendored Pink URDF must keep the spawned USD's movable joints and limits [rad].""" + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + cfg = PickPlaceGR1T2EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + usd_limits = {} + for prim in Usd.PrimRange(stage.GetPseudoRoot(), Usd.TraverseInstanceProxies()): + if prim.IsA(UsdPhysics.RevoluteJoint): + joint = UsdPhysics.RevoluteJoint(prim) + usd_limits[prim.GetName()] = ( + math.radians(joint.GetLowerLimitAttr().Get()), + math.radians(joint.GetUpperLimitAttr().Get()), + ) + elif prim.IsA(UsdPhysics.Joint) and not prim.IsA(UsdPhysics.FixedJoint): + pytest.fail(f"Unexpected non-revolute movable joint {prim.GetPath()}") + + urdf_path = ( + Path(__import__("isaaclab_tasks").__file__).parent + / "contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + urdf_limits = {} + for joint in ET.parse(urdf_path).getroot().findall("joint"): + if joint.get("type") == "fixed": + continue + limit = joint.find("limit") + urdf_limits[joint.get("name")] = (float(limit.get("lower")), float(limit.get("upper"))) + + assert set(urdf_limits) == set(usd_limits) + for name, (lower, upper) in usd_limits.items(): + assert urdf_limits[name] == pytest.approx((lower, upper), abs=1.0e-4), name From 4fe2ad143d7d56104d7ba909f437f74fea440ee5 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:27:23 -0400 Subject: [PATCH 015/145] Keep humanoid Mimic tasks on Isaac Sim PhysX The GR1T2 and G1 locomanipulation Mimic configurations inherited the teleop tasks' ovphysx/newton_mjwarp selectors, but their Mimic environments never run the kitless preparation, so those selections built unprepared environments. Pin them to Isaac Sim PhysX; the selectors now fail with the preset resolver's error. Review: #8789-L6 --- .../changelog.d/kitless-robot-tasks.fixed.rst | 2 ++ .../envs/locomanipulation_g1_mimic_env_cfg.py | 6 ++++ .../envs/pickplace_gr1t2_mimic_env_cfg.py | 6 ++++ ...place_gr1t2_waist_enabled_mimic_env_cfg.py | 6 ++++ .../test/test_mimic_humanoid_physics.py | 31 +++++++++++++++++++ 5 files changed, 51 insertions(+) create mode 100644 source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst create mode 100644 source/isaaclab_mimic/test/test_mimic_humanoid_physics.py diff --git a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst new file mode 100644 index 00000000000..fd13f958c31 --- /dev/null +++ b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst @@ -0,0 +1,2 @@ +* Kept the GR1T2 and G1 locomanipulation Mimic tasks on Isaac Sim PhysX. Their environments do + not run the kitless backend preparation the base teleoperation tasks now offer. diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py index 17bfc63f68b..5721a05d811 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/locomanipulation_g1_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import ( @@ -16,6 +19,9 @@ class LocomanipulationG1MimicEnvCfg(LocomanipulationG1EnvCfg, MimicEnvCfg): """Configuration for G1 Locomanipulation Mimic environment.""" class_type: type | str = "{DIR}.locomanipulation_g1_mimic_env:LocomanipulationG1MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Call parent post-init diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py index d1daf6d240b..60d47b76676 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg @@ -14,6 +17,9 @@ class PickPlaceGR1T2MimicEnvCfg(PickPlaceGR1T2EnvCfg, MimicEnvCfg): """Configuration for GR1T2 Pick Place Mimic environment.""" class_type: type | str = "{DIR}.pickplace_gr1t2_mimic_env:PickPlaceGR1T2MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Calling post init of parents diff --git a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py index ff326c66e41..c9636a3e885 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/envs/pickplace_gr1t2_waist_enabled_mimic_env_cfg.py @@ -3,7 +3,10 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg + from isaaclab.envs.mimic_env_cfg import MimicEnvCfg, SubTaskConfig +from isaaclab.sim import SimulationCfg from isaaclab.utils import configclass from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import ( @@ -16,6 +19,9 @@ class PickPlaceGR1T2WaistEnabledMimicEnvCfg(PickPlaceGR1T2WaistEnabledEnvCfg, Mi """Configuration for GR1T2 Pick Place Waist Enabled Mimic environment.""" class_type: type | str = "{DIR}.pickplace_gr1t2_mimic_env:PickPlaceGR1T2MimicEnv" + sim: SimulationCfg = SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the Mimic environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" def __post_init__(self): # Calling post init of parents diff --git a/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py new file mode 100644 index 00000000000..f889df9e9a8 --- /dev/null +++ b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py @@ -0,0 +1,31 @@ +# Copyright (c) 2024-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +"""Humanoid Mimic tasks keep Isaac Sim PhysX; their environments skip kitless preparation.""" + +import pytest + + +@pytest.mark.parametrize( + "task", + [ + "Isaac-PickPlace-GR1T2-Abs-Mimic-v0", + "Isaac-PickPlace-GR1T2-WaistEnabled-Abs-Mimic-v0", + "Isaac-Locomanipulation-G1-Abs-Mimic-v0", + ], +) +def test_humanoid_mimic_tasks_offer_only_isaac_sim_physx(task): + from isaaclab_physx.physics import PhysxCfg + + import isaaclab_mimic.envs # noqa: F401 -- registers the Mimic task IDs + + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg(task, device="cpu", num_envs=1) + assert type(cfg.sim.physics) is PhysxCfg + for selector in ("physics=ovphysx", "physics=newton_mjwarp"): + with pytest.raises(ValueError, match="did not match a selectable preset"): + parse_env_cfg(task, device="cpu", num_envs=1, overrides=[selector]) From 92450db5a46b2c18448911884477b22f485e68d4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:27:23 -0400 Subject: [PATCH 016/145] Complete 8789 changelog, docs and test order - Record the HumanoidTeleopEnv entry point, the GR1T2 action term type and the fixed-base G1 reset re-renders as changed; drop the review history line from added. - List the WaistEnabled task in the kitless selector table and fix the example and fragment below it. - Exercise task preparation through the live entrypoint's real validate-then-prepare order. - Clarify the quaternion start value in the wrist rotation test. Review: #8789-L4 --- .../changelog.d/kitless-robot-tasks.added.rst | 1 - .../kitless-robot-tasks.changed.rst | 11 +++++++++ source/isaaclab_teleop/docs/kitless_xr.md | 6 ++--- .../test/test_teleop_entrypoint_units.py | 23 ++++++------------- 4 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst index 1748c52da86..5407015deaa 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst @@ -4,5 +4,4 @@ standalone backend preparation without optional teleoperation configuration. Registered construction prepares the selected backend and preserves prepared configuration copies and serialized spawn callbacks. -* Batched G1 joint-limit preparation and device validation without changing action safety bounds. * Prepared packing-table mesh collisions for standalone physics. diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst new file mode 100644 index 00000000000..93040cc5ec9 --- /dev/null +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -0,0 +1,11 @@ +* Changed the gym entry point of ``IsaacContrib-PickPlace-GR1T2-Abs``, + ``IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs``, ``IsaacContrib-PickPlace-Locomanipulation-G1-Abs`` + and ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` to + :class:`~isaaclab_tasks.contrib.teleop_env.HumanoidTeleopEnv`, which prepares the selected physics + backend before construction. Code that constructs these configurations directly should call + :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment` first. +* Changed the GR1T2 upper-body action term to + :class:`~isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. + With Isaac Sim PhysX it behaves as the previous Pink term. +* Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), + which adds render work per reset when cameras are rendered. diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index e27492c2b06..7c222ba60ea 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -55,6 +55,7 @@ The same registered-task selectors apply to live input and MCAP replay: | Task | Physics selectors | | --- | --- | | `IsaacContrib-PickPlace-GR1T2-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | | `IsaacContrib-PickPlace-Locomanipulation-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | | `IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | | `IsaacContrib-Stack-Cube-SO101-IK-Abs-v0` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | @@ -76,11 +77,10 @@ For example, replace the task in this command with any row above: ```bash uv run --python 3.12 --extra ovxr --locked isaaclab teleop run \ --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ - --device cpu --physics ovphysx --ovxr --cloudxr_env cloudxrjs + --device cpu --physics ovphysx --ovxr ``` - `--ovxr` selects OVStage -and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported +`--ovxr` selects OVStage and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported CloudXR client, then use Play, Pause and Reset as in Kit-based teleoperation. For desktop IWER, select **VR** and set **Reference Space** to **local-floor** diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 0776bd836dd..705f4c1351f 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -642,31 +642,22 @@ def test_product_xr_default_resolves_cloudxrjs_and_autolaunch(monkeypatch): ], ) def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch, task_name, physics): - """Task preparation must reach launch validation without losing its backend settings.""" - from isaaclab.app.sim_launcher import validate_launch - - from isaaclab_tasks.utils import parse_env_cfg - - agent = _load_teleop_agent(monkeypatch, "--task", task_name, "--physics", physics) - env_cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=[f"physics={physics}"]) + """The entrypoint validates the launch, then prepares the task, keeping its backend settings.""" + agent = _load_teleop_agent(monkeypatch, "--task", task_name, "--physics", physics, "--disable_external_cameras") + env_cfg, _teleop_cfg, _use_isaac_teleop = agent._resolve_env_cfg() physics_cfg = env_cfg.sim.physics - env_cfg.isaac_teleop.prepare_environment(env_cfg) + assert not hasattr(agent.args_cli, "physics") if physics == "newton_mjwarp": if "Locomanipulation-G1" in task_name: assert physics_cfg.num_substeps == 2 assert physics_cfg.solver_cfg.integrator == "implicitfast" - else: - assert physics_cfg.solver_cfg.nconmax == 4096 - assert physics_cfg.solver_cfg.njmax == 1536 + assert physics_cfg.solver_cfg.nconmax >= 4096 + assert physics_cfg.solver_cfg.njmax >= 1536 else: from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg assert isinstance(env_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) == (physics == "ovphysx") - - configs, _ = validate_launch(SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)), vars(agent.args_cli).copy()) - assert not hasattr(agent.args_cli, "physics") - assert env_cfg.sim.physics is physics_cfg - assert any(cfg is physics_cfg for cfg in configs) + assert getattr(env_cfg, "_prepared_kitless_backend", None) == (None if physics == "isaacsim_physx" else physics) def test_task_action_hook_validates_before_and_after_conditioning(monkeypatch): From ec3d6e3b183eb7e118b5072b73879faf67911be6 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:32:35 -0400 Subject: [PATCH 017/145] Preserve authored partition visibility in exports Re-apply the export change moved out of #8787 at its consumer: the XR scene partition authors showAllPartitionsByDefault=False to hide excluded head meshes from the eye cameras, and export used to force it back to True. Record the behavior change for ordinary exports. Review: #8787-6 --- .../changelog.d/leased-camera-products.changed.rst | 4 ++++ source/isaaclab_ov/isaaclab_ov/stage.py | 9 ++++++--- source/isaaclab_ov/test/test_ovrtx_usd.py | 12 +++++++++--- 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst new file mode 100644 index 00000000000..32eff1ffc41 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.changed.rst @@ -0,0 +1,4 @@ +* Changed render-stage exports to preserve an authored ``showAllPartitionsByDefault`` value instead + of always forcing it on, so the OVXR scene partition can hide excluded meshes. This also applies + to exports without an external lease: scenes that relied on export forcing every partition + visible should author ``showAllPartitionsByDefault=True`` explicitly. diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index 36e991849a0..701b5ba04e5 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -287,13 +287,16 @@ def export_stage_to_string( export_session.subLayerPaths = [stage.GetSessionLayer().identifier] export_stage = Usd.Stage.Open(stage.GetRootLayer(), export_session) export_stage.SetEditTarget(export_session) - # Perspective cameras see every partition; sensor cameras retain their explicit partition. + # Perspective cameras see every partition unless the scene authors otherwise (the XR scene + # partition hides excluded meshes this way); sensor cameras retain their explicit partition. settings_path = export_stage.GetMetadata("renderSettingsPrimPath") or "/Render/IsaacLabSettings" settings = UsdRender.Settings.Define(export_stage, settings_path) settings.GetPrim().AddAppliedSchema("OmniRtxSettingsGlobalCommonAPI_1") - settings.GetPrim().CreateAttribute( + show_all = settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ).Set(True) + ) + if not show_all.HasAuthoredValueOpinion(): + show_all.Set(True) export_stage.SetMetadata("renderSettingsPrimPath", settings_path) prim_paths = [] if len(clone_env_paths) > 1: diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 522a4d5d479..54ddb03a01a 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -528,14 +528,20 @@ def test_export_stage_heterogeneous_keeps_multiple_sources(): assert 'def Camera "Camera"' not in exported -@pytest.mark.parametrize("settings_path", [None, "/Render/AuthoredSettings"]) -def test_export_stage_preserves_authored_scene(settings_path): +@pytest.mark.parametrize( + "settings_path, authored_show_all", + [(None, None), ("/Render/AuthoredSettings", False), ("/Render/AuthoredSettings", True)], +) +def test_export_stage_preserves_authored_scene(settings_path, authored_show_all): """Renderer settings and clone trimming are confined to the exported scene.""" num_envs = 4 stage = _make_multi_env_stage(num_envs) if settings_path is not None: settings = UsdRender.Settings.Define(stage, settings_path) settings.CreateResolutionAttr(Gf.Vec2i(1280, 720)) + settings.GetPrim().CreateAttribute( + "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool + ).Set(authored_show_all) stage.SetMetadata("renderSettingsPrimPath", settings_path) before = stage.ExportToString() layer = Sdf.Layer.CreateAnonymous() @@ -548,7 +554,7 @@ def test_export_stage_preserves_authored_scene(settings_path): assert exported.HasAuthoredMetadata("renderSettingsPrimPath") settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) show_all = settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault") - assert show_all and show_all.Get() is True + assert show_all and show_all.Get() is (True if authored_show_all is None else authored_show_all) if settings_path is not None: assert settings.GetPath() == settings_path assert settings.GetAttribute("resolution").Get() == (1280, 720) From b920bf742a82499f92ac9b20eab1ddc35022b01e Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:44:25 -0400 Subject: [PATCH 018/145] Make fixed-base G1 image observations opt-in The wrist cameras and head/wrist XR feeds stay, but the three RGB policy terms move behind presets=rgb. OVXR PiP never needed them (it strips camera observation terms and reads the render products), and as a default they broke every state-only consumer of the task and rendered three cameras into the observation buffer on non-XR runs. image_obs_list keeps the camera names; robomimic uses only names present in the policy group. The change is no longer breaking, so the bump drops to minor. Update the canonical teleop feature page, which still claimed a state-only schema and a single panel, and the task README. Assert that remove_camera_configs(keep=...) keeps selected sensors and drops their observation terms. Review: #8790-M1, #8790-M2, #8790-L3 --- docs/source/features/isaac_teleop.rst | 12 +++++---- .../ovxr-robot-camera-observations.added.rst | 3 +++ ...ovxr-robot-camera-observations.changed.rst | 10 +++---- ...r => ovxr-robot-camera-observations.minor} | 0 source/isaaclab_tasks/docs/README.md | 27 +++++++------------ .../fixed_base_upper_body_ik_g1_env_cfg.py | 21 +++++++++------ .../test/core/test_xr_camera_feed_presets.py | 16 +++++++---- .../test/test_xr_camera_feed.py | 19 +++++++++++++ 8 files changed, 66 insertions(+), 42 deletions(-) rename source/isaaclab_tasks/changelog.d/{ovxr-robot-camera-observations.major => ovxr-robot-camera-observations.minor} (100%) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 7240c44e311..3b7ffe1bcb2 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1447,8 +1447,10 @@ vertical, and grid layouts. The following registered tasks enable PiP by default * ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` Both G1 tasks present the calibrated head camera in a ``head_locked`` panel that follows headset -position and orientation. Locomanipulation retains the camera as a recorded policy observation; -fixed-base G1 uses it only for PiP and keeps its policy observation schema unchanged. +position and orientation. Fixed-base G1 also shows its left and right wrist cameras in panels +0.43 m to either side of the head panel. Locomanipulation retains the head camera as a recorded +policy observation. Fixed-base G1 keeps a state-only policy observation schema by default; add +``presets=rgb`` to record its head and wrist images as policy observations. To run fixed-base G1 upper-body teleoperation with its default head-locked PiP: @@ -1465,9 +1467,9 @@ use the GPU. ``teleop_se3_agent.py`` and ``record_demos.py`` show every enabled feed when an Isaac Capture-enabled environment runs with ``--xr``. PiP is absent unless the task explicitly selects an existing -``CameraCfg`` through ``xr_camera_feeds``. Except for fixed-base G1, the selected -``robot_pov_cam`` is also a policy image observation, so the normal demonstration recorder stores -the same view shown to the operator. Each camera is parented to a physical robot body link, so the +``CameraCfg`` through ``xr_camera_feeds``. Except for fixed-base G1 without ``presets=rgb``, the +selected cameras are also policy image observations, so the normal demonstration recorder stores +the same views shown to the operator. Each camera is parented to a physical robot body link, so the camera view follows robot motion: .. figure:: ../_static/teleop/xr-camera-pip.jpg diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst index ecf50db66d4..d197eef489f 100644 --- a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.added.rst @@ -1 +1,4 @@ * Added fixed-base G1 wrist cameras and isolated robot head visuals for OVXR views. +* Added an ``rgb`` preset to ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` that adds the + head and wrist RGB images to its policy observations (``presets=rgb``). The default policy + observations stay state-only. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst index c55f11e4a58..1920a658d8b 100644 --- a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst @@ -1,6 +1,4 @@ -* **Breaking:** Added head and wrist RGB terms to the default policy observation space of - ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs``, including non-teleoperation consumers. - Existing state-only policies must set ``robot_pov_cam``, ``left_wrist_cam``, and - ``right_wrist_cam`` to ``None`` in both ``env_cfg.scene`` and ``env_cfg.observations.policy``, - and clear ``env_cfg.image_obs_list`` before environment construction. Policies using the - new defaults must accept the three image terms and configure camera rendering. +* Changed Kit ``--xr`` camera panels for the fixed-base G1 task: it shows head and wrist panels, + and head-panel uploads are capped at 20 Hz (wrist panels at 10 Hz). +* Changed GR1T2 Kit ``--xr`` camera panels to a world-fixed layout. OVXR presents head-locked + panels only; pass ``--ovxr_pip_head_locked`` there. diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.major b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.minor similarity index 100% rename from source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.major rename to source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.minor diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 699b28823e0..b5dc60e261b 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -77,28 +77,19 @@ cameras: `robot_pov_cam` at 20 Hz, and `left_wrist_cam` and `right_wrist_cam` at are mounted on their respective wrist-yaw links and look along wrist-local +X with +Z up. Their offsets are configurable in the task's scene configuration. -These cameras and image observations are task defaults, including outside XR. -They add image keys to the policy observation space and require camera rendering. +The cameras feed the task's XR camera panels. The policy observation space stays +state-only by default. Select `presets=rgb` to add the three images to the policy +group, for example to record demonstrations with them: -To retain the previous state-only observation space for an existing policy or -checkpoint, remove the three sensors, their policy terms, and their image metadata -from the resolved environment configuration before calling `gym.make()` or -`instantiate(cfg)`. This migration does not require the teleoperation package: - -```python -env_cfg.scene.robot_pov_cam = None -env_cfg.scene.left_wrist_cam = None -env_cfg.scene.right_wrist_cam = None -env_cfg.observations.policy.robot_pov_cam = None -env_cfg.observations.policy.left_wrist_cam = None -env_cfg.observations.policy.right_wrist_cam = None -env_cfg.image_obs_list = [] +```bash +uv run isaaclab teleop record --task IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs \ + --xr presets=rgb ``` -To use the new defaults, update the policy's observation schema to consume all -three image terms and configure a supported camera renderer. +`image_obs_list` names the three cameras; image consumers such as robomimic use +only the names present in the policy observations. -The same names identify policy image observations and the task's XR camera +The same names identify the policy image observations and the task's XR camera feeds, with the head panel centered and wrist panels to its left and right. Sensor `update_period` controls image sampling in simulation time; feed `max_update_hz` caps uploads. Actual delivery also depends on simulation and diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index 95c643fee9d..08ec7863bf8 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -27,6 +27,7 @@ check_kitless_backend_preparation, prepare_humanoid_teleop_environment, ) +from isaaclab_tasks.utils import preset from isaaclab_assets.robots.unitree import G1_29DOF_CFG @@ -328,11 +329,11 @@ class ActionsCfg: @configclass class ObservationsCfg: - """State and camera observations for fixed-base manipulation.""" + """State observations, with opt-in head and wrist RGB images (``presets=rgb``).""" @configclass class PolicyCfg(ObsGroup): - """Policy observations with state values and head/wrist RGB images.""" + """Policy observations with state values.""" actions = ObsTerm(func=manip_mdp.last_action) robot_joint_pos = ObsTerm( @@ -358,6 +359,14 @@ class PolicyCfg(ObsGroup): params={"left_eef_link_name": "left_wrist_yaw_link", "right_eef_link_name": "right_wrist_yaw_link"}, ) + def __post_init__(self): + self.enable_corruption = False + self.concatenate_terms = False + + @configclass + class RgbPolicyCfg(PolicyCfg): + """Policy observations with state values and head/wrist RGB images [uint8, H x W x 3].""" + robot_pov_cam = ObsTerm( func=base_mdp.image_rgb, params={ @@ -380,12 +389,8 @@ class PolicyCfg(ObsGroup): }, ) - def __post_init__(self): - self.enable_corruption = False - self.concatenate_terms = False - - # observation groups - policy: PolicyCfg = PolicyCfg() + # observation groups; the XR camera panels do not need the image terms + policy: PolicyCfg = preset(default=PolicyCfg(), rgb=RgbPolicyCfg()) @configclass diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py index 7282af0dc51..0acf45030c2 100644 --- a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -58,14 +58,20 @@ def test_g1_xr_camera_uses_calibration_and_head_locked_panel(env_cfg_type): assert camera_cfg.renderer_cfg.isaacsim_rtx == IsaacRtxRendererCfg() -def test_fixed_base_g1_pip_preserves_policy_observations_and_fixed_root(): - """Retain the existing three-camera observation contract and fixed-base root.""" - cfg = FixedBaseUpperBodyIKG1EnvCfg() +@pytest.mark.parametrize("selection", [[], ["presets=rgb"]]) +def test_fixed_base_g1_pip_keeps_state_policy_unless_rgb_selected_and_fixed_root(selection): + """Three cameras feed XR panels; their policy image terms are opt-in through ``presets=rgb``.""" + import isaaclab_tasks # noqa: F401 + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg( + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", device="cpu", num_envs=1, overrides=selection + ) assert cfg.image_obs_list == ["robot_pov_cam", "left_wrist_cam", "right_wrist_cam"] - assert hasattr(cfg.scene, "robot_pov_cam") for name in ("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"): - assert hasattr(cfg.observations.policy, name) + assert hasattr(cfg.scene, name) + assert hasattr(cfg.observations.policy, name) is bool(selection) assert cfg.scene.robot.spawn.fix_root_link is True assert [feed.camera_name for feed in cfg.isaac_teleop.xr_camera_feeds] == [ "robot_pov_cam", diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index d46d13820f6..89af03bd5a0 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -814,3 +814,22 @@ def test_ovxr_automatic_panels_use_shared_layout_without_mutating_feeds(): assert panels[0].position == pytest.approx((-0.15, -0.2, -1.2)) assert panels[1].position == pytest.approx((0.35, -0.2, -1.2)) assert feeds[0].offset_m == XrCameraFeedCfg().offset_m + + +def test_remove_camera_configs_keeps_selected_sensors_but_drops_their_observations(): + """OVXR keeps selected PiP sensors for its renderer while no observation requests a second render.""" + import isaaclab_tasks # noqa: F401 + from isaaclab_teleop import remove_camera_configs + + from isaaclab_tasks.utils import parse_env_cfg + + cfg = parse_env_cfg( + "IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", device="cpu", num_envs=1, overrides=["presets=rgb"] + ) + assert hasattr(cfg.observations.policy, "left_wrist_cam") + remove_camera_configs(cfg, keep=("robot_pov_cam", "left_wrist_cam")) + assert hasattr(cfg.scene, "robot_pov_cam") and hasattr(cfg.scene, "left_wrist_cam") + assert not hasattr(cfg.scene, "right_wrist_cam") + for name in ("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"): + assert not hasattr(cfg.observations.policy, name) + assert hasattr(cfg.observations.policy, "robot_joint_pos") From c112629fa49cdc769beadd21bdc2f74808fd15b2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:44:25 -0400 Subject: [PATCH 019/145] Check XR head exclusions against the real assets The G1 pattern matches the asset's only head visual mesh and the GR1T2 pattern all three head visual meshes, with no collision geometry (verified on the Isaac 6.2 USDs). Pin that with the asset-backed tests, because OVXR attach fails closed when a pattern matches nothing. Review: #8790-M3 --- .../test/contrib/test_teleop_task_assets.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py index 64c63976fe9..719ad8ea32f 100644 --- a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -95,3 +95,39 @@ def test_gr1t2_kitless_kinematics_urdf_matches_the_spawned_asset(): assert set(urdf_limits) == set(usd_limits) for name, (lower, upper) in usd_limits.items(): assert urdf_limits[name] == pytest.approx((lower, upper), abs=1.0e-4), name + + +@pytest.mark.parametrize("task", ["fixed_base_g1", "gr1t2", "gr1t2_waist"]) +def test_xr_head_exclusions_match_only_visual_head_meshes(task): + """OVXR attach fails closed when an exclusion matches nothing; check the patterns on the real assets.""" + from pxr import UsdGeom + + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import ( + PickPlaceGR1T2WaistEnabledEnvCfg, + ) + + cfg_type = { + "fixed_base_g1": FixedBaseUpperBodyIKG1EnvCfg, + "gr1t2": PickPlaceGR1T2EnvCfg, + "gr1t2_waist": PickPlaceGR1T2WaistEnabledEnvCfg, + }[task] + cfg = cfg_type() + if cfg.xr is None or not cfg.xr.excluded_visual_meshes: + pytest.skip("The teleoperation configuration is unavailable") + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + for pattern in cfg.xr.excluded_visual_meshes: + assert pattern.startswith("/World/envs/env_0/Robot/"), pattern + meshes = [ + prim + for prim in _robot_paths(stage, _ENV_ROBOT + pattern[len("/World/envs/env_0/Robot") :]) + if prim.IsA(UsdGeom.Mesh) + ] + assert meshes, pattern + for prim in meshes: + assert "collisions" not in str(prim.GetPath()).split("/") + assert not prim.HasAPI(UsdPhysics.CollisionAPI) + assert "head" in str(prim.GetPath()) From 2ee31c6c3d9ba84e6aab34671288b82ae2d99954 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:44:25 -0400 Subject: [PATCH 020/145] Import the desktop mirror only when enabled - Import Capture's mirror modules only when the mirror is enabled, so --headless never depends on the optional desktop-mirror selection. - Reject a missing selected camera before native Stage binding. - Pin the Camera._update_poses -> renderer.update_camera contract the adapter relies on, and document auxiliary_render_product_paths. Review: #8790-L1, #8790-L4, #8790-N5, #8790-N6 --- .../test_camera_pose_renderer_notification.py | 38 +++++++++++++++++++ .../isaaclab_ov/renderers/render_lease.py | 6 +++ .../isaaclab_teleop/ov_libraries_xr.py | 16 +++++--- .../test/test_ov_libraries_xr.py | 12 ++++++ 4 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py diff --git a/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py new file mode 100644 index 00000000000..24f778cd4a8 --- /dev/null +++ b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py @@ -0,0 +1,38 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Pin the camera pose-refresh contract that external renderer borrowers rely on.""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import warp as wp + +from isaaclab.sensors.camera.camera import Camera + + +def test_camera_pose_refresh_notifies_its_renderer(): + """``Camera._update_poses`` must forward the refreshed world pose to its renderer. + + The OVXR adapter refreshes selected PiP camera poses through this method before each scene + publication; a refactor that stops notifying the renderer would leave those panels stale. + """ + camera = Camera.__new__(Camera) + pose = SimpleNamespace(warp=wp.zeros(1, dtype=wp.vec3f, device="cpu")) + orientation = SimpleNamespace(warp=wp.array([[0.0, 0.0, 0.0, 1.0]], dtype=wp.quatf, device="cpu")) + camera._sensor_prims = [object()] + camera._view = SimpleNamespace(get_world_poses=lambda _env_ids: (pose, orientation)) + camera._update_camera_state = MagicMock() + camera._render_data = object() + camera._renderer = MagicMock() + camera._data = SimpleNamespace(pos_w="positions", quat_w_world="orientations", intrinsic_matrices="intrinsics") + + camera._update_poses() + + camera._update_camera_state.assert_called_once() + assert camera._update_camera_state.call_args.kwargs["update_pose"] is True + camera._renderer.update_camera.assert_called_once_with( + camera._render_data, "positions", "orientations", "intrinsics" + ) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index c28e57d8e29..a9f96548247 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -74,6 +74,12 @@ def __exit__(self, *_exc: object) -> None: @property def auxiliary_render_product_paths(self) -> tuple[str, ...]: + """RenderProduct paths of the selected auxiliary cameras, in configuration order. + + Raises: + RuntimeError: If the lease is not bound and live, or a selected camera did not + materialize exactly one product. + """ self.validate_live() return cast("RenderLeaseController", self._controller).product_paths(self) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index ebdb3b29efe..28634517e6a 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -292,6 +292,12 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: self._bootstrap.claim_render_owner_transfer(capability) unwrapped = getattr(self._env, "unwrapped", self._env) + if self._auxiliary_cameras: + # Fail before native Stage binding and partition export. + sensors = unwrapped.scene.sensors + for camera in self._auxiliary_cameras: + if sensors.get(camera.name) is None: + raise RuntimeError(f"Configured auxiliary camera {camera.name!r} is not present in the environment") _apply_hidden_visual_prims(self._env, self._hidden_visual_prims) _declare_ovxr_runtime_prims(self._env, self._cfg) from .xr_scene_partition import partition_visual_meshes @@ -301,10 +307,6 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: self._lease = self._bootstrap.bind_environment(self._env) auxiliary_paths = () if self._auxiliary_cameras: - sensors = unwrapped.scene.sensors - for camera in self._auxiliary_cameras: - if sensors.get(camera.name) is None: - raise RuntimeError(f"Configured auxiliary camera {camera.name!r} is not present in the environment") auxiliary_paths = self._lease.auxiliary_render_product_paths with self._lease.ovstage_owner_lane(): self._lease.configure_external_camera_scene_partitions(_ENVIRONMENT_ROOT_PATH, self._cfg.camera_paths) @@ -329,8 +331,6 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: setup_error.add_note(f"Anchor query cleanup also failed: {cleanup_error!r}") raise from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig, CameraPanelConsumer - from isaaccapture.ovxr_session_manager.desktop_mirror import DesktopMirrorConfig - from isaaccapture.ovxr_session_manager.owned_mirror import MirrorProduct, OwnedDesktopMirror panels = tuple( CameraPanelConfig( @@ -346,6 +346,10 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: ) consumers = [CameraPanelConsumer(panels)] if panels else [] if self._cfg.desktop_mirror_enabled: + # Headless sessions never import the optional mirror modules. + from isaaccapture.ovxr_session_manager.desktop_mirror import DesktopMirrorConfig + from isaaccapture.ovxr_session_manager.owned_mirror import MirrorProduct, OwnedDesktopMirror + self.desktop_mirror = OwnedDesktopMirror( DesktopMirrorConfig( max_fps=self._cfg.desktop_mirror_max_fps, diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 91ddd57e918..f0b0a24fe61 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -505,6 +505,18 @@ def test_host_rejects_a_missing_selected_auxiliary_camera(xr_module, monkeypatch with pytest.raises(RuntimeError, match="not present"): host.attach_on_render_thread(host.offer_render_owner_transfer()) + # The selection fails before native Stage binding and partition export. + assert not any(isinstance(event, tuple) and event[0] == "bind" for event in bootstrap.events) + + +def test_headless_attach_does_not_import_the_desktop_mirror(xr_module, monkeypatch): + """``--headless`` must work without the optional desktop-mirror modules.""" + host, _bootstrap, _env = _make_host(xr_module, monkeypatch) + monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager.owned_mirror", None) + monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager.desktop_mirror", None) + assert not host._cfg.desktop_mirror_enabled + binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) + assert binding.frame_consumers == () def test_selected_camera_poses_follow_motion_and_reset_before_publication(xr_module, monkeypatch): From fa1f569aac6e96f53ea141bf015c44448eafd71a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 09:44:25 -0400 Subject: [PATCH 021/145] Share OVXR presentation options between live and replay Replay lacked --ovxr_pip_fps, --ovxr_pip_view_position and --ovxr_desktop_mirror_fps (hard-coding 30 Hz), always demanded --ovxr_pip_head_locked, and rebuilt the PiP selection by hand. Both entrypoints now share one parser/validator and layout rule, and replay reuses its validated select_ovxr_camera_feeds result. Also restore the scene check in XrCameraFeedSession, fix stale camera comments, and restore the Isaac Capture branding in the README. Review: #8790-L2, #8790-L3, #8790-N1, #8790-N4, #8790-N7 --- .../leased-camera-products.added.rst | 1 - .../changelog.d/ovxr-camera-panels.added.rst | 9 +- .../ovxr-camera-panels.changed.rst | 2 + source/isaaclab_teleop/docs/README.md | 4 +- .../isaaclab_teleop/camera_feed.py | 2 +- .../entrypoints/_ovxr_launch.py | 86 +++++++++++++++++++ .../entrypoints/teleop_replay_agent.py | 66 ++++++-------- .../entrypoints/teleop_se3_agent.py | 75 +++------------- .../test/test_teleop_replay_bootstrap.py | 64 ++++++++++++++ 9 files changed, 201 insertions(+), 108 deletions(-) create mode 100644 source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst index af673174ba2..6dd14498977 100644 --- a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst @@ -1,2 +1 @@ * Added leased camera render products and independent camera-panel scheduling to external consumers. -* Registered visual materials before deferred scene population and bound their addresses before publication. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst index ae5bcc606fd..3c8cac94a78 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -1,7 +1,12 @@ * Added optional OVXR camera panels and a desktop mirror as application-configured Isaac Capture frame consumers using completed render products, with OVUI's default - Vulkan backend for the mirror. + Vulkan backend for the mirror. :class:`~isaaclab_teleop.OvLibrariesXrCameraConfig` selects a panel. * Enabled the mirror by default in OVXR live and MCAP replay entrypoints; ``--headless`` disables the local window while preserving XR and camera-panel rendering. * Supported task-configured automatic layouts for head-locked panels - in live teleoperation and replay. + in live teleoperation and replay. Replay accepts the live ``--ovxr_pip_fps``, + ``--ovxr_pip_view_position`` and ``--ovxr_desktop_mirror_fps`` options. +* Added :attr:`~isaaclab_teleop.XrCfg.excluded_visual_meshes`, which hides matching visual meshes + from the OVXR eye cameras only. +* Added the ``keep`` argument to :func:`~isaaclab_teleop.remove_camera_configs` to retain selected + cameras while removing their observation terms. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst new file mode 100644 index 00000000000..d9472f15c61 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst @@ -0,0 +1,2 @@ +* Changed MCAP replay PiP to follow the task's camera layout like live teleoperation; + ``--ovxr_pip_head_locked`` is required only when that layout is not head-locked. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index f5af33f7df2..0f7f1b64ad3 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -166,7 +166,7 @@ rendering without blocking. | `require_initial_bimanual_hand_tracking` | `bool` | `False` | Wait for initial hand presence before stepping | | `action_conditioner` | `Callable \| None` | `None` | Task-owned stateful pre-step action conditioning | | `action_validator` | `Callable \| None` | `None` | Task-owned fail-closed action validation | -| `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the IsaacTeleop session | +| `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the Isaac Capture session | The 25 ms `DeadlinePacingConfig` safety margin staggers Isaac Capture's Python work behind Isaac Lab's step Python, giving native work such as rendering time to overlap instead of having both Python stacks @@ -224,7 +224,7 @@ startup and client connection. For OVXR, use the separate - **`isaaclab`** -- core Isaac Lab framework - **`isaaclab_ov`** -- required by the optional OVXR session adapter; the standalone `isaaclab_teleop` package does not install this backend automatically -- **`isaacteleop`** -- IsaacTeleop retargeting engine, device I/O, and session management +- **`isaacteleop`** -- Isaac Capture retargeting engine, device I/O, and session management - **`isaacsim`** -- required for the Kit XR bridge, not for the OVXR session path - **OVXR / OVRTX / OVStage** -- matched native runtime and stock peers for OVXR - **OVUI** -- optional desktop-mirror extra; headset-only operation does not import it diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index eb629ff7dd4..e2ac1091127 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -139,7 +139,7 @@ def _prepare_camera_feed_cfgs(env_cfg: Any, cfgs: list[XrCameraFeedCfg]) -> list if duplicates: raise ValueError(f"XR camera feeds must have unique camera names. Duplicates: {duplicates}.") - scene = env_cfg.scene + scene = getattr(env_cfg, "scene", None) if scene is None: raise ValueError("XR camera feeds require an environment configuration with a scene.") for cfg in enabled_cfgs: diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 0efc91498e3..ab2fefbd7bb 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -7,6 +7,8 @@ from __future__ import annotations +import argparse +import math from typing import Any @@ -27,3 +29,87 @@ def require_kitless_physics(env_cfg: Any) -> None: "--ovxr requires a kitless physics preset, but the task leaves sim.physics unset (Isaac Sim" " PhysX through Kit). Select one, for example physics=ovphysx or physics=newton_mjwarp." ) + + +def add_ovxr_presentation_args(parser: argparse.ArgumentParser) -> None: + """Add the OVXR camera-panel and desktop-mirror options shared by live teleoperation and replay.""" + parser.add_argument( + "--ovxr_desktop_mirror", + action="store_true", + default=False, + help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", + ) + parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") + parser.add_argument( + "--ovxr_desktop_mirror_fps", + type=float, + default=30.0, + help="Maximum desktop-mirror update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_camera", + type=str, + nargs="+", + default=None, + help="Show up to three task-configured cameras as application-owned XR panels.", + ) + parser.add_argument( + "--ovxr_pip_fps", + type=float, + default=30.0, + help="Maximum OVXR PiP update rate (default: 30 Hz).", + ) + parser.add_argument( + "--ovxr_pip_head_locked", + action="store_true", + help="Place the application camera quads in the OpenXR VIEW space, overriding the task layout.", + ) + parser.add_argument( + "--ovxr_pip_view_position", + type=float, + nargs=3, + metavar=("X", "Y", "Z"), + help="Optional VIEW-space camera panel center in meters; requires --ovxr_pip_head_locked.", + ) + + +def validate_ovxr_presentation_args( + parser: argparse.ArgumentParser, args: argparse.Namespace, *, using_ovxr: bool +) -> None: + """Validate the shared OVXR presentation options and resolve the desktop-mirror default. + + Args: + parser: Parser used to report errors. + args: Parsed namespace; ``ovxr_desktop_mirror`` is set to the resolved selection. + using_ovxr: Whether the command selected ``--ovxr``. + """ + if args.ovxr_desktop_mirror and not using_ovxr: + parser.error("--ovxr_desktop_mirror requires --ovxr") + if args.ovxr_pip_camera is not None and not using_ovxr: + parser.error("--ovxr_pip_camera requires --ovxr") + if (args.ovxr_pip_head_locked or args.ovxr_pip_view_position is not None) and not using_ovxr: + parser.error("OVXR presentation options require --ovxr") + args.ovxr_desktop_mirror = using_ovxr and not args.headless + if args.ovxr_pip_camera is not None and getattr(args, "disable_external_cameras", False): + parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") + if not math.isfinite(args.ovxr_pip_fps) or args.ovxr_pip_fps <= 0.0: + parser.error("--ovxr_pip_fps must be finite and positive") + if args.ovxr_pip_view_position is not None: + if not args.ovxr_pip_head_locked or args.ovxr_pip_camera is None: + parser.error("--ovxr_pip_view_position requires a head-locked PiP camera") + if len(args.ovxr_pip_camera) > 1: + parser.error("A PiP position override requires one camera; multiple panels use task layout") + if not all(math.isfinite(value) for value in args.ovxr_pip_view_position): + parser.error("--ovxr_pip_view_position must be finite") + if not math.isfinite(args.ovxr_desktop_mirror_fps) or args.ovxr_desktop_mirror_fps <= 0: + parser.error("--ovxr_desktop_mirror_fps must be finite and positive") + + +def ovxr_camera_layout(env_cfg: Any, head_locked: bool) -> Any: + """Return the task's camera-panel layout, forced to head-locked placement when requested.""" + from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg + + layout = env_cfg.isaac_teleop.xr_camera_feed_layout + if head_locked: + layout = (layout or XrCameraFeedLayoutCfg()).replace(placement="head_locked") + return layout diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 27ac1a713e7..e5251b1328d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -14,9 +14,11 @@ OVXR viewer: ``--ovxr`` presents replay through an independently owned OVXR session while - tracker and control input remain sourced from MCAP. Optional camera quads - use ``--ovxr_pip_camera NAME --ovxr_pip_head_locked``. The desktop mirror is enabled - by default; ``--headless`` disables it. See ``source/isaaclab_teleop/docs/kitless_xr.md`` + tracker and control input remain sourced from MCAP. Optional camera quads use + ``--ovxr_pip_camera NAME`` with the same rate, placement and position options as live + teleoperation (``--ovxr_pip_head_locked`` overrides a task layout that is not head-locked). + The desktop mirror is enabled by default; ``--headless`` disables it. See + ``source/isaaclab_teleop/docs/kitless_xr.md`` for installation and launch instructions. Inputs: @@ -296,13 +298,20 @@ poll_control_events, ) from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action -from ._ovxr_launch import require_kitless_physics +from ._ovxr_launch import ( + add_ovxr_presentation_args, + ovxr_camera_layout, + require_kitless_physics, + validate_ovxr_presentation_args, +) logger = logging.getLogger(__name__) args_cli: argparse.Namespace """Parsed command line, set by :func:`_parse_args`.""" hydra_overrides: list[str] +ovxr_pip_selection: list = [] +"""OVXR PiP cameras selected while preparing the environment configuration.""" """Unrecognized arguments, forwarded as Hydra task config overrides.""" @@ -319,14 +328,7 @@ def _build_parser() -> argparse.ArgumentParser: parser.add_argument( "--ovxr", action="store_true", help="Present recorded motion through kitless OVXR; input stays MCAP." ) - parser.add_argument("--ovxr_pip_camera", nargs="+", default=None, help="Task camera names to show as OVXR panels.") - parser.add_argument("--ovxr_pip_head_locked", action="store_true", help="Use the supported head-locked PiP layout.") - parser.add_argument( - "--ovxr_desktop_mirror", - action="store_true", - help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", - ) - parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") + add_ovxr_presentation_args(parser) parser.add_argument("--ovxr_session_mode", choices=("ar", "vr"), default="ar") parser.add_argument( "--replay_file", @@ -454,8 +456,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = _build_parser() args_cli, hydra_overrides = setup_preset_cli(parser) - if args_cli.ovxr: - args_cli.ovxr_desktop_mirror = not args_cli.headless + validate_ovxr_presentation_args(parser, args_cli, using_ovxr=args_cli.ovxr) sys.argv = [sys.argv[0]] + hydra_overrides # Enable external camera rendering by default so the replay mimics the production teleop env (perf @@ -1131,8 +1132,9 @@ def _prepare_env_cfg( ) if hasattr(env_cfg.terminations, "time_out"): env_cfg.terminations.time_out = None - # Keep camera configs when external cameras are enabled (defaulted on) so the replay - # renders them for production parity; otherwise strip them for a lighter headless replay. + # OVXR renders only through its leased renderer: keep just the selected PiP cameras. + # Otherwise keep camera configs when external cameras are enabled (defaulted on) so the + # replay renders them for production parity, or strip them for a lighter headless replay. if args_cli.ovxr: if args_cli.num_envs != 1 or args_cli.xr: raise ValueError("--ovxr requires one environment and no Kit XR or local visualizer") @@ -1142,16 +1144,12 @@ def _prepare_env_cfg( from ..camera_feed import select_ovxr_camera_feeds + global ovxr_pip_selection names = args_cli.ovxr_pip_camera or [] - if names and args_cli.disable_external_cameras: - raise ValueError("OVXR PiP conflicts with --disable_external_cameras") - if names and not args_cli.ovxr_pip_head_locked: - raise ValueError("OVXR replay PiP requires --ovxr_pip_head_locked") - select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) + # The presentation reuses this validated selection and its shared renderer configuration. + ovxr_pip_selection = select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) env_cfg = remove_camera_configs(env_cfg, keep=tuple(names)) require_kitless_physics(env_cfg) - elif args_cli.ovxr_pip_camera or args_cli.ovxr_desktop_mirror or args_cli.ovxr_pip_head_locked: - raise ValueError("OVXR presentation options require --ovxr") elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. @@ -1243,22 +1241,13 @@ def __init__(self, env_cfg): from .. import OvLibrariesXrConfig, OvLibrariesXrSession from ..camera_feed import resolve_ovxr_camera_configs - from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg - - selected = [ - ( - name, - getattr(env_cfg.scene, name), - next( - feed for feed in env_cfg.isaac_teleop.xr_camera_feeds if feed.enabled and feed.camera_name == name - ), - ) - for name in args_cli.ovxr_pip_camera or [] - ] - layout = (env_cfg.isaac_teleop.xr_camera_feed_layout or XrCameraFeedLayoutCfg()).replace( - placement="head_locked" + + cameras = resolve_ovxr_camera_configs( + ovxr_pip_selection, + ovxr_camera_layout(env_cfg, args_cli.ovxr_pip_head_locked), + max_fps=args_cli.ovxr_pip_fps, + position_override=args_cli.ovxr_pip_view_position, ) - cameras = resolve_ovxr_camera_configs(selected, layout, max_fps=30.0) self.session = OvLibrariesXrSession.prepare( env_cfg.sim, OvLibrariesXrConfig( @@ -1267,6 +1256,7 @@ def __init__(self, env_cfg): cloudxr_env_file=args_cli.cloudxr_env, auto_launch_cloudxr=args_cli.auto_launch_cloudxr, desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, + desktop_mirror_max_fps=args_cli.ovxr_desktop_mirror_fps, auxiliary_cameras=cameras, ), cameras[0].camera_cfg.renderer_cfg if cameras else OVRTXRendererCfg(), diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index f9d6dc2cf9b..7c66064400f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -54,7 +54,12 @@ ) from ..action_policy import prepare_teleop_action from ..device_selection import create_teleop_device, select_teleop_cfg -from ._ovxr_launch import require_kitless_physics +from ._ovxr_launch import ( + add_ovxr_presentation_args, + ovxr_camera_layout, + require_kitless_physics, + validate_ovxr_presentation_args, +) logger = logging.getLogger(__name__) @@ -136,44 +141,7 @@ def _build_parser() -> argparse.ArgumentParser: default="ar", help="OVXR session mode. Defaults to 'ar'; pass 'vr' for an opaque background.", ) - parser.add_argument( - "--ovxr_desktop_mirror", - action="store_true", - default=False, - help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", - ) - parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") - parser.add_argument( - "--ovxr_desktop_mirror_fps", - type=float, - default=30.0, - help="Maximum desktop-mirror update rate (default: 30 Hz).", - ) - parser.add_argument( - "--ovxr_pip_camera", - type=str, - nargs="+", - default=None, - help="Show up to three task-configured cameras as application-owned XR panels.", - ) - parser.add_argument( - "--ovxr_pip_fps", - type=float, - default=30.0, - help="Maximum OVXR PiP update rate (default: 30 Hz).", - ) - parser.add_argument( - "--ovxr_pip_head_locked", - action="store_true", - help="Place the application camera quad in the OpenXR VIEW space.", - ) - parser.add_argument( - "--ovxr_pip_view_position", - type=float, - nargs=3, - metavar=("X", "Y", "Z"), - help="Optional VIEW-space camera panel center in meters; requires --ovxr_pip_head_locked.", - ) + add_ovxr_presentation_args(parser) add_launcher_args(parser) return parser @@ -192,22 +160,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser.error("--step_hz must be finite and non-negative; 0 disables active-control pacing") if args_cli.step_hz > 0.0 and not args_cli.using_ovxr: parser.error("--step_hz requires --ovxr") - if args_cli.ovxr_desktop_mirror and not args_cli.using_ovxr: - parser.error("--ovxr_desktop_mirror requires --ovxr") - args_cli.ovxr_desktop_mirror = args_cli.using_ovxr and not args_cli.headless - if args_cli.ovxr_pip_camera is not None and not args_cli.using_ovxr: - parser.error("--ovxr_pip_camera requires --ovxr") - if args_cli.ovxr_pip_camera is not None and args_cli.disable_external_cameras: - parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") - if not math.isfinite(args_cli.ovxr_pip_fps) or args_cli.ovxr_pip_fps <= 0.0: - parser.error("--ovxr_pip_fps must be finite and positive") - if args_cli.ovxr_pip_view_position is not None: - if not args_cli.ovxr_pip_head_locked or args_cli.ovxr_pip_camera is None: - parser.error("--ovxr_pip_view_position requires a head-locked PiP camera") - if not all(math.isfinite(value) for value in args_cli.ovxr_pip_view_position): - parser.error("--ovxr_pip_view_position must be finite") - if not math.isfinite(args_cli.ovxr_desktop_mirror_fps) or args_cli.ovxr_desktop_mirror_fps <= 0: - parser.error("--ovxr_desktop_mirror_fps must be finite and positive") + validate_ovxr_presentation_args(parser, args_cli, using_ovxr=args_cli.using_ovxr) # Call an external callback if requested. if args_cli.external_callback: @@ -383,13 +336,11 @@ def _resolve_env_cfg() -> tuple[ from ..camera_feed import select_ovxr_camera_feeds names = args_cli.ovxr_pip_camera - if len(names) > 1 and args_cli.ovxr_pip_view_position is not None: - raise ValueError("A PiP position override requires one camera; multiple panels use task layout") pip_selection = select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) keep_cameras = tuple(names) - # The leased OVRTX renderer is the only render path in OVXR. Strip - # every task camera before the launcher scans the config. + # The leased OVRTX renderer is the only render path in OVXR. Strip every task camera + # except the selected PiP cameras before the launcher scans the config. if args_cli.using_ovxr or args_cli.disable_external_cameras: from .. import remove_camera_configs @@ -552,14 +503,10 @@ def run_teleoperation( # noqa: C901 from .. import OvLibrariesXrConfig, OvLibrariesXrSession from ..camera_feed import resolve_ovxr_camera_configs - from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg - layout = env_cfg.isaac_teleop.xr_camera_feed_layout - if args_cli.ovxr_pip_head_locked: - layout = (layout or XrCameraFeedLayoutCfg()).replace(placement="head_locked") selected_cameras = resolve_ovxr_camera_configs( pip_selection, - layout, + ovxr_camera_layout(env_cfg, args_cli.ovxr_pip_head_locked), max_fps=args_cli.ovxr_pip_fps, position_override=args_cli.ovxr_pip_view_position, ) diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index fc3b5959c11..dfd68a53ed8 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -64,18 +64,31 @@ def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_present original_layout = config.isaac_teleop.xr_camera_feed_layout original_layout_fields = original_layout.to_dict() if original_layout is not None else None original_feeds = [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop.camera_feed import select_ovxr_camera_feeds + monkeypatch.setitem( replay_presentation_type.__init__.__globals__, "args_cli", SimpleNamespace( ovxr_pip_camera=names, + ovxr_pip_head_locked=True, + ovxr_pip_fps=30.0, + ovxr_pip_view_position=None, task="Test", ovxr_session_mode="ar", cloudxr_env="none", auto_launch_cloudxr=False, ovxr_desktop_mirror=False, + ovxr_desktop_mirror_fps=30.0, ), ) + # Environment preparation records the validated selection that the presentation reuses. + monkeypatch.setitem( + replay_presentation_type.__init__.__globals__, + "ovxr_pip_selection", + select_ovxr_camera_feeds(config, names, OVRTXRendererCfg()), + ) prepare = Mock() monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) presentation = replay_presentation_type(config) @@ -404,3 +417,54 @@ def condition(_env, action, state): assert conditioning_frames == ([0, 1, 2, 3, 4, 0, 1, 2] if start_on_replay else [0, 1, 0, 1]) assert stats.active_iterations == len(applied) assert stats.outcome == "incomplete" + + +@pytest.mark.parametrize( + ("arguments", "message"), + [ + (["--ovxr_pip_head_locked"], "OVXR presentation options require --ovxr"), + (["--ovxr", "--ovxr_pip_fps", "0"], "--ovxr_pip_fps must be finite and positive"), + ( + [ + "--ovxr", + "--ovxr_pip_camera", + "a", + "b", + "--ovxr_pip_head_locked", + "--ovxr_pip_view_position", + "0", + "0", + "-1", + ], + "A PiP position override requires one camera", + ), + ], +) +def test_replay_and_live_share_ovxr_presentation_validation(monkeypatch, capsys, arguments, message): + """Replay accepts the live OVXR presentation options and rejects the same invalid selections.""" + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + for module, required in ( + (teleop_replay_agent, ["--task", "Test", "--replay_file", "recording.mcap"]), + (teleop_se3_agent, ["--task", "Test"]), + ): + monkeypatch.setattr(sys, "argv", ["agent.py", *required, *arguments]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + with pytest.raises(SystemExit): + module._parse_args() + assert message in capsys.readouterr().err + + +def test_replay_accepts_live_ovxr_rate_and_position_options(monkeypatch): + from isaaclab_teleop.entrypoints import teleop_replay_agent + + arguments = ["--task", "Test", "--replay_file", "recording.mcap", "--ovxr", "--ovxr_pip_camera", "head"] + arguments += ["--ovxr_pip_head_locked", "--ovxr_pip_view_position", "0", "-0.1", "-0.8"] + arguments += ["--ovxr_pip_fps", "15", "--ovxr_desktop_mirror_fps", "20", "--headless"] + monkeypatch.setattr(sys, "argv", ["teleop_replay_agent.py", *arguments]) + monkeypatch.setattr(teleop_replay_agent, "args_cli", None, raising=False) + args = teleop_replay_agent._parse_args() + assert args.ovxr_pip_fps == 15.0 + assert args.ovxr_desktop_mirror_fps == 20.0 + assert args.ovxr_pip_view_position == [0.0, -0.1, -0.8] + assert args.ovxr_desktop_mirror is False From 291de87060106816f4ac5f1f6640d1bbb5c5371e Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 10:43:56 -0400 Subject: [PATCH 022/145] Resolve preset defaults for directly built humanoid configs The fixed-base G1 rgb observation preset left a directly constructed configuration (as in test_g1_kitless_ovphysx) with an unresolved policy group, which the observation manager rejects. Humanoid teleop preparation now resolves every remaining preset to its default before construction, as task resolution would. Review: #8790-M1 (follow-up) --- .../isaaclab_tasks/contrib/teleop_physics_cfg.py | 7 +++++-- .../test/core/test_xr_camera_feed_presets.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py index 933e76586c7..da6b0754d6e 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py @@ -16,7 +16,7 @@ from isaaclab.physics import PhysxAutoCfg from isaaclab.utils import configclass -from isaaclab_tasks.utils import PresetCfg +from isaaclab_tasks.utils import PresetCfg, resolve_presets if TYPE_CHECKING: from isaaclab.envs import ManagerBasedRLEnvCfg @@ -27,7 +27,9 @@ def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: Dispatches to the configuration's ``stabilize_for_kitless_ovphysx`` or ``stabilize_for_kitless_newton_mjwarp`` for the selected standalone backend. Kit PhysX - needs no preparation. Repeated calls on a prepared configuration are no-ops. + needs no preparation. Repeated calls on a prepared configuration are no-ops. A + configuration constructed directly, without task resolution, uses the default of every + remaining preset. Args: cfg: Resolved humanoid teleoperation task configuration. @@ -35,6 +37,7 @@ def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: Raises: ValueError: If ``cfg`` was already prepared for a different physics backend. """ + resolve_presets(cfg) physics = cfg.sim.physics if isinstance(physics, OvPhysxCfg): cfg.stabilize_for_kitless_ovphysx() diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py index 0acf45030c2..92161b66db8 100644 --- a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -130,3 +130,16 @@ def test_xr_camera_reference_renderer_resolves_for_supported_backends(): assert default == expected_rtx assert isaacsim_rtx == expected_rtx assert isinstance(ovrtx, OVRTXRendererCfg) + + +def test_fixed_base_g1_direct_construction_prepares_the_state_policy_default(): + """Registered construction without task resolution uses the state-only policy preset default.""" + from isaaclab.managers import ObservationGroupCfg + + from isaaclab_tasks.contrib.teleop_physics_cfg import prepare_humanoid_teleop_environment + + cfg = FixedBaseUpperBodyIKG1EnvCfg() + prepare_humanoid_teleop_environment(cfg) + assert isinstance(cfg.observations.policy, ObservationGroupCfg) + assert not hasattr(cfg.observations.policy, "robot_pov_cam") + assert hasattr(cfg.observations.policy, "robot_joint_pos") From f4cfbe63894c160a029a81d0ce4fd2a4c581af29 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 11:23:06 -0400 Subject: [PATCH 023/145] Sort imports in the camera keep-list test --- source/isaaclab_teleop/test/test_xr_camera_feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab_teleop/test/test_xr_camera_feed.py b/source/isaaclab_teleop/test/test_xr_camera_feed.py index 89af03bd5a0..87db8788aa9 100644 --- a/source/isaaclab_teleop/test/test_xr_camera_feed.py +++ b/source/isaaclab_teleop/test/test_xr_camera_feed.py @@ -818,9 +818,9 @@ def test_ovxr_automatic_panels_use_shared_layout_without_mutating_feeds(): def test_remove_camera_configs_keeps_selected_sensors_but_drops_their_observations(): """OVXR keeps selected PiP sensors for its renderer while no observation requests a second render.""" - import isaaclab_tasks # noqa: F401 from isaaclab_teleop import remove_camera_configs + import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import parse_env_cfg cfg = parse_env_cfg( From 2551f659004f8227744372c907aac3a70939fd1e Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:02:10 -0400 Subject: [PATCH 024/145] Mock Isaac Capture for the OVXR session API docs --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index bd62975a2e5..da4c74c9eea 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -294,6 +294,7 @@ def _read_pinned_versions() -> dict: "ipywidgets", "mpl_toolkits", "isaacteleop", + "isaaccapture", "scipy", "hydra", "hydra.core", From bef60faa0ee9f58f1e66f53a7aa5d885b1872d16 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:03:38 -0400 Subject: [PATCH 025/145] Run OVXR config tests without Isaac Capture Gate only the adapter tests on Capture's OVXR session module, and check the configuration import in a fresh process where Capture and the native OV packages are absent, so the check cannot pass on a cached module. --- .../test/test_ov_libraries_xr.py | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 52013f92047..55d6b04ffe4 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -10,6 +10,7 @@ import builtins import contextlib import importlib +import subprocess import sys import threading from contextlib import contextmanager @@ -18,9 +19,7 @@ import pytest -# The adapter needs a Capture release with OVXR session support. The Kit ``teleop`` extra -# selects an older release without it; skip rather than error in that environment. -pytest.importorskip("isaaccapture.ovxr_session_manager") +from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig class _FakeLease: @@ -131,6 +130,9 @@ def xr_module(monkeypatch, real_lease_type): The stub-bound adapter is removed from ``sys.modules`` and its package on teardown so later tests import the real module. """ + # The adapter needs a Capture release with OVXR session support. The Kit ``teleop`` extra + # selects an older release without it; skip rather than error in that environment. + pytest.importorskip("isaaccapture.ovxr_session_manager") import isaaclab_teleop isaaclab_ov = ModuleType("isaaclab_ov") @@ -692,15 +694,15 @@ def create_bootstrap(*_args): xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(renderer_settings=settings)) -def test_config_rejects_unknown_session_mode(xr_module): +def test_config_rejects_unknown_session_mode(): with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): - xr_module.OvLibrariesXrConfig(session_mode="automatic") + OvLibrariesXrConfig(session_mode="automatic") @pytest.mark.parametrize("scale", [0.0, -1.0, float("nan"), float("inf")]) -def test_config_rejects_invalid_scene_scale(xr_module, scale): +def test_config_rejects_invalid_scene_scale(scale): with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): - xr_module.OvLibrariesXrConfig(meters_per_unit=scale) + OvLibrariesXrConfig(meters_per_unit=scale) def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): @@ -885,9 +887,14 @@ def test_start_rejects_anchor_state_without_the_protocol_before_owner_creation(x assert isinstance(_FakeAnchorState(), xr_module.OvLibrariesXrSession.ExternalAnchorState) -def test_config_imports_without_native_ovxr(monkeypatch): - """Launchers build the configuration without importing Capture's OVXR session stack.""" - monkeypatch.setitem(sys.modules, "isaaccapture.ovxr_session_manager", None) - from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig - - assert OvLibrariesXrConfig(session_mode="vr").session_mode == "vr" +def test_config_imports_without_native_ovxr(): + """Launchers build the configuration in a fresh process where Capture and the native OV packages are absent.""" + script = ( + "import sys\n" + "for name in ('isaaccapture', 'ovxr', 'ovrtx', 'ovstage'):\n" + " sys.modules[name] = None\n" + "from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig\n" + "assert OvLibrariesXrConfig(session_mode='vr').session_mode == 'vr'\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr From 48d1297372b325aca4a44361b89760c9dcfd2e35 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:05:09 -0400 Subject: [PATCH 026/145] Document that OVRTX material sync requires CUDA Leased and ordinary OVRTX renderers both reject CPU material buffers, so a CPU environment with VisualMaterial assets fails at reset either way. --- source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py | 4 ++++ source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 40323737203..e86f0f12a60 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -391,6 +391,10 @@ def visual_material_writer(self): A leased pair registers its writer before the borrower binds the Stage, because environment reset finalizes material consumers first; addresses compile on the first publication after population. + + The writer requires material buffers on one CUDA device. On a CPU simulation device, + an environment with :class:`~isaaclab.assets.VisualMaterial` assets therefore fails at + reset, with or without a lease. """ return ( self._create_visual_material_writer diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 3078be76127..a4c8118f5d6 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -64,6 +64,10 @@ The adapter retains the environment if lease retirement cannot establish complet default `OVRTXRendererCfg()`) is rejected while the lease is active. - Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its scene publisher writes the same physics state at the reserved ordinal. +- CPU simulation devices publish scene transforms, but visual-material + publication requires CUDA material buffers, as it does for ordinary OVRTX + cameras. With `--device cpu`, an environment that has `VisualMaterial` assets + fails at `sim.reset()`. - Dynamic anchoring takes any object implementing `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects other objects before the worker claims the lease. From a02fbf4f72aa98491a74ca14f2dd737b5542f4cf Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:08:14 -0400 Subject: [PATCH 027/145] Validate the OVXR environment count and device before the claim Stage deferral moved the leased one-environment check onto the render worker, after it claims the lease. Reject num_envs != 1 in validate_environment so start_render_worker fails on the caller, and resolve a bare 'cuda' environment device the way SimulationContext does instead of comparing strings literally. --- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 5 +- .../isaaclab_teleop/ovxr_session.py | 29 ++++--- .../isaaclab_teleop/test/test_ovxr_session.py | 75 ++++++++++++++++++- 3 files changed, 96 insertions(+), 13 deletions(-) diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index a4c8118f5d6..b4672a71a2e 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -52,8 +52,9 @@ The adapter retains the environment if lease retirement cannot establish complet - `OvLibrariesXrSession.prepare()` creates the `SimulationContext` from the `sim_cfg` it receives, and the environment reuses that context. Pass the - environment's own `env_cfg.sim`; `start_render_worker()` rejects an - environment whose simulation device or time step differs. + environment's own `env_cfg.sim`; before the worker claims the lease, + `start_render_worker()` rejects an environment whose simulation device or + time step differs, or whose `num_envs` is not 1. - Only environments that accept a pre-created simulation context are supported. Manager-based environments do; `DirectRLEnv` always creates its own context and fails with "Simulation context already exists". diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index ef1c7a54848..a693ab8432d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -37,8 +37,8 @@ class OvxrSessionBootstrap: The bootstrap creates the :class:`~isaaclab.sim.SimulationContext` from the ``sim_cfg`` given here, and the environment reuses it. Only environments that accept a pre-created context are supported (manager-based environments; :class:`~isaaclab.envs.DirectRLEnv` - always creates its own). :meth:`validate_environment` rejects an environment whose - simulation device or time step disagrees with the prepared context. + always creates its own). :meth:`validate_environment` requires ``num_envs == 1`` and + rejects a simulation device or time step that disagrees with the prepared context. """ def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None = None) -> None: @@ -104,7 +104,7 @@ def validate_environment(self, env: Any) -> None: Raises: RuntimeError: If the environment did not reuse the prepared context, has no - completed stage or a non-positive environment count, or its simulation + completed stage or not exactly one environment, or its simulation configuration selects a different device or physics time step [s]. """ self._require_open() @@ -113,17 +113,26 @@ def validate_environment(self, env: Any) -> None: raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") if unwrapped.scene.stage is None: raise RuntimeError("Environment has no completed InteractiveScene Stage") - num_envs = unwrapped.num_envs - if not isinstance(num_envs, int) or num_envs <= 0: - raise RuntimeError(f"Environment has invalid num_envs: {num_envs!r}") + if unwrapped.num_envs != 1: + raise RuntimeError( + f"The OVXR render lease requires exactly one environment; got num_envs={unwrapped.num_envs!r}" + ) env_sim_cfg = getattr(getattr(unwrapped, "cfg", None), "sim", None) prepared_cfg = self._sim.cfg if env_sim_cfg is not None and env_sim_cfg is not prepared_cfg: - for field in ("device", "dt"): - if getattr(env_sim_cfg, field, None) != getattr(prepared_cfg, field, None): + env_device = env_sim_cfg.device + if env_device == "cuda": + # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. + cuda_device = self._sim.get_setting("/physics/cudaDevice") + env_device = f"cuda:{max(0, int(cuda_device) if cuda_device is not None else 0)}" + for field, env_value, prepared_value in ( + ("device", env_device, prepared_cfg.device), + ("dt", env_sim_cfg.dt, prepared_cfg.dt), + ): + if env_value != prepared_value: raise RuntimeError( - f"Environment sim.{field}={getattr(env_sim_cfg, field, None)!r} differs from the OVXR" - f" prepared simulation context ({getattr(prepared_cfg, field, None)!r}); prepare the" + f"Environment sim.{field}={getattr(env_sim_cfg, field)!r} differs from the OVXR" + f" prepared simulation context ({prepared_value!r}); prepare the" " session with the environment's sim configuration" ) diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py index ec31863ca6b..eb8d6262a4b 100644 --- a/source/isaaclab_teleop/test/test_ovxr_session.py +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -5,11 +5,12 @@ """OVXR bootstrap retirement across the teleoperation/renderer boundary.""" +import builtins from types import SimpleNamespace import pytest -from isaaclab.sim import SimulationContext +from isaaclab.sim import SimulationCfg, SimulationContext pytest.importorskip("ovrtx") from isaaclab_ov.renderers import OVRTXRenderer, OVRTXRendererCfg @@ -55,3 +56,75 @@ def wrong_owner(): getattr(bootstrap, operation)() assert renderer._render_lease.active is None assert cleared == ([True] if operation == "abort" else []) + + +@pytest.fixture +def bootstrap(monkeypatch): + """Run the real bootstrap constructor and renderer lease; only the physics-backed context is replaced.""" + from isaaclab_teleop import ovxr_session + + from isaaclab.renderers import RenderContext + + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + monkeypatch.delattr(builtins, ovxr_session._TERMINAL_MODE_SENTINEL, raising=False) + renderer = OVRTXRenderer( + OVRTXRendererCfg(), sim=SimpleNamespace(device="cpu", get_scene_data_provider=lambda: None) + ) + + class Context: + current = None + + def __init__(self, cfg): + self.cfg = cfg + self.render_context = RenderContext([(renderer.cfg, renderer)]) + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + cls.current = None + + def get_or_create_backend(self, _cfg): + return renderer + + def get_setting(self, path): + # SimulationContext resolves a bare "cuda" device from this setting. + return 1 if path == "/physics/cudaDevice" else None + + monkeypatch.setattr(ovxr_session, "SimulationContext", Context) + bootstrap = ovxr_session.OvxrSessionBootstrap(SimulationCfg(device="cuda:1", dt=0.01)) + yield bootstrap + bootstrap.abort() + assert renderer._render_lease.active is None + + +def _environment(bootstrap, *, num_envs=1, device="cuda:1", dt=0.01): + return SimpleNamespace( + sim=bootstrap.simulation_context, + scene=SimpleNamespace(stage=object()), + num_envs=num_envs, + cfg=SimpleNamespace(sim=SimulationCfg(device=device, dt=dt)), + ) + + +@pytest.mark.parametrize( + ("num_envs", "device", "dt", "error"), + [ + (1, "cuda", 0.01, None), + (2, "cuda:1", 0.01, "exactly one environment"), + (1, "cuda:0", 0.01, r"sim\.device='cuda:0'"), + (1, "cuda:1", 0.02, r"sim\.dt=0\.02"), + ], +) +def test_environment_validation_resolves_devices_and_requires_one_environment(bootstrap, num_envs, device, dt, error): + """The caller-side check resolves a bare CUDA device like SimulationContext and rejects late worker failures.""" + env = _environment(bootstrap, num_envs=num_envs, device=device, dt=dt) + if error is None: + bootstrap.validate_environment(env) + else: + with pytest.raises(RuntimeError, match=error): + bootstrap.validate_environment(env) From 8776b7808adeb35c5b0b413fb79a34e7b800ab4d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:09:52 -0400 Subject: [PATCH 028/145] Require the claimed owner transfer before binding the OVXR Stage The bootstrap always defers Stage materialization, so an unclaimed bind without camera sensors deferred again and failed with 'not bound to a prepared Stage'. Reject materialization before the claim whenever deferral is selected, and document the precondition and start_render_worker's validation errors. --- .../isaaclab_ov/renderers/render_lease.py | 9 +++++++-- .../isaaclab_teleop/ov_libraries_xr.py | 4 ++++ .../isaaclab_teleop/ovxr_session.py | 16 +++++++++++++++- source/isaaclab_teleop/test/test_ovxr_session.py | 6 ++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 62bb53ce179..384b64d4973 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -111,6 +111,10 @@ def materialize_deferred_stage(self, stage: Any, num_envs: int) -> bool: Returns: Whether a deferred preparation was completed. ``False`` means nothing was deferred. + + Raises: + RuntimeError: If called off the render-owner thread, or deferral is selected and + the owner transfer has not been claimed. """ return cast("RenderLeaseController", self._controller).materialize_deferred_stage(self, stage, num_envs) @@ -484,12 +488,13 @@ def prepare_stage(self, num_envs: int) -> bool: def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: """Materialize a deferred Stage after render-thread ownership transfer.""" self.require_owner_thread("materialize a deferred Stage") + # Unclaimed, a fallback prepare_stage would only defer again; reject even if nothing is deferred yet. + if self.defer_until_transfer and not self._transfer_claimed: + raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") if not self.stage_deferred: return False if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: raise RuntimeError("Deferred Stage is not eligible for materialization") - if not self._transfer_claimed: - raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") self._renderer.prepare_stage(stage, num_envs) return True diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index 6f1dd4aa54d..0519c76c263 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -580,7 +580,11 @@ def start_render_worker( external_anchor_state: Dynamic anchor policy seeded from committed scene publications. Raises: + RuntimeError: If the session is closed or already started, or ``env`` does not reuse the + prepared simulation context, has no completed stage or not exactly one environment, + or selects a different simulation device or physics time step. TypeError: If ``external_anchor_state`` does not implement :class:`ExternalAnchorState`. + ValueError: If ``max_blank_frames`` is not a nonnegative integer. """ self._require_open() if self._owner is not None: diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index a693ab8432d..c93e89c7a30 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -137,7 +137,21 @@ def validate_environment(self, env: Any) -> None: ) def bind_environment(self, env: Any) -> OvLibrariesRenderLease: - """Prepare and bind the completed environment Stage to the reserved pair.""" + """Prepare and bind the completed environment Stage to the reserved pair. + + The bootstrap defers Stage materialization to the render owner, so call this on the + worker after :meth:`claim_render_owner_transfer`. + + Args: + env: Constructed environment, possibly wrapped. + + Returns: + The bound lease. + + Raises: + RuntimeError: If the bootstrap is closed or already bound, the owner transfer has not + been claimed, or :meth:`validate_environment` rejects the environment. + """ self._require_open() if self._bound: raise RuntimeError("OVXR bootstrap is already bound") diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py index eb8d6262a4b..3f1b57d17ad 100644 --- a/source/isaaclab_teleop/test/test_ovxr_session.py +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -128,3 +128,9 @@ def test_environment_validation_resolves_devices_and_requires_one_environment(bo else: with pytest.raises(RuntimeError, match=error): bootstrap.validate_environment(env) + + +def test_binding_requires_the_claimed_owner_transfer(bootstrap): + """Binding before the claim names the missing claim, even when no camera deferred the Stage yet.""" + with pytest.raises(RuntimeError, match="requires a claimed owner transfer"): + bootstrap.bind_environment(_environment(bootstrap)) From ab024b50d47beb798dbbfdd0b1e4d2a2767b3ebd Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:10:05 -0400 Subject: [PATCH 029/145] Drop the XrAnchorManager claim from the anchor protocol docstring XrAnchorManager implements ExternalAnchorState only once the dynamic-anchor layer lands; until then passing one is rejected. --- source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index 0519c76c263..c4b38f87986 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -475,11 +475,7 @@ class OvLibrariesXrSession: @runtime_checkable class ExternalAnchorState(Protocol): - """Dynamic XR-anchor policy sealed together with complete scene publications. - - :class:`~isaaclab_teleop.xr_anchor_manager.XrAnchorManager` implements this protocol - when it is constructed for an OVXR session. - """ + """Dynamic XR-anchor policy sealed together with complete scene publications.""" @property def external_anchor_paths(self) -> tuple[str, ...]: From 00ee188dbd670d151a5b6241a08d6f8da3f865a4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:11:55 -0400 Subject: [PATCH 030/145] Sort imports in the OVXR adapter test --- source/isaaclab_teleop/test/test_ov_libraries_xr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 55d6b04ffe4..8f6e4cdb396 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -18,7 +18,6 @@ from types import ModuleType, SimpleNamespace import pytest - from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig From e6cdb5922aa4919efe2645e23bf0ca73696f5de7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:15:47 -0400 Subject: [PATCH 031/145] Point the OVXR unset-physics error at --xr instead of a selector A task that leaves sim.physics unset declares no physics preset, so the suggested physics=ovphysx selector failed in preset resolution. Name the task and point to --xr or a kitless physics preset in its configuration. --- .../isaaclab_teleop/entrypoints/_ovxr_launch.py | 12 ++++++++---- .../entrypoints/teleop_replay_agent.py | 2 +- .../isaaclab_teleop/entrypoints/teleop_se3_agent.py | 2 +- .../test/test_teleop_entrypoint_units.py | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 0efc91498e3..500fcf81939 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -10,20 +10,24 @@ from typing import Any -def require_kitless_physics(env_cfg: Any) -> None: +def require_kitless_physics(env_cfg: Any, task: str) -> None: """Reject a task whose physics selection needs the Kit runtime before an OVXR launch. A task that leaves ``sim.physics`` unset selects Isaac Sim PhysX, which needs Kit. - The kitless OVXR session cannot run it, so fail before launch with the selector to use. + The kitless OVXR session cannot run it, so fail before launch. Such a task normally + declares no physics preset, so the error points at ``--xr`` or the task configuration + rather than at a ``physics=`` selector. Args: env_cfg: Resolved environment configuration. + task: Task name, for the error message. Raises: ValueError: If ``env_cfg.sim.physics`` is ``None``. """ if env_cfg.sim.physics is None: raise ValueError( - "--ovxr requires a kitless physics preset, but the task leaves sim.physics unset (Isaac Sim" - " PhysX through Kit). Select one, for example physics=ovphysx or physics=newton_mjwarp." + f"--ovxr needs a task that selects kitless physics (OvPhysxCfg or NewtonCfg), but task '{task}' leaves" + " sim.physics unset, which runs Isaac Sim PhysX through Kit. Run this task with --xr instead, or give" + " its configuration a kitless physics preset." ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index bbe3207011b..dde3813fb4b 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -1123,7 +1123,7 @@ def _prepare_env_cfg( if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") env_cfg = remove_camera_configs(env_cfg) - require_kitless_physics(env_cfg) + require_kitless_physics(env_cfg, task) elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 15199bcbdcf..501e8a07acb 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -323,7 +323,7 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg env_cfg = remove_camera_configs(env_cfg) if args_cli.using_ovxr: - require_kitless_physics(env_cfg) + require_kitless_physics(env_cfg, args_cli.task) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) if args_cli.using_ovxr: diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index d310b6d86da..883e8087f55 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -869,6 +869,7 @@ def validate_launch(selected_cfg, _args): _UNSET_PHYSICS_TASK = "IsaacContrib-PickPlace-GR1T2-Abs" +_UNSET_PHYSICS_OVXR_ERROR = rf"task '{_UNSET_PHYSICS_TASK}' leaves sim.physics unset.*Run this task with --xr" def _unset_task_physics(monkeypatch, module): @@ -899,7 +900,7 @@ def test_live_ovxr_rejects_unset_task_physics_before_launch(monkeypatch): monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, "--ovxr") _unset_task_physics(monkeypatch, agent) - with pytest.raises(ValueError, match="kitless physics preset"): + with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): agent._resolve_env_cfg() @@ -917,7 +918,7 @@ def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monk replay._parse_args() _unset_task_physics(monkeypatch, replay) if ovxr: - with pytest.raises(ValueError, match="kitless physics preset"): + with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) return env_cfg, _success = replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) From 40eb1df8a85db0db74a333d6c60b9ac89f61bfee Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:17:26 -0400 Subject: [PATCH 032/145] Drive the real live loop in the rejected OVXR frame test The test called the frame helper with render=False itself, so dropping the keyword at the live-loop call site would still pass. Run the live owner with a rejecting validator and assert no render, no step, no republication and one paced wait. --- .../test/test_teleop_entrypoint_units.py | 52 ++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 883e8087f55..011a1f8f784 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -946,11 +946,49 @@ def test_rtx_gate_reads_the_launcher_kit_visualizer(monkeypatch, arguments, expe assert agent._rtx_rendering_requested(agent.args_cli) is expected -def test_rejected_ovxr_frame_does_not_render_on_the_control_thread(monkeypatch): - agent = _load_teleop_agent(monkeypatch) - env = SimpleNamespace(sim=SimpleNamespace(render=lambda: pytest.fail("rendered under OVXR"))) - stops = [] - assert not agent._apply_teleop_frame( - env, torch.zeros(1, 2), "missing left hand tracking", lambda: None, lambda: stops.append(True), render=False +def test_rejected_ovxr_frame_does_not_render_on_the_control_thread(monkeypatch, run_live_workflow): + """OVXR owns rendering, so a rejected frame holds the scene without a control-thread render or step.""" + from isaaclab_teleop.entrypoints import teleop_se3_agent as agent + + advances = 0 + renders = [] + steps = [] + requests = [] + waits = [] + + def advance(): + nonlocal advances + advances += 1 + return torch.ones((1, 2)) + + def request(identity, *, force): + requests.append(force) + return 1 + + monkeypatch.setattr(agent, "time", SimpleNamespace(perf_counter=lambda: 0.0, sleep=waits.append)) + env = SimpleNamespace( + num_envs=1, + step_dt=0.02, + reset=lambda: None, + step=steps.append, + sim=SimpleNamespace( + is_running=lambda: advances < 1, + get_physics_step_count=lambda: len(steps), + get_physics_dt=lambda: 0.005, + render=lambda: renders.append(True), + ), + ) + run_live_workflow( + env, + SimpleNamespace( + advance=advance, + reset=lambda: None, + control_events=lambda: SimpleNamespace(is_active=True, should_reset=False), + ), + SimpleNamespace(shutdown_requested=False, request_render=request, wait_for_shutdown=waits.append), + validator=lambda _action: "rejected", ) - assert stops == [True] + assert advances == 1 + assert renders == [] and steps == [] + assert requests == [True] # Only the reset publication; the held scene is not republished. + assert waits == [0.02] # A rejected frame waits one action period, like a paused one. From f28a8530284e61caf79858c19cb72166a2520afa Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:20:02 -0400 Subject: [PATCH 033/145] Describe the OvPhysX contact settings against the PhysxSceneAPI schema The bounce threshold docstring claimed every contact bounces by default; the schema default of 0 only removes the speed threshold, and restitution still decides. Quote the schema defaults for all three contact fields and say in the class docstring and backend page that OvPhysxCfg can author them onto the PhysicsScene prim. --- .../concepts/physics-backends/ovphysx.rst | 5 ++++- .../physics/ovphysx_manager_cfg.py | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/source/concepts/physics-backends/ovphysx.rst b/docs/source/concepts/physics-backends/ovphysx.rst index 06400250103..1f4e9c669b7 100644 --- a/docs/source/concepts/physics-backends/ovphysx.rst +++ b/docs/source/concepts/physics-backends/ovphysx.rst @@ -14,7 +14,10 @@ OvPhysX OvPhysX runs PhysX without Omniverse Kit. It reads scene-level solver settings from the USD ``PhysicsScene`` prim. :class:`~isaaclab_ov.physics.OvPhysxCfg` holds -GPU buffer settings that USD does not represent. +GPU buffer settings and can also author selected contact settings onto that prim: +``bounce_threshold_velocity``, ``friction_correlation_distance`` and +``solve_articulation_contact_last``. Contact settings left at ``None`` keep the +stage value. OvPhysX is selected through :class:`~isaaclab_ov.physics.OvPhysxCfg`: diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py index 02b56a2f5a4..a19d5e2683d 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py @@ -43,9 +43,13 @@ class OvPhysxBackendCfg(BackendCfg): class OvPhysxCfg(PhysicsCfg): """Configuration for the ovphysx physics manager. - PhysX scene-level parameters (solver iterations, GPU buffer sizes, etc.) are - read from the USD PhysicsScene prim. Only ovphysx-specific settings that are - not captured in USD live here. + PhysX scene-level parameters that this config does not set, such as solver iterations, + are read from the USD PhysicsScene prim. Before loading, the manager authors the scene + settings held here onto that prim: determinism, external forces, GPU buffer sizes (GPU + only) and the optional contact parameters :attr:`bounce_threshold_velocity`, + :attr:`friction_correlation_distance` and :attr:`solve_articulation_contact_last`. A + contact parameter left at ``None`` keeps the stage value, or the ``PhysxSceneAPI`` + schema default when the stage does not author it. """ class_type = "{DIR}.ovphysx_manager:OvPhysxManager" @@ -92,22 +96,23 @@ class OvPhysxCfg(PhysicsCfg): bounce_threshold_velocity: float | None = None """Relative contact velocity below which contacts do not bounce [m/s]. - ``None`` leaves the stage value (OvPhysX's default is effectively 0, so every contact bounces). + ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is 0, which sets no speed + threshold, so contact restitution alone decides whether a contact bounces. Authored as ``physxScene:bounceThreshold``. """ friction_correlation_distance: float | None = None """Distance below which contact points merge into one friction anchor [m]. - ``None`` leaves the stage value (OvPhysX's default is 0.025 / metersPerUnit). - Authored as ``physxScene:frictionCorrelationDistance``. + ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is ``0.025 * tolerancesLength``, + which is 0.025 m on a meter-based stage. Authored as ``physxScene:frictionCorrelationDistance``. """ solve_articulation_contact_last: bool | None = None """Whether the solver resolves articulation contacts after other constraints. Enable it so object contacts can stall position-driven grippers before they tunnel through a - grasp. ``None`` leaves the stage value (off by default). Authored as + grasp. ``None`` leaves the stage value. The ``PhysxSceneAPI`` schema default is off. Authored as ``physxScene:solveArticulationContactLast``. """ From 1baae4d43b1d01a3e21aa60d6380881ae50222a8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:20:02 -0400 Subject: [PATCH 034/145] Run the OvPhysX contact settings test on one device The contact attributes are authored outside the GPU-only branch, so the cpu and gpu cases exercised the same path. --- .../test/physics/test_ovphysx_manager_lifecycle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py index 797058724b4..b382280a087 100644 --- a/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py +++ b/source/isaaclab_ov/test/physics/test_ovphysx_manager_lifecycle.py @@ -555,8 +555,7 @@ def test_scene_settings(monkeypatch, manager, device, rate, override, expected): assert prim.GetAttribute("physxScene:enableSceneQuerySupport").Get() is sim_cfg.enable_scene_query_support -@pytest.mark.parametrize("device", ["cpu", "gpu"]) -def test_scene_contact_settings_are_authored_only_when_selected(monkeypatch, manager, device): +def test_scene_contact_settings_are_authored_only_when_selected(monkeypatch, manager): """Optional contact tuning reaches the scene prim; unset fields keep the stage value.""" from isaaclab_ov.physics.ovphysx_manager_cfg import OvPhysxCfg @@ -567,14 +566,15 @@ def test_scene_contact_settings_are_authored_only_when_selected(monkeypatch, man ) stage = Usd.Stage.CreateInMemory() default_prim = stage.DefinePrim("/World/DefaultScene", "PhysicsScene") - manager._configure_physx_scene_prim(default_prim, OvPhysxCfg(), device) + # Contact settings are authored the same way on every device, so one device covers them. + manager._configure_physx_scene_prim(default_prim, OvPhysxCfg(), "cpu") for attr in ("bounceThreshold", "frictionCorrelationDistance", "solveArticulationContactLast"): assert not default_prim.GetAttribute(f"physxScene:{attr}").HasAuthoredValue() prim = stage.DefinePrim("/World/PhysicsScene", "PhysicsScene") cfg = OvPhysxCfg( bounce_threshold_velocity=0.01, friction_correlation_distance=0.00625, solve_articulation_contact_last=True ) - manager._configure_physx_scene_prim(prim, cfg, device) + manager._configure_physx_scene_prim(prim, cfg, "cpu") assert prim.GetAttribute("physxScene:bounceThreshold").Get() == pytest.approx(0.01) assert prim.GetAttribute("physxScene:frictionCorrelationDistance").Get() == pytest.approx(0.00625) assert prim.GetAttribute("physxScene:solveArticulationContactLast").Get() is True From 9ac5284df036802cbe2d0c2245102f1b67f069f1 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:22:05 -0400 Subject: [PATCH 035/145] Skip the SO101 pipeline test when Capture lacks cache execution The guard only checked that the core-types module exists, so a Capture release without ExecutionCache, GraphTime or execute_pipeline_with_cache failed collection instead of skipping. --- .../test/contrib/test_so101_teleop_pipeline.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py index e7f42bfeec1..e3dec65bc14 100644 --- a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py +++ b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py @@ -9,8 +9,14 @@ import pytest from scipy.spatial.transform import Rotation -# The test drives Capture's retargeting-engine internals; skip releases that do not ship them. -pytest.importorskip("isaacteleop.retargeting_engine.interface.retargeter_core_types") +# The test drives Capture's retargeting-engine cache execution; skip releases without the parts it uses. +_core_types = pytest.importorskip("isaacteleop.retargeting_engine.interface.retargeter_core_types") +if not ( + hasattr(_core_types, "ExecutionCache") + and hasattr(_core_types, "GraphTime") + and hasattr(getattr(_core_types, "GraphExecutable", None), "execute_pipeline_with_cache") +): + pytest.skip("Capture does not provide retargeting-engine cache execution", allow_module_level=True) from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource from isaacteleop.retargeting_engine.interface import ( From 49e9febb8a2bc1c3d8357dbb99202a25df9885ab Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:22:05 -0400 Subject: [PATCH 036/145] Compare the SO101 OvPhysX tuning with its Isaac Sim PhysX preset The ovphysx assertions restated the tuning literals. Resolve the same task with physics=isaacsim_physx and require matching contact settings. --- source/isaaclab_tasks/test/core/test_preset_cli.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index 28ce5dd527b..68dda9ded85 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -295,7 +295,8 @@ def test_teleop_tasks_resolve_public_physics_selectors(physics): if physics == "newton_mjwarp": assert isinstance(cfg.sim.physics.solver_cfg, MJWarpSolverCfg) if physics == "ovphysx": - # The kitless OvPhysX selection keeps the stack family's PhysX grasp tuning. - assert cfg.sim.physics.bounce_threshold_velocity == pytest.approx(0.01) - assert cfg.sim.physics.friction_correlation_distance == pytest.approx(0.00625) - assert cfg.sim.physics.solve_articulation_contact_last is True + # The kitless OvPhysX selection keeps the task's Isaac Sim PhysX grasp tuning. + _, physx_selections = setup_preset_cli(_make_parser(), ["--task", task_name, "--physics", "isaacsim_physx"]) + physx_cfg = parse_env_cfg(task_name, device="cpu", num_envs=1, overrides=physx_selections).sim.physics + for name in ("bounce_threshold_velocity", "friction_correlation_distance", "solve_articulation_contact_last"): + assert getattr(cfg.sim.physics, name) == getattr(physx_cfg, name), name From 1812334bccea03c41b326ef792b629691a276064 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:22:49 -0400 Subject: [PATCH 037/145] Clarify the SO101 reset and empty Warp array changelog entries State the previous SO101 joint noise std for the migration note, and phrase the empty CPU Warp array conversion entry as a fix. --- source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst | 3 ++- source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst index 27950c93cdb..6df62986cc7 100644 --- a/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst +++ b/source/isaaclab/changelog.d/kitless-ovxr-lifecycle.fixed.rst @@ -1 +1,2 @@ -* Supported empty CPU Warp arrays in Torch conversion. +* Fixed ``wp.to_torch`` failing on empty CPU Warp arrays, whose null data pointer NumPy rejects. + They now convert through DLPack. diff --git a/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst index 0ca6527bba7..bdc60b1dcec 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-teleop.changed.rst @@ -1,3 +1,4 @@ * Changed the SO101 IK-Abs stack task to reset the arm to its nominal joint pose without joint noise and to use a full-pose clutch home. Policies or datasets that relied on randomized - initial arm joints should restore ``events.randomize_joint_state.params["std"]``. + initial arm joints should restore ``events.randomize_joint_state.params["std"]`` to its + previous value of ``0.02``. From b44dc09886f0482bd813b8043152d9cd12f8d233 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:23:52 -0400 Subject: [PATCH 038/145] Name XrAnchorManager as the anchor protocol implementation This layer gives XrAnchorManager the external anchor members, so restore the docstring pointer and state when the device exposes it. --- source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index c4b38f87986..4d031c00f66 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -475,7 +475,12 @@ class OvLibrariesXrSession: @runtime_checkable class ExternalAnchorState(Protocol): - """Dynamic XR-anchor policy sealed together with complete scene publications.""" + """Dynamic XR-anchor policy sealed together with complete scene publications. + + :class:`~isaaclab_teleop.xr_anchor_manager.XrAnchorManager` implements this protocol. It + drives the anchor when constructed for an OVXR session with ``XrCfg.anchor_prim_path`` set, + which is when :attr:`~isaaclab_teleop.IsaacTeleopDevice.external_anchor_state` returns it. + """ @property def external_anchor_paths(self) -> tuple[str, ...]: From fc02a4cf6378b294731c21963db4ffdb15764e7a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:26:30 -0400 Subject: [PATCH 039/145] Skip live OVXR loop tests without Capture OVXR session support The live-workflow fixture imports the OVXR session adapter, which needs the kitless Capture selection. Under the Kit teleop extra its tests failed with ModuleNotFoundError instead of skipping. --- source/isaaclab_teleop/test/test_teleop_entrypoint_units.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 011a1f8f784..cda8c0e44c7 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -191,6 +191,9 @@ def run_live_workflow(monkeypatch): """Run the imported live owner with finite input, renderer, and simulation endpoints.""" import contextlib + # The live OVXR owner imports the session adapter, which needs a Capture release with OVXR + # session support. The Kit ``teleop`` extra selects an older release; skip rather than error there. + pytest.importorskip("isaaccapture.ovxr_session_manager") import isaaclab_teleop from isaaclab_teleop.entrypoints import teleop_se3_agent as agent From 963adc2dfc061d2e189556d275cf488df5c96498 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:27:58 -0400 Subject: [PATCH 040/145] Skip the OVXR task-ordering test without OVRTX --- source/isaaclab_teleop/test/test_teleop_entrypoint_units.py | 1 + 1 file changed, 1 insertion(+) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index cda8c0e44c7..86d0666b53c 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -790,6 +790,7 @@ def stop(): @pytest.mark.parametrize("entrypoint", ["live", "replay"]) def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoint): """Both actual workflows finalize the selected config before OVXR allocates its context.""" + pytest.importorskip("ovrtx") from contextlib import contextmanager import isaaclab_teleop From f39bc37948a83b9232ba2bade7786a2669b353a8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:36:32 -0400 Subject: [PATCH 041/145] Describe the Mimic PhysX pin as a change Released Mimic configurations already ran on Isaac Sim PhysX, so pinning them fixes nothing a user had; it keeps the selectors their base tasks gained in this change out of the Mimic tasks. --- .../isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst | 3 +++ .../isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst delete mode 100644 source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst diff --git a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst new file mode 100644 index 00000000000..ad147de3c46 --- /dev/null +++ b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst @@ -0,0 +1,3 @@ +* Kept the GR1T2 and G1 locomanipulation Mimic tasks on Isaac Sim PhysX without the ``ovphysx`` and + ``newton_mjwarp`` selectors of their base teleoperation tasks, because the Mimic environments do not + run the kitless backend preparation. diff --git a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst deleted file mode 100644 index fd13f958c31..00000000000 --- a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Kept the GR1T2 and G1 locomanipulation Mimic tasks on Isaac Sim PhysX. Their environments do - not run the kitless backend preparation the base teleoperation tasks now offer. From 429c6fb0eb3c162a964b22e2ab9c022c023feeb0 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:38:43 -0400 Subject: [PATCH 042/145] Keep the G1 locomanipulation SDG task on Isaac Sim PhysX G1LocomanipulationSDGEnvCfg inherited the locomanipulation task's ovphysx/newton_mjwarp selectors and its HumanoidTeleopEnv class type, but the registered SDG environment never runs the kitless preparation, and generate_data.py forwards physics=... overrides to the task. Pin it to Isaac Sim PhysX like the Mimic configurations and declare its own environment class. --- .../changelog.d/kitless-robot-tasks.changed.rst | 7 ++++--- .../envs/g1_locomanipulation_sdg_env_cfg.py | 6 ++++++ source/isaaclab_mimic/test/test_mimic_humanoid_physics.py | 4 +++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst index ad147de3c46..0463675e973 100644 --- a/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_mimic/changelog.d/kitless-robot-tasks.changed.rst @@ -1,3 +1,4 @@ -* Kept the GR1T2 and G1 locomanipulation Mimic tasks on Isaac Sim PhysX without the ``ovphysx`` and - ``newton_mjwarp`` selectors of their base teleoperation tasks, because the Mimic environments do not - run the kitless backend preparation. +* Kept the GR1T2 and G1 locomanipulation Mimic tasks and the G1 locomanipulation SDG task + (``Isaac-G1-SteeringWheel-Locomanipulation``) on Isaac Sim PhysX without the ``ovphysx`` and + ``newton_mjwarp`` selectors of their base teleoperation tasks, because their environments do not run + the kitless backend preparation. diff --git a/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py b/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py index fbe822afda2..55e78fe2ccc 100644 --- a/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py +++ b/source/isaaclab_mimic/isaaclab_mimic/locomanipulation_sdg/envs/g1_locomanipulation_sdg_env_cfg.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 +from isaaclab_physx.physics import PhysxCfg from isaaclab_physx.renderers import IsaacRtxRendererCfg, IsaacRtxRendererGlobalSettingsCfg import isaaclab.sim as sim_utils @@ -125,6 +126,11 @@ class PolicyCfg(ObservationsCfg.PolicyCfg): class G1LocomanipulationSDGEnvCfg(LocomanipulationG1EnvCfg, LocomanipulationSDGEnvCfg): """Configuration for the G1 29DoF environment.""" + class_type: type | str = "{DIR}.g1_locomanipulation_sdg_env:G1LocomanipulationSDGEnv" + sim: sim_utils.SimulationCfg = sim_utils.SimulationCfg(physics=PhysxCfg()) + """Isaac Sim PhysX only: the SDG environment does not run the teleoperation task's kitless + backend preparation, so the inherited ``ovphysx``/``newton_mjwarp`` selectors are not offered.""" + # Scene settings scene: G1LocomanipulationSDGSceneCfg = G1LocomanipulationSDGSceneCfg(num_envs=1, env_spacing=2.5) recorders: LocomanipulationSDGRecorderManagerCfg = LocomanipulationSDGRecorderManagerCfg() diff --git a/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py index f889df9e9a8..a617eca7e9c 100644 --- a/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py +++ b/source/isaaclab_mimic/test/test_mimic_humanoid_physics.py @@ -3,7 +3,7 @@ # # SPDX-License-Identifier: Apache-2.0 -"""Humanoid Mimic tasks keep Isaac Sim PhysX; their environments skip kitless preparation.""" +"""Humanoid Mimic and SDG tasks keep Isaac Sim PhysX; their environments skip kitless preparation.""" import pytest @@ -14,12 +14,14 @@ "Isaac-PickPlace-GR1T2-Abs-Mimic-v0", "Isaac-PickPlace-GR1T2-WaistEnabled-Abs-Mimic-v0", "Isaac-Locomanipulation-G1-Abs-Mimic-v0", + "Isaac-G1-SteeringWheel-Locomanipulation", ], ) def test_humanoid_mimic_tasks_offer_only_isaac_sim_physx(task): from isaaclab_physx.physics import PhysxCfg import isaaclab_mimic.envs # noqa: F401 -- registers the Mimic task IDs + import isaaclab_mimic.locomanipulation_sdg.envs # noqa: F401 -- registers the SDG task ID import isaaclab_tasks # noqa: F401 from isaaclab_tasks.utils import parse_env_cfg From 06f7ea1340b77cd067037859fe5af788270183d1 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:39:41 -0400 Subject: [PATCH 043/145] Describe the G1 contact-path change as Newton-specific Isaac Sim PhysX and OvPhysX contact sensors split the path into a parent and a leaf name and search all descendants, so the flat G1 hand pattern already resolved the nested links there. Only Newton matches whole body paths. File the path change under changed, not as a Kit fix. --- .../isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst | 3 +++ .../isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst | 2 -- source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst index 93040cc5ec9..adfe2d64111 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -9,3 +9,6 @@ With Isaac Sim PhysX it behaves as the previous Pink term. * Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), which adds render work per reset when cameras are rendered. +* Changed the G1 hand contact-sensor paths to name the ``left_hand``/``right_hand`` parent of the hand + links in the Isaac 6.2 ``g1.usd``. Newton matches whole body paths, so it needs that parent; Isaac Sim + PhysX and OvPhysX resolve the same links by name as before. diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst deleted file mode 100644 index 2a5564f291c..00000000000 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.fixed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Fixed the G1 hand contact sensors to match the hand links nested under ``left_hand``/``right_hand`` - in the Isaac 6.2 ``g1.usd``. The previous flat pattern matched no prims, including on Kit PhysX. diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py index 64c63976fe9..1672a3a5988 100644 --- a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -47,7 +47,7 @@ def _robot_paths(stage: Usd.Stage, pattern: str) -> list[Usd.Prim]: @pytest.mark.parametrize("task", ["fixed_base", "locomanipulation"]) def test_g1_hand_contact_sensors_resolve_to_rigid_hand_links(task): - """The Isaac 6.2 G1 nests its hand links under ``left_hand``/``right_hand``; flat paths match nothing.""" + """Newton matches whole paths, so the patterns must name the ``left_hand``/``right_hand`` parent prims.""" from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( FixedBaseUpperBodyIKG1EnvCfg, ) From 589e727f9f478089fde0b655c32010a0fbeb92f3 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:46:31 -0400 Subject: [PATCH 044/145] Skip the teleop asset checks only when the asset is unavailable retrieve_file_path reports a missing asset or an unreachable server as FileNotFoundError and a failed download as RuntimeError. Skip on those alone, so a misconfigured job, such as one without omni.client, fails instead of silently skipping both asset checks. --- source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py index 1672a3a5988..ed99ab3805d 100644 --- a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -28,7 +28,8 @@ def _open_asset(usd_path: str) -> Usd.Stage: try: local_path = retrieve_file_path(usd_path) - except Exception as exc: # The asset server is optional for local test runs. + except (FileNotFoundError, RuntimeError) as exc: + # The asset server is optional for local test runs: it may be unreachable or fail the download. pytest.skip(f"Robot asset is unavailable: {exc}") stage = Usd.Stage.Open(local_path) assert stage is not None, local_path From ce96626c9fa69d0d114700130acad8d95451f54c Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:48:36 -0400 Subject: [PATCH 045/145] Resolve preset defaults for directly built humanoid configs A humanoid task configuration constructed directly keeps the TeleopPhysicsCfg preset and the robot camera renderer preset, so preparing it treated the preset as Kit PhysX and simulation construction then rejected the unresolved physics. Humanoid teleop preparation now resolves every remaining preset to its default before construction, as task resolution would, and the changelog says when to call it. The code matches the #8790 follow-up 291de87060, so that layer merges it without change. --- .../changelog.d/kitless-robot-tasks.changed.rst | 5 +++-- .../isaaclab_tasks/contrib/teleop_physics_cfg.py | 7 +++++-- .../isaaclab_tasks/test/core/test_preset_cli.py | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst index adfe2d64111..e9dc7d55877 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -2,8 +2,9 @@ ``IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs``, ``IsaacContrib-PickPlace-Locomanipulation-G1-Abs`` and ``IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs`` to :class:`~isaaclab_tasks.contrib.teleop_env.HumanoidTeleopEnv`, which prepares the selected physics - backend before construction. Code that constructs these configurations directly should call - :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment` first. + backend before construction. Code that constructs the environment without that entry point should + first call :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment`, + which also applies the preset defaults to a configuration built without task resolution. * Changed the GR1T2 upper-body action term to :class:`~isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. With Isaac Sim PhysX it behaves as the previous Pink term. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py index 933e76586c7..da6b0754d6e 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/teleop_physics_cfg.py @@ -16,7 +16,7 @@ from isaaclab.physics import PhysxAutoCfg from isaaclab.utils import configclass -from isaaclab_tasks.utils import PresetCfg +from isaaclab_tasks.utils import PresetCfg, resolve_presets if TYPE_CHECKING: from isaaclab.envs import ManagerBasedRLEnvCfg @@ -27,7 +27,9 @@ def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: Dispatches to the configuration's ``stabilize_for_kitless_ovphysx`` or ``stabilize_for_kitless_newton_mjwarp`` for the selected standalone backend. Kit PhysX - needs no preparation. Repeated calls on a prepared configuration are no-ops. + needs no preparation. Repeated calls on a prepared configuration are no-ops. A + configuration constructed directly, without task resolution, uses the default of every + remaining preset. Args: cfg: Resolved humanoid teleoperation task configuration. @@ -35,6 +37,7 @@ def prepare_humanoid_teleop_environment(cfg: ManagerBasedRLEnvCfg) -> None: Raises: ValueError: If ``cfg`` was already prepared for a different physics backend. """ + resolve_presets(cfg) physics = cfg.sim.physics if isinstance(physics, OvPhysxCfg): cfg.stabilize_for_kitless_ovphysx() diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index 0ccdd7c4872..ae10a560332 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -436,6 +436,22 @@ def inspect_before_simulation(self, cfg, *args, **kwargs): gym.make(task_name, cfg=prepared) +def test_humanoid_preparation_resolves_directly_built_configs(): + """A configuration built without task resolution is prepared with the default of every preset.""" + from isaaclab_physx.physics import PhysxCfg + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( + FixedBaseUpperBodyIKG1EnvCfg, + ) + from isaaclab_tasks.contrib.teleop_physics_cfg import prepare_humanoid_teleop_environment + + cfg = FixedBaseUpperBodyIKG1EnvCfg() + prepare_humanoid_teleop_environment(cfg) + assert type(cfg.sim.physics) is PhysxCfg + assert type(cfg.scene.robot_pov_cam.renderer_cfg) is IsaacRtxRendererCfg + + def _spawn_task_roundtrip_fixture(path, _cfg, **_kwargs): """Module-level source spawner, preserving the public callable serialization contract.""" from pxr import UsdGeom, UsdPhysics From 8f8aad0ffca15045f17d2eda484f78175d1b3e69 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:48:46 -0400 Subject: [PATCH 046/145] Point the GR1T2 action changelog entry at its defining module GR1T2KitlessPinkInverseKinematicsActionCfg is defined in gr1t2_kitless_action_cfg; pickplace_gr1t2_env_cfg only imports it. --- .../isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst index e9dc7d55877..65f84d13c4b 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -6,7 +6,7 @@ first call :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment`, which also applies the preset defaults to a configuration built without task resolution. * Changed the GR1T2 upper-body action term to - :class:`~isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. + :class:`~isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. With Isaac Sim PhysX it behaves as the previous Pink term. * Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), which adds render work per reset when cameras are rendered. From 75645c5352f2007291dca025ba3a1ce2c4e48c44 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 17:51:13 -0400 Subject: [PATCH 047/145] Re-seed the G1 wrist ramp after first-step terminations The ramp detected auto-resets by environment 0's episode counter going backwards, which misses an episode that terminates on its first step: the counter goes 0 -> 1 -> 0 between two frames. Track the global step at which the episode began instead, which moves on every reset. Read the step counters directly from the environment, which the launchers always pass unwrapped, and give the test fake both counters. --- .../locomanip_pick_place/g1_kitless_newton.py | 19 ++++----- .../test/contrib/test_g1_kitless_newton.py | 40 +++++++++++++------ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py index 6b146a2ed80..2a968e2875c 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -181,29 +181,24 @@ def _step_quaternion_towards(current: torch.Tensor, target: torch.Tensor, max_st _WRIST_RAMP_KEYS = ("left_pos", "left_quat", "right_pos", "right_quat") -def _episode_step(env) -> int | None: - """Return environment 0's episode step, or ``None`` when the environment has no counter.""" - buffer = getattr(getattr(env, "unwrapped", env), "episode_length_buf", None) - return None if buffer is None else int(buffer[0]) - - def condition_kitless_newton_g1_action(env, action: torch.Tensor, state: dict[str, Any]) -> torch.Tensor: """Ramp both wrist targets from the current G1 poses before Pink solves IK. - The ramp re-seeds from the measured wrist poses whenever the episode restarts, including - termination auto-resets inside ``env.step`` that the teleoperation loop does not observe. + The ramp re-seeds from the measured wrist poses whenever environment 0 starts a new episode, + including termination auto-resets inside ``env.step`` that the teleoperation loop does not observe. """ flat = action.detach().flatten() if flat.numel() != EXPECTED_ACTION_DIM: state.clear() return action - episode_step = _episode_step(env) - previous_step = state.get("_episode_step") - if episode_step is not None and previous_step is not None and episode_step < previous_step: + # The global step at which environment 0's episode began changes with every reset, including a + # termination on the episode's first step, which leaves the episode counter at 0 between frames. + episode_start = env.common_step_counter - int(env.episode_length_buf[0]) + if state.get("_episode_start") != episode_start: for key in _WRIST_RAMP_KEYS: state.pop(key, None) - state["_episode_step"] = episode_step + state["_episode_start"] = episode_start conditioned = action.clone() conditioned_flat = conditioned.flatten() diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py index dcf320127f0..6cd1e60fa24 100644 --- a/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_newton.py @@ -76,7 +76,10 @@ def _fake_env(): ) ) ) - return SimpleNamespace(scene={"robot": robot}, cfg=cfg), left, right + env = SimpleNamespace( + scene={"robot": robot}, cfg=cfg, common_step_counter=0, episode_length_buf=torch.zeros(1, dtype=torch.long) + ) + return env, left, right def test_g1_newton_contract_is_backend_scoped_and_complete() -> None: @@ -162,22 +165,32 @@ def test_recorded_first_action_is_ramped_and_finite() -> None: def test_wrist_ramp_reseeds_after_termination_auto_reset() -> None: """A success/drop auto-reset inside env.step moves the wrists; the ramp must restart from there.""" - env, left, right = _fake_env() - env.episode_length_buf = torch.tensor([0]) + env, left, _ = _fake_env() + poses = env.scene["robot"].data.body_link_pose_w + + def step(terminated: bool = False) -> None: + # Advance the counters like ManagerBasedRLEnv.step; a terminated episode resets to a lower pose. + env.common_step_counter += 1 + env.episode_length_buf += 1 + if terminated: + env.episode_length_buf[0] = 0 + poses[0, :, :3] += torch.tensor([0.0, 0.0, -0.2]) + state = {} target = RECORDED_FIRST_ACTION.clone() - for step in range(1, 60): + conditioned = condition_kitless_newton_g1_action(env, target, state) + for _ in range(58): + step() conditioned = condition_kitless_newton_g1_action(env, target, state) - env.episode_length_buf[0] = step # The ramp has converged near the operator's hands, far from the reset pose. assert float(torch.linalg.norm(conditioned[:3] - left[:3])) > 2 * POSITION_MAX_STEP_M - # Auto-reset: the robot returns to a different pose and the episode counter restarts. - poses = env.scene["robot"].data.body_link_pose_w - poses[0, :, :3] += torch.tensor([0.0, 0.0, -0.2]) - env.episode_length_buf[0] = 0 - conditioned = condition_kitless_newton_g1_action(env, target, state) - assert float(torch.linalg.norm(conditioned[:3] - poses[0, 0, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 - assert float(torch.linalg.norm(conditioned[7:10] - poses[0, 1, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + # The episode terminates, then the next one terminates on its first step: its counter goes + # 0 -> 1 -> 0 between two frames without decreasing. + for _ in range(2): + step(terminated=True) + conditioned = condition_kitless_newton_g1_action(env, target, state) + assert float(torch.linalg.norm(conditioned[:3] - poses[0, 0, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 + assert float(torch.linalg.norm(conditioned[7:10] - poses[0, 1, :3])) <= POSITION_MAX_STEP_M + 1.0e-6 @pytest.mark.parametrize( @@ -213,7 +226,8 @@ def test_invalid_actions_fail_closed() -> None: assert teleop_action_target_issue(oversized) == "unreasonable hand joint target magnitude" assert "unexpected G1 controller action size" in teleop_action_target_issue(torch.zeros(27)) - missing_wrist_env = SimpleNamespace(scene={}, cfg=SimpleNamespace()) + missing_wrist_env, _, _ = _fake_env() + missing_wrist_env.scene = {} with pytest.raises(RuntimeError, match="could not resolve the task-selected G1 wrist poses"): condition_kitless_newton_g1_action(missing_wrist_env, RECORDED_FIRST_ACTION, {}) From f6f8f6186ce76a0ed420c415bccb38ae92269f2a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:09:05 -0400 Subject: [PATCH 048/145] Regenerate the environment browser for the fixed-base G1 rgb preset The fixed-base G1 task now offers presets=rgb. The regenerated rows also drop newton_mjwarp from SO101-IK-Abs-v0, which the generator excludes for that task since the stack added its OvPhysX preset. --- docs/source/_static/css/environment-browser.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 87cfbbb378f..6b4ee228b6e 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -98,7 +98,7 @@ ["IsaacContrib-Open-Drawer-Franka-IK-Abs", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-Franka-IK-Rel", "rsl_rl", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/franka_open_drawer.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], ["IsaacContrib-Open-Drawer-OpenArm", "rl_games,rsl_rl", "", "", "", "environment-previews/openarm_uni_open_drawer.jpg"], - ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], + ["IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "rgb", "environment-previews/g1_pick_place_fixed_base.jpg", false, {}, {}, "", null, ["||", "||rgb", "|isaacsim_rtx|", "|isaacsim_rtx|rgb", "|newton_renderer|", "|newton_renderer|rgb", "isaacsim_physx||", "isaacsim_physx||rgb", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|isaacsim_rtx|rgb", "isaacsim_physx|newton_renderer|", "isaacsim_physx|newton_renderer|rgb", "newton_mjwarp||", "newton_mjwarp||rgb", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|isaacsim_rtx|rgb", "newton_mjwarp|newton_renderer|", "newton_mjwarp|newton_renderer|rgb", "newton_mjwarp|ovrtx|", "newton_mjwarp|ovrtx|rgb", "ovphysx|newton_renderer|", "ovphysx|newton_renderer|rgb", "ovphysx|ovrtx|", "ovphysx|ovrtx|rgb"]], ["IsaacContrib-PickPlace-G1-InspireFTP-Abs", "", "", "", "", "environment-previews/g1_pick_place.jpg"], ["IsaacContrib-PickPlace-GR1T2-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "isaacsim_rtx,newton_renderer,ovrtx", "", "environment-previews/gr_1_pick_place.jpg", false, {}, {}, "", null, ["||", "|isaacsim_rtx|", "|newton_renderer|", "isaacsim_physx||", "isaacsim_physx|isaacsim_rtx|", "isaacsim_physx|newton_renderer|", "newton_mjwarp||", "newton_mjwarp|isaacsim_rtx|", "newton_mjwarp|newton_renderer|", "newton_mjwarp|ovrtx|", "ovphysx|newton_renderer|", "ovphysx|ovrtx|"]], ["IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/gr_1_pick_place_waist.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], @@ -129,7 +129,7 @@ ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka-IK-Rel", "", "", "", "", "environment-previews/franka_instance_randomize_stack.jpg"], ["IsaacContrib-Stack-Cube-RedGreen-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-RedGreenBlue-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], - ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], + ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "ovphysx||"]], ["IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-SO101-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/ur10_stack_surface_gripper.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], From 0a39e24c837b15640e29dfae88f4df7998cffc47 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:15:06 -0400 Subject: [PATCH 049/145] Resolve OVXR PiP panels before launch Live teleoperation and MCAP replay now select and resolve the PiP panels while preparing the task configuration, and pass the resolved panels to session construction. A task layout that is not head-locked fails there, before launch, with an error that names --ovxr_pip_head_locked. Replay no longer carries the selection through a module global, which also restores the hydra_overrides docstring. Tests drive both workflows from the command line to the session configuration and check the panel rate, position override, mirror rate and the pre-launch rejection. --- source/isaaclab_teleop/docs/kitless_xr.md | 4 +- .../entrypoints/_ovxr_launch.py | 48 +++++- .../entrypoints/teleop_replay_agent.py | 56 +++---- .../entrypoints/teleop_se3_agent.py | 30 ++-- .../isaaclab_teleop/test/test_entrypoints.py | 6 +- .../test/test_teleop_entrypoint_units.py | 139 +++++++++++++++++- .../test/test_teleop_replay_bootstrap.py | 56 ++++--- 7 files changed, 246 insertions(+), 93 deletions(-) diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 91d786776e1..7b66d12eaa8 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -149,7 +149,9 @@ uv run --extra ovxr isaaclab teleop run --ovxr \ `--ovxr_pip_fps` limits panel updates. `--ovxr_pip_view_position X Y Z` overrides one panel's position; multiple panels use the task layout. Panel cameras must be enabled in the task's `xr_camera_feeds`. World and viewer-start layouts are not -supported by the OVXR presenter. The session snapshots its camera selection at +supported by the OVXR presenter: `--ovxr_pip_head_locked` places such a layout's +panels in the OpenXR VIEW space, and without it the command fails before launch. +The session snapshots its camera selection at preparation; create a new session to change it. Panels use the same renderer as the headset and do not require a desktop display. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index fecd5fd92f2..8e225882c60 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -9,7 +9,10 @@ import argparse import math -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from ..ovxr_cfg import OvLibrariesXrCameraConfig def require_kitless_physics(env_cfg: Any, task: str) -> None: @@ -109,11 +112,42 @@ def validate_ovxr_presentation_args( parser.error("--ovxr_desktop_mirror_fps must be finite and positive") -def ovxr_camera_layout(env_cfg: Any, head_locked: bool) -> Any: - """Return the task's camera-panel layout, forced to head-locked placement when requested.""" +def resolve_ovxr_pip_cameras( + env_cfg: Any, args: argparse.Namespace, renderer_cfg: Any +) -> tuple[OvLibrariesXrCameraConfig, ...]: + """Select the requested PiP cameras and resolve their OVXR panels before launch. + + Selection assigns ``renderer_cfg`` to each selected camera, so the OVXR session and its panels + share one OVRTX configuration. Resolution creates no native resources, so an invalid selection or + task layout fails before the simulation launches. + + Args: + env_cfg: Resolved environment configuration. + args: Parsed namespace with the shared OVXR presentation options. + renderer_cfg: OVRTX configuration shared by the OVXR session and its PiP cameras. + + Returns: + The resolved panels in selection order, or an empty tuple without ``--ovxr_pip_camera``. + + Raises: + ValueError: If a selected camera or its feed is invalid, or if the task layout is not + head-locked and ``--ovxr_pip_head_locked`` is not given. + """ + if args.ovxr_pip_camera is None: + return () + + from ..camera_feed import resolve_ovxr_camera_configs, select_ovxr_camera_feeds from ..isaac_teleop_cfg import XrCameraFeedLayoutCfg - layout = env_cfg.isaac_teleop.xr_camera_feed_layout - if head_locked: - layout = (layout or XrCameraFeedLayoutCfg()).replace(placement="head_locked") - return layout + selected = select_ovxr_camera_feeds(env_cfg, args.ovxr_pip_camera, renderer_cfg) + layout = env_cfg.isaac_teleop.xr_camera_feed_layout or XrCameraFeedLayoutCfg() + if args.ovxr_pip_head_locked: + layout = layout.replace(placement="head_locked") + elif layout.placement != "head_locked": + raise ValueError( + f"OVXR camera panels are head-locked, but the task places them with {layout.placement!r} placement." + " Pass --ovxr_pip_head_locked to place them in the OpenXR VIEW space." + ) + return resolve_ovxr_camera_configs( + selected, layout, max_fps=args.ovxr_pip_fps, position_override=args.ovxr_pip_view_position + ) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index aac053a48b5..c3551639218 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -294,14 +294,15 @@ CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopDevice, + OvLibrariesXrCameraConfig, create_isaac_teleop_device, poll_control_events, ) from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action from ._ovxr_launch import ( add_ovxr_presentation_args, - ovxr_camera_layout, require_kitless_physics, + resolve_ovxr_pip_cameras, validate_ovxr_presentation_args, ) @@ -310,8 +311,6 @@ args_cli: argparse.Namespace """Parsed command line, set by :func:`_parse_args`.""" hydra_overrides: list[str] -ovxr_pip_selection: list = [] -"""OVXR PiP cameras selected while preparing the environment configuration.""" """Unrecognized arguments, forwarded as Hydra task config overrides.""" @@ -1074,7 +1073,7 @@ def _prepare_env_cfg( num_envs: int, device: str, overrides: Sequence[str] = (), -) -> tuple[ManagerBasedRLEnvCfg, object | None]: +) -> tuple[ManagerBasedRLEnvCfg, object | None, tuple[OvLibrariesXrCameraConfig, ...]]: """Build and tweak an env config suitable for non-interactive replay. Mirrors the env-config mutations performed by ``record_demos.py``'s @@ -1110,8 +1109,10 @@ def _prepare_env_cfg( overrides: Hydra-style ``key=value`` overrides passed to :func:`resolve_task_config`. Returns: - Tuple ``(env_cfg, success_term)``. ``success_term`` is ``None`` when - the env doesn't define a ``success`` termination term. + Tuple ``(env_cfg, success_term, selected_cameras)``. ``success_term`` is + ``None`` when the env doesn't define a ``success`` termination term. + ``selected_cameras`` holds the resolved OVXR PiP panels, empty without + ``--ovxr_pip_camera``. """ overrides = list(overrides) env_cfg, _ = resolve_task_config(task, "", overrides=overrides) @@ -1135,6 +1136,7 @@ def _prepare_env_cfg( # OVXR renders only through its leased renderer: keep just the selected PiP cameras. # Otherwise keep camera configs when external cameras are enabled (defaulted on) so the # replay renders them for production parity, or strip them for a lighter headless replay. + selected_cameras = () if args_cli.ovxr: if args_cli.num_envs != 1 or args_cli.xr: raise ValueError("--ovxr requires one environment and no Kit XR or local visualizer") @@ -1142,13 +1144,9 @@ def _prepare_env_cfg( raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") from isaaclab_ov.renderers import OVRTXRendererCfg - from ..camera_feed import select_ovxr_camera_feeds - - global ovxr_pip_selection - names = args_cli.ovxr_pip_camera or [] - # The presentation reuses this validated selection and its shared renderer configuration. - ovxr_pip_selection = select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) - env_cfg = remove_camera_configs(env_cfg, keep=tuple(names)) + # Resolve the panels now so that an invalid selection or task layout fails before launch. + selected_cameras = resolve_ovxr_pip_cameras(env_cfg, args_cli, OVRTXRendererCfg()) + env_cfg = remove_camera_configs(env_cfg, keep=tuple(args_cli.ovxr_pip_camera or ())) require_kitless_physics(env_cfg, task) elif args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) @@ -1162,7 +1160,7 @@ def _prepare_env_cfg( args_cli.require_kit = bool(kit_sources) if env_cfg.isaac_teleop is not None and env_cfg.isaac_teleop.prepare_environment is not None: env_cfg.isaac_teleop.prepare_environment(env_cfg) - return (env_cfg, success_term) + return (env_cfg, success_term, selected_cameras) def _replay_requires_hard_sim_reset(physics_backend: str | None) -> bool: @@ -1236,18 +1234,11 @@ def _process_success_condition( class _ReplayPresentation: """Own a batch's viewer independently of each recorded-input session.""" - def __init__(self, env_cfg): + def __init__(self, env_cfg, selected_cameras: tuple[OvLibrariesXrCameraConfig, ...]): from isaaclab_ov.renderers import OVRTXRendererCfg from .. import OvLibrariesXrConfig, OvLibrariesXrSession - from ..camera_feed import resolve_ovxr_camera_configs - cameras = resolve_ovxr_camera_configs( - ovxr_pip_selection, - ovxr_camera_layout(env_cfg, args_cli.ovxr_pip_head_locked), - max_fps=args_cli.ovxr_pip_fps, - position_override=args_cli.ovxr_pip_view_position, - ) self.session = OvLibrariesXrSession.prepare( env_cfg.sim, OvLibrariesXrConfig( @@ -1257,9 +1248,9 @@ def __init__(self, env_cfg): auto_launch_cloudxr=args_cli.auto_launch_cloudxr, desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, desktop_mirror_max_fps=args_cli.ovxr_desktop_mirror_fps, - auxiliary_cameras=cameras, + auxiliary_cameras=selected_cameras, ), - cameras[0].camera_cfg.renderer_cfg if cameras else OVRTXRendererCfg(), + selected_cameras[0].camera_cfg.renderer_cfg if selected_cameras else OVRTXRendererCfg(), ) self.xr_cfg = env_cfg.isaac_teleop.xr_cfg self.device = None @@ -1751,7 +1742,7 @@ def main() -> int: if args_cli.num_replays < 1: raise ValueError(f"--num_replays must be >= 1; got {args_cli.num_replays}") - env_cfg, success_term = _prepare_env_cfg( + env_cfg, success_term, selected_cameras = _prepare_env_cfg( args_cli.task, args_cli.num_envs, args_cli.device, @@ -1765,13 +1756,22 @@ def main() -> int: ) with launch_simulation(env_cfg, args_cli, cleanup_context=not args_cli.ovxr): - exit_code = _run_replay_batch(env_cfg, success_term) + exit_code = _run_replay_batch(env_cfg, success_term, selected_cameras) return exit_code -def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None) -> int: +def _run_replay_batch( + env_cfg: ManagerBasedRLEnvCfg, + success_term: object | None, + selected_cameras: tuple[OvLibrariesXrCameraConfig, ...], +) -> int: """Build the env, run the ``--num_replays`` replays, and report the stats; see :func:`main`. + Args: + env_cfg: Environment configuration prepared by :func:`_prepare_env_cfg`. + success_term: Extracted ``success`` termination term, or ``None``. + selected_cameras: OVXR PiP panels resolved by :func:`_prepare_env_cfg`. + Returns: The host process exit code, see :func:`main`. """ @@ -1794,7 +1794,7 @@ def _run_replay_batch(env_cfg: ManagerBasedRLEnvCfg, success_term: object | None # work in XR-active mode without losing the OpenXR runtime # IPC between runs. if args_cli.ovxr: - presentation = _ReplayPresentation(env_cfg) + presentation = _ReplayPresentation(env_cfg, selected_cameras) def request_shutdown(_signum, _frame): nonlocal shutting_down diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 8dc8d05ee1b..2b776e9b6c5 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -45,6 +45,7 @@ CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg, + OvLibrariesXrCameraConfig, se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, @@ -56,8 +57,8 @@ from ..device_selection import create_teleop_device, select_teleop_cfg from ._ovxr_launch import ( add_ovxr_presentation_args, - ovxr_camera_layout, require_kitless_physics, + resolve_ovxr_pip_cameras, validate_ovxr_presentation_args, ) @@ -271,7 +272,7 @@ def _ovxr_publication_identity( def _resolve_env_cfg() -> tuple[ - ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, list[tuple[str, object, object]] + ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, tuple[OvLibrariesXrCameraConfig, ...] ]: """Resolve task configuration and teleop ownership before runtime launch.""" overrides = list(sys.argv[1:]) @@ -328,16 +329,14 @@ def _resolve_env_cfg() -> tuple[ if stage_mode.strip() != "1": raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") - pip_selection = [] + selected_cameras = () keep_cameras: tuple[str, ...] = () if args_cli.ovxr_pip_camera is not None: from isaaclab_ov.renderers import OVRTXRendererCfg - from ..camera_feed import select_ovxr_camera_feeds - - names = args_cli.ovxr_pip_camera - pip_selection = select_ovxr_camera_feeds(env_cfg, names, OVRTXRendererCfg()) - keep_cameras = tuple(names) + # Resolve the panels now so that an invalid selection or task layout fails before launch. + selected_cameras = resolve_ovxr_pip_cameras(env_cfg, args_cli, OVRTXRendererCfg()) + keep_cameras = tuple(args_cli.ovxr_pip_camera) # The leased OVRTX renderer is the only render path in OVXR. Strip every task camera # except the selected PiP cameras before the launcher scans the config. @@ -357,7 +356,7 @@ def _resolve_env_cfg() -> tuple[ raise ValueError("--ovxr requires no Kit runtime or local visualizer") if use_isaac_teleop and teleop_cfg.prepare_environment is not None: teleop_cfg.prepare_environment(env_cfg) - return env_cfg, teleop_cfg, use_isaac_teleop, pip_selection + return env_cfg, teleop_cfg, use_isaac_teleop, selected_cameras def _install_ovxr_shutdown_handlers(ov_libraries_xr_session) -> dict: @@ -459,7 +458,7 @@ def run_teleoperation( # noqa: C901 env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, use_isaac_teleop: bool, - pip_selection: list[tuple[str, object, object]], + selected_cameras: tuple[OvLibrariesXrCameraConfig, ...], ) -> None: """Run teleoperation through the selected Kit or OVXR runtime.""" @@ -502,14 +501,7 @@ def run_teleoperation( # noqa: C901 from isaaclab_ov.renderers import OVRTXRendererCfg from .. import OvLibrariesXrConfig, OvLibrariesXrSession - from ..camera_feed import resolve_ovxr_camera_configs - selected_cameras = resolve_ovxr_camera_configs( - pip_selection, - ovxr_camera_layout(env_cfg, args_cli.ovxr_pip_head_locked), - max_fps=args_cli.ovxr_pip_fps, - position_override=args_cli.ovxr_pip_view_position, - ) ov_libraries_xr_session = OvLibrariesXrSession.prepare( env_cfg.sim, OvLibrariesXrConfig( @@ -817,9 +809,9 @@ def run_loop(): def main() -> None: """Resolve the task and run teleoperation with its selected runtime.""" - resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, pip_selection = _resolve_env_cfg() + resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, selected_cameras = _resolve_env_cfg() with launch_simulation(resolved_env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): - run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, pip_selection) + run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop, selected_cameras) def run_teleop_se3_agent_cli(argv: list[str] | None = None) -> int: diff --git a/source/isaaclab_teleop/test/test_entrypoints.py b/source/isaaclab_teleop/test/test_entrypoints.py index 2dd220b02e6..efffdbcdf57 100644 --- a/source/isaaclab_teleop/test/test_entrypoints.py +++ b/source/isaaclab_teleop/test/test_entrypoints.py @@ -20,9 +20,9 @@ def test_live_entrypoint_preserves_resolved_camera_selection(monkeypatch): """The public workflow carries its resolved PiP selection into the selected runtime.""" from isaaclab_teleop.entrypoints import teleop_se3_agent as workflow - config, teleop_config, camera = object(), object(), object() - selection = [("robot_pov_cam", camera, object())] - resolved = (config, teleop_config, True, selection) + config, teleop_config = object(), object() + selected_cameras = (object(),) + resolved = (config, teleop_config, True, selected_cameras) monkeypatch.setattr(workflow, "_resolve_env_cfg", lambda: resolved) calls = [] diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index ff022e562e4..de49319f502 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -249,7 +249,7 @@ def last_control_events(self): monkeypatch.setattr(agent, "_install_ovxr_shutdown_handlers", lambda *_args: {}) monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", lambda *_args, **_kwargs: camera) monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", lambda *_args, **_kwargs: session) - agent.run_teleoperation(cfg, cfg.isaac_teleop, True, []) + agent.run_teleoperation(cfg, cfg.isaac_teleop, True, ()) return run @@ -483,9 +483,11 @@ def resolve(task, agent, *, overrides): monkeypatch.setattr(replay, "resolve_task_config", resolve) monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) monkeypatch.setattr(replay, "remove_camera_configs", isaaclab_teleop.remove_camera_configs) - prepared, success = namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) + prepared, success, cameras = namespace["_prepare_env_cfg"]( + "test-task", 1, None, overrides=namespace["hydra_overrides"] + ) assert resolutions == [("test-task", "", ["physics=ovphysx"])] - assert prepared is cfg and success is not None + assert prepared is cfg and success is not None and cameras == () assert cfg.sim.physics is physics assert namespace["args_cli"].require_kit is False assert not namespace["args_cli"].start_on_replay @@ -873,10 +875,10 @@ def prepare(_sim, xr_cfg, *_args, **_kwargs): lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), ) with pytest.raises(PreparationReached): - module.run_teleoperation(cfg, cfg.isaac_teleop, True, []) + module.run_teleoperation(cfg, cfg.isaac_teleop, True, ()) else: with pytest.raises(PreparationReached): - module._ReplayPresentation(cfg) + module._ReplayPresentation(cfg, ()) assert received == [expected] assert isaaclab_teleop.OvLibrariesXrConfig().desktop_mirror_enabled is False @@ -907,6 +909,129 @@ def test_non_ovxr_cli_keeps_mirror_disabled(monkeypatch, entrypoint, flags): assert args.headless is ("--headless" in flags) +def _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, flags, calls): + """Run an entrypoint's ``main`` with one PiP camera until it prepares the OVXR session. + + Task resolution and launch are stubbed; CLI parsing, pre-launch preparation and session + construction are production code. ``calls`` records the kept cameras, launch and preparation. + + Returns: + The task configuration and the arguments passed to session preparation. + """ + from contextlib import contextmanager, suppress + + import isaaclab_teleop + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent + + from isaaclab.sensors import CameraCfg + from isaaclab.sim import SimulationContext + + config = SimpleNamespace( + sim=SimpleNamespace(device="cpu", physics=OvPhysxCfg()), + scene=SimpleNamespace(num_envs=1, head_cam=CameraCfg(prim_path="/World/head_cam", width=640, height=480)), + terminations=SimpleNamespace(success=None, time_out=None), + isaac_teleop=isaaclab_teleop.IsaacTeleopCfg( + xr_camera_feeds=[isaaclab_teleop.XrCameraFeedCfg(camera_name="head_cam", offset_m=(0.1, -0.15))], + xr_camera_feed_layout=layout, + ), + ) + prepared = [] + + class SessionReached(RuntimeError): + pass + + @contextmanager + def launch(selected_cfg, _args, *, cleanup_context): + assert selected_cfg is config + calls.append("launch") + yield + + def prepare(sim_cfg, xr_cfg, renderer_cfg=None): + assert sim_cfg is config.sim + calls.append("prepare") + prepared.append((xr_cfg, renderer_cfg)) + raise SessionReached + + def remove_camera_configs(selected_cfg, *, keep=()): + calls.append(("keep", keep)) + return selected_cfg + + module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent + monkeypatch.setattr(sys, "argv", ["teleop-test"]) + monkeypatch.setattr(module, "args_cli", None, raising=False) + monkeypatch.setattr(module, "ManagerBasedRLEnvCfg", SimpleNamespace) + monkeypatch.setattr(module, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) + monkeypatch.setattr(module, "validate_launch", lambda *_args: ((), ())) + monkeypatch.setattr(module, "launch_simulation", launch) + monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", remove_camera_configs) + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + monkeypatch.setattr(SimulationContext, "instance", lambda: None) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + arguments = ["--task", "Test", "--ovxr", "--cloudxr_env", "none", "--ovxr_pip_camera", "head_cam", *flags] + if entrypoint == "live": + monkeypatch.setattr(module, "has_kit", lambda: False) + monkeypatch.setattr( + isaaclab_teleop.XrCameraFeedSession, + "prepare", + lambda *_args, **_kwargs: SimpleNamespace(close=lambda: None), + ) + else: + monkeypatch.setattr(module, "hydra_overrides", None, raising=False) + monkeypatch.setattr(module, "remove_camera_configs", remove_camera_configs) + arguments += ["--replay_file", "recording.mcap"] + module._parse_args(arguments) + with suppress(SessionReached): + module.main() + return config, prepared + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize( + "placement, flags, position", + [ + # A head-locked task layout keeps its panel placement without the flag. + ("head_locked", [], (0.1, -0.15, -0.8)), + # The flag places a world layout in the VIEW space, at the requested position. + ("world", ["--ovxr_pip_head_locked", "--ovxr_pip_view_position", "0", "-0.1", "-0.6"], (0.0, -0.1, -0.6)), + ], +) +def test_ovxr_pip_cli_reaches_session_construction(monkeypatch, entrypoint, placement, flags, position): + """The PiP and mirror options of both workflows reach the OVXR session configuration.""" + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop import XrCameraFeedLayoutCfg + + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(1.0, 0.0, 1.5)) + rates = ["--ovxr_pip_fps", "15", "--ovxr_desktop_mirror_fps", "20"] + calls = [] + config, prepared = _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, flags + rates, calls) + + assert calls == [("keep", ("head_cam",)), "launch", "prepare"] + [(xr_cfg, renderer_cfg)] = prepared + assert xr_cfg.desktop_mirror_enabled is True + assert xr_cfg.desktop_mirror_max_fps == 20.0 + [camera] = xr_cfg.auxiliary_cameras + assert camera.name == "head_cam" and camera.camera_cfg is config.scene.head_cam + assert camera.max_fps == 15.0 + assert camera.position == pytest.approx(position) + assert isinstance(renderer_cfg, OVRTXRendererCfg) and renderer_cfg is config.scene.head_cam.renderer_cfg + # Forcing head-locked placement leaves the task layout unchanged. + assert config.isaac_teleop.xr_camera_feed_layout.placement == placement + + +@pytest.mark.parametrize("entrypoint", ["live", "replay"]) +@pytest.mark.parametrize("placement", ["viewer_start", "world"]) +def test_ovxr_pip_rejects_task_layout_that_is_not_head_locked_before_launch(monkeypatch, entrypoint, placement): + """Without ``--ovxr_pip_head_locked``, a task layout OVXR cannot present fails before launch.""" + from isaaclab_teleop import XrCameraFeedLayoutCfg + + layout = XrCameraFeedLayoutCfg(placement=placement, world_position_m=(1.0, 0.0, 1.5)) + calls = [] + with pytest.raises(ValueError, match=rf"{placement!r} placement.*--ovxr_pip_head_locked"): + _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, [], calls) + assert calls == [] + + @pytest.mark.parametrize("entrypoint", ["live", "replay"]) def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoint): """Both actual workflows finalize the selected config before OVXR allocates its context.""" @@ -1045,7 +1170,9 @@ def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monk with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) return - env_cfg, _success = replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) + env_cfg, _success, _cameras = replay._prepare_env_cfg( + _UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides + ) assert env_cfg.sim.physics is None assert replay.args_cli.require_kit diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index dfd68a53ed8..d579e6f53f0 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -36,7 +36,7 @@ def test_replay_failed_input_drain_requests_stop_without_releasing_borrowers(rep @pytest.mark.parametrize("layout_mode", ["horizontal", "manual", None]) def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_presentation_type, layout_mode): - """The actual viewer constructor changes placement without discarding task packing.""" + """The forced head-locked placement keeps task packing, and the viewer receives the CLI rates.""" import isaaclab_teleop from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg @@ -65,35 +65,32 @@ def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_present original_layout_fields = original_layout.to_dict() if original_layout is not None else None original_feeds = [feed.to_dict() for feed in config.isaac_teleop.xr_camera_feeds] from isaaclab_ov.renderers import OVRTXRendererCfg - from isaaclab_teleop.camera_feed import select_ovxr_camera_feeds - - monkeypatch.setitem( - replay_presentation_type.__init__.__globals__, - "args_cli", - SimpleNamespace( - ovxr_pip_camera=names, - ovxr_pip_head_locked=True, - ovxr_pip_fps=30.0, - ovxr_pip_view_position=None, - task="Test", - ovxr_session_mode="ar", - cloudxr_env="none", - auto_launch_cloudxr=False, - ovxr_desktop_mirror=False, - ovxr_desktop_mirror_fps=30.0, - ), - ) - # Environment preparation records the validated selection that the presentation reuses. - monkeypatch.setitem( - replay_presentation_type.__init__.__globals__, - "ovxr_pip_selection", - select_ovxr_camera_feeds(config, names, OVRTXRendererCfg()), + from isaaclab_teleop.entrypoints._ovxr_launch import resolve_ovxr_pip_cameras + + args = SimpleNamespace( + ovxr_pip_camera=names, + ovxr_pip_head_locked=True, + ovxr_pip_fps=15.0, + ovxr_pip_view_position=None, + task="Test", + ovxr_session_mode="ar", + cloudxr_env="none", + auto_launch_cloudxr=False, + ovxr_desktop_mirror=True, + ovxr_desktop_mirror_fps=20.0, ) + monkeypatch.setitem(replay_presentation_type.__init__.__globals__, "args_cli", args) + # Environment preparation resolves the panels that the presentation receives. + cameras = resolve_ovxr_pip_cameras(config, args, OVRTXRendererCfg()) prepare = Mock() monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) - presentation = replay_presentation_type(config) + presentation = replay_presentation_type(config, cameras) assert presentation.session is prepare.return_value session_config = prepare.call_args.args[1] + assert session_config.desktop_mirror_enabled is True + assert session_config.desktop_mirror_max_fps == 20.0 + assert [camera.max_fps for camera in session_config.auxiliary_cameras] == [15.0, 15.0] + assert prepare.call_args.args[2] is config.scene.left.renderer_cfg expected = ( [(-0.15, -0.2, -1.2), (0.35, -0.2, -1.2)] if layout_mode == "horizontal" @@ -241,7 +238,8 @@ def clear_context(): ) class Presentation: - def __init__(self, cfg): + def __init__(self, cfg, cameras): + assert cameras == () self.session = session self.started = False @@ -291,15 +289,15 @@ def replay(**kwargs): monkeypatch.setattr(replay_agent, name, value, raising=False) if stop_at.startswith("startup"): with pytest.raises(KeyboardInterrupt): - replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) assert calls == ["presentation-close"] elif stop_at.startswith("construction"): with pytest.raises(RuntimeError, match="environment construction failed"): - replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) assert calls == ["presentation-close"] + (["context-clear"] if stop_at == "construction-new" else []) assert context_state["current"] is (None if stop_at == "construction-new" else previous) else: - result = replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None) + result = replay_agent._run_replay_batch(SimpleNamespace(isaac_teleop=None), None, ()) assert result == (0 if requested == 1 else 1) assert reports[0]["num_replays"] == 1 assert reports[0]["requested_num_replays"] == requested From 2717aef70e06275cb823bfdd595a085992e64ce3 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:15:19 -0400 Subject: [PATCH 050/145] Fold the replay PiP layout note into the added changelog entry Replay PiP is new in this release, so the note about it following the task layout belongs with the entry that adds it. --- .../isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst | 3 ++- .../isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) delete mode 100644 source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst index 3c8cac94a78..c8617fe78ae 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -4,7 +4,8 @@ * Enabled the mirror by default in OVXR live and MCAP replay entrypoints; ``--headless`` disables the local window while preserving XR and camera-panel rendering. * Supported task-configured automatic layouts for head-locked panels - in live teleoperation and replay. Replay accepts the live ``--ovxr_pip_fps``, + in live teleoperation and replay. ``--ovxr_pip_head_locked`` places a task layout that is + not head-locked in the OpenXR VIEW space. Replay accepts the live ``--ovxr_pip_fps``, ``--ovxr_pip_view_position`` and ``--ovxr_desktop_mirror_fps`` options. * Added :attr:`~isaaclab_teleop.XrCfg.excluded_visual_meshes`, which hides matching visual meshes from the OVXR eye cameras only. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst deleted file mode 100644 index d9472f15c61..00000000000 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.changed.rst +++ /dev/null @@ -1,2 +0,0 @@ -* Changed MCAP replay PiP to follow the task's camera layout like live teleoperation; - ``--ovxr_pip_head_locked`` is required only when that layout is not head-locked. From a744125374dc9c5ae922a4c0d0cc295c80d063cb Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:15:38 -0400 Subject: [PATCH 051/145] Document OvLibrariesXrCameraConfig and the camera keep list Add the public OVXR camera-panel configuration to the isaaclab_teleop API reference and mention the keep argument in the README utilities list. --- docs/source/api/lab_teleop/isaaclab_teleop.rst | 4 ++++ source/isaaclab_teleop/docs/README.md | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 149ef166652..0c71f12719a 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -23,6 +23,7 @@ isaaclab_teleop XrAnchorRotationMode XrAnchorSynchronizer OvLibrariesXrConfig + OvLibrariesXrCameraConfig OvLibrariesXrSession .. rubric:: Functions @@ -69,6 +70,9 @@ Kitless OVXR Session .. autoclass:: OvLibrariesXrConfig :members: +.. autoclass:: OvLibrariesXrCameraConfig + :members: + .. autoclass:: OvLibrariesXrSession :members: diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 0f7f1b64ad3..2e0a13a5b0d 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -208,8 +208,9 @@ recorders or rolling-shutter playback. ## Utilities -- **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy - observation terms. +- **`remove_camera_configs(env_cfg, keep=())`** -- strips camera sensors and their associated policy + observation terms. Cameras named in `keep` stay in the scene, but their observation terms are + still removed. ## Run with Docker From fbcbdc0db50ea1e33e0635eb1573a1565f8596b7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:16:20 -0400 Subject: [PATCH 052/145] Treat a blocked partition-visibility value as unauthored in exports HasAuthoredValueOpinion is deprecated and counts a value block as an authored opinion, so a blocked showAllPartitionsByDefault skipped the export default. HasAuthoredValue keeps the default for a block. --- source/isaaclab_ov/isaaclab_ov/stage.py | 2 +- source/isaaclab_ov/test/test_ovrtx_usd.py | 22 ++++++++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index 701b5ba04e5..ddc61a9d5d0 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -295,7 +295,7 @@ def export_stage_to_string( show_all = settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool ) - if not show_all.HasAuthoredValueOpinion(): + if not show_all.HasAuthoredValue(): show_all.Set(True) export_stage.SetMetadata("renderSettingsPrimPath", settings_path) prim_paths = [] diff --git a/source/isaaclab_ov/test/test_ovrtx_usd.py b/source/isaaclab_ov/test/test_ovrtx_usd.py index 54ddb03a01a..3f4de05d119 100644 --- a/source/isaaclab_ov/test/test_ovrtx_usd.py +++ b/source/isaaclab_ov/test/test_ovrtx_usd.py @@ -529,19 +529,29 @@ def test_export_stage_heterogeneous_keeps_multiple_sources(): @pytest.mark.parametrize( - "settings_path, authored_show_all", - [(None, None), ("/Render/AuthoredSettings", False), ("/Render/AuthoredSettings", True)], + "settings_path, authored_show_all, expected_show_all", + [ + (None, None, True), + ("/Render/AuthoredSettings", False, False), + ("/Render/AuthoredSettings", True, True), + # A value block authors no value, so the export keeps every partition visible. + ("/Render/AuthoredSettings", "blocked", True), + ], ) -def test_export_stage_preserves_authored_scene(settings_path, authored_show_all): +def test_export_stage_preserves_authored_scene(settings_path, authored_show_all, expected_show_all): """Renderer settings and clone trimming are confined to the exported scene.""" num_envs = 4 stage = _make_multi_env_stage(num_envs) if settings_path is not None: settings = UsdRender.Settings.Define(stage, settings_path) settings.CreateResolutionAttr(Gf.Vec2i(1280, 720)) - settings.GetPrim().CreateAttribute( + show_all = settings.GetPrim().CreateAttribute( "omni:rtx:scenePartitioning:showAllPartitionsByDefault", Sdf.ValueTypeNames.Bool - ).Set(authored_show_all) + ) + if authored_show_all == "blocked": + show_all.Block() + else: + show_all.Set(authored_show_all) stage.SetMetadata("renderSettingsPrimPath", settings_path) before = stage.ExportToString() layer = Sdf.Layer.CreateAnonymous() @@ -554,7 +564,7 @@ def test_export_stage_preserves_authored_scene(settings_path, authored_show_all) assert exported.HasAuthoredMetadata("renderSettingsPrimPath") settings = exported.GetPrimAtPath(exported.GetMetadata("renderSettingsPrimPath")) show_all = settings.GetAttribute("omni:rtx:scenePartitioning:showAllPartitionsByDefault") - assert show_all and show_all.Get() is (True if authored_show_all is None else authored_show_all) + assert show_all and show_all.Get() is expected_show_all if settings_path is not None: assert settings.GetPath() == settings_path assert settings.GetAttribute("resolution").Get() == (1280, 720) From ece342bf20311511d0f019d18141c735c14b1abc Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:16:43 -0400 Subject: [PATCH 053/145] Keep the camera pose contract test destructor-safe The Camera built with __new__ lacked the callback handles and view close that its destructor uses, so pytest reported an unraisable exception. Also mark the file as a unit test like its sibling sensor unit tests. --- .../sensors/test_camera_pose_renderer_notification.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py index 24f778cd4a8..06401e6079d 100644 --- a/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py +++ b/source/isaaclab/test/sensors/test_camera_pose_renderer_notification.py @@ -8,10 +8,13 @@ from types import SimpleNamespace from unittest.mock import MagicMock +import pytest import warp as wp from isaaclab.sensors.camera.camera import Camera +pytestmark = pytest.mark.unit + def test_camera_pose_refresh_notifies_its_renderer(): """``Camera._update_poses`` must forward the refreshed world pose to its renderer. @@ -20,10 +23,13 @@ def test_camera_pose_refresh_notifies_its_renderer(): publication; a refactor that stops notifying the renderer would leave those panels stale. """ camera = Camera.__new__(Camera) + # ``__new__`` skips ``__init__``; set the callback handles that the destructor clears. + camera._initialize_handle = camera._invalidate_initialize_handle = camera._prim_deletion_handle = None + camera._debug_vis_handle = None pose = SimpleNamespace(warp=wp.zeros(1, dtype=wp.vec3f, device="cpu")) orientation = SimpleNamespace(warp=wp.array([[0.0, 0.0, 0.0, 1.0]], dtype=wp.quatf, device="cpu")) camera._sensor_prims = [object()] - camera._view = SimpleNamespace(get_world_poses=lambda _env_ids: (pose, orientation)) + camera._view = SimpleNamespace(get_world_poses=lambda _env_ids: (pose, orientation), close=lambda: None) camera._update_camera_state = MagicMock() camera._render_data = object() camera._renderer = MagicMock() From 36cb1f9fd4ef6a57f377eb6c6be798a332a59bc9 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:17:45 -0400 Subject: [PATCH 054/145] Stop editing the unresolved fixed-base G1 policy preset in a test The fixed-base G1 policy observations are a preset whose default is state-only, so setting camera terms on the unresolved wrapper did nothing. Only the scene cameras need removing for that task. --- .../isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py index 55372fb44e1..7b48a6d59d1 100644 --- a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py @@ -61,12 +61,12 @@ def test_g1_wheel_remains_supported_on_ovphysx_cpu(cfg_type, task) -> None: env_cfg.sim.device = "cpu" env_cfg.sim.physics = OvPhysxCfg() env_cfg.scene.robot_pov_cam = None - env_cfg.observations.policy.robot_pov_cam = None if isinstance(env_cfg, FixedBaseUpperBodyIKG1EnvCfg): + # The default policy observations are state-only, so only the scene cameras are removed. env_cfg.scene.left_wrist_cam = None env_cfg.scene.right_wrist_cam = None - env_cfg.observations.policy.left_wrist_cam = None - env_cfg.observations.policy.right_wrist_cam = None + else: + env_cfg.observations.policy.robot_pov_cam = None env_cfg.image_obs_list = [] env_cfg.stabilize_for_kitless_ovphysx() From 4ab3ae5bc0d5a07e85f9d3cfdcb9ef8ac39242c5 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:19:32 -0400 Subject: [PATCH 055/145] Partition the real robot assets in the head-exclusion test Reference each task's robot asset at the spawned env_0 path in an in-memory stage and call partition_visual_meshes, instead of re-implementing its matcher. The test now also covers de-instancing, the authored partition primvar and removal of the temporary layer. --- .../test/contrib/test_teleop_task_assets.py | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py index f0fffb3ec55..11fbf8170e1 100644 --- a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -25,12 +25,16 @@ _ENV_ROBOT = "{ENV_REGEX_NS}/Robot" -def _open_asset(usd_path: str) -> Usd.Stage: +def _retrieve_asset(usd_path: str) -> str: try: - local_path = retrieve_file_path(usd_path) + return retrieve_file_path(usd_path) except (FileNotFoundError, RuntimeError) as exc: # The asset server is optional for local test runs: it may be unreachable or fail the download. pytest.skip(f"Robot asset is unavailable: {exc}") + + +def _open_asset(usd_path: str) -> Usd.Stage: + local_path = _retrieve_asset(usd_path) stage = Usd.Stage.Open(local_path) assert stage is not None, local_path return stage @@ -99,8 +103,8 @@ def test_gr1t2_kitless_kinematics_urdf_matches_the_spawned_asset(): @pytest.mark.parametrize("task", ["fixed_base_g1", "gr1t2", "gr1t2_waist"]) -def test_xr_head_exclusions_match_only_visual_head_meshes(task): - """OVXR attach fails closed when an exclusion matches nothing; check the patterns on the real assets.""" +def test_xr_head_exclusions_partition_only_visual_head_meshes(task): + """OVXR attach fails closed when an exclusion matches nothing; partition the real spawned asset.""" from pxr import UsdGeom from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( @@ -119,16 +123,23 @@ def test_xr_head_exclusions_match_only_visual_head_meshes(task): cfg = cfg_type() if cfg.xr is None or not cfg.xr.excluded_visual_meshes: pytest.skip("The teleoperation configuration is unavailable") - stage = _open_asset(cfg.scene.robot.spawn.usd_path) - for pattern in cfg.xr.excluded_visual_meshes: - assert pattern.startswith("/World/envs/env_0/Robot/"), pattern - meshes = [ - prim - for prim in _robot_paths(stage, _ENV_ROBOT + pattern[len("/World/envs/env_0/Robot") :]) - if prim.IsA(UsdGeom.Mesh) - ] - assert meshes, pattern - for prim in meshes: - assert "collisions" not in str(prim.GetPath()).split("/") + from isaaclab_teleop.xr_scene_partition import partition_visual_meshes + + # Reference the robot where the single-environment scene spawns it. + stage = Usd.Stage.CreateInMemory() + stage.DefinePrim("/World/envs/env_0/Robot").GetReferences().AddReference( + _retrieve_asset(cfg.scene.robot.spawn.usd_path) + ) + session_sublayers = list(stage.GetSessionLayer().subLayerPaths) + with partition_visual_meshes(stage, cfg.xr.excluded_visual_meshes) as paths: + assert paths + for path in paths: + prim = stage.GetPrimAtPath(path) + assert prim.IsA(UsdGeom.Mesh), path + assert "head" in path and "collisions" not in path.split("/") assert not prim.HasAPI(UsdPhysics.CollisionAPI) - assert "head" in str(prim.GetPath()) + assert UsdGeom.PrimvarsAPI(prim).GetPrimvar("omni:scenePartition").Get() == "xr_excluded" + # The partition is temporary: exiting removes its session sublayer and the authored primvars. + assert list(stage.GetSessionLayer().subLayerPaths) == session_sublayers + for path in paths: + assert not UsdGeom.PrimvarsAPI(stage.GetPrimAtPath(path)).HasPrimvar("omni:scenePartition") From feae00647824c48200c911763319077127840386 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 18:26:06 -0400 Subject: [PATCH 056/145] Skip OVXR session tests without Capture session support The Kit teleop extra installs a Capture release without OVXR sessions, where patching OvLibrariesXrSession raises ModuleNotFoundError. Skip the tests that build a session there, as run_live_workflow already does, and keep the pre-launch layout rejection test running in both environments. --- .../test/test_teleop_entrypoint_units.py | 10 +++++++++- .../test/test_teleop_replay_bootstrap.py | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index de49319f502..51dddaac161 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -843,6 +843,9 @@ def test_auxiliary_camera_pip_cli_is_explicit_and_bounded(monkeypatch): ) def test_ovxr_mirror_cli_reaches_session_construction(monkeypatch, entrypoint, flags, expected): """The real entrypoint parser controls the config passed to XR session preparation.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") import isaaclab_teleop from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent @@ -965,7 +968,9 @@ def remove_camera_configs(selected_cfg, *, keep=()): monkeypatch.setattr(module, "validate_launch", lambda *_args: ((), ())) monkeypatch.setattr(module, "launch_simulation", launch) monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", remove_camera_configs) - monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) + # The Kit Capture release has no OVXR session; layout rejection never reaches the session. + with suppress(ModuleNotFoundError): + monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", prepare) monkeypatch.setattr(SimulationContext, "instance", lambda: None) monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") arguments = ["--task", "Test", "--ovxr", "--cloudxr_env", "none", "--ovxr_pip_camera", "head_cam", *flags] @@ -998,6 +1003,9 @@ def remove_camera_configs(selected_cfg, *, keep=()): ) def test_ovxr_pip_cli_reaches_session_construction(monkeypatch, entrypoint, placement, flags, position): """The PiP and mirror options of both workflows reach the OVXR session configuration.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") from isaaclab_ov.renderers import OVRTXRendererCfg from isaaclab_teleop import XrCameraFeedLayoutCfg diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index d579e6f53f0..8fdd5fe7f8c 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -37,6 +37,9 @@ def test_replay_failed_input_drain_requests_stop_without_releasing_borrowers(rep @pytest.mark.parametrize("layout_mode", ["horizontal", "manual", None]) def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_presentation_type, layout_mode): """The forced head-locked placement keeps task packing, and the viewer receives the CLI rates.""" + # Session construction needs a Capture release with OVXR session support; the Kit + # ``teleop`` extra selects an older release, so skip there like ``run_live_workflow``. + pytest.importorskip("isaaccapture.ovxr_session_manager") import isaaclab_teleop from isaaclab_teleop.isaac_teleop_cfg import IsaacTeleopCfg, XrCameraFeedCfg, XrCameraFeedLayoutCfg From de2dec60d41fa8d335752b8a916e2a05b240590b Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sat, 31 Oct 2026 19:38:34 -0400 Subject: [PATCH 057/145] Regenerate the environment browser for the SO101 IK-Abs task --- docs/source/_static/css/environment-browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/_static/css/environment-browser.js b/docs/source/_static/css/environment-browser.js index 022715b3db3..d76a57f400e 100644 --- a/docs/source/_static/css/environment-browser.js +++ b/docs/source/_static/css/environment-browser.js @@ -129,7 +129,7 @@ ["IsaacContrib-Stack-Cube-Instance-Randomize-Franka-IK-Rel", "", "", "", "", "environment-previews/franka_instance_randomize_stack.jpg"], ["IsaacContrib-Stack-Cube-RedGreen-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-RedGreenBlue-Franka-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/franka_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], - ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,newton_mjwarp,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "newton_mjwarp||", "ovphysx||"]], + ["IsaacContrib-Stack-Cube-SO101-IK-Abs-v0", "", "isaacsim_physx,ovphysx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||", "ovphysx||"]], ["IsaacContrib-Stack-Cube-SO101-Joint-Teleop-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-SO101-v0", "", "isaacsim_physx", "", "", "environment-previews/so101_stack.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], ["IsaacContrib-Stack-Cube-UR10-Long-Suction-IK-Rel", "", "isaacsim_physx", "", "", "environment-previews/ur10_stack_surface_gripper.jpg", false, {}, {}, "", null, ["||", "isaacsim_physx||"]], From cc90610597cd059286c109c6ad0588e631e5a470 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:19:15 -0500 Subject: [PATCH 058/145] Let a missing Isaac Capture raise its original import error The contribution guide asks optional dependencies to raise Python's own ModuleNotFoundError rather than a wrapped installation hint. The prefix check could also mislabel a broken nested Capture module as a missing release. The source guide keeps the installation instructions. --- .../isaaclab_teleop/ov_libraries_xr.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index c4b38f87986..fad2d42a34b 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -16,17 +16,7 @@ from dataclasses import replace from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable -try: - from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger -except ModuleNotFoundError as exc: - if not (exc.name or "").startswith("isaaccapture"): - raise - raise ModuleNotFoundError( - "OvLibrariesXrSession requires an Isaac Capture release with OVXR session support" - " (isaaccapture[ovxr]>=1.7.93a1). The 'teleop' extra selects the Kit release; install the" - " kitless selection described in source/isaaclab_teleop/docs/kitless_xr_source.md.", - name=exc.name, - ) from exc +from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger from isaaclab_ov.renderers import OVRTXRendererCfg from .ovxr_cfg import OvLibrariesXrConfig From 7314947b36623f8e4500ee0658ec58dca8248795 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:19:15 -0500 Subject: [PATCH 059/145] Document and close the OVStage scene publisher contract publication_required() now rejects a closed publisher under the same lock as write_complete_scene(), so it cannot report a write that the writer would then refuse. The returned publisher is rendered in the API docs, with the publication order, guard, ordinal sealing, identity and failure rules it expects of a borrower. The renderer imports the lease and publication-buffer modules relatively, and the controller's private ordinal helper follows its public methods. --- .../api/lab_ov/isaaclab_ov.renderers.rst | 8 +- .../isaaclab_ov/renderers/ovrtx_renderer.py | 5 +- .../isaaclab_ov/renderers/render_lease.py | 94 +++++++++++++++---- .../test/test_ovrtx_renderer_contract.py | 18 ++++ 4 files changed, 102 insertions(+), 23 deletions(-) diff --git a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst index 05cc82ff841..73fbec26cef 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst @@ -35,12 +35,16 @@ An external consumer, such as the OVXR teleoperation adapter, can borrow one exa renderer and its private OVStage with :meth:`OVRTXRenderer.acquire_ov_libraries_render_lease` before clone preparation. While the lease is active, ordinary camera rendering, scene writes and renderer/stage destruction are rejected; eager scene updates are left to the lease's scene -publisher. A failed leased publication poisons publication only; the borrower can still retire -the pair, after which the renderer must be recreated. +publisher, which :meth:`OvLibrariesRenderLease.capture_scene_publisher` returns. A failed leased +publication poisons publication only; the borrower can still retire the pair, after which the +renderer must be recreated. .. autoclass:: OvLibrariesRenderLease :members: +.. autoclass:: isaaclab_ov.renderers.render_lease.OvLibrariesScenePublisher + :members: + Simulation-owned stages ----------------------- diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index e86f0f12a60..92bd5e56045 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -102,8 +102,6 @@ build_render_product_as_string, render_var_prim_names_by_source, ) -from isaaclab_ov.renderers.publication_buffers import _AsyncWriteBuffers -from isaaclab_ov.renderers.render_lease import OvLibrariesRenderLease, RenderLeaseController from isaaclab_ov.renderers.visual_materials import OVRTXVisualMaterialWriter from isaaclab_ov.stage import ( export_stage_to_string, @@ -112,6 +110,9 @@ xform_tensor_from_warp, ) +from .publication_buffers import _AsyncWriteBuffers +from .render_lease import OvLibrariesRenderLease, RenderLeaseController + if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, FrameOutput, Operation, PendingFetch, RenderVarOutput diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 384b64d4973..d529d536d29 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -119,7 +119,20 @@ def materialize_deferred_stage(self, stage: Any, num_envs: int) -> bool: return cast("RenderLeaseController", self._controller).materialize_deferred_stage(self, stage, num_envs) def capture_scene_publisher(self) -> OvLibrariesScenePublisher: - """Capture the application-thread scene publisher before render ownership transfers.""" + """Capture the application-thread scene publisher before render ownership transfers. + + Call this on the owner thread while the lease is unbound and before + :meth:`offer_owner_transfer`. The publisher stays bound to the calling thread; see + :class:`~isaaclab_ov.renderers.render_lease.OvLibrariesScenePublisher` for the publication + protocol. + + Returns: + The lease's only scene publisher. + + Raises: + RuntimeError: If called off the owner thread or while a transfer is offered, after the + Stage is bound, or when the lease already has a publisher. + """ return cast("RenderLeaseController", self._controller).capture_scene_publisher(self) @contextlib.contextmanager @@ -189,7 +202,26 @@ def _validate_active(self) -> None: class OvLibrariesScenePublisher: - """Application-thread writer for complete provider-derived OVStage batches.""" + """Application-thread writer for complete provider-derived OVStage batches. + + Obtain it with :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.capture_scene_publisher`. + While the lease is active, the renderer skips its own eager transform, geometry and material + writes, and the borrower publishes each complete physics state through this object instead: + + 1. On the application thread, call :meth:`publication_required` with the identity of the + current physics state. + 2. If it returns ``True``, reserve a nonzero Stage ordinal and call + :meth:`write_complete_scene` with it. The write does not advance the Stage write floor: + the borrower seals that ordinal afterwards, or abandons it if the write fails. + 3. Hold one application publication guard around both calls and around every operation the + render owner performs in :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.ovstage_owner_lane`, so that scene + writes and native frame operations never interleave. + + A failed write poisons the pair: later publication and ordinary renderer use are rejected, + but the borrower can still retire its consumers and release the lease. The renderer must then + be recreated. :meth:`close` stops publication; releasing the lease also closes the publisher. + The OVXR teleoperation adapter, ``isaaclab_teleop.OvLibrariesXrSession``, follows this protocol. + """ def __init__(self, owner: RenderLeaseController, lease: OvLibrariesRenderLease) -> None: self._controller = owner @@ -202,17 +234,41 @@ def publication_required(self, physics_step_id: Hashable) -> bool: """Return whether a new physics step or dirty material requires a scene write. Args: - physics_step_id: Hashable identity of the complete physics state to publish. + physics_step_id: Hashable identity of the complete physics state to publish. Only the + last written identity is remembered, so the identity must change whenever the + state does, including after a reset that does not advance the physics step count. + + Returns: + Whether ``physics_step_id`` differs from the last written batch or a visual material + has pending updates. Raises: - RuntimeError: If the lease is not bound, the publisher is inactive, or a failed - write poisoned the pair. + RuntimeError: If called off the captured application thread, the publisher is closed + or inactive, the lease is not bound, or a failed write poisoned the pair. + TypeError: If ``physics_step_id`` is not hashable. """ self._require_application_thread() - return self._controller.producer_publication_required(self._lease, physics_step_id) + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + return self._controller.producer_publication_required(self._lease, physics_step_id) def write_complete_scene(self, ordinal: int, physics_step_id: Hashable) -> None: - """Write, but do not publish, one complete batch at ``ordinal``.""" + """Write, but do not publish, one complete batch at ``ordinal``. + + Writes the current transforms, geometries and pending visual materials at ``ordinal`` and + records ``physics_step_id`` as written. The caller seals the ordinal afterwards. + + Args: + ordinal: Nonzero Stage ordinal reserved by the caller for this batch. + physics_step_id: The identity passed to the preceding :meth:`publication_required` call. + + Raises: + ValueError: If ``ordinal`` is not positive. + RuntimeError: If called off the captured application thread, the publisher is closed or + inactive, the state needs no publication, or the write failed. A failed write + poisons the pair. + """ self._require_application_thread() with self._lock: if self._closed: @@ -220,7 +276,7 @@ def write_complete_scene(self, ordinal: int, physics_step_id: Hashable) -> None: self._controller.write_producer_scene(self._lease, ordinal, physics_step_id) def close(self) -> None: - """Stop scene publication and wait for an in-flight batch to finish.""" + """Stop scene publication after any in-flight batch finishes. Safe to call repeatedly.""" with self._lock: self._closed = True @@ -422,17 +478,6 @@ def owns_scene_updates(self) -> bool: """Whether an active lease's scene publisher owns transform and geometry writes.""" return self.active is not None - def _next_ordinal(self) -> int: - """Return ``floor + 1`` from the exact renderer-owned Stage.""" - if self._renderer.scene.stage is None: - raise RuntimeError("Cannot query an OVStage floor before scene materialization") - with self._renderer.scene.stage.get_attribute_write_floor() as floor_query: - floor_query.wait() - floor = int(floor_query.fetch()) - if floor >= (1 << 64) - 1: - raise RuntimeError("OVStage ordinal range is exhausted") - return floor + 1 - def producer_publication_required(self, lease: OvLibrariesRenderLease, physics_step_id: Hashable) -> bool: """Check publication on the application thread without entering the render thread.""" if self.active is not lease or self._publisher is None: @@ -517,3 +562,14 @@ def reserve_pair(self) -> None: raise RuntimeError("The render lease requires an unused private OVStage") renderer.backend._render_lease_owner = renderer renderer.scene._render_lease_owner = renderer + + def _next_ordinal(self) -> int: + """Return ``floor + 1`` from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None: + raise RuntimeError("Cannot query an OVStage floor before scene materialization") + with self._renderer.scene.stage.get_attribute_write_floor() as floor_query: + floor_query.wait() + floor = int(floor_query.fetch()) + if floor >= (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + return floor + 1 diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index b2474e0e963..f06f565b791 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1564,6 +1564,24 @@ def test_render_lease_complete_publication_deduplicates_and_poison_fails_closed( lease.release() +def test_render_lease_closed_publisher_reports_no_pending_publication(leased_renderer, monkeypatch): + """A closed publisher must not report a write that :meth:`write_complete_scene` would then reject.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: None) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + assert publisher.publication_required("step-1") + publisher.close() + publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.publication_required("step-1") + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.write_complete_scene(18, "step-1") + lease.release() + + def test_render_lease_poison_blocks_publication_but_not_retirement(leased_renderer, monkeypatch): """A failed producer write must not prevent the borrower from retiring the pair (Capture's close order).""" renderer = leased_renderer From 689cbcb54076859525ebdabe8c2137c32fcb2f93 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:28:36 -0500 Subject: [PATCH 060/145] Check the GR1T2 kinematics URDF against the asset's joint frames Matching joint names and limits would not catch a moved joint origin or a reversed axis. The test composes the spawned USD's revolute joint frames from base_link at an asymmetric pose and compares every URDF link pose from Pinocchio. It also checks that the pose exposes a reversed axis on each of the 14 arm joints at its Pink wrist frame. A reversed elbow or wrist axis and a 1 mm shoulder origin shift each fail it. --- .../test/contrib/test_teleop_task_assets.py | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py index ed99ab3805d..a4c14a6a212 100644 --- a/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py +++ b/source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py @@ -96,3 +96,98 @@ def test_gr1t2_kitless_kinematics_urdf_matches_the_spawned_asset(): assert set(urdf_limits) == set(usd_limits) for name, (lower, upper) in usd_limits.items(): assert urdf_limits[name] == pytest.approx((lower, upper), abs=1.0e-4), name + + +def test_gr1t2_kitless_kinematics_urdf_reproduces_the_asset_forward_kinematics(): + """Every URDF link must sit where the spawned USD's joint frames place it at an asymmetric pose [m, rad]. + + Matching joint names and limits would not catch a moved joint origin or a reversed joint axis. + """ + import numpy as np + + pin = pytest.importorskip("pinocchio") + from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg + + cfg = PickPlaceGR1T2EnvCfg() + stage = _open_asset(cfg.scene.robot.spawn.usd_path) + + def joint_frame(position, orientation) -> np.ndarray: + transform = np.eye(4) + quaternion = pin.Quaternion(orientation.GetReal(), *orientation.GetImaginary()) + transform[:3, :3] = quaternion.normalized().toRotationMatrix() + transform[:3, 3] = position + return transform + + # Child link -> (joint name, parent link, joint frame on parent, joint frame on child, joint axis). + joints, limits = {}, {} + for prim in Usd.PrimRange(stage.GetPseudoRoot(), Usd.TraverseInstanceProxies()): + if prim.IsA(UsdPhysics.RevoluteJoint): + joint = UsdPhysics.RevoluteJoint(prim) + joints[joint.GetBody1Rel().GetTargets()[0].name] = ( + prim.GetName(), + joint.GetBody0Rel().GetTargets()[0].name, + joint_frame(joint.GetLocalPos0Attr().Get(), joint.GetLocalRot0Attr().Get()), + joint_frame(joint.GetLocalPos1Attr().Get(), joint.GetLocalRot1Attr().Get()), + np.eye(3)["XYZ".index(joint.GetAxisAttr().Get())], + ) + limits[prim.GetName()] = ( + math.radians(joint.GetLowerLimitAttr().Get()), + math.radians(joint.GetUpperLimitAttr().Get()), + ) + + def usd_link_poses(angles: dict[str, float]) -> dict[str, np.ndarray]: + """Compose ``parent @ frame0 @ R(axis, angle) @ inv(frame1)`` from ``base_link``.""" + poses = {"base_link": np.eye(4)} + + def pose(link: str) -> np.ndarray: + if link not in poses: + name, parent, frame0, frame1, axis = joints[link] + rotation = np.eye(4) + rotation[:3, :3] = pin.AngleAxis(angles[name], axis).toRotationMatrix() + poses[link] = pose(parent) @ frame0 @ rotation @ np.linalg.inv(frame1) + return poses[link] + + for link in joints: + pose(link) + return poses + + def pose_change(a: np.ndarray, b: np.ndarray) -> float: + return float(np.linalg.norm(a[:3, 3] - b[:3, 3]) + np.linalg.norm(pin.log3(a[:3, :3].T @ b[:3, :3]))) + + # Spread the joints over their ranges at golden-ratio fractions, so neighbouring joints and the + # two sides take different angles. + angles = { + name: lower + (upper - lower) * (0.2 + 0.6 * ((index * 0.618034) % 1.0)) + for index, (name, (lower, upper)) in enumerate(sorted(limits.items())) + } + expected = usd_link_poses(angles) + + urdf_path = ( + Path(__import__("isaaclab_tasks").__file__).parent + / "contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + model = pin.buildModelFromUrdf(str(urdf_path)) + data = model.createData() + configuration = pin.neutral(model) + for name, angle in angles.items(): + assert model.existJointName(name), name + joint = model.joints[model.getJointId(name)] + assert joint.nq == 1, name + configuration[joint.idx_q] = angle + pin.framesForwardKinematics(model, data, configuration) + base_inverse = data.oMf[model.getFrameId("base_link")].inverse() + for link, usd_pose in expected.items(): + assert model.existFrame(link), link + urdf_pose = (base_inverse * data.oMf[model.getFrameId(link)]).homogeneous + assert urdf_pose[:3, 3] == pytest.approx(usd_pose[:3, 3], abs=1.0e-5), link + assert np.linalg.norm(pin.log3(urdf_pose[:3, :3].T @ usd_pose[:3, :3])) < 5.0e-4, link + + # The pose must expose a reversed arm axis: negating any one arm joint moves its Pink wrist frame. + wrists = cfg.actions.upper_body_ik.target_eef_link_names + for side in ("left", "right"): + wrist = wrists[f"{side}_wrist"] + arm_joints = [name for name in angles if re.fullmatch(rf"{side}_(shoulder|elbow|wrist)_\w+_joint", name)] + assert len(arm_joints) == 7, arm_joints + for name in arm_joints: + reversed_pose = usd_link_poses(angles | {name: -angles[name]})[wrist] + assert pose_change(reversed_pose, expected[wrist]) > 1.0e-2, name From 3104bc9ed34e9797fc2116cb77c3179054100f13 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:28:36 -0500 Subject: [PATCH 061/145] Regenerate the GR1T2 kinematics URDF with Kit's exporter The vendored model came from an unrecorded conversion. It is now the exact output of isaacsim.asset.exporter.urdf 2.2.10, through the convert_usd_to_urdf call Kit's Pink path makes, with link visuals and collisions removed. The header records the source asset's SHA-256, the exporter version, the command and the post-processing. Kinematics are unchanged: the new forward-kinematics test passes on both files, which differ only in rounding and an extra root link and FixedJoint in the old one. --- .../GR1T2_fourier_hand_6dof_kinematics.urdf | 1277 +++++++++-------- 1 file changed, 670 insertions(+), 607 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf index 9269c43f21c..4b911c3c036 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/assets/GR1T2_fourier_hand_6dof_kinematics.urdf @@ -7,779 +7,842 @@ SPDX-License-Identifier: BSD-3-Clause Kinematics-only model of the Isaac 6.2 asset Robots/FourierIntelligence/GR-1/GR1T2_fourier_hand_6dof/GR1T2_fourier_hand_6dof.usd, the robot -spawned by GR1T2_CFG. Kit regenerates the equivalent URDF from that USD at runtime; kitless Pink -uses this copy. Regenerate it whenever the asset changes: -source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py checks its joints and limits against -the spawned USD. +spawned by GR1T2_CFG. Kit converts that USD at runtime for Pink; kitless Pink uses this copy. + +Generated from the asset with SHA-256 +4c10726748b53e431bcc80d5b872fd0457e5458ec2714f735e8940cfb25d4215 by Isaac Sim's +isaacsim.asset.exporter.urdf 2.2.10, through the same call Kit's Pink path makes, inside a +Kit application (for example after AppLauncher(headless=True)): + + isaaclab.controllers.utils.convert_usd_to_urdf(PickPlaceGR1T2EnvCfg().scene.robot.spawn.usd_path, out_dir) + +followed by removing every link's and elements, which reference the +exported meshes, and adding this header. Regenerate it the same way whenever the asset changes; +source/isaaclab_tasks/test/contrib/test_teleop_task_assets.py checks its joints, limits and +forward kinematics against the spawned USD. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From add4f9cf0493bf5c83d5ecd951af2eea244335d6 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:28:36 -0500 Subject: [PATCH 062/145] Document the G1 kitless Newton helper contracts The public action, joint-target and configuration helpers now state their tensor shapes, joint order, units, wrist frame, caller-owned state and batch-wide failure behaviour, and the configuration helpers name their parameter types. --- .../locomanip_pick_place/g1_kitless_newton.py | 84 +++++++++++++++++-- 1 file changed, 76 insertions(+), 8 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py index 2a968e2875c..72348cc3412 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -7,13 +7,19 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np import torch from isaaclab.envs.mdp.actions.pink_task_space_actions import PinkInverseKinematicsAction +if TYPE_CHECKING: + from isaaclab.assets import ArticulationCfg + from isaaclab.envs import ManagerBasedRLEnv + + from .fixed_base_upper_body_ik_g1_env_cfg import FixedBaseUpperBodyIKG1EnvCfg + POSITION_MAX_STEP_M = 0.10 """Maximum Cartesian wrist-target change accepted in one active frame.""" @@ -44,7 +50,16 @@ def teleop_action_target_issue(action: torch.Tensor) -> str | None: - """Return why an incoming G1 controller action must not reach Pink/Newton.""" + """Return why an incoming G1 controller action must not reach Pink/Newton. + + Args: + action: One G1 action of :data:`EXPECTED_ACTION_DIM` elements, in any shape that flattens to + it: left wrist position [m] and quaternion (x, y, z, w), right wrist position [m] and + quaternion, then fourteen hand joint targets [rad]. + + Returns: + A description of the first problem found, or ``None`` when the action is acceptable. + """ flat = action.detach().flatten() if flat.numel() != EXPECTED_ACTION_DIM: return f"unexpected G1 controller action size {flat.numel()} (expected {EXPECTED_ACTION_DIM})" @@ -73,7 +88,25 @@ def bound_pink_joint_targets( lower: torch.Tensor, upper: torch.Tensor, ) -> torch.Tensor: - """Keep one Pink update finite, inside limits, and close to current state.""" + """Keep one Pink update finite, inside limits, and close to current state. + + Each target moves at most :data:`JOINT_TARGET_MAX_STEP_RAD` from ``current`` and stays + :data:`JOINT_LIMIT_MARGIN_RAD` inside ``[lower, upper]``. + + Args: + current: Current joint positions [rad]. Shape is (num_envs, num_joints), in the Pink action's + Isaac Lab controlled-joint order. + proposed: Pink joint targets [rad], with the same shape and joint order. + lower: Lower joint limits [rad], with the same shape and joint order. + upper: Upper joint limits [rad], with the same shape and joint order. + + Returns: + The bounded targets [rad], with the same shape and joint order. + + Raises: + RuntimeError: If any input element is non-finite, or the margin-reduced limits of any joint do + not overlap. The check covers the whole batch, so no environment's targets are returned. + """ safe_lower = lower + JOINT_LIMIT_MARGIN_RAD safe_upper = upper - JOINT_LIMIT_MARGIN_RAD flags = ( @@ -181,11 +214,30 @@ def _step_quaternion_towards(current: torch.Tensor, target: torch.Tensor, max_st _WRIST_RAMP_KEYS = ("left_pos", "left_quat", "right_pos", "right_quat") -def condition_kitless_newton_g1_action(env, action: torch.Tensor, state: dict[str, Any]) -> torch.Tensor: +def condition_kitless_newton_g1_action( + env: ManagerBasedRLEnv, action: torch.Tensor, state: dict[str, Any] +) -> torch.Tensor: """Ramp both wrist targets from the current G1 poses before Pink solves IK. + Each call moves the left and right wrist targets at most :data:`POSITION_MAX_STEP_M` and + :data:`ROTATION_MAX_STEP_RAD` toward the requested poses; hand targets pass through unchanged. The ramp re-seeds from the measured wrist poses whenever environment 0 starts a new episode, including termination auto-resets inside ``env.step`` that the teleoperation loop does not observe. + + Args: + env: The single-environment G1 teleoperation environment. + action: One G1 action in the layout described by :func:`teleop_action_target_issue`. The Pink + action reads its wrist poses relative to the environment origin, while the ramp seeds from + the measured world-frame wrist poses, so environment 0's origin must be the world origin. + state: Mutable dictionary owned by the caller; pass the same dictionary to every call of one + session, starting empty. The ramp stores its targets and cached wrist body indices there. + + Returns: + A conditioned copy of ``action`` with the same shape. An action whose size is not + :data:`EXPECTED_ACTION_DIM` is returned unchanged and clears ``state``. + + Raises: + RuntimeError: If the task-selected wrist bodies cannot be resolved. ``state`` is cleared. """ flat = action.detach().flatten() if flat.numel() != EXPECTED_ACTION_DIM: @@ -224,8 +276,15 @@ def condition_kitless_newton_g1_action(env, action: torch.Tensor, state: dict[st return conditioned -def apply_g1_newton_actuator_profile(robot_cfg) -> None: - """Configure the G1 actuator groups used by teleoperation.""" +def apply_g1_newton_actuator_profile(robot_cfg: ArticulationCfg) -> None: + """Apply :data:`ACTUATOR_PROFILE` to the G1 actuator groups used by teleoperation. + + Args: + robot_cfg: The G1 articulation configuration, modified in place. + + Raises: + RuntimeError: If the configuration lacks the ``waist``, ``arms`` or ``hands`` actuator group. + """ groups = robot_cfg.actuators if not set(ACTUATOR_PROFILE).issubset(groups): raise RuntimeError(f"Expected G1 waist/arms/hands actuator groups; found {sorted(groups)}") @@ -235,8 +294,17 @@ def apply_g1_newton_actuator_profile(robot_cfg) -> None: setattr(actuator, key, value) -def configure_g1_for_kitless_newton(env_cfg) -> None: - """Configure the controller, asset and solver for kitless Newton.""" +def configure_g1_for_kitless_newton(env_cfg: FixedBaseUpperBodyIKG1EnvCfg) -> None: + """Configure the controller, asset and solver for kitless Newton. + + Args: + env_cfg: The G1 environment configuration with a Newton MJWarp physics selection, modified + in place. + + Raises: + RuntimeError: If the physics configuration has no MJWarp solver configuration, or the robot + lacks an expected actuator group. + """ from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions from isaaclab_tasks.contrib.steering_wheel import condition_newton_steering_wheel_detail_collisions From 54e23b26bd1dd50b1682027997bdb22371882096 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:28:36 -0500 Subject: [PATCH 063/145] List SO101 without Newton in the kitless XR guide The environment catalog disables SO101 IK with newton_mjwarp after the environment smoke matrix found runtime constraints, and the generated environment browser omits it, so the guide no longer advertises it. --- source/isaaclab_teleop/docs/kitless_xr.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 7c222ba60ea..e94da75ac1e 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -58,14 +58,17 @@ The same registered-task selectors apply to live input and MCAP replay: | `IsaacContrib-PickPlace-GR1T2-WaistEnabled-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | | `IsaacContrib-PickPlace-Locomanipulation-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | | `IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | -| `IsaacContrib-Stack-Cube-SO101-IK-Abs-v0` | `ovphysx`, `newton_mjwarp`, `isaacsim_physx` | +| `IsaacContrib-Stack-Cube-SO101-IK-Abs-v0` | `ovphysx`, `isaacsim_physx` | Omitting `--physics` retains the Kit/Isaac Sim default. Use `--physics ovphysx --device cpu` for standalone CPU physics, or `--physics newton_mjwarp --device cuda:0` for Newton; these standalone selections support `--ovxr`. Kit's `isaacsim_physx` selection uses its separate `--xr` workflow and Isaac Sim installation, not `--ovxr`. The task presets retain backend-specific solver -settings and, for SO101, choose the matching USD Physics variant. +settings and, for SO101, choose the matching USD Physics variant. SO101 also +defines a `newton_mjwarp` preset, but the environment catalog +(`tools/environ_docs.py`) disables that combination after the environment smoke +matrix found runtime constraints, so it is not listed here. The SO101 IK task resets its joints to the configured nominal pose without joint-position noise, giving the clutch a repeatable reset pose. Its configured From da1f578efd3a4a0cbb9d33fa6d0bcae7a5ae1335 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:32:12 -0500 Subject: [PATCH 064/145] Keep a leased camera's product until the lease is released Releasing camera render data removes its RenderProduct reference from the Stage and releases its pose query. For an authorized auxiliary camera that Stage is the one the lease's borrower owns, so cleanup() now rejects it while the lease is active, as other ordinary scene writes already are. The camera stays registered and close() releases it after the lease. --- .../isaaclab_ov/renderers/ovrtx_renderer.py | 6 ++++ .../test/test_ovrtx_renderer_contract.py | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index ec3e7ce7a60..63b0302b19a 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -1322,9 +1322,15 @@ def cleanup(self, render_data: OVRTXCameraRenderData | None) -> None: Each camera owns its product and pose binding. Scene and physics bindings remain alive until :meth:`close`, so other cameras can continue rendering. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + Releasing a product removes it from the borrowed Stage, so the camera keeps its + resources until the lease is released; :meth:`close` releases them afterwards. """ if render_data is None: return + self._render_lease.reject_ordinary_operation("release camera render data", retiring=True) try: self.reset(render_data) finally: diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index f5b31ec2667..1ae6d5ec73e 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1978,6 +1978,36 @@ def consume_product(spec, render_data, **kwargs): renderer.create_render_data(spec) +def test_leased_camera_cleanup_waits_for_lease_release(leased_renderer, monkeypatch): + """Releasing a leased camera would remove its product from the borrowed Stage; it waits for release.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = "/World/Camera" + lease.configure_auxiliary_camera(cfg) + _prepare_leased_renderer(renderer, bind=False) + renderer.prepare_stage(renderer._sim.stage, 1) + data = renderer.create_render_data(CameraRenderSpec(cfg, "cpu", 1, (cfg.prim_path,), 1)) + population = ovrtx_renderer_module.ovstage.population + removed = [] + monkeypatch.setattr(population, "add_usd_reference_from_string", lambda *args: 7) + monkeypatch.setattr(population, "apply_usd_changes", lambda *args, **kwargs: None) + monkeypatch.setattr(population, "remove_usd", lambda _stage, reference: removed.append(reference)) + lease.claim_owner_transfer(lease.offer_owner_transfer()) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + + with pytest.raises(RuntimeError, match="release camera render data while an OvLibrariesRenderLease is active"): + renderer.cleanup(data) + assert removed == [] + assert data in renderer._camera_render_data + assert data.camera_xform_binding is not None + + lease.release() + renderer.close() + assert removed == [7] + assert data not in renderer._camera_render_data + + def test_external_camera_native_rate_is_authored_only_on_export(leased_renderer): from pxr import Usd, UsdGeom From 4c2ee0f2e9023b25c3901f118fbb57b310f481d7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:32:13 -0500 Subject: [PATCH 065/145] Document auxiliary camera matching and order lease helpers configure_auxiliary_camera() now states the exact matching rule, the max_fps unit and what None means, and its errors. The controller's private helpers follow its public methods. --- .../isaaclab_ov/renderers/render_lease.py | 269 ++++++++++-------- 1 file changed, 144 insertions(+), 125 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 82efa14a96c..6a047edf6c1 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -96,7 +96,26 @@ def ovstage(self) -> OvStage: return cast("OvStage", self._stage) def configure_auxiliary_camera(self, camera_cfg: Any, *, max_fps: float | None = None) -> None: - """Authorize an exact camera before Stage materialization.""" + """Authorize an exact camera before Stage materialization. + + The leased renderer then accepts the one camera sensor whose specification matches + ``camera_cfg`` exactly: a single camera view with the same width, height and ordered + ``data_types``, at ``prim_path`` (``{ENV_REGEX_NS}`` matches any environment) or its + ``camera`` child prim. Every other camera on the leased renderer remains rejected. + + Args: + camera_cfg: Configuration of the camera sensor to authorize, such as a + :class:`~isaaclab.sensors.camera.CameraCfg`. Its ``prim_path`` must be unique among + the lease's cameras. + max_fps: Maximum native rate of the camera's frames [Hz], authored on its prim in the + exported Stage. ``None`` authors no rate, so the camera renders with every frame of + its render product. + + Raises: + RuntimeError: If the lease is released or already bound to a Stage, or ``prim_path`` is + already configured. + ValueError: If ``camera_cfg`` is ``None``, or ``max_fps`` is not finite and positive. + """ self._validate_active() if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: raise RuntimeError("Auxiliary camera configuration must precede Stage materialization") @@ -636,16 +655,6 @@ def product_paths(self, lease: OvLibrariesRenderLease) -> tuple[str, ...]: raise RuntimeError("Auxiliary camera did not materialize exactly one RenderProduct per selection") return paths - def _write_scene_components(self, ordinal: int, *, partition_lease: OvLibrariesRenderLease | None) -> None: - """Write selected scene components; the caller owns validation and commit.""" - self._renderer.scene.ordinal = ordinal - with self._renderer._publish_visual_materials(): - self._renderer._update_transforms() - self._renderer._update_geometries() - self._update_cameras() - if partition_lease is not None: - self._write_camera_partitions(partition_lease, ordinal) - def configure_camera_partitions( self, lease: OvLibrariesRenderLease, @@ -674,83 +683,6 @@ def configure_camera_partitions( lease._external_camera_scene_partitions_pending = True return token - def _write_camera_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> None: - """Write the external camera tokens inside the complete scene batch.""" - if self._renderer.scene.stage is None or self._renderer.scene.paths is None: - raise RuntimeError("Cannot write external camera partitions before Stage materialization") - camera_scene_partitions = lease._external_camera_scene_partitions - if not camera_scene_partitions: - raise RuntimeError("No external camera scene partitions are configured") - camera_paths = [path for path, _token in camera_scene_partitions] - token_ids = np.array( - [self._renderer.scene.paths.intern_token(token) for _path, token in camera_scene_partitions], - dtype=np.uint64, - ) - camera_paths_list = self._renderer.scene.paths.create_path_list_from_strings(camera_paths) - camera_query = None - try: - camera_query = self._renderer.scene.stage.query_from_path_list(camera_paths_list) - if camera_query is None: - raise RuntimeError("External camera scene-partition query is None") - self._renderer.scene.stage.write_attribute( - camera_query, - "omni:scenePartition", - ordinal=ordinal, - tensors=token_ids, - is_array=False, - semantic=ovstage.AttributeSemantic.TOKEN_ID, - ).wait() - finally: - if camera_query is not None: - self._renderer.scene.stage.release_query(camera_query).wait() - self._renderer.scene.paths.destroy_path_list(camera_paths_list) - - def _read_partition_tokens( - self, paths: list[str], attributes: tuple[str, ...], ordinal: int - ) -> dict[tuple[str, str], str]: - """Read resolved partition tokens from the exact renderer-owned Stage.""" - if self._renderer.scene.stage is None or self._renderer.scene.paths is None: - raise RuntimeError("Cannot read scene partitions before Stage materialization") - path_list = self._renderer.scene.paths.create_path_list_from_strings(paths) - attribute_tokens = [self._renderer.scene.paths.intern_token(attribute) for attribute in attributes] - resolved: dict[tuple[str, str], str] = {} - try: - with self._renderer.scene.stage.query_from_path_list(path_list) as query: - query.wait() - with self._renderer.scene.stage.read_attributes( - query, attribute_tokens, ovstage.OrdinalRange.latest(ordinal) - ) as read: - read.wait() - for group in read.groups(): - try: - if group.is_delete or group.is_array or group.tensor_count != 1: - continue - attribute = self._renderer.scene.paths.token_to_string(group.attribute) - if attribute not in attributes: - continue - group_paths = self._renderer.scene.paths.get_path_strings(group.prim_list) - try: - values = np.asarray(group.array(0).copy()).reshape(-1) - except NotImplementedError: - values = torch.from_dlpack(group.dlpack(0)).detach().cpu().numpy().copy().reshape(-1) - logical_count = min(group.prim_count, group.data_count) - for local_index in range(logical_count): - prim_index = group.prim_index(local_index) - data_row_index = group.data_row_index(local_index) - if prim_index >= len(group_paths) or data_row_index >= len(values): - raise RuntimeError("OVStage scene-partition read escaped its group-owned rows") - token_id = int(values[data_row_index]) - token = self._renderer.scene.paths.token_to_string(token_id) if token_id else "" - key = (group_paths[prim_index], attribute) - if key in resolved and resolved[key] != token: - raise RuntimeError(f"OVStage returned conflicting scene-partition rows for {key}") - resolved[key] = token - finally: - self._renderer.scene.stage.release_group(group) - finally: - self._renderer.scene.paths.destroy_path_list(path_list) - return resolved - def validate_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> dict[str, str]: """Prove the configured root and external cameras resolve to one valid token.""" self.validate_active(lease) @@ -782,43 +714,6 @@ def validate_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> di raise RuntimeError("External scene-partition proof failed: " + "; ".join(mismatches)) return partitions - def _author_camera_schedules(self, usd: str, lease: OvLibrariesRenderLease) -> str: - """Author selected native camera rates on the private export, preserving the application stage.""" - - layer = Sdf.Layer.CreateAnonymous("borrowed-renderer.usda") - if not layer.ImportFromString(usd): - raise RuntimeError("Cannot parse the exported leased Stage") - stage = Usd.Stage.Open(layer) - for camera in lease._auxiliary_cameras: - if camera.max_fps is None: - continue - if camera.spec is None: - raise RuntimeError("Native-rate auxiliary camera was not prepared") - prim = stage.GetPrimAtPath(camera.spec.camera_prim_paths[0]) - if not prim.IsValid(): - raise RuntimeError("Native-rate auxiliary camera is absent from the exported Stage") - prim.CreateAttribute("omni:sensor:frameRate", Sdf.ValueTypeNames.Double2).Set(Gf.Vec2d(camera.max_fps, 1.0)) - prim.CreateAttribute("omni:sensor:Core:partialOutputs", Sdf.ValueTypeNames.Bool).Set(False) - return stage.ExportToString() - - def _materialize_camera_products(self, lease: OvLibrariesRenderLease) -> None: - for camera in lease._auxiliary_cameras: - if camera.spec is None or camera.render_data is None: - raise RuntimeError("Selected auxiliary camera was not prepared") - self._renderer._register_camera(camera.spec, camera.render_data) - camera.product_path = camera.render_data.render_product_path - if lease._auxiliary_cameras: - self._renderer.scene.commit() - - def _update_cameras(self) -> None: - lease = self.active - if lease is None: - return - for camera in lease._auxiliary_cameras: - if camera.state is None or camera.render_data.camera_xform_binding is None: - raise RuntimeError("Auxiliary camera has no live world-pose sample") - self._renderer._update_camera(*camera.state) - def prepare_stage(self, num_envs: int) -> bool: """Validate or defer leased preparation; ordinary rendering is immediately allowed. @@ -911,3 +806,127 @@ def _next_ordinal(self) -> int: if floor >= (1 << 64) - 1: raise RuntimeError("OVStage ordinal range is exhausted") return floor + 1 + + def _write_scene_components(self, ordinal: int, *, partition_lease: OvLibrariesRenderLease | None) -> None: + """Write selected scene components; the caller owns validation and commit.""" + self._renderer.scene.ordinal = ordinal + with self._renderer._publish_visual_materials(): + self._renderer._update_transforms() + self._renderer._update_geometries() + self._update_cameras() + if partition_lease is not None: + self._write_camera_partitions(partition_lease, ordinal) + + def _write_camera_partitions(self, lease: OvLibrariesRenderLease, ordinal: int) -> None: + """Write the external camera tokens inside the complete scene batch.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot write external camera partitions before Stage materialization") + camera_scene_partitions = lease._external_camera_scene_partitions + if not camera_scene_partitions: + raise RuntimeError("No external camera scene partitions are configured") + camera_paths = [path for path, _token in camera_scene_partitions] + token_ids = np.array( + [self._renderer.scene.paths.intern_token(token) for _path, token in camera_scene_partitions], + dtype=np.uint64, + ) + camera_paths_list = self._renderer.scene.paths.create_path_list_from_strings(camera_paths) + camera_query = None + try: + camera_query = self._renderer.scene.stage.query_from_path_list(camera_paths_list) + if camera_query is None: + raise RuntimeError("External camera scene-partition query is None") + self._renderer.scene.stage.write_attribute( + camera_query, + "omni:scenePartition", + ordinal=ordinal, + tensors=token_ids, + is_array=False, + semantic=ovstage.AttributeSemantic.TOKEN_ID, + ).wait() + finally: + if camera_query is not None: + self._renderer.scene.stage.release_query(camera_query).wait() + self._renderer.scene.paths.destroy_path_list(camera_paths_list) + + def _read_partition_tokens( + self, paths: list[str], attributes: tuple[str, ...], ordinal: int + ) -> dict[tuple[str, str], str]: + """Read resolved partition tokens from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None or self._renderer.scene.paths is None: + raise RuntimeError("Cannot read scene partitions before Stage materialization") + path_list = self._renderer.scene.paths.create_path_list_from_strings(paths) + attribute_tokens = [self._renderer.scene.paths.intern_token(attribute) for attribute in attributes] + resolved: dict[tuple[str, str], str] = {} + try: + with self._renderer.scene.stage.query_from_path_list(path_list) as query: + query.wait() + with self._renderer.scene.stage.read_attributes( + query, attribute_tokens, ovstage.OrdinalRange.latest(ordinal) + ) as read: + read.wait() + for group in read.groups(): + try: + if group.is_delete or group.is_array or group.tensor_count != 1: + continue + attribute = self._renderer.scene.paths.token_to_string(group.attribute) + if attribute not in attributes: + continue + group_paths = self._renderer.scene.paths.get_path_strings(group.prim_list) + try: + values = np.asarray(group.array(0).copy()).reshape(-1) + except NotImplementedError: + values = torch.from_dlpack(group.dlpack(0)).detach().cpu().numpy().copy().reshape(-1) + logical_count = min(group.prim_count, group.data_count) + for local_index in range(logical_count): + prim_index = group.prim_index(local_index) + data_row_index = group.data_row_index(local_index) + if prim_index >= len(group_paths) or data_row_index >= len(values): + raise RuntimeError("OVStage scene-partition read escaped its group-owned rows") + token_id = int(values[data_row_index]) + token = self._renderer.scene.paths.token_to_string(token_id) if token_id else "" + key = (group_paths[prim_index], attribute) + if key in resolved and resolved[key] != token: + raise RuntimeError(f"OVStage returned conflicting scene-partition rows for {key}") + resolved[key] = token + finally: + self._renderer.scene.stage.release_group(group) + finally: + self._renderer.scene.paths.destroy_path_list(path_list) + return resolved + + def _author_camera_schedules(self, usd: str, lease: OvLibrariesRenderLease) -> str: + """Author selected native camera rates on the private export, preserving the application stage.""" + + layer = Sdf.Layer.CreateAnonymous("borrowed-renderer.usda") + if not layer.ImportFromString(usd): + raise RuntimeError("Cannot parse the exported leased Stage") + stage = Usd.Stage.Open(layer) + for camera in lease._auxiliary_cameras: + if camera.max_fps is None: + continue + if camera.spec is None: + raise RuntimeError("Native-rate auxiliary camera was not prepared") + prim = stage.GetPrimAtPath(camera.spec.camera_prim_paths[0]) + if not prim.IsValid(): + raise RuntimeError("Native-rate auxiliary camera is absent from the exported Stage") + prim.CreateAttribute("omni:sensor:frameRate", Sdf.ValueTypeNames.Double2).Set(Gf.Vec2d(camera.max_fps, 1.0)) + prim.CreateAttribute("omni:sensor:Core:partialOutputs", Sdf.ValueTypeNames.Bool).Set(False) + return stage.ExportToString() + + def _materialize_camera_products(self, lease: OvLibrariesRenderLease) -> None: + for camera in lease._auxiliary_cameras: + if camera.spec is None or camera.render_data is None: + raise RuntimeError("Selected auxiliary camera was not prepared") + self._renderer._register_camera(camera.spec, camera.render_data) + camera.product_path = camera.render_data.render_product_path + if lease._auxiliary_cameras: + self._renderer.scene.commit() + + def _update_cameras(self) -> None: + lease = self.active + if lease is None: + return + for camera in lease._auxiliary_cameras: + if camera.state is None or camera.render_data.camera_xform_binding is None: + raise RuntimeError("Auxiliary camera has no live world-pose sample") + self._renderer._update_camera(*camera.state) From 7b834a380fc1ab4096846e2e429a46ed3aaabab5 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 08:32:13 -0500 Subject: [PATCH 066/145] Say which cameras the XR head exclusions hide from Excluded meshes leave the env_0 partition that environment cameras select, so camera panels and recorded camera images omit them too, not only the OVXR eye cameras. --- .../isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst index c8617fe78ae..8dabe03c359 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -8,6 +8,7 @@ not head-locked in the OpenXR VIEW space. Replay accepts the live ``--ovxr_pip_fps``, ``--ovxr_pip_view_position`` and ``--ovxr_desktop_mirror_fps`` options. * Added :attr:`~isaaclab_teleop.XrCfg.excluded_visual_meshes`, which hides matching visual meshes - from the OVXR eye cameras only. + from the OVXR eye cameras and from environment cameras selecting the ``env_0`` partition, such + as camera panels and recorded camera images. * Added the ``keep`` argument to :func:`~isaaclab_teleop.remove_camera_configs` to retain selected cameras while removing their observation terms. From 9577aef839ff4fd66b20a7a4b0590bc49a6ecc47 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:20:32 -0500 Subject: [PATCH 067/145] Keep CPU OVRTX support out of the renderer lease The lease and the OVXR adapter do not need OVRTX on a CPU simulation device, and the leased renderer rejects every camera path, so the CPU branches only changed ordinary OVRTX rendering. Restore the upstream render-product device selection, render-var mapping, scene and camera write streams, their tests and the matching docs. Kitless CPU scene publication and CPU camera buffers return with the layers that first use them. --- .../changelog.d/renderer-lease.added.rst | 1 - .../isaaclab_ov/renderers/ovrtx_renderer.py | 47 ++---- .../isaaclab_ov/renderers/ovrtx_usd.py | 7 +- .../isaaclab_ov/test/test_ovrtx_clone_plan.py | 3 +- .../test/test_ovrtx_deformable_bindings.py | 6 +- .../test/test_ovrtx_renderer_contract.py | 137 +----------------- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 4 - 7 files changed, 25 insertions(+), 180 deletions(-) diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst index 2060408dcb7..d7bcd91f081 100644 --- a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst +++ b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst @@ -2,5 +2,4 @@ :meth:`~isaaclab_ov.renderers.OVRTXRenderer.acquire_ov_libraries_render_lease` for application-owned OVRTX/OVStage borrowing by external consumers, with private-stage selection, capability-checked render-thread ownership transfer, deferred Stage materialization, scene publication, and ordered cleanup. -* Supported CPU scene transforms and camera buffers alongside GPU rendering. * Published dirty visual materials with leased scene updates, including material-only changes. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 92bd5e56045..d2f134dd5fa 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -392,10 +392,6 @@ def visual_material_writer(self): A leased pair registers its writer before the borrower binds the Stage, because environment reset finalizes material consumers first; addresses compile on the first publication after population. - - The writer requires material buffers on one CUDA device. On a CPU simulation device, - an environment with :class:`~isaaclab.assets.VisualMaterial` assets therefore fails at - reset, with or without a lease. """ return ( self._create_visual_material_writer @@ -634,10 +630,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: render_data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) scope, product_path = render_data.render_scope_name, render_data.render_product_path usd = build_render_product_as_string( - spec, - render_data, - device_id=self._warp_device.ordinal if self._warp_device.is_cuda else None, - enable_shadows=self.cfg.enable_shadows, + spec, render_data, device_id=self._warp_device.ordinal, enable_shadows=self.cfg.enable_shadows ) try: # Registration mutates the shared scene; finish renders using its existing products first. @@ -804,7 +797,7 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Wait for rendering on the consuming stream, or on the host on Linux unless :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` is ``1``. Consume the zero-copy view inside the context. Unmapping records the consuming stream; native release waits for its queued - reads and the last view to be dropped. CPU consumers wait on the host and release without a CUDA stream. + reads and the last view to be dropped. Args: render_var: OVRTX ``RenderVarOutput`` to map (looked up from ``frame.render_vars``). @@ -812,14 +805,6 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Yields: The render var's contents as a Warp array, valid for the duration of the context. """ - if not self._warp_device.is_cuda: - mapping = render_var.map(device=Device.CPU) - try: - mapping.wait() - yield wp.from_dlpack(mapping) - finally: - mapping.unmap() - return gpu_side_sync = _gpu_side_render_var_sync_enabled() # OVRTX uses 0 for no synchronization and 1 for Torch's legacy default stream (CUDA handle 0). stream = self._warp_device.stream.cuda_stream or 1 @@ -1071,7 +1056,7 @@ def _update_transforms(self) -> None: timestamp = self._sdp.backend.transforms_timestamp if self._transforms_timestamp == timestamp: return - stream = self._warp_device.stream if self._warp_device.is_cuda else None + stream = self._warp_device.stream asynchronous = self.cfg.async_rendering and not self._use_ovstage output = self._transform_writes.acquire(stream) if not self._sdp.get_transforms(output, allow_passthrough=not asynchronous, scales=self._object_scales): @@ -1087,16 +1072,12 @@ def _update_transforms(self) -> None: tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + cuda_stream=stream.cuda_stream or 1, ).wait() elif asynchronous: self._transform_writes.submit(binding, matrices, stream) else: - binding.write( - matrices, - data_access=DataAccess.ASYNC, - cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, - ) + binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) self._transforms_timestamp = timestamp def update_geometries(self) -> None: @@ -1128,7 +1109,7 @@ def _update_geometries(self) -> None: binding = self._geometry_points_binding if binding is None: return - stream = self._warp_device.stream if self._warp_device.is_cuda else None + stream = self._warp_device.stream asynchronous = self.cfg.async_rendering and not self._use_ovstage if asynchronous: output = self._geometry_writes.acquire(stream) @@ -1149,16 +1130,12 @@ def _update_geometries(self) -> None: tensors=[points_tensor_from_warp(array) for array in points], is_array=True, semantic=ovstage.AttributeSemantic.POINT, - cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + cuda_stream=stream.cuda_stream or 1, ).wait() elif asynchronous: self._geometry_writes.submit(binding, points, stream) else: - binding.write( - points, - data_access=DataAccess.ASYNC, - cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, - ) + binding.write(points, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) self._geometry_timestamp = timestamp def update_camera( @@ -1173,7 +1150,7 @@ def update_camera( binding = render_data.camera_xform_binding if binding is None: return - stream = self._warp_device.stream if self._warp_device.is_cuda else None + stream = self._warp_device.stream matrices = render_data.camera_writes.acquire(stream) wp.launch( create_camera_transforms_kernel, len(matrices), [positions, orientations, matrices], device=self._device @@ -1186,7 +1163,7 @@ def update_camera( tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + cuda_stream=stream.cuda_stream or 1, ).wait() else: operation = render_data.camera_writes.submit(binding, matrices, stream) @@ -1194,14 +1171,14 @@ def update_camera( operation.wait() def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsics: wp.array, parameters: wp.array): - """Publish calibration columns from their producing device into the renderer-owned scene.""" + """Publish calibration columns from GPU memory into the renderer-owned scene.""" # A runtime calibration change is a scene write for this camera's bindings, so its # in-flight renders must finish first. Other cameras keep their pipelines. self._render_lease.reject_ordinary_operation("update ordinary camera intrinsics") errors = self.drain_pending_renders((render_data,)) if errors: raise ExceptionGroup("OVRTX renders failed before calibration update", errors) - stream = (wp.get_stream(parameters.device).cuda_stream or 1) if parameters.device.is_cuda else None + stream = wp.get_stream(parameters.device).cuda_stream or 1 if self._use_ovstage: self.scene.stage.write_attributes( render_data.camera_xform_binding, diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 30094ae61f2..85032bcbd42 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -144,7 +144,7 @@ def build_render_product_as_string( spec: CameraRenderSpec, render_data: OVRTXCameraRenderData, *, - device_id: int | None = None, + device_id: int, enable_shadows: bool = False, ) -> str: """Build a complete render product USD layer as a string. @@ -160,7 +160,6 @@ def build_render_product_as_string( render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. - When None, leave native GPU selection automatic for CPU consumers. enable_shadows: Whether lights cast shadows. Defaults to False. Only honored for the ``simple_shading_*`` data types, which are the ones that select RTX Minimal mode. @@ -211,8 +210,6 @@ def build_render_product_as_string( for _, name, source in render_var_configs ) - device_ids_line = "" if device_id is None else f"uint[] deviceIds = [{device_id}]" - layer = f'''#usda 1.0 (defaultPrim = "{render_data.render_scope_name}") @@ -222,7 +219,7 @@ def RenderProduct "RenderProduct" ( prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] ) {{ rel camera = [<{render_data.camera_paths[0]}>] - {device_ids_line} + uint[] deviceIds = [{device_id}] {bg_type_line} {render_mode_block} token[] omni:rtx:waitForEvents = ["AllLoadingFinished", "OnlyOnFirstRequest"] diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index 1d85cb0d570..e82e0768d4f 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -105,7 +105,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: write_array_attribute=lambda *args, **kwargs: None, write_attribute=lambda *args, **kwargs: None, ) - renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True) + renderer._warp_device = SimpleNamespace(ordinal=0) renderer.scene.next_camera_id = 0 return renderer @@ -285,7 +285,6 @@ def test_create_render_data_writes_combined_stage_on_the_simulation_device(tmp_p class Device: ordinal = 1 - is_cuda = True def __str__(self): return "cuda:1" diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index edc27f8e13b..1dad8fc7c55 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -46,7 +46,7 @@ def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer.scene = renderer.backend renderer.scene.query = OvstageBackend.query.__get__(renderer.scene) renderer._device = "cpu" - renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True, stream=SimpleNamespace(cuda_stream=99)) + renderer._warp_device = SimpleNamespace(ordinal=0, stream=SimpleNamespace(cuda_stream=99)) return renderer, renderer.backend.renderer @@ -57,7 +57,7 @@ def test_geometry_bindings_follow_mixed_sdp_points_and_pointer_swaps(mode): renderer.cfg.async_rendering = mode == "async" use_ovstage = mode == "ovstage" renderer._use_ovstage = use_ovstage - renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=42)) + renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=42)) points = { path: wp.array(np.arange(count * 3, dtype=np.float32).reshape(count, 3), dtype=wp.vec3f, device="cpu") for path, count in ( @@ -189,7 +189,7 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc stage = Usd.Stage.CreateInMemory() UsdGeom.Xform.Define(stage, paths[0]).AddScaleOp().Set(Gf.Vec3d(2, 3, 4) if scaled else Gf.Vec3d(1)) renderer.scene.clone_copies = [(paths[0], [])] - renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=0)) + renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=0)) renderer._use_ovstage = use_ovstage renderer.scene.ordinal = 5 writes = [] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index f06f565b791..ef5c2b6d2e8 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1042,7 +1042,7 @@ def test_ovrtx_render_var_sync_rejects_non_boolean_values(monkeypatch, value): @pytest.mark.parametrize( ("gpu_side", "cuda_stream", "expected_stream", "consumer_fails"), - [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False), (False, None, None, True)], + [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False)], ) def test_ovrtx_map_render_var_orders_the_read_against_render_completion( monkeypatch, gpu_side, cuda_stream, expected_stream, consumer_fails @@ -1050,17 +1050,15 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( """Order reads after rendering and release after reads, including when the consumer raises.""" events = [] mapping = types.SimpleNamespace( - wait=lambda: events.append("wait"), unmap=lambda *, stream=None: events.append(("release", stream)) + wait=lambda: events.append("wait"), unmap=lambda *, stream: events.append(("release", stream)) ) render_var = types.SimpleNamespace(map=MagicMock(return_value=mapping)) sentinel = object() monkeypatch.setattr(ovrtx_renderer_module, "_gpu_side_render_var_sync_enabled", lambda: gpu_side) - monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: events.append("wrap") or sentinel) + monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: sentinel) renderer = _make_ovrtx_renderer_without_backend() - renderer._warp_device = types.SimpleNamespace( - is_cuda=cuda_stream is not None, stream=types.SimpleNamespace(cuda_stream=cuda_stream) - ) + renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=cuda_stream)) with pytest.raises(ValueError, match="consumer failed") if consumer_fails else contextlib.nullcontext(): with renderer._map_render_var_to_dlpack(render_var) as array: assert array is sentinel @@ -1068,11 +1066,8 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( if consumer_fails: raise ValueError("consumer failed") - if cuda_stream is None: - assert render_var.map.call_args.kwargs == {"device": ovrtx_renderer_module.Device.CPU} - else: - assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) - assert events == ([] if gpu_side else ["wait"]) + ["wrap", "consume", ("release", expected_stream)] + assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) + assert events == ([] if gpu_side else ["wait"]) + ["consume", ("release", expected_stream)] @pytest.mark.parametrize( @@ -1194,7 +1189,7 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs assert not vars(renderer).keys() & {"_camera_xform_binding", "_camera_xform_query", "_camera_paths_list"} monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(wp, "synchronize_stream", lambda stream: None) - renderer._warp_device = types.SimpleNamespace(is_cuda=True, stream=types.SimpleNamespace(cuda_stream=99)) + renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(ovrtx_renderer_module, "xform_tensor_from_warp", lambda value: value) positions = wp.array([[1, 2, 3], [4, 5, 6]], dtype=wp.vec3f, device="cpu") rotations = np.array([[1, 2, 3, 4], [0, 0, 0, 1]], dtype=np.float32) @@ -1218,7 +1213,6 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs renderer.update_camera_intrinsics(cameras[1], wp.zeros(2, dtype=wp.mat33f, device="cpu"), parameters) if use_ovstage: assert renderer.scene.stage.write_attributes.call_args.args[0] is cameras[1].camera_xform_binding - assert all(desc.cuda_stream is None for desc in renderer.scene.stage.write_attributes.call_args.args[1]) bound_paths = [call.args[0] for call in renderer.scene.paths.create_path_list_from_strings.call_args_list] assert bound_paths == [ entry @@ -1728,120 +1722,3 @@ def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monke lease.release() for writer in context._visual_material_writers: writer.close() - - -@pytest.mark.parametrize("use_ovstage, asynchronous", [(True, False), (False, False), (False, True)]) -def test_cpu_scene_publication_uses_host_tensors_without_a_cuda_stream(use_ovstage, asynchronous): - renderer = _make_ovrtx_renderer_without_backend() - renderer._use_ovstage = use_ovstage - renderer.cfg = OVRTXRendererCfg(async_rendering=asynchronous) - renderer._warp_device = wp.get_device("cpu") - renderer._device = "cpu" - renderer._object_xform_binding = MagicMock() - renderer._geometry_points_binding = MagicMock() - renderer._transforms_timestamp = renderer._geometry_timestamp = None - renderer._object_scales = None - renderer._geometry_paths = ["/World/Object"] - matrices = wp.array(np.eye(4, dtype=np.float64)[None], dtype=wp.mat44d, device="cpu") - points = wp.array([[1.0, 2.0, 3.0]], dtype=wp.vec3f, device="cpu") - renderer._transform_writes = _AsyncWriteBuffers([types.SimpleNamespace(matrices=matrices)]) - renderer._geometry_writes = _AsyncWriteBuffers([wp.empty_like(points)]) - renderer._geometry_offsets = {"/World/Object": 0} - - def get_points(*, output=None, offsets=None): - if output is None: - return {"/World/Object": points} - wp.copy(output, points) - return None - - renderer._sdp = types.SimpleNamespace( - backend=types.SimpleNamespace(transforms_timestamp=1, geometry_timestamp=1), - get_transforms=lambda *_args, **_kwargs: True, - get_geometry_points=get_points, - ) - stage = MagicMock() - renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) - renderer.update_transforms() - renderer.update_geometries() - bindings = (renderer._object_xform_binding, renderer._geometry_points_binding) - writes = [getattr(binding, "write_async" if asynchronous else "write") for binding in bindings] - calls = stage.write_attribute.call_args_list if use_ovstage else [write.call_args for write in writes] - assert len(calls) == 2 and all(c.kwargs["cuda_stream"] is None for c in calls) - if use_ovstage: - assert [c.args[1] for c in calls] == ["omni:xform", "points"] - assert all(c.kwargs["ordinal"] == 11 for c in calls) - else: - np.testing.assert_array_equal(calls[0].args[0].numpy(), matrices.numpy()) - np.testing.assert_array_equal(calls[1].args[0][0].numpy(), points.numpy()) - # A repeated publication of unchanged state must not duplicate either write. - renderer.update_transforms() - renderer.update_geometries() - assert stage.write_attribute.call_count == 2 if use_ovstage else all(write.call_count == 1 for write in writes) - camera = _make_ovrtx_camera_render_data() - camera.camera_xform_binding = MagicMock() - camera.camera_writes = _AsyncWriteBuffers([wp.empty(2, dtype=wp.mat44d, device="cpu")]) - positions = wp.array([[1.0, 2.0, 3.0]] * 2, dtype=wp.vec3f, device="cpu") - orientations = wp.array([[0.0, 0.0, 0.0, 1.0]] * 2, dtype=wp.quatf, device="cpu") - update = renderer.update_camera - update(camera, positions, orientations, None) - call = stage.write_attribute.call_args if use_ovstage else camera.camera_xform_binding.write_async.call_args - assert call.kwargs["cuda_stream"] is None - if use_ovstage: - assert call.args == (camera.camera_xform_binding, "omni:xform") - assert call.kwargs["ordinal"] == 11 - else: - # Reusing the camera ring waits for previous writes without a CUDA stream. - update(camera, positions, orientations, None) - update(camera, positions, orientations, None) - camera.camera_writes.close() - renderer._transform_writes.close() - renderer._geometry_writes.close() - - -@pytest.mark.parametrize("cuda_ordinal", [None, 2]) -def test_camera_registration_authors_valid_render_device_selection(monkeypatch, cuda_ordinal): - """CPU camera consumers leave GPU selection automatic; CUDA consumers retain their device.""" - from pxr import Usd - - class AuthoredProduct(Exception): - pass - - renderer = _make_ovrtx_renderer_without_backend() - renderer.cfg = OVRTXRendererCfg() - - class Device: - is_cuda = cuda_ordinal is not None - ordinal = cuda_ordinal - - def __str__(self): - return f"cuda:{cuda_ordinal}" if self.is_cuda else "cpu" - - renderer._warp_device = Device() - renderer._device = str(renderer._warp_device) - monkeypatch.setattr(ovrtx_renderer_module.wp, "get_device", lambda _device: renderer._warp_device) - renderer.scene.clone_env_paths = [] - renderer.drain_pending_renders = lambda: [] - spec = types.SimpleNamespace( - cfg=_make_camera_cfg(["rgb"]), - device=renderer._device, - num_instances=1, - camera_prim_paths=["/World/envs/env_0/Camera"], - render_settings={}, - ) - build_product = ovrtx_renderer_module.build_render_product_as_string - - def consume_product(spec, render_data, **kwargs): - stage = Usd.Stage.CreateInMemory() - assert stage.GetRootLayer().ImportFromString(build_product(spec, render_data, **kwargs)) - product = stage.GetPrimAtPath(render_data.render_product_path) - devices = product.GetAttribute("deviceIds") - if cuda_ordinal is None: - assert not devices.HasAuthoredValueOpinion() - else: - assert list(devices.Get()) == [cuda_ordinal] - # Native population is outside this parser/selection regression. - raise AuthoredProduct - - monkeypatch.setattr(ovrtx_renderer_module, "build_render_product_as_string", consume_product) - with pytest.raises(AuthoredProduct): - renderer.create_render_data(spec) diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index b4672a71a2e..7e8585132a2 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -65,10 +65,6 @@ The adapter retains the environment if lease retirement cannot establish complet default `OVRTXRendererCfg()`) is rejected while the lease is active. - Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its scene publisher writes the same physics state at the reserved ordinal. -- CPU simulation devices publish scene transforms, but visual-material - publication requires CUDA material buffers, as it does for ordinary OVRTX - cameras. With `--device cpu`, an environment that has `VisualMaterial` assets - fails at `sim.reset()`. - Dynamic anchoring takes any object implementing `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects other objects before the worker claims the lease. From 2982179b2c54958a2fd338aab0bdf372151c76c7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:21:26 -0500 Subject: [PATCH 068/145] Keep the async write buffers in the OVRTX renderer module The lease does not need a separate publication-buffer module, so return _AsyncWriteBuffers to its upstream place and text at the bottom of ovrtx_renderer.py with its imports, and drop the module and its doc references. --- .../isaaclab_ov/renderers/ovrtx_renderer.py | 45 ++++++++++++-- .../renderers/publication_buffers.py | 62 ------------------- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 3 +- 3 files changed, 42 insertions(+), 68 deletions(-) delete mode 100644 source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index d2f134dd5fa..f6609f265f9 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -16,8 +16,6 @@ - **render_lease.py**: External borrowing, ownership transfer and scene publication ordering. -- **publication_buffers.py**: Producer-buffer retention and native write completion. - - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. - **ovrtx_usd.py**: USD helpers for OVRTX: render var config, camera injection, etc. @@ -34,9 +32,10 @@ import tempfile import weakref from builtins import ExceptionGroup +from collections import deque from collections.abc import Callable, Iterable, Iterator, Sequence from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Generic, TypeVar logger = logging.getLogger(__name__) @@ -110,7 +109,6 @@ xform_tensor_from_warp, ) -from .publication_buffers import _AsyncWriteBuffers from .render_lease import OvLibrariesRenderLease, RenderLeaseController if TYPE_CHECKING: @@ -1649,3 +1647,42 @@ def close(self) -> None: raise ExceptionGroup("OVRTX renders failed before marker cleanup", errors) self._transforms = self._selection = self._stage = None self._local_transforms = None + + +_BufferT = TypeVar("_BufferT", wp.array, SceneDataFormat.TransposedMatrix44d) + + +class _AsyncWriteBuffers(Generic[_BufferT]): + """Retain GPU buffers until native writes finish; reuse them in submission order.""" + + def __init__(self, buffers: Iterable[_BufferT] = ()): + self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( + (buffer, None, None) for buffer in buffers + ) + + def acquire(self, stream: wp.Stream) -> _BufferT: + """Wait until the next buffer is writable on the given stream.""" + buffer, operation, producer = self._writes[0] + if operation is not None: + operation.wait() + if producer != stream: + stream.wait_stream(producer) + return buffer + + def submit( + self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream + ) -> Operation[bool]: + """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" + operation = binding.write_async(values, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + self._writes[0] = (self._writes[0][0], operation, stream) + self._writes.rotate(-1) + return operation + + def close(self) -> None: + """Complete every pending write before releasing storage, including on failure.""" + with contextlib.ExitStack() as writes: + writes.callback(self._writes.clear) + for _, operation, producer in self._writes: + if operation is not None: + writes.callback(wp.synchronize_stream, producer) + writes.callback(operation.wait) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py b/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py deleted file mode 100644 index 950990d3bd2..00000000000 --- a/source/isaaclab_ov/isaaclab_ov/renderers/publication_buffers.py +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). -# All rights reserved. -# -# SPDX-License-Identifier: BSD-3-Clause - - -"""Producer storage retained until native scene publication completes.""" - -from __future__ import annotations - -import contextlib -from collections import deque -from collections.abc import Iterable -from typing import TYPE_CHECKING, Generic, TypeVar - -import warp as wp -from ovrtx import DataAccess - -from isaaclab.scene_data import SceneDataFormat - -if TYPE_CHECKING: - from ovrtx import AttributeBinding, Operation -_BufferT = TypeVar("_BufferT", wp.array, SceneDataFormat.TransposedMatrix44d) - - -class _AsyncWriteBuffers(Generic[_BufferT]): - """Retain buffers until native writes finish; reuse them in submission order.""" - - def __init__(self, buffers: Iterable[_BufferT] = ()): - self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( - (buffer, None, None) for buffer in buffers - ) - - def acquire(self, stream: wp.Stream | None) -> _BufferT: - """Wait until the next buffer is writable on the given stream.""" - buffer, operation, producer = self._writes[0] - if operation is not None: - operation.wait() - if stream is not None and producer is not None and producer != stream: - stream.wait_stream(producer) - return buffer - - def submit( - self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream | None - ) -> Operation[bool]: - """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" - operation = binding.write_async( - values, data_access=DataAccess.ASYNC, cuda_stream=(stream.cuda_stream or 1) if stream is not None else None - ) - self._writes[0] = (self._writes[0][0], operation, stream) - self._writes.rotate(-1) - return operation - - def close(self) -> None: - """Complete every pending write before releasing storage, including on failure.""" - with contextlib.ExitStack() as writes: - writes.callback(self._writes.clear) - for _, operation, producer in self._writes: - if operation is not None: - if producer is not None: - writes.callback(wp.synchronize_stream, producer) - writes.callback(operation.wait) diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 7e8585132a2..4047a29bc6a 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -74,8 +74,7 @@ The adapter retains the environment if lease retirement cannot establish complet The ordinary renderer remains in `isaaclab_ov.renderers.ovrtx_renderer`. `render_lease.RenderLeaseController` owns the lease state, thread-transfer capability and per-owner operation guards; it borrows the renderer's native resources -and delegates component writes back to that renderer. `publication_buffers` -retains producer storage until native writes complete. Public renderer and lease +and delegates component writes back to that renderer. Public renderer and lease imports remain available from `isaaclab_ov.renderers`. Acquiring a lease selects a private rendering OVStage without changing process From 28883575242f6621b36955f16463b0a6bd07695f Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:21:43 -0500 Subject: [PATCH 069/145] Keep the upstream material drain comment and names Extracting the material publish/drain block for leased publication is required, but the move dropped the upstream comment and renamed its variables. Restore both; only the log text changes, because the helper now also serves leased scene updates. --- .../isaaclab_ov/renderers/ovrtx_renderer.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index f6609f265f9..3a7c901ab09 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -406,20 +406,22 @@ def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ... @contextlib.contextmanager def _publish_visual_materials(self) -> Iterator[None]: """Publish dirty materials and complete their reads before buffers may change.""" - writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None + material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None try: - if writer is not None: - writer.publish() + if material_writer is not None: + material_writer.publish() yield finally: - if writer is not None: + if material_writer is not None: + # When another exception is already propagating, log the drain failure instead of + # replacing it. Losing it silently would hide a failed material write. primary_active = sys.exc_info()[0] is not None try: - writer.drain() - except Exception as exc: + material_writer.drain() + except Exception as e: if not primary_active: raise - logger.warning("Error draining material writes after a failed scene update: %s", exc, exc_info=True) + logger.warning("Error draining material writes after a failed scene update: %s", e, exc_info=True) def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" From 452a41a1432ddf621a6e89bc0365031d7f3a0bb8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:21:50 -0500 Subject: [PATCH 070/145] Say when a leased material writer compiles its addresses A leased pair registers its material writer before the Stage is populated, so "after scene population" no longer describes the writer factory. Say that it retains the buffers and that deferred scenes compile addresses after population. --- source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 3a7c901ab09..4093c4d430d 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -398,7 +398,7 @@ def visual_material_writer(self): ) def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> OVRTXVisualMaterialWriter: - """Create the detached-scene material writer after scene population.""" + """Retain material buffers; deferred scenes compile addresses after population.""" writer = OVRTXVisualMaterialWriter(self, batches) self._visual_material_writer_ref = weakref.ref(writer) return writer From c3434a3663e0301b511e8a541f6f9c4886cde531 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:22:25 -0500 Subject: [PATCH 071/145] Drop hidden visual prims from the OVXR adapter No task hides prims through the adapter, and later layers hide the robot head from the XR eyes with a render-export partition instead. Remove the persistent USD-visibility helper, its start parameter and its test. --- .../isaaclab_teleop/ov_libraries_xr.py | 55 +------------------ .../test/test_ov_libraries_xr.py | 49 +---------------- 2 files changed, 4 insertions(+), 100 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index fad2d42a34b..7625baaad70 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -9,7 +9,6 @@ import contextlib import logging -import re import threading import time from collections.abc import Callable, Hashable, Iterable @@ -44,51 +43,6 @@ _DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" -def _apply_hidden_visual_prims(env: Any, patterns: Iterable[str]) -> tuple[str, ...]: - """Author persistent application-stage visibility before materialization. - - This affects every view of the application scene, not just the XR eyes. - Use scene partitions when source visibility must remain unchanged. - """ - patterns = tuple(patterns) - if not patterns: - return () - - from pxr import UsdGeom - - unwrapped = getattr(env, "unwrapped", env) - stage = unwrapped.scene.stage - if stage is None: - raise RuntimeError("Configured hidden visual prims require an environment Stage") - - compiled = [(pattern, re.compile(pattern)) for pattern in patterns] - matches: dict[str, list[Any]] = {pattern: [] for pattern in patterns} - for prim in stage.Traverse(): - if not prim.IsA(UsdGeom.Imageable): - continue - path = str(prim.GetPath()) - for pattern, expression in compiled: - if expression.fullmatch(path): - matches[pattern].append(prim) - - missing = [pattern for pattern, prims in matches.items() if not prims] - if missing: - raise RuntimeError(f"Hidden visual prim patterns matched no imageable prims: {missing}") - - hidden_paths: list[str] = [] - hidden_path_set: set[str] = set() - for pattern in patterns: - for prim in matches[pattern]: - path = str(prim.GetPath()) - if path in hidden_path_set: - continue - UsdGeom.Imageable(prim).MakeInvisible() - hidden_paths.append(path) - hidden_path_set.add(path) - logger.info("Hid %d prim(s) from all camera views (%s)", len(hidden_paths), list(patterns)) - return tuple(hidden_paths) - - def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrConfig) -> None: """Declare type-only OVXR prims for the application Stage's lifetime.""" if not cfg.create_render_products: @@ -224,13 +178,11 @@ def __init__( bootstrap: OvxrSessionBootstrap, env: Any, cfg: OvLibrariesXrConfig, - hidden_visual_prims: Iterable[str], external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, ) -> None: self._bootstrap = bootstrap self._env = env self._cfg = cfg - self._hidden_visual_prims = tuple(hidden_visual_prims) self._publication_lock = threading.Lock() self._scene_publisher = None self._lease = None @@ -253,7 +205,6 @@ def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding self._bootstrap.claim_render_owner_transfer(capability) - _apply_hidden_visual_prims(self._env, self._hidden_visual_prims) _declare_ovxr_runtime_prims(self._env, self._cfg) self._lease = self._bootstrap.bind_environment(self._env) with self._lease.ovstage_owner_lane(): @@ -549,7 +500,6 @@ def start_render_worker( *, tracking_anchor_path: str | None = None, required_openxr_extensions: Iterable[str] = (), - hidden_visual_prims: Iterable[str] = (), max_blank_frames: int = 1, timeout_s: float = 30.0, external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, @@ -560,7 +510,6 @@ def start_render_worker( env: Constructed environment sharing the prepared simulation context. tracking_anchor_path: Override the configured USD tracking-anchor path. required_openxr_extensions: Extensions required by the input pipeline. - hidden_visual_prims: Patterns hidden persistently from every scene view. max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. timeout_s: Maximum worker startup wait [s]. external_anchor_state: Dynamic anchor policy seeded from committed scene publications. @@ -588,9 +537,7 @@ def start_render_worker( from .session_lifecycle import cloudxr_eula_accepted anchor_path = self._cfg.tracking_anchor_path if tracking_anchor_path is None else tracking_anchor_path - host = _IsaacLabOvxrApplicationInterface( - self._bootstrap, env, self._cfg, hidden_visual_prims, external_anchor_state - ) + host = _IsaacLabOvxrApplicationInterface(self._bootstrap, env, self._cfg, external_anchor_state) self._host = host self._owner = OvxrSession( host, diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 8f6e4cdb396..878c1894627 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -296,56 +296,13 @@ def GetTypeName(self): xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig()) -def test_hidden_visual_prims_fullmatch_imageables_and_fail_closed(xr_module, monkeypatch): - class FakePrim: - def __init__(self, path): - self.path = path - self.visible = True - - def GetPath(self): - return self.path - - def IsA(self, _schema): - return True - - class FakeImageable: - def __init__(self, prim): - self.prim = prim - - def MakeInvisible(self): - self.prim.visible = False - - pxr = ModuleType("pxr") - pxr.UsdGeom = SimpleNamespace(Imageable=FakeImageable) - monkeypatch.setitem(sys.modules, "pxr", pxr) - head = FakePrim("/World/envs/env_0/Robot/head_yaw_link") - body = FakePrim("/World/envs/env_0/Robot/base_link") - stage = SimpleNamespace(Traverse=lambda: iter((head, body))) - env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) - - assert xr_module._apply_hidden_visual_prims(env, ("/World/envs/env_.*/Robot/head_.*",)) == (head.path,) - assert not head.visible and body.visible - with pytest.raises(RuntimeError, match="matched no imageable prims"): - xr_module._apply_hidden_visual_prims(env, ("/World/missing/.*",)) - - def _make_host(xr_module, monkeypatch): bootstrap = _FakeBootstrap() env = SimpleNamespace(scene=SimpleNamespace(stage=None), close=lambda: bootstrap.events.append("environment-close")) monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) - monkeypatch.setattr( - xr_module, - "_apply_hidden_visual_prims", - lambda _env, _patterns: bootstrap.events.append("hide"), - ) monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda _env, _cfg: bootstrap.events.append("declare")) - host = xr_module._IsaacLabOvxrApplicationInterface( - bootstrap, - env, - xr_module.OvLibrariesXrConfig(), - ("/hidden",), - ) + host = xr_module._IsaacLabOvxrApplicationInterface(bootstrap, env, xr_module.OvLibrariesXrConfig()) return host, bootstrap, env @@ -360,7 +317,7 @@ def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monk assert binding.renderer is bootstrap.lease.ovrtx_renderer assert binding.stage is bootstrap.lease.ovstage assert binding.ordinal_ledger.stage is bootstrap.lease.ovstage - assert bootstrap.events == ["capture", "offer", ("claim", "capability"), "hide", "declare", ("bind", env)] + assert bootstrap.events == ["capture", "offer", ("claim", "capability"), "declare", ("bind", env)] assert ordinal == 11 assert bootstrap.publisher.calls[0][:2] == (11, "step-1") host.validate_first_published_scene(ordinal) @@ -823,7 +780,7 @@ def _make_anchor_host(xr_module, monkeypatch, anchor): bootstrap = _FakeBootstrap() bootstrap.get_world_transform = lambda _path: np.eye(4) host = xr_module._IsaacLabOvxrApplicationInterface( - bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrConfig(), (), external_anchor_state=anchor + bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrConfig(), external_anchor_state=anchor ) host._scene_publisher = bootstrap.publisher host._ordinal_ledger = _FakeOrdinalLedger(None) From dc4b12f8dff2f83f4bbb0d6672080be8f4fbeaa3 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:22:31 -0500 Subject: [PATCH 072/145] Drop a later layer's attribute from the OVXR adapter test fake The render lease in this layer never reads _geometry_publication, which a later layer adds to the renderer. --- source/isaaclab_teleop/test/test_ov_libraries_xr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 878c1894627..0772f2773ba 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -367,7 +367,6 @@ def test_host_reconciles_failed_publication_without_reusing_generation(xr_module _initialized_scene=False, backend=None, scene=None, - _geometry_publication=None, _update_transforms=lambda: None, _update_geometries=lambda: None, _visual_material_writer_ref=None, From b5a8dc87682c8d74b9d58392e4e542ebb0c87286 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:22:53 -0500 Subject: [PATCH 073/145] Rename OvLibrariesXrConfig to OvLibrariesXrCfg The configuration lives in ovxr_cfg beside XrCfg, IsaacTeleopCfg and the other *Cfg classes, so use the package's existing naming. The class has not been released, so no deprecation alias is needed. --- .../source/api/lab_teleop/isaaclab_teleop.rst | 4 +-- .../changelog.d/kitless-ovxr.added.rst | 2 +- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 2 +- .../isaaclab_teleop/__init__.pyi | 4 +-- .../isaaclab_teleop/ov_libraries_xr.py | 12 +++---- .../isaaclab_teleop/ovxr_cfg.py | 2 +- .../test/test_ov_libraries_xr.py | 36 +++++++++---------- 7 files changed, 31 insertions(+), 31 deletions(-) diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 149ef166652..3400a2dc749 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -22,7 +22,7 @@ isaaclab_teleop XrCfg XrAnchorRotationMode XrAnchorSynchronizer - OvLibrariesXrConfig + OvLibrariesXrCfg OvLibrariesXrSession .. rubric:: Functions @@ -66,7 +66,7 @@ XR Camera Feedback Kitless OVXR Session -------------------- -.. autoclass:: OvLibrariesXrConfig +.. autoclass:: OvLibrariesXrCfg :members: .. autoclass:: OvLibrariesXrSession diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst index f6bd105ca1c..0565d24bd9f 100644 --- a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst +++ b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst @@ -2,5 +2,5 @@ bootstrap that borrow the simulation-owned OVRTX renderer and OVStage, publish scene updates, and coordinate render-thread ownership and shutdown with Isaac Capture. Publication failures close admission, and failed lease retirement retains borrowed resources. -* Added :class:`~isaaclab_teleop.OvLibrariesXrConfig` in :mod:`isaaclab_teleop.ovxr_cfg`, which +* Added :class:`~isaaclab_teleop.OvLibrariesXrCfg` in :mod:`isaaclab_teleop.ovxr_cfg`, which imports without the native OVXR packages. diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 4047a29bc6a..b1863f0103b 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -35,7 +35,7 @@ Ownership transfer is single-use. Failed retirement must preserve resources for the owning thread to finish cleanup; a failed drain is not successful shutdown. Importing `OvLibrariesXrSession` requires a Capture release with OVXR session support; see [Installing kitless XR](kitless_xr_source.md). The -`OvLibrariesXrConfig` dataclass lives in `isaaclab_teleop.ovxr_cfg` and imports +`OvLibrariesXrCfg` dataclass lives in `isaaclab_teleop.ovxr_cfg` and imports without those packages. A failed scene write or seal stops further publication. Recreate the session diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index 1f0da9f8be1..79656e7f087 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,7 +15,7 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", - "OvLibrariesXrConfig", + "OvLibrariesXrCfg", "OvLibrariesXrSession", "SupportsControlEvents", "SystemCheckItem", @@ -69,7 +69,7 @@ from .isaac_teleop_cfg import ( ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device from .ov_libraries_xr import OvLibrariesXrSession -from .ovxr_cfg import OvLibrariesXrConfig +from .ovxr_cfg import OvLibrariesXrCfg from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index 7625baaad70..a6d4515ee5f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -18,7 +18,7 @@ from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger from isaaclab_ov.renderers import OVRTXRendererCfg -from .ovxr_cfg import OvLibrariesXrConfig +from .ovxr_cfg import OvLibrariesXrCfg from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation if TYPE_CHECKING: @@ -28,7 +28,7 @@ from isaaclab.sim import SimulationCfg -__all__ = ["OvLibrariesXrConfig", "OvLibrariesXrSession"] +__all__ = ["OvLibrariesXrCfg", "OvLibrariesXrSession"] logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ _DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" -def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrConfig) -> None: +def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrCfg) -> None: """Declare type-only OVXR prims for the application Stage's lifetime.""" if not cfg.create_render_products: return @@ -177,7 +177,7 @@ def __init__( self, bootstrap: OvxrSessionBootstrap, env: Any, - cfg: OvLibrariesXrConfig, + cfg: OvLibrariesXrCfg, external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, ) -> None: self._bootstrap = bootstrap @@ -457,7 +457,7 @@ def commit_external_pose( """ ... - def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrConfig) -> None: + def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrCfg) -> None: self._bootstrap = bootstrap self._cfg = cfg self._owner = None @@ -468,7 +468,7 @@ def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrConfig) -> def prepare( cls, sim_cfg: SimulationCfg, - cfg: OvLibrariesXrConfig, + cfg: OvLibrariesXrCfg, renderer_cfg: OVRTXRendererCfg | None = None, ) -> OvLibrariesXrSession: """Snapshot session selection, create simulation and reserve its OVRTX lease. diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py index 676f399b7e8..3c4fee43a3f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -22,7 +22,7 @@ @dataclass -class OvLibrariesXrConfig: +class OvLibrariesXrCfg: """Isaac Lab configuration for an IsaacTeleop-owned OVXR session.""" application_name: str = "isaaclab-kitless-teleop" diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 0772f2773ba..0a74dc8c2c2 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -18,7 +18,7 @@ from types import ModuleType, SimpleNamespace import pytest -from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig +from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg class _FakeLease: @@ -267,7 +267,7 @@ def DefinePrim(self, path, type_name): stage = FakeStage() env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) - xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig()) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) assert stage.definitions == [ ("/OVXRGeneratedCameras/OVXR_LeftEyeCamera", "Camera"), @@ -291,9 +291,9 @@ def GetTypeName(self): stage = SimpleNamespace(GetPrimAtPath=lambda _path: ExistingPrim(), DefinePrim=lambda *_args: None) env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) - xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig(create_render_products=False)) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg(create_render_products=False)) with pytest.raises(RuntimeError, match="incompatible type"): - xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrConfig()) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) def _make_host(xr_module, monkeypatch): @@ -302,7 +302,7 @@ def _make_host(xr_module, monkeypatch): monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda _env, _cfg: bootstrap.events.append("declare")) - host = xr_module._IsaacLabOvxrApplicationInterface(bootstrap, env, xr_module.OvLibrariesXrConfig()) + host = xr_module._IsaacLabOvxrApplicationInterface(bootstrap, env, xr_module.OvLibrariesXrCfg()) return host, bootstrap, env @@ -525,9 +525,9 @@ def close(self): external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") monkeypatch.setattr(external, "OvxrSession", FakeOwner) bootstrap = _FakeBootstrap() - assert xr_module.OvLibrariesXrConfig().extra_openxr_extensions == () + assert xr_module.OvLibrariesXrCfg().extra_openxr_extensions == () session = xr_module.OvLibrariesXrSession( - bootstrap, xr_module.OvLibrariesXrConfig(extra_openxr_extensions=("XR_EXT_debug_utils",)) + bootstrap, xr_module.OvLibrariesXrCfg(extra_openxr_extensions=("XR_EXT_debug_utils",)) ) env = SimpleNamespace() assert session.start_render_worker( @@ -600,7 +600,7 @@ def test_failed_start_releases_lease_before_environment_and_context(xr_module, b module, context, events = bootstrap_context bootstrap = module.OvxrSessionBootstrap(object()) if route == "before_owner": - session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) session.close() else: # Capture calls host.abort when validation fails before its worker exists. @@ -646,22 +646,22 @@ def create_bootstrap(*_args): return bootstrap monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", create_bootstrap) - xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrConfig(renderer_settings=settings)) + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(renderer_settings=settings)) def test_config_rejects_unknown_session_mode(): with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): - OvLibrariesXrConfig(session_mode="automatic") + OvLibrariesXrCfg(session_mode="automatic") @pytest.mark.parametrize("scale", [0.0, -1.0, float("nan"), float("inf")]) def test_config_rejects_invalid_scene_scale(scale): with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): - OvLibrariesXrConfig(meters_per_unit=scale) + OvLibrariesXrCfg(meters_per_unit=scale) def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): - cfg = xr_module.OvLibrariesXrConfig() + cfg = xr_module.OvLibrariesXrCfg() cfg.meters_per_unit = float("nan") monkeypatch.setitem( sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) @@ -676,7 +676,7 @@ def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) bootstrap = _FakeBootstrap() - session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) with pytest.raises(ValueError, match="max_blank_frames must be a non-negative integer"): session.start_render_worker(SimpleNamespace(), max_blank_frames=count) assert bootstrap.events == [] @@ -685,7 +685,7 @@ def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, def test_public_xr_exports_resolve_to_implemented_types(xr_module): import isaaclab_teleop - assert isaaclab_teleop.OvLibrariesXrConfig is xr_module.OvLibrariesXrConfig + assert isaaclab_teleop.OvLibrariesXrCfg is xr_module.OvLibrariesXrCfg assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession @@ -779,7 +779,7 @@ def _make_anchor_host(xr_module, monkeypatch, anchor): bootstrap = _FakeBootstrap() bootstrap.get_world_transform = lambda _path: np.eye(4) host = xr_module._IsaacLabOvxrApplicationInterface( - bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrConfig(), external_anchor_state=anchor + bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrCfg(), external_anchor_state=anchor ) host._scene_publisher = bootstrap.publisher host._ordinal_ledger = _FakeOrdinalLedger(None) @@ -835,7 +835,7 @@ def test_start_rejects_anchor_state_without_the_protocol_before_owner_creation(x external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) bootstrap = _FakeBootstrap() - session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrConfig()) + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) with pytest.raises(TypeError, match="ExternalAnchorState"): session.start_render_worker(SimpleNamespace(), external_anchor_state=SimpleNamespace(reset=lambda: None)) assert bootstrap.events == [] @@ -848,8 +848,8 @@ def test_config_imports_without_native_ovxr(): "import sys\n" "for name in ('isaaccapture', 'ovxr', 'ovrtx', 'ovstage'):\n" " sys.modules[name] = None\n" - "from isaaclab_teleop.ovxr_cfg import OvLibrariesXrConfig\n" - "assert OvLibrariesXrConfig(session_mode='vr').session_mode == 'vr'\n" + "from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg\n" + "assert OvLibrariesXrCfg(session_mode='vr').session_mode == 'vr'\n" ) result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=60) assert result.returncode == 0, result.stderr From e845280810d60dadc70e811976ca66c752907bbf Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:22:58 -0500 Subject: [PATCH 074/145] Drop the repeated word from two render-lease test names --- source/isaaclab_ov/test/test_ovrtx_renderer_contract.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index ef5c2b6d2e8..af3f9ede545 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1448,7 +1448,7 @@ def _prepare_leased_renderer(renderer, *, bind=True): renderer.prepare_stage(renderer._sim.stage, 1) -def test_render_lease_lease_borrows_registry_pair_and_blocks_shared_users(leased_renderer, monkeypatch): +def test_render_lease_borrows_registry_pair_and_blocks_shared_users(leased_renderer, monkeypatch): renderer = leased_renderer shared = SimulationContext.instance().get_or_create_backend(renderer.cfg) assert shared is renderer @@ -1488,7 +1488,7 @@ def test_render_lease_rejects_acquisition_after_clone_preparation(leased_rendere assert renderer.backend._render_lease_owner is None -def test_render_lease_lease_rejects_calibration_before_scene_access(leased_renderer, monkeypatch): +def test_render_lease_rejects_calibration_before_scene_access(leased_renderer, monkeypatch): """An ordinary calibration write cannot bypass the external owner's publication lane.""" renderer = leased_renderer lease = renderer.acquire_ov_libraries_render_lease() From 04f271661ba451e1e349aa88962a759eda04e1db Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:23:12 -0500 Subject: [PATCH 075/145] Read the environment's simulation configuration directly Manager-based environments always have cfg.sim, so a getattr default only hid a missing configuration and silently skipped the device and time-step check. --- source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index c93e89c7a30..c006ebd9659 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -117,9 +117,9 @@ def validate_environment(self, env: Any) -> None: raise RuntimeError( f"The OVXR render lease requires exactly one environment; got num_envs={unwrapped.num_envs!r}" ) - env_sim_cfg = getattr(getattr(unwrapped, "cfg", None), "sim", None) + env_sim_cfg = unwrapped.cfg.sim prepared_cfg = self._sim.cfg - if env_sim_cfg is not None and env_sim_cfg is not prepared_cfg: + if env_sim_cfg is not prepared_cfg: env_device = env_sim_cfg.device if env_device == "cuda": # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. From b2d4e47683b97922dc599eeb8ecd0aa7fcb3a591 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:23:45 -0500 Subject: [PATCH 076/145] Import NumPy, Warp and SceneDataFormat at module level They are hard dependencies, not optional backends, so the OVXR session and adapter need no local imports for them. --- .../isaaclab_teleop/ov_libraries_xr.py | 3 +-- .../isaaclab_teleop/isaaclab_teleop/ovxr_session.py | 11 +++-------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index a6d4515ee5f..dbbad5342f9 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -15,6 +15,7 @@ from dataclasses import replace from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +import numpy as np from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger from isaaclab_ov.renderers import OVRTXRendererCfg @@ -22,7 +23,6 @@ from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation if TYPE_CHECKING: - import numpy as np from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding from isaacteleop.oxr import OpenXRSessionHandles @@ -299,7 +299,6 @@ def commit(matrices: np.ndarray) -> None: return ordinal def _write_anchor_matrices(self, matrices: np.ndarray, ordinal: int) -> None: - import numpy as np import ovstage from isaaclab_ov.stage import xform_tensor_from_numpy diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index c006ebd9659..f75afecdcd3 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -10,14 +10,16 @@ import builtins from typing import TYPE_CHECKING, Any +import numpy as np +import warp as wp from isaaclab_ov.renderers import OvLibrariesRenderLease, OVRTXRenderer, OVRTXRendererCfg +from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext _TERMINAL_MODE_SENTINEL = "ISAAC_LAUNCHED_FROM_TERMINAL" if TYPE_CHECKING: - import numpy as np from isaaclab_ov.renderers.render_lease import OvLibrariesScenePublisher from isaaclab.sim import SimulationCfg @@ -194,11 +196,6 @@ def get_world_transform(self, prim_path: str) -> np.ndarray: if not isinstance(prim_path, str) or not prim_path.startswith("/"): raise ValueError(f"Target-frame prim path must be absolute: {prim_path!r}") - import numpy as np - import warp as wp - - from isaaclab.scene_data import SceneDataFormat - provider = self._sim.get_scene_data_provider() cached = self._target_frame_readers.get(prim_path) if cached is None or cached[0] is not provider: @@ -232,8 +229,6 @@ def get_world_transform(self, prim_path: str) -> np.ndarray: def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: """Read world-to-target transform [4, 4], float32, with translation in scene units.""" - import numpy as np - world_T_target = self.get_world_transform(prim_path) try: target_T_world = np.linalg.inv(world_T_target) From 58bd7566da9415278cb27a901817f3e37cf2367d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:25:00 -0500 Subject: [PATCH 077/145] Resolve a bare CUDA device through one OVXR session helper Environment validation re-implemented the /physics/cudaDevice resolution that SimulationContext and launch_simulation apply to a bare "cuda". Give the OVXR session one resolve_cuda_device helper that the teleop entrypoints can reuse, and read the setting from the settings manager in its test. --- .../isaaclab_teleop/ovxr_session.py | 26 +++++++++++++++---- .../isaaclab_teleop/test/test_ovxr_session.py | 20 ++++++++------ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index f75afecdcd3..94e44651e48 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -14,6 +14,7 @@ import warp as wp from isaaclab_ov.renderers import OvLibrariesRenderLease, OVRTXRenderer, OVRTXRendererCfg +from isaaclab.app import get_settings_manager from isaaclab.scene_data import SceneDataFormat from isaaclab.sim import SimulationContext @@ -33,6 +34,24 @@ def validate_ovxr_session_preparation() -> None: raise RuntimeError("OVXR bootstrap cannot reuse a context in extension mode") +def resolve_cuda_device(device: str) -> str: + """Resolve a bare ``"cuda"`` device to the GPU that the simulation selects for it. + + :class:`~isaaclab.sim.SimulationContext` and :func:`~isaaclab.app.launch_simulation` read + the ``/physics/cudaDevice`` setting the same way. + + Args: + device: Simulation device, for example ``"cpu"``, ``"cuda"`` or ``"cuda:1"``. + + Returns: + ``"cuda:"`` for a bare ``"cuda"``; any other device unchanged. + """ + if device != "cuda": + return device + cuda_device = get_settings_manager().get("/physics/cudaDevice") + return f"cuda:{max(0, int(cuda_device) if cuda_device is not None else 0)}" + + class OvxrSessionBootstrap: """Reserve an OVRTX/OVStage lease before a real environment is constructed. @@ -122,11 +141,8 @@ def validate_environment(self, env: Any) -> None: env_sim_cfg = unwrapped.cfg.sim prepared_cfg = self._sim.cfg if env_sim_cfg is not prepared_cfg: - env_device = env_sim_cfg.device - if env_device == "cuda": - # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. - cuda_device = self._sim.get_setting("/physics/cudaDevice") - env_device = f"cuda:{max(0, int(cuda_device) if cuda_device is not None else 0)}" + # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. + env_device = resolve_cuda_device(env_sim_cfg.device) for field, env_value, prepared_value in ( ("device", env_device, prepared_cfg.device), ("dt", env_sim_cfg.dt, prepared_cfg.dt), diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py index 3f1b57d17ad..101d6a72330 100644 --- a/source/isaaclab_teleop/test/test_ovxr_session.py +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -63,6 +63,7 @@ def bootstrap(monkeypatch): """Run the real bootstrap constructor and renderer lease; only the physics-backed context is replaced.""" from isaaclab_teleop import ovxr_session + from isaaclab.app import get_settings_manager from isaaclab.renderers import RenderContext monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") @@ -91,15 +92,18 @@ def clear_instance(cls): def get_or_create_backend(self, _cfg): return renderer - def get_setting(self, path): - # SimulationContext resolves a bare "cuda" device from this setting. - return 1 if path == "/physics/cudaDevice" else None - monkeypatch.setattr(ovxr_session, "SimulationContext", Context) - bootstrap = ovxr_session.OvxrSessionBootstrap(SimulationCfg(device="cuda:1", dt=0.01)) - yield bootstrap - bootstrap.abort() - assert renderer._render_lease.active is None + # SimulationContext resolves a bare "cuda" device from this setting. + settings = get_settings_manager() + previous_cuda_device = settings.get("/physics/cudaDevice") + settings.set("/physics/cudaDevice", 1) + try: + bootstrap = ovxr_session.OvxrSessionBootstrap(SimulationCfg(device="cuda:1", dt=0.01)) + yield bootstrap + bootstrap.abort() + assert renderer._render_lease.active is None + finally: + settings.set("/physics/cudaDevice", previous_cuda_device) def _environment(bootstrap, *, num_envs=1, device="cuda:1", dt=0.01): From dfbdf32fc388250b11d4b04a8fe358ea539646d8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:25:18 -0500 Subject: [PATCH 078/145] Document the render lease's effect on OVRTX clone routing An active lease makes OVStage rendering the unset default for every OVRTX renderer in the simulation and rejects an explicit OVStage opt-out or shared stage, which the routing guide did not say. --- docs/source/api/lab_ov/isaaclab_ov.cloner.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst index de123825d15..84518c38e1d 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst @@ -23,6 +23,10 @@ both OVPhysX and Newton. OVPhysX also retains native physics cloning. ``0`` selects native OVRTX cloning. Set the flag before creating the simulation. Reset retains the selected route. This flag alone never enables physics/rendering stage sharing. +An active :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` makes OVStage rendering the +unset default for every OVRTX renderer in that simulation, and rejects +``ISAAC_LAB_OVRTX_USE_OVSTAGE=0`` and ``ISAAC_LAB_SHARE_OVSTAGE=1``. + To share one OVStage with OVPhysX and OVRTX, set ``ISAAC_LAB_SHARE_OVSTAGE=1``. Sharing requires OVPhysX and selects OVStage rendering regardless of ``ISAAC_LAB_OVRTX_USE_OVSTAGE``. From 3cc5aa1e57fe57a33c73a386eed056c6b228045a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:25:44 -0500 Subject: [PATCH 079/145] Put the material writer's pending-update property after its constructor Properties follow the constructor. The address-compilation helper stays next to the constructor it was extracted from, so the upstream binding lines keep their place. --- .../isaaclab_ov/renderers/visual_materials.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index ee1c75580fc..921b640bc1a 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -67,6 +67,11 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, if renderer._initialized_scene: self._compile_addresses(renderer) + @property + def has_pending_updates(self) -> bool: + """Whether material changes require a scene publication even without a physics step.""" + return bool(self._dirty_channels) + def _compile_addresses(self, renderer: OVRTXRenderer) -> None: """Bind addresses only after the leased or ordinary scene has been populated.""" if not renderer._initialized_scene: @@ -91,11 +96,6 @@ def _compile_addresses(self, renderer: OVRTXRenderer) -> None: self._addresses = addresses self._pending_groups = [] - @property - def has_pending_updates(self) -> bool: - """Whether material changes require a scene publication even without a physics step.""" - return bool(self._dirty_channels) - def __call__(self, material_offsets: dict[str, Any] | None = None, env_ids: Any | None = None) -> None: """Mark channels dirty; OVRTX currently copies each dirty channel's full device buffer.""" del env_ids From 5af4912f66e4b943985b447dea44af22be9b5404 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:27:30 -0500 Subject: [PATCH 080/145] Cover the scene publisher's component and authority rules Replace the single publication test with a fixture that doubles each component endpoint, and check that the material, transform, geometry and material-drain writes share the supplied ordinal, that a failure in any of them poisons the pair without moving the commit boundary, and that thread, hashability, ordinal and closed-publisher checks reject a publication before any component runs. --- .../test/test_ovrtx_renderer_contract.py | 121 ++++++++++++++++-- 1 file changed, 107 insertions(+), 14 deletions(-) diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index af3f9ede545..a0773735ec7 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1535,27 +1535,120 @@ def owner(): assert renderer.backend is renderer.scene is None -def test_render_lease_complete_publication_deduplicates_and_poison_fails_closed(leased_renderer, monkeypatch): +@pytest.fixture +def complete_scene_publication(leased_renderer, monkeypatch): + """Exercise the application publisher with native component endpoints doubled.""" renderer = leased_renderer lease = renderer.acquire_ov_libraries_render_lease() publisher = lease.capture_scene_publisher() _prepare_leased_renderer(renderer) - renderer.scene.stage.advance_write_floor.reset_mock() - transforms = MagicMock() - monkeypatch.setattr(renderer, "_update_transforms", transforms) - monkeypatch.setattr(renderer, "_update_geometries", lambda: None) - assert publisher.publication_required("step-1") - publisher.write_complete_scene(18, "step-1") - assert not publisher.publication_required("step-1") - transforms.assert_called_once_with() + stage = renderer.scene.stage + stage.get_attribute_write_floor.reset_mock() + stage.advance_write_floor.reset_mock() + state = types.SimpleNamespace( + renderer=renderer, + lease=lease, + publisher=publisher, + ordinal=23, + events=[], + failure=None, + cause=ValueError("component endpoint failed"), + initial_id=renderer._render_lease._last_step_id, + ) + + def component(name): + state.events.append((name, renderer.scene.ordinal)) + if state.failure == name: + raise state.cause + + writer = types.SimpleNamespace( + has_pending_updates=False, + publish=lambda: component("material"), + drain=lambda: component("material-drain"), + ) + monkeypatch.setattr(renderer, "_visual_material_writer_ref", lambda: writer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: component("transform")) + monkeypatch.setattr(renderer, "_update_geometries", lambda: component("geometry")) + state.publish = lambda step_id="step": publisher.write_complete_scene(state.ordinal, step_id) + try: + yield state + finally: + lease.release() + + +def test_render_lease_complete_scene_components_share_the_supplied_ordinal(complete_scene_publication): + state = complete_scene_publication + renderer = state.renderer + assert state.publisher.publication_required("step") + assert state.publish() is None + names = ["material", "transform", "geometry", "material-drain"] + assert state.events == [(name, state.ordinal) for name in names] + assert renderer._render_lease._last_step_id == "step" + assert renderer.scene.ordinal == state.ordinal + assert not state.publisher.publication_required("step") + with pytest.raises(RuntimeError, match="publication is not required"): + state.publish() + assert state.events == [(name, state.ordinal) for name in names] + renderer.scene.stage.get_attribute_write_floor.assert_not_called() renderer.scene.stage.advance_write_floor.assert_not_called() - transforms.side_effect = ValueError("write failed") - with pytest.raises(RuntimeError, match="scene publication failed"): - publisher.write_complete_scene(19, "step-2") + assert not renderer._render_lease._poisoned + + +@pytest.mark.parametrize("failure", ["material", "transform", "geometry", "material-drain"]) +def test_render_lease_complete_scene_failure_preserves_commit_boundary(complete_scene_publication, failure): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + state.failure = failure + with pytest.raises(RuntimeError, match="Producer scene publication failed") as caught: + state.publish() + assert caught.value.__cause__ is state.cause + order = ["material", "transform", "geometry", "material-drain"] + expected = order[: order.index(failure) + 1] + if failure != "material-drain": + expected.append("material-drain") + assert state.events == [(name, state.ordinal) for name in expected] + assert renderer._render_lease._last_step_id == state.initial_id + assert renderer.scene.ordinal == state.ordinal + assert renderer._render_lease._poisoned with pytest.raises(RuntimeError, match="poisoned"): _ = lease.ovstage - publisher.close() - lease.release() + previous_events = list(state.events) + with pytest.raises(RuntimeError, match="unavailable|poisoned"): + state.publish("next-step") + assert state.events == previous_events + renderer.scene.stage.advance_write_floor.assert_not_called() + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + + +def test_render_lease_complete_scene_public_authority_precedes_components(complete_scene_publication): + import threading + + state = complete_scene_publication + errors = [] + + def foreign_thread(): + try: + state.publish() + except BaseException as exc: + errors.append(exc) + + thread = threading.Thread(target=foreign_thread) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert "thread" in str(errors[0]) + with pytest.raises(TypeError, match="physics_step_id must be hashable"): + state.publish([]) + with pytest.raises(ValueError, match="ordinal must be nonzero"): + state.publisher.write_complete_scene(0, "step") + state.publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + state.publish() + assert not state.events + state.renderer.scene.stage.get_attribute_write_floor.assert_not_called() + state.renderer.scene.stage.advance_write_floor.assert_not_called() + assert not state.renderer._render_lease._poisoned def test_render_lease_closed_publisher_reports_no_pending_publication(leased_renderer, monkeypatch): From 3c06330840485203c5153fd0ed9c5052bb84043d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:28:35 -0500 Subject: [PATCH 081/145] Cover OVXR session preparation checks in the adapter tests Test that preparation rejects an existing simulation context and extension mode before it changes renderer globals, that the configuration requires exactly two stereo camera paths, and that a failed session construction aborts the bootstrap and keeps the first error. --- .../test/test_ov_libraries_xr.py | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 0a74dc8c2c2..f7b3b163437 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -89,6 +89,7 @@ def __init__(self): self.publisher = _FakePublisher() self.events = [] self.closed = False + self.aborted = False def capture_scene_publisher(self): self.events.append("capture") @@ -114,6 +115,9 @@ def get_target_frame_T_world(self, path): def close(self): self.closed = True + def abort(self): + self.aborted = True + @pytest.fixture def real_lease_type(): @@ -649,6 +653,31 @@ def create_bootstrap(*_args): xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(renderer_settings=settings)) +def test_prepare_rejects_existing_context_before_global_settings(xr_module, monkeypatch): + from isaaclab.sim import SimulationContext + + monkeypatch.setattr(SimulationContext, "instance", lambda: object()) + monkeypatch.setitem( + sys.modules, + "ovxr", + SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed existing renderer settings")), + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated second bootstrap")) + with pytest.raises(RuntimeError, match="must create the SimulationContext"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + +@pytest.mark.parametrize("terminal_mode", [False, None, 1]) +def test_prepare_rejects_extension_mode_before_global_settings(xr_module, monkeypatch, terminal_mode): + monkeypatch.setattr(builtins, "ISAAC_LAUNCHED_FROM_TERMINAL", terminal_mode, raising=False) + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(RuntimeError, match="extension mode"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + def test_config_rejects_unknown_session_mode(): with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): OvLibrariesXrCfg(session_mode="automatic") @@ -660,6 +689,12 @@ def test_config_rejects_invalid_scene_scale(scale): OvLibrariesXrCfg(meters_per_unit=scale) +@pytest.mark.parametrize("camera_paths", [("/Camera/mono",), ("/Camera/left", "")]) +def test_config_rejects_non_stereo_camera_paths(camera_paths): + with pytest.raises(ValueError, match="exactly two"): + OvLibrariesXrCfg(camera_paths=camera_paths) + + def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): cfg = xr_module.OvLibrariesXrCfg() cfg.meters_per_unit = float("nan") @@ -682,6 +717,38 @@ def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, assert bootstrap.events == [] +@pytest.mark.parametrize("abort_fails", [False, True]) +def test_prepare_aborts_partial_bootstrap_and_preserves_first_error(xr_module, monkeypatch, abort_fails): + bootstrap = _FakeBootstrap() + first_error = RuntimeError("preparation failed") + original_cause = ValueError("invalid session input") + first_error.add_note("Original preparation diagnostic") + abort_error = RuntimeError("abort failed") + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + + def fail(*_args, **_kwargs): + raise first_error from original_cause + + def abort(): + bootstrap.aborted = True + if abort_fails: + raise abort_error + + monkeypatch.setattr(bootstrap, "abort", abort) + monkeypatch.setattr(xr_module.OvLibrariesXrSession, "__init__", fail) + + with pytest.raises(RuntimeError) as caught: + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + assert bootstrap.aborted + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert caught.value.__notes__ == ["Original preparation diagnostic"] + ( + [f"Cleanup after failed XR preparation also failed: {abort_error!r}"] if abort_fails else [] + ) + + def test_public_xr_exports_resolve_to_implemented_types(xr_module): import isaaclab_teleop From 6e9caf6a3dda0203b76287140a5ccb718760746f Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:29:09 -0500 Subject: [PATCH 082/145] Describe the OVXR adapter's ownership and dependencies in the README Say which kitless XR work isaaclab_teleop owns and what it borrows from the renderer lease, and list the isaaclab_ov and native OVXR, OVRTX and OVStage dependencies that the adapter needs but the package does not install. --- source/isaaclab_teleop/docs/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index df4a0adfc7f..05e7848dcba 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -36,6 +36,11 @@ renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; | `TeleopSessionLifecycle` | Pipeline building, OpenXR handle acquisition, session create/destroy, action-tensor extraction | | `CommandHandler` | Callback registration and XR message-bus command dispatch | +For kitless XR, `isaaclab_teleop` owns session preparation and tracking anchors. +It borrows the application renderer through `isaaclab_ov`'s `OvLibrariesRenderLease` +interface. The renderer retains ownership of native resources, thread synchronization +and GPU scene publication. + ## Usage ### 1. Configure Your Environment @@ -184,5 +189,7 @@ run automatically. ## Dependencies - **`isaaclab`** -- core Isaac Lab framework +- **`isaaclab_ov`** -- required by the optional OVXR session adapter; the standalone `isaaclab_teleop` package does not install this backend automatically - **`isaacteleop`** -- Isaac Capture retargeting engine, device I/O, and session management - **`isaacsim`** -- Isaac Sim runtime (provides the Kit XR bridge for OpenXR handle acquisition) +- **OVXR / OVRTX / OVStage** -- matched native runtime and stock peers for OVXR From b85423d366678341af6c201fe630314000347b30 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:32:39 -0500 Subject: [PATCH 083/145] Add an opt-in native harness for the OVRTX owner transfer The adapter and lease tests double the native endpoints, so nothing in this layer exercised the transfer protocol against real OVRTX and OVStage. Run the same-thread and cross-thread owner-transfer cases on the upstream Isaac-Cartpole task with its OvPhysX preset, each in a child process, behind ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1, and document the command in the adapter guide. --- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 16 ++ .../test_ovrtx_transfer_native_harness.py | 182 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index b1863f0103b..2e7c91fbf4f 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -84,3 +84,19 @@ ordinary renderer operations and does not start an XR session. While a consumer holds the lease, ordinary camera rendering, scene writes and renderer destruction are rejected to prevent concurrent access to the borrowed resources. The same generic borrowing interface is available to applications outside teleoperation. + +## Native validation + +The adapter and lease unit tests double the native OVRTX and OVStage endpoints. +An opt-in harness runs the render-owner transfer and complete scene publication +against the real libraries, on the manager-based `Isaac-Cartpole` task with its +`ovphysx` physics preset and one environment on `cuda:0`. It needs a CUDA GPU, +OVRTX, OVStage, OvPhysX and an Isaac Capture release with OVXR session support; +it starts no XR session or CloudXR runtime. Each case runs in a child process: + +```bash +ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1 uv run python -m pytest \ + source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py +``` + +Without `ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1`, both cases are skipped. diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py new file mode 100644 index 00000000000..a70bd378a34 --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Native scene publication and renderer ownership transfer.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from dataclasses import dataclass +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1", + reason="native OVRTX transfer harness is explicitly opt-in", + ), +] + +_TASK = "Isaac-Cartpole" +"""Manager-based task with an ``ovphysx`` physics preset and no camera sensors.""" + + +@dataclass +class _CaseResult: + offer_thread: int + claim_thread: int | None = None + renderer_id: int | None = None + stage_id: int | None = None + ordinal: int | None = None + error: BaseException | None = None + + +def _make_env_and_bootstrap(task: str): + import gymnasium as gym + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap + + import isaaclab_tasks # noqa: F401 -- registers the task + from isaaclab_tasks.utils.hydra import resolve_presets + from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry + + # parse_env_cfg resolves every preset to its default, so select OvPhysX while resolving. + cfg = resolve_presets(load_cfg_from_registry(task, "env_cfg_entry_point"), selected=("ovphysx",)) + assert isinstance(cfg.sim.physics, OvPhysxCfg) + cfg.sim.device = "cuda:0" + cfg.scene.num_envs = 1 + bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) + try: + return bootstrap, gym.make(task, cfg=cfg).unwrapped + except BaseException: + bootstrap.abort() + raise + + +def _run_case(task: str, *, transfer: bool) -> _CaseResult: + from isaaclab_teleop.ov_libraries_xr import _IsaacLabOvxrApplicationInterface + from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg + + bootstrap, env = _make_env_and_bootstrap(task) + result = _CaseResult(offer_thread=threading.get_native_id()) + host = _IsaacLabOvxrApplicationInterface(bootstrap, env, OvLibrariesXrCfg(auto_launch_cloudxr=False)) + capability = host.offer_render_owner_transfer() + ready, published = threading.Event(), threading.Event() + + def attach(): + binding = host.attach_on_render_thread(capability) + result.renderer_id, result.stage_id = id(binding.renderer), id(binding.stage) + return binding + + def update_stage(binding): + with host.publication_guard(), host.owner_lane(): + binding.renderer.update_from_stage(result.ordinal) + + def close(): + host.close_publication() + try: + host.close_on_render_thread() + finally: + host.close_environment_on_render_thread() + + if not transfer: + try: + binding = attach() + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-control", 0)) + update_stage(binding) + finally: + close() + return result + + with pytest.raises(RuntimeError, match="transfer is pending"): + _ = bootstrap._lease.ovstage + + def worker(): + try: + result.claim_thread = threading.get_native_id() + binding = attach() + ready.set() + if not published.wait(60): + raise TimeoutError("application did not complete its scene publication") + if result.error is None: + update_stage(binding) + with pytest.raises(RuntimeError, match="capability is invalid"): + bootstrap.claim_render_owner_transfer(capability) + except BaseException as exc: + if result.error is None: + result.error = exc + finally: + ready.set() + try: + close() + except BaseException as exc: + if result.error is None: + result.error = exc + + thread = threading.Thread(target=worker, name="ovrtx-native-transfer-worker") + thread.start() + try: + if not ready.wait(90): + raise TimeoutError("renderer claimant did not finish binding") + if result.error is not None: + raise result.error + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-transfer", 0)) + except BaseException as exc: + result.error = exc + finally: + published.set() + thread.join(timeout=90) + assert not thread.is_alive(), "native transfer worker timed out" + if result.error is not None: + raise result.error + assert result.claim_thread is not None and result.claim_thread != result.offer_thread + with pytest.raises(RuntimeError, match="(owner thread|released)"): + _ = bootstrap._lease.ovstage + return result + + +def _run_native_subprocess(task: str, transfer: bool) -> None: + # A native non-daemon thread cannot be bounded by Thread.join alone. + # Keep the native lifetime in a child that the standard subprocess timeout owns. + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), task, "transfer" if transfer else "same-thread"], + env={**os.environ, "ISAAC_LAB_OVRTX_USE_OVSTAGE": "1"}, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + print(completed.stdout) + assert completed.returncode == 0, completed.stderr + + +def test_same_thread_ovxr_session_native_control(): + """One-thread production scene publication remains viable.""" + _run_native_subprocess(_TASK, False) + + +def test_cross_thread_preuse_owner_transfer_native(): + """The application publishes while the render thread binds, renders and closes the pair.""" + _run_native_subprocess(_TASK, True) + + +if __name__ == "__main__": + if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": + raise RuntimeError("native transfer harness requires its explicit opt-in") + result = _run_case(sys.argv[1], transfer=sys.argv[2] == "transfer") + assert result.renderer_id is not None and result.stage_id is not None + assert result.ordinal is not None + print( + f"offer={result.offer_thread} claim={result.claim_thread} " + f"renderer={result.renderer_id} stage={result.stage_id} ordinal={result.ordinal}" + ) From 1b2b6796e334eb413ecb2ee2583a3eb9f9c3c407 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:46:52 -0500 Subject: [PATCH 084/145] Drop the unused XrCfg.hidden_visual_prims field --- .../changelog.d/ovxr-live-replay.added.rst | 4 ++-- source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py | 10 ---------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst index 9b8f9553348..262ae4817ac 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -3,8 +3,8 @@ * Preserved recorded replay controls by default, with explicit ``--start_on_replay`` for recordings without start events. * Added task hooks to :class:`~isaaclab_teleop.IsaacTeleopCfg`: ``prepare_environment``, ``action_conditioner``, ``action_validator`` and ``require_initial_bimanual_hand_tracking``. -* Added ``render_anchor_pos``, ``render_anchor_rot`` and ``hidden_visual_prims`` to - :class:`~isaaclab_teleop.XrCfg` for a render-only anchor offset and globally hidden prims. +* Added ``render_anchor_pos`` and ``render_anchor_rot`` to :class:`~isaaclab_teleop.XrCfg` for a + render-only anchor offset. * Added external OVXR inputs to :class:`~isaaclab_teleop.IsaacTeleopDevice`, together with the ``target_frame_prim_path``, ``xr_anchor_path``, ``xr_render_anchor_path``, ``external_anchor_state``, ``required_openxr_extensions``, ``session_cleanup_failed`` and diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py index f3dd7fceb16..86ab849d006 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py @@ -68,16 +68,6 @@ class XrCfg: effective only when either render-anchor override is configured. """ - hidden_visual_prims: list[str] = [] - """Prim-path regex patterns hidden from the kitless OVXR view. - - Tasks can use this to hide robot geometry that encloses the eye cameras. - The OVXR session authors visibility on the live stage before - renderer materialization and fails if a configured pattern matches no - imageable prim. Visibility persists for the application Stage's lifetime - and affects all cameras, including PiP; it is not restored on XR close. - """ - anchor_prim_path: str | None = None """Specifies the prim path to attach the XR anchor to for dynamic positioning. From 63d34e45c9c88630c8a305babf7aa76af7d2fd1e Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:47:16 -0500 Subject: [PATCH 085/145] Drop the unused IsaacTeleopDevice.xr_anchor_path property --- .../changelog.d/ovxr-live-replay.added.rst | 6 +++--- .../isaaclab_teleop/isaac_teleop_device.py | 9 --------- .../test/test_xr_anchor_manager_contract.py | 1 - 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst index 262ae4817ac..73b3a24a1e6 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -6,6 +6,6 @@ * Added ``render_anchor_pos`` and ``render_anchor_rot`` to :class:`~isaaclab_teleop.XrCfg` for a render-only anchor offset. * Added external OVXR inputs to :class:`~isaaclab_teleop.IsaacTeleopDevice`, together with the - ``target_frame_prim_path``, ``xr_anchor_path``, ``xr_render_anchor_path``, - ``external_anchor_state``, ``required_openxr_extensions``, ``session_cleanup_failed`` and - ``hand_tracking_presence`` properties. + ``target_frame_prim_path``, ``xr_render_anchor_path``, ``external_anchor_state``, + ``required_openxr_extensions``, ``session_cleanup_failed`` and ``hand_tracking_presence`` + properties. diff --git a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py index e6d0841d4ea..635a53d0be9 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py +++ b/source/isaaclab_teleop/isaaclab_teleop/isaac_teleop_device.py @@ -313,15 +313,6 @@ def target_frame_prim_path(self) -> str | None: """Concrete target frame for external scene readers, after environment-path expansion.""" return self._cfg.target_frame_prim_path - @property - def xr_anchor_path(self) -> str: - """Return the tracking-anchor prim path that maps OpenXR poses into the scene. - - External OVXR frames its cameras from :attr:`xr_render_anchor_path`; the two paths differ - only when :attr:`~isaaclab_teleop.XrCfg.render_anchor_pos` or ``render_anchor_rot`` is set. - """ - return self._anchor_manager.anchor_headset_path - @property def xr_render_anchor_path(self) -> str: """Return the anchor prim path used to frame XR cameras.""" diff --git a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py index d4d8ccea836..0ba5e45e65e 100644 --- a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py +++ b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py @@ -112,7 +112,6 @@ def test_public_device_exposes_dynamic_anchor_only_for_ovxr_session(monkeypatch, kwargs.update(ovxr_session=True, mcap_replay_path="recording.mcap") with use_stage(Usd.Stage.CreateInMemory()): device = isaac_teleop_device.IsaacTeleopDevice(cfg, **kwargs) - assert device.xr_anchor_path == "/World/Robot/XRAnchor" assert (device.external_anchor_state is not None) == (capability != "none") lifecycle.assert_called_once() From f754a3b82c8fdc6973f99e9e8e567029938ce081 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:50:23 -0500 Subject: [PATCH 086/145] Publish OVRTX scene updates from a CPU simulation device --- .../ovphysx-contact-tuning.added.rst | 2 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 39 +++++++++----- .../test/test_ovrtx_deformable_bindings.py | 6 +-- .../test/test_ovrtx_renderer_contract.py | 52 +++++++++++++++++++ source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 4 ++ 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst index 0400b82d7de..e2a512611be 100644 --- a/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst +++ b/source/isaaclab_ov/changelog.d/ovphysx-contact-tuning.added.rst @@ -1,3 +1,5 @@ * Added ``bounce_threshold_velocity``, ``friction_correlation_distance`` and ``solve_articulation_contact_last`` to :class:`~isaaclab_ov.physics.OvPhysxCfg`. Unset values keep the stage's scene settings. +* Added CPU simulation-device support to OVRTX scene transform and geometry publication, so kitless + OVXR can present CPU physics. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 4093c4d430d..559a8393b2e 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -390,6 +390,10 @@ def visual_material_writer(self): A leased pair registers its writer before the borrower binds the Stage, because environment reset finalizes material consumers first; addresses compile on the first publication after population. + + The writer requires material buffers on one CUDA device. On a CPU simulation device, + an environment with :class:`~isaaclab.assets.VisualMaterial` assets therefore fails at + reset, with or without a lease. """ return ( self._create_visual_material_writer @@ -1056,7 +1060,7 @@ def _update_transforms(self) -> None: timestamp = self._sdp.backend.transforms_timestamp if self._transforms_timestamp == timestamp: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage output = self._transform_writes.acquire(stream) if not self._sdp.get_transforms(output, allow_passthrough=not asynchronous, scales=self._object_scales): @@ -1072,12 +1076,16 @@ def _update_transforms(self) -> None: tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._transform_writes.submit(binding, matrices, stream) else: - binding.write(matrices, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + matrices, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._transforms_timestamp = timestamp def update_geometries(self) -> None: @@ -1109,7 +1117,7 @@ def _update_geometries(self) -> None: binding = self._geometry_points_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None asynchronous = self.cfg.async_rendering and not self._use_ovstage if asynchronous: output = self._geometry_writes.acquire(stream) @@ -1130,12 +1138,16 @@ def _update_geometries(self) -> None: tensors=[points_tensor_from_warp(array) for array in points], is_array=True, semantic=ovstage.AttributeSemantic.POINT, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() elif asynchronous: self._geometry_writes.submit(binding, points, stream) else: - binding.write(points, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + binding.write( + points, + data_access=DataAccess.ASYNC, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, + ) self._geometry_timestamp = timestamp def update_camera( @@ -1655,27 +1667,29 @@ def close(self) -> None: class _AsyncWriteBuffers(Generic[_BufferT]): - """Retain GPU buffers until native writes finish; reuse them in submission order.""" + """Retain buffers until native writes finish; reuse them in submission order.""" def __init__(self, buffers: Iterable[_BufferT] = ()): self._writes: deque[tuple[_BufferT, Operation[bool] | None, wp.Stream | None]] = deque( (buffer, None, None) for buffer in buffers ) - def acquire(self, stream: wp.Stream) -> _BufferT: + def acquire(self, stream: wp.Stream | None) -> _BufferT: """Wait until the next buffer is writable on the given stream.""" buffer, operation, producer = self._writes[0] if operation is not None: operation.wait() - if producer != stream: + if stream is not None and producer is not None and producer != stream: stream.wait_stream(producer) return buffer def submit( - self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream + self, binding: AttributeBinding, values: wp.array | list[wp.array], stream: wp.Stream | None ) -> Operation[bool]: """Submit a write and retain its inputs; a failed submission does not advance the buffers.""" - operation = binding.write_async(values, data_access=DataAccess.ASYNC, cuda_stream=stream.cuda_stream or 1) + operation = binding.write_async( + values, data_access=DataAccess.ASYNC, cuda_stream=(stream.cuda_stream or 1) if stream is not None else None + ) self._writes[0] = (self._writes[0][0], operation, stream) self._writes.rotate(-1) return operation @@ -1686,5 +1700,6 @@ def close(self) -> None: writes.callback(self._writes.clear) for _, operation, producer in self._writes: if operation is not None: - writes.callback(wp.synchronize_stream, producer) + if producer is not None: + writes.callback(wp.synchronize_stream, producer) writes.callback(operation.wait) diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 1dad8fc7c55..edc27f8e13b 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -46,7 +46,7 @@ def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer.scene = renderer.backend renderer.scene.query = OvstageBackend.query.__get__(renderer.scene) renderer._device = "cpu" - renderer._warp_device = SimpleNamespace(ordinal=0, stream=SimpleNamespace(cuda_stream=99)) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True, stream=SimpleNamespace(cuda_stream=99)) return renderer, renderer.backend.renderer @@ -57,7 +57,7 @@ def test_geometry_bindings_follow_mixed_sdp_points_and_pointer_swaps(mode): renderer.cfg.async_rendering = mode == "async" use_ovstage = mode == "ovstage" renderer._use_ovstage = use_ovstage - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=42)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=42)) points = { path: wp.array(np.arange(count * 3, dtype=np.float32).reshape(count, 3), dtype=wp.vec3f, device="cpu") for path, count in ( @@ -189,7 +189,7 @@ def test_update_transforms_consumes_sdp_matrices_once_per_publication(monkeypatc stage = Usd.Stage.CreateInMemory() UsdGeom.Xform.Define(stage, paths[0]).AddScaleOp().Set(Gf.Vec3d(2, 3, 4) if scaled else Gf.Vec3d(1)) renderer.scene.clone_copies = [(paths[0], [])] - renderer._warp_device = SimpleNamespace(stream=SimpleNamespace(cuda_stream=0)) + renderer._warp_device = SimpleNamespace(is_cuda=True, stream=SimpleNamespace(cuda_stream=0)) renderer._use_ovstage = use_ovstage renderer.scene.ordinal = 5 writes = [] diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index a0773735ec7..96412f0fb03 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1815,3 +1815,55 @@ def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monke lease.release() for writer in context._visual_material_writers: writer.close() + + +@pytest.mark.parametrize("use_ovstage, asynchronous", [(True, False), (False, False), (False, True)]) +def test_cpu_scene_publication_uses_host_tensors_without_a_cuda_stream(use_ovstage, asynchronous): + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = use_ovstage + renderer.cfg = OVRTXRendererCfg(async_rendering=asynchronous) + renderer._warp_device = wp.get_device("cpu") + renderer._device = "cpu" + renderer._object_xform_binding = MagicMock() + renderer._geometry_points_binding = MagicMock() + renderer._transforms_timestamp = renderer._geometry_timestamp = None + renderer._object_scales = None + renderer._geometry_paths = ["/World/Object"] + matrices = wp.array(np.eye(4, dtype=np.float64)[None], dtype=wp.mat44d, device="cpu") + points = wp.array([[1.0, 2.0, 3.0]], dtype=wp.vec3f, device="cpu") + renderer._transform_writes = _AsyncWriteBuffers([types.SimpleNamespace(matrices=matrices)]) + renderer._geometry_writes = _AsyncWriteBuffers([wp.empty_like(points)]) + renderer._geometry_offsets = {"/World/Object": 0} + + def get_points(*, output=None, offsets=None): + if output is None: + return {"/World/Object": points} + wp.copy(output, points) + return None + + renderer._sdp = types.SimpleNamespace( + backend=types.SimpleNamespace(transforms_timestamp=1, geometry_timestamp=1), + get_transforms=lambda *_args, **_kwargs: True, + get_geometry_points=get_points, + ) + stage = MagicMock() + renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) + renderer.update_transforms() + renderer.update_geometries() + bindings = (renderer._object_xform_binding, renderer._geometry_points_binding) + writes = [getattr(binding, "write_async" if asynchronous else "write") for binding in bindings] + calls = stage.write_attribute.call_args_list if use_ovstage else [write.call_args for write in writes] + assert len(calls) == 2 and all(c.kwargs["cuda_stream"] is None for c in calls) + if use_ovstage: + assert [c.args[1] for c in calls] == ["omni:xform", "points"] + assert all(c.kwargs["ordinal"] == 11 for c in calls) + else: + np.testing.assert_array_equal(calls[0].args[0].numpy(), matrices.numpy()) + np.testing.assert_array_equal(calls[1].args[0][0].numpy(), points.numpy()) + # A repeated publication of unchanged state must not duplicate either write. + renderer.update_transforms() + renderer.update_geometries() + assert stage.write_attribute.call_count == 2 if use_ovstage else all(write.call_count == 1 for write in writes) + # Draining the retained buffers waits for previous writes without a CUDA stream. + renderer._transform_writes.close() + renderer._geometry_writes.close() diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 2e7c91fbf4f..eafd110b1eb 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -65,6 +65,10 @@ The adapter retains the environment if lease retirement cannot establish complet default `OVRTXRendererCfg()`) is rejected while the lease is active. - Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its scene publisher writes the same physics state at the reserved ordinal. +- CPU simulation devices publish scene transforms, but visual-material + publication requires CUDA material buffers, as it does for ordinary OVRTX + cameras. With `--device cpu`, an environment that has `VisualMaterial` assets + fails at `sim.reset()`. - Dynamic anchoring takes any object implementing `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects other objects before the worker claims the lease. From a72ca75d527d7fd5e3084f647695ec4a21b2db10 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:53:49 -0500 Subject: [PATCH 087/145] Run the native OVRTX transfer harness with CPU physics --- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 10 ++++---- .../test_ovrtx_transfer_native_harness.py | 24 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index eafd110b1eb..306845f60bd 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -94,13 +94,15 @@ generic borrowing interface is available to applications outside teleoperation. The adapter and lease unit tests double the native OVRTX and OVStage endpoints. An opt-in harness runs the render-owner transfer and complete scene publication against the real libraries, on the manager-based `Isaac-Cartpole` task with its -`ovphysx` physics preset and one environment on `cuda:0`. It needs a CUDA GPU, -OVRTX, OVStage, OvPhysX and an Isaac Capture release with OVXR session support; -it starts no XR session or CloudXR runtime. Each case runs in a child process: +`ovphysx` physics preset and one environment on `cuda:0`. The transfer case +also runs with CPU physics, which publishes host scene tensors to the GPU +renderer. It needs a CUDA GPU, OVRTX, OVStage, OvPhysX and an Isaac Capture +release with OVXR session support; it starts no XR session or CloudXR runtime. +Each case runs in a child process: ```bash ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1 uv run python -m pytest \ source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py ``` -Without `ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1`, both cases are skipped. +Without `ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1`, every case is skipped. diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py index a70bd378a34..70d8f94593d 100644 --- a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -27,6 +27,9 @@ _TASK = "Isaac-Cartpole" """Manager-based task with an ``ovphysx`` physics preset and no camera sensors.""" +_TRANSFER_DEVICES = ("cuda:0", "cpu") +"""Simulation devices for the transfer case; CPU physics publishes host scene tensors.""" + @dataclass class _CaseResult: @@ -38,7 +41,7 @@ class _CaseResult: error: BaseException | None = None -def _make_env_and_bootstrap(task: str): +def _make_env_and_bootstrap(task: str, device: str): import gymnasium as gym from isaaclab_ov.physics import OvPhysxCfg from isaaclab_ov.renderers import OVRTXRendererCfg @@ -51,7 +54,7 @@ def _make_env_and_bootstrap(task: str): # parse_env_cfg resolves every preset to its default, so select OvPhysX while resolving. cfg = resolve_presets(load_cfg_from_registry(task, "env_cfg_entry_point"), selected=("ovphysx",)) assert isinstance(cfg.sim.physics, OvPhysxCfg) - cfg.sim.device = "cuda:0" + cfg.sim.device = device cfg.scene.num_envs = 1 bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) try: @@ -61,11 +64,11 @@ def _make_env_and_bootstrap(task: str): raise -def _run_case(task: str, *, transfer: bool) -> _CaseResult: +def _run_case(task: str, device: str, *, transfer: bool) -> _CaseResult: from isaaclab_teleop.ov_libraries_xr import _IsaacLabOvxrApplicationInterface from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg - bootstrap, env = _make_env_and_bootstrap(task) + bootstrap, env = _make_env_and_bootstrap(task, device) result = _CaseResult(offer_thread=threading.get_native_id()) host = _IsaacLabOvxrApplicationInterface(bootstrap, env, OvLibrariesXrCfg(auto_launch_cloudxr=False)) capability = host.offer_render_owner_transfer() @@ -145,11 +148,11 @@ def worker(): return result -def _run_native_subprocess(task: str, transfer: bool) -> None: +def _run_native_subprocess(task: str, device: str, transfer: bool) -> None: # A native non-daemon thread cannot be bounded by Thread.join alone. # Keep the native lifetime in a child that the standard subprocess timeout owns. completed = subprocess.run( - [sys.executable, str(Path(__file__).resolve()), task, "transfer" if transfer else "same-thread"], + [sys.executable, str(Path(__file__).resolve()), task, device, "transfer" if transfer else "same-thread"], env={**os.environ, "ISAAC_LAB_OVRTX_USE_OVSTAGE": "1"}, capture_output=True, text=True, @@ -162,18 +165,19 @@ def _run_native_subprocess(task: str, transfer: bool) -> None: def test_same_thread_ovxr_session_native_control(): """One-thread production scene publication remains viable.""" - _run_native_subprocess(_TASK, False) + _run_native_subprocess(_TASK, "cuda:0", False) -def test_cross_thread_preuse_owner_transfer_native(): +@pytest.mark.parametrize("device", _TRANSFER_DEVICES) +def test_cross_thread_preuse_owner_transfer_native(device): """The application publishes while the render thread binds, renders and closes the pair.""" - _run_native_subprocess(_TASK, True) + _run_native_subprocess(_TASK, device, True) if __name__ == "__main__": if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": raise RuntimeError("native transfer harness requires its explicit opt-in") - result = _run_case(sys.argv[1], transfer=sys.argv[2] == "transfer") + result = _run_case(sys.argv[1], sys.argv[2], transfer=sys.argv[3] == "transfer") assert result.renderer_id is not None and result.stage_id is not None assert result.ordinal is not None print( From 098b046525a0272aa1fc649bfea87b3248b3fb1f Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:54:10 -0500 Subject: [PATCH 088/145] Keep the environment open when the XR camera feed fails to close --- .../isaaclab_teleop/entrypoints/teleop_se3_agent.py | 4 ++-- source/isaaclab_teleop/test/test_teleop_failure_exit.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index b2546b8eec2..ec198ef56ec 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -403,7 +403,7 @@ def report_cleanup_error(error: Exception) -> None: except Exception as error: camera_drained = False report_cleanup_error(error) - if env is not None and xr_drained: + if env is not None and xr_drained and camera_drained: if ov_libraries_xr_session is None or not ov_libraries_xr_session.async_worker_owns_environment: try: env.close() @@ -427,7 +427,7 @@ def report_cleanup_error(error: Exception) -> None: report_cleanup_error(error) if cleanup_failure is not None: raise cleanup_failure - if env is not None and xr_drained: + if env is not None and xr_drained and camera_drained: print("Environment closed") diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py index eac106fe1a1..7123edf438d 100644 --- a/source/isaaclab_teleop/test/test_teleop_failure_exit.py +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -109,7 +109,7 @@ def test_input_cleanup_failure_prevents_borrowed_owner_release(entry_namespace): @pytest.mark.parametrize( ("cleanup_stage", "has_primary_error"), - [("xr", True), ("environment", True), ("camera", False), ("signals", False)], + [("xr", True), ("environment", True), ("camera", False), ("camera", True), ("signals", False)], ) def test_cleanup_preserves_first_error_and_never_frees_undrained_stage( entry_namespace, cleanup_stage, has_primary_error @@ -140,7 +140,7 @@ def exercise(): exercise() assert caught.value is (primary_error if has_primary_error else cleanup_error) assert events == ( - ["xr", "camera", "signals"] if cleanup_stage == "xr" else ["xr", "camera", "environment", "signals"] + ["xr", "camera", "signals"] if cleanup_stage in ("xr", "camera") else ["xr", "camera", "environment", "signals"] ) if has_primary_error: assert any("cleanup failed" in note for note in primary_error.__notes__) From 0cfc2cd2f14bc31585ac8c64266bde690724500d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:54:24 -0500 Subject: [PATCH 089/145] Correct the kitless guide's client controls and SO101 checks --- source/isaaclab_teleop/docs/kitless_xr.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 665441e43a6..7a0b9113663 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -52,19 +52,23 @@ uv run --extra ovxr isaaclab teleop run \ Use `--physics ovphysx --device cpu` for CPU OVPhysX. `--ovxr` selects OVStage and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported -CloudXR client, then use Play, Pause and Reset as in Kit-based teleoperation. +CloudXR client, then use its Play and Reset controls. Pause/resume is available +only on clients that expose it; Reset is a separate action. The SO101 IK task resets its joints to the configured nominal pose without joint-position noise, giving the clutch a repeatable reset pose. Its configured home transform supplies the clutch calibration. This applies to its Kit and kitless backends. The separate joint-control task retains its randomized reset. +For SO101, check right-controller grip/squeeze engagement, release and +re-engagement, proportional trigger control and Reset separately from cube +grasping. Object retention and cube stacking also depend on contact properties +and grasp placement; validate them separately from the input controls. + For desktop IWER, select **VR** and set **Reference Space** to **local-floor** under **Teleoperator adjustment** before connecting. This keeps the emulated head and controllers relative to the floor. Between trials, end the previous XR session and reload the client page before reconnecting; recheck both settings. -A blank or below-scene main view with a working camera panel warrants checking -this client origin before changing the renderer. `--ovxr` and Kit's `--xr` select separate XR runtimes and are mutually exclusive. An explicit `ISAAC_LAB_OVRTX_USE_OVSTAGE=0` is incompatible with OVXR. @@ -76,8 +80,7 @@ client to establish its OpenXR session. Installing Capture and resolving Python packages do not warm application-specific RTX shaders or Newton/Warp physics kernels. First runtime preparation and the first physics step can take longer than subsequent runs. Retain startup logs and distinguish stage/render -preparation from client connection and active simulation; do not treat a warm -retry as proof that an earlier timeout was fixed. +preparation from client connection and active simulation. Use `--step_hz 20` to cap live control at 20 updates per second while XR rendering runs independently. Time spent reading input, stepping physics and publishing From 62ed4d33ed783bb4faaf42ef9ef99af30867c9d9 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:55:05 -0500 Subject: [PATCH 090/145] Restore the upstream teleop README examples and Docker section --- source/isaaclab_teleop/docs/README.md | 37 +++++++++---------- .../test/test_teleop_entrypoint_units.py | 27 -------------- 2 files changed, 18 insertions(+), 46 deletions(-) diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 27624f076eb..74487cf4de7 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -81,8 +81,7 @@ class MyEnvCfg(ManagerBasedRLEnvCfg): Create a function that builds your Isaac Capture retargeting pipeline. The builder should return an `OutputCombiner` with an `"action"` key containing the flattened action tensor (typically via -`TensorReorderer`). A setup factory may also return retargeters for the tuning UI; -unpack those before assigning the two configuration callbacks: +`TensorReorderer`). Optionally return a list of retargeters to expose in the tuning UI: ```python from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource @@ -101,12 +100,6 @@ def my_pipeline_builder(): ### 3. Run Teleoperation -The upstream device factory also supports keyboard, gamepad and SpaceMouse input -without Kit when the selected task and visualizers do not require it. Kit XR uses -`--xr` and its own installation selection; it continues updating a headless Kit -application without requiring a desktop viewport. Anchor helpers defer native -USD imports until use so that the selected runtime initializes its own provider. - The existing teleop scripts automatically detect `isaac_teleop` in the environment config: ```bash @@ -124,11 +117,7 @@ reference tasks, camera selection, layout, placement, renderer, disable, and kit ```python from isaaclab_teleop import IsaacTeleopCfg, IsaacTeleopDevice -pipeline, retargeters = my_pipeline_builder() -cfg = IsaacTeleopCfg( - pipeline_builder=lambda: pipeline, - retargeters_to_tune=lambda: retargeters, -) +cfg = IsaacTeleopCfg(pipeline_builder=my_pipeline_builder) with IsaacTeleopDevice(cfg) as device: device.add_callback("RESET", env.reset) @@ -182,12 +171,22 @@ contend for the GIL at the start of the step. ## Run with Docker -For Kit XR, follow the canonical -[CloudXR installation guide](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#install-isaac-capture) -and its [Docker instructions](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html#run-with-docker). -They cover the current `teleop` dependency selection, container setup, runtime -startup and client connection. For OVXR, use the separate -[kitless XR installation and runtime guide](kitless_xr.md). +Teleoperation with Isaac Lab runs in a **single container**. Build the image yourself and run a single container. **Do not use Docker Compose** for this workflow (no multi-container setup). Everything runs inside one container with Isaac Lab. + +Inside the container: install Isaac Capture once (`uv pip install 'isaacteleop[retargeters,cloudxr]~=1.0.0' --extra-index-url https://pypi.nvidia.com`), then start the CloudXR runtime with `--accept-eula` so there is no interactive EULA prompt, and run your teleop script. Example: + +```bash +uv run python -m isaacteleop.cloudxr --accept-eula & +source ~/.cloudxr/run/cloudxr.env +uv run python scripts/tools/record_demos.py --task IsaacContrib-PickPlace-Locomanipulation-G1-Abs --num_demos 5 --dataset_file ./datasets/dataset.hdf5 --xr --visualizer kit +``` + +In the Isaac Sim UI, set the AR panel to **System OpenXR Runtime** and click **Start XR**. For the full flow and options, see the [CloudXR teleoperation how-to](https://isaac-sim.github.io/IsaacLab/main/source/how-to/cloudxr_teleoperation.html) and [Isaac Capture Quick Start](https://nvidia.github.io/IsaacCapture/main/getting_started/quick_start.html). + +For a fully headless experience, omit `--visualizer kit` when running Docker, and the XR teleop session will +run automatically. + +For OVXR, use the separate [kitless XR installation and runtime guide](kitless_xr.md). ## Dependencies diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 86d0666b53c..fe4980a3b74 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -5,7 +5,6 @@ """In-process teleoperation entrypoint, configuration, and control-loop contracts.""" -import ast import os import re import signal @@ -68,32 +67,6 @@ def test_xr_experience_declares_only_shipped_extensions(): ) -def test_teleop_readme_programmatic_config_returns_pipeline_not_factory_tuple(): - """Execute the documented public config, without entering an XR session.""" - from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource - from isaacteleop.retargeting_engine.interface import OutputCombiner - - readme = ROOT / "source/isaaclab_teleop/docs/README.md" - section = readme.read_text().split("### 4. Programmatic Usage", 1)[1] - example = re.search(r"```python\n(.*?)```", section, re.DOTALL) - assert example is not None - tree = ast.parse(example.group(1)) - config_statements = [] - for statement in tree.body: - if isinstance(statement, ast.With): - break - config_statements.append(statement) - source = ControllersSource(name="documentation_controllers") - pipeline = OutputCombiner({"action": source.output(ControllersSource.RIGHT)}) - retargeters = [] - namespace = {"my_pipeline_builder": lambda: (pipeline, retargeters)} - exec(compile(ast.Module(body=config_statements, type_ignores=[]), str(readme), "exec"), namespace) - cfg = namespace["cfg"] - assert cfg.pipeline_builder() is pipeline - assert cfg.pipeline_builder().output("action") is not None - assert cfg.retargeters_to_tune() is retargeters - - @pytest.mark.parametrize( ("kitless", "selection", "expected", "reject"), [ From 55b69a1bb7004288df24654293c6a4ecc41b0368 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:55:31 -0500 Subject: [PATCH 091/145] Document the kitless teleop hooks, render anchors and MCAP replay --- docs/source/features/isaac_teleop.rst | 20 ++++++++++++++++++++ docs/source/how-to/cloudxr_teleoperation.rst | 4 ++++ source/isaaclab_teleop/docs/README.md | 17 +++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 5d1fc11eb33..f36bc490850 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1344,6 +1344,26 @@ through this variable -- later runs no longer prompt. The variable applies to ev that launches the runtime, including the process-scoped launcher in ``teleop_replay_agent.py``. +Replay MCAP with a Kitless OVXR Viewer +-------------------------------------- + +The MCAP replay entrypoint can present recorded input through an independent +OVXR viewer. Follow the :ref:`kitless XR guide ` to install Isaac +Capture with its OVXR extra and select the compatible rendering dependencies. + +.. code-block:: bash + + uv run --extra ovxr python \ + scripts/environments/teleoperation/teleop_replay_agent.py \ + --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --physics ovphysx \ + --device cpu --num_envs 1 --ovxr \ + --replay_file /absolute/path/to/session.mcap + +MCAP supplies tracker and control input; the connected client supplies the view. +A recorded STOP, task success or failure, or the replay duration limit ends the +run. The separate ``isaaclab teleop replay`` command replays HDF5 demonstrations +through Kit. + .. _isaac-teleop-xr-anchor: Configure the XR Anchor diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index 066f2f4f01d..2a41ddd0fe8 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -150,6 +150,10 @@ to capture demonstrations, and ``replay`` to play a dataset back. :ref:`The next bundled Isaac Sim pins ``packaging==26.0`` while those runtimes require ``<24``. Install the OV runtimes separately when you need them. +For XR teleoperation without Isaac Sim, follow the :ref:`kitless XR guide `. It +explains how to select a public Isaac Capture wheel and its optional OVXR dependencies. The Kit +setup below uses the published ``teleop`` extra. + Complete these steps first: #. Install the system libraries required by the CloudXR runtime: diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 74487cf4de7..7dfc9f2e263 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -39,7 +39,8 @@ renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; For kitless XR, `isaaclab_teleop` owns session preparation and tracking anchors. It borrows the application renderer through `isaaclab_ov`'s `OvLibrariesRenderLease` interface. The renderer retains ownership of native resources, thread synchronization -and GPU scene publication. +and GPU scene publication. Robot-specific action conditioning and physics preparation +remain in the task package. ## Usage @@ -145,6 +146,9 @@ rendering without blocking. | `sim_device` | `str` | `"cuda:0"` | Torch device for output action tensors | | `retargeting_execution` | `RetargetingExecutionConfig \| None` | `None` (resolved at session start to `mode="pipelined", pacing=DeadlinePacingConfig(safety_margin_s=0.025)`) | Isaac Capture retargeting execution settings; deferred so the config imports without `isaacteleop` | | `teleoperation_active_default` | `bool` | `False` | Whether teleoperation is active on session start | +| `require_initial_bimanual_hand_tracking` | `bool` | `False` | Wait for initial hand presence before stepping | +| `action_conditioner` | `Callable \| None` | `None` | Task-owned stateful pre-step action conditioning | +| `action_validator` | `Callable \| None` | `None` | Task-owned fail-closed action validation | | `app_name` | `str` | `"IsaacLabTeleop"` | Application name for the Isaac Capture session | The 25 ms `DeadlinePacingConfig` safety margin staggers Isaac Capture's Python work behind Isaac Lab's @@ -157,13 +161,22 @@ contend for the GIL at the start of the step. |---|---|---|---| | `anchor_pos` | `tuple[float, float, float]` | `(0, 0, 0)` | XR anchor position in world frame | | `anchor_rot` | `tuple[float, float, float, float]` | `(0, 0, 0, 1)` | XR anchor rotation (quaternion xyzw) | -| `anchor_prim_path` | `str \| None` | `None` | Prim to attach anchor to for dynamic positioning | +| `render_anchor_pos` | `tuple[float, float, float] \| None` | `None` | Optional render-only anchor position | +| `render_anchor_rot` | `tuple[float, float, float, float] \| None` | `None` | Optional render-only anchor rotation | +| `anchor_prim_path` | `str \| None` | `None` | Prim to attach anchor to for dynamic positioning; OVXR reads committed physics poses and shares the same anchor policy | | `anchor_rotation_mode` | `XrAnchorRotationMode` | `FIXED` | How anchor rotation tracks the reference prim | | `anchor_rotation_smoothing_time` | `float` | `1.0` | Slerp time constant (seconds) for `FOLLOW_PRIM_SMOOTHED` mode | | `anchor_rotation_custom_func` | `Callable` | identity | Custom rotation function for `CUSTOM` mode | | `near_plane` | `float` | `0.15` | Near clipping plane distance for the XR device | | `fixed_anchor_height` | `bool` | `True` | Fix anchor height to initial value of the reference prim | +## Capture and replay boundaries + +MCAP records DeviceIO tracker/control input, not historical simulation scenes. +Environment/control reset does not rewind MCAP input. HDF5 +demonstration scripts use the Kit/Isaac Sim application path; these OVXR +commands do not qualify HDF5 recording/replay. + ## Utilities - **`remove_camera_configs(env_cfg)`** -- strips camera sensors and their associated policy From a671f348b1377540e790e43ba52f42860c53a920 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:55:53 -0500 Subject: [PATCH 092/145] Drop later layers' attributes from the teleop entrypoint test fakes --- source/isaaclab_teleop/test/test_teleop_entrypoint_units.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index fe4980a3b74..1f8e827646b 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -138,9 +138,6 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( teleop_device=None, sensitivity=1.0, using_ovxr=kitless, - ovxr_pip_camera=None, - geometry_publication_device=None, - ovrtx_active_cuda_gpus=None, disable_external_cameras=False, physics=None, ), @@ -324,7 +321,6 @@ def condition(_env, value, state): shutdown_requested=False, wait_for_shutdown=sleep, request_render=request, - report_host_timing=lambda *args: None, ), conditioner=condition, validator=lambda value: None, From 8818d398f985c5a89952e7d5ec2c72d8495d69d8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:58:36 -0500 Subject: [PATCH 093/145] Restore upstream helpers and texts that the rebase reintroduced --- .../entrypoints/teleop_replay_agent.py | 4 +- .../entrypoints/teleop_se3_agent.py | 86 ++++++------------- .../isaaclab_teleop/session_lifecycle.py | 6 +- .../test/test_teleop_entrypoint_units.py | 39 +++------ .../test/test_teleop_failure_exit.py | 10 ++- 5 files changed, 47 insertions(+), 98 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 4d59d537534..c29732be782 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -1071,7 +1071,7 @@ def _prepare_env_cfg( ``--num_success_steps``. This matches record_demos.py's pattern of manually counting consecutive success steps before resetting. * The ``time_out`` term is cleared for the same reason it is cleared in - :file:`scripts/tools/record_demos.py` and + :mod:`isaaclab_teleop.entrypoints.record_demos` and :file:`scripts/imitation_learning/robomimic/play.py`: a recorded trajectory often exceeds ``episode_length_s`` (pick-place is 20s by default; a successful operator demo can easily run 25-30s). With the @@ -1158,7 +1158,7 @@ def _reset_replay_environment(env: object, *, hard_sim_reset: bool, reset_record def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice, *, hard_sim_reset: bool) -> None: """Run the full env+teleop reset cycle used by ``record_demos.py``. - Mirrors :func:`scripts.tools.record_demos.handle_reset` (sans the + Mirrors :func:`isaaclab_teleop.entrypoints.record_demos.handle_reset` (sans the instruction-display update, which the headless replay agent doesn't own). The optional hard physics reset keeps Pink seeded against fresh articulation views on PhysX backends. Newton instead retains the solver diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index ec198ef56ec..e62e10f9539 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -27,6 +27,7 @@ import gymnasium as gym import torch +from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks @@ -202,44 +203,7 @@ def _pace_ovxr_control_loop( wait_for_shutdown(remaining_s) -def _rtx_rendering_requested(args: argparse.Namespace) -> bool: - """Return whether the CLI selects a renderer that actually drives RTX rendering. - - The RTX/DLSS global settings are only meaningful when something renders through RTX. - That happens when the Kit visualizer is enabled (``--viz kit`` or a task-declared Kit - visualizer), when external cameras are rendered (on by default; see - ``--disable_external_cameras``), or in XR mode (``--xr``). A pure-headless session with - none of these renders nothing. - - This reads the resolved launcher namespace rather than any Kit/carb runtime state. - :func:`~isaaclab.app.launch_simulation` records ``kit_visualizer`` on it; ``--viz`` itself - is consumed by the preset CLI and is not kept on the namespace. - """ - external_cameras = not getattr(args, "disable_external_cameras", False) - kit_visualizer = bool(getattr(args, "kit_visualizer", False)) - return external_cameras or kit_visualizer or bool(getattr(args, "xr", False)) - - -def _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop: bool): - """Return ``(update, stop)`` callables driving controller haptics, or no-ops. - - Keeps haptics opt-in without branching in the main loop: both callables are - no-ops unless the active device is an Isaac Capture device and the env declares - a ``haptic_feedback`` config. ``update`` renders the current contact force; - ``stop`` zeroes it so a stale pulse does not persist while teleop is paused. - """ - noop = lambda: None # noqa: E731 - if not use_isaac_teleop: - return noop, noop - from .. import create_haptic_feedback_driver - - driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) - if driver is None: - return noop, noop - return driver.update, driver.stop - - -def _apply_teleop_frame(env, action, issue: str | None, haptic_update, haptic_stop, *, render: bool = True) -> bool: +def _apply_teleop_frame(env, action, issue: str | None, haptic_driver, *, render: bool = True) -> bool: """Step one valid action, or hold the scene while an unsafe tracker frame is rejected. ``render`` is false under OVXR, which owns rendering and republishes the unchanged scene. @@ -247,10 +211,16 @@ def _apply_teleop_frame(env, action, issue: str | None, haptic_update, haptic_st if issue is not None: if render: env.sim.render() - haptic_stop() + if haptic_driver is not None: + haptic_driver.stop() return False - env.step(action.repeat(env.num_envs, 1)) - haptic_update() + # process actions + actions = action.repeat(env.num_envs, 1) + # apply actions + env.step(actions) + # render controller haptics from post-step contact forces + if haptic_driver is not None: + haptic_driver.update() return True @@ -454,14 +424,11 @@ def run_teleoperation( # noqa: C901 ov_libraries_xr_session = None previous_shutdown_handlers = {} try: - # Apply the RTX/DLSS global settings when an RTX render pipeline will run (Kit visualizer, - # external cameras, or XR). ``apply_isaac_rtx_global_settings`` uses ``omni.replicator``, - # which some experiences do not preload, so ensure it is loaded first. - if has_kit() and _rtx_rendering_requested(args_cli): - from isaaclab_physx.renderers import IsaacRtxRendererGlobalSettingsCfg + # Apply the RTX/DLSS global settings when Kit runs an RTX render pipeline (Kit visualizer, + # external cameras, or XR). + if has_kit() and (not args_cli.disable_external_cameras or args_cli.kit_visualizer or args_cli.xr): from isaaclab_physx.renderers.isaac_rtx_renderer_utils import apply_isaac_rtx_global_settings - # The renderer's public helper loads Replicator when its settings need it. apply_isaac_rtx_global_settings( IsaacRtxRendererGlobalSettingsCfg( antialiasing_mode="DLSS", @@ -574,9 +541,11 @@ def stop_teleoperation() -> None: print(f"Using teleop device: {teleop_interface}") - # Optional controller haptics: no-ops unless the env declares a - # ``haptic_feedback`` config and the device can render it (Isaac Capture). - haptic_update, haptic_stop = _make_haptic_io(env, teleop_interface, env_cfg, use_isaac_teleop) + haptic_driver = None + if use_isaac_teleop: + from .. import create_haptic_feedback_driver + + haptic_driver = create_haptic_feedback_driver(env.unwrapped, teleop_interface, env_cfg) require_initial_bimanual_tracking = ( env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking if use_isaac_teleop else False @@ -659,12 +628,13 @@ def run_loop(): # action is None when Isaac Capture session hasn't started yet # (e.g. waiting for user to click "Start AR") - if action is None: + if action is None or not teleoperation_active: if ov_libraries_xr_session is None: env.sim.render() - haptic_stop() + if haptic_driver is not None: + haptic_driver.stop() action_applied = False - elif teleoperation_active: + else: issue = ( _required_hand_tracking_issue(teleop_interface, initialized_hand_tracking) if require_initial_bimanual_tracking @@ -675,7 +645,7 @@ def run_loop(): env, action, action_conditioner, action_validator, action_state ) action_applied = _apply_teleop_frame( - env, action, issue, haptic_update, haptic_stop, render=ov_libraries_xr_session is None + env, action, issue, haptic_driver, render=ov_libraries_xr_session is None ) if issue is not None: skipped_tracking_frames += 1 @@ -685,12 +655,6 @@ def run_loop(): issue, skipped_tracking_frames, ) - else: - if ov_libraries_xr_session is None: - env.sim.render() - # not stepping: zero haptics so a paused grip stops buzzing - haptic_stop() - action_applied = False state_changed = action_applied force_publication = False @@ -742,7 +706,7 @@ def run_loop(): # Install after environment/device initialization because embedded native # libraries may replace inherited signal dispositions during startup. previous_shutdown_handlers = _install_ovxr_shutdown_handlers(ov_libraries_xr_session) - # IsaacTeleop requires a context manager, native devices do not. + # Isaac Capture requires a context manager, native devices don't if use_isaac_teleop: if ov_libraries_xr_session is not None: # Capture releases borrowed handles before the outer cleanup diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 292754ab3c3..6a645aa271e 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -1226,7 +1226,7 @@ def _try_start_session(self) -> bool: oxr_handles = self._external_oxr_handles_provider() if oxr_handles is None: if not self._session_start_deferred_logged: - logger.info("External OpenXR handles are not ready; IsaacTeleop session creation deferred") + logger.info("External OpenXR handles are not ready; Isaac Capture session creation deferred") self._session_start_deferred_logged = True return False else: @@ -1421,7 +1421,7 @@ def step( self._external_oxr_handles_provider is not None or self._kit_xr_session_is_active() ): logger.warning( - "IsaacTeleop retargeting step failed (owned/existing XR session); " + "Isaac Capture retargeting step failed (pipeline error, XR session still active); " f"restarting the teleop session in {self._STEP_FAILURE_RESTART_COOLDOWN_S:.0f}s: {e}" ) self._restart_holdoff_until = time.monotonic() + self._STEP_FAILURE_RESTART_COOLDOWN_S @@ -1498,7 +1498,7 @@ def _teardown_dead_session(self) -> None: self._last_step_result = None self._last_control_events = _NO_OP_EVENTS self._last_control_event_frame_id = None - logger.info("IsaacTeleop session torn down after external XR shutdown") + logger.info("Isaac Capture session torn down after external XR shutdown") # ------------------------------------------------------------------ # External input building diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 1f8e827646b..a5289ada5b1 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -196,6 +196,7 @@ def last_control_events(self): session.close = lambda: None session.async_worker_owns_environment = False env.close = lambda: None + env.unwrapped = env env.scene = SimpleNamespace(env_prim_paths=["/World/envs/env_0"]) env.sim.device = "cpu" env.sim.visualizers = () @@ -215,7 +216,7 @@ def last_control_events(self): monkeypatch.setattr(agent, "has_kit", lambda: False) monkeypatch.setattr(agent.gym, "make", lambda *_args, **_kwargs: SimpleNamespace(unwrapped=env)) monkeypatch.setattr(agent, "create_teleop_device", lambda *_args, **_kwargs: Device()) - monkeypatch.setattr(agent, "_make_haptic_io", lambda *_args: (lambda: None, lambda: None)) + monkeypatch.setattr(isaaclab_teleop, "create_haptic_feedback_driver", lambda *_args: None) monkeypatch.setattr(agent, "_install_ovxr_shutdown_handlers", lambda *_args: {}) monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", lambda *_args, **_kwargs: camera) monkeypatch.setattr(isaaclab_teleop.OvLibrariesXrSession, "prepare", lambda *_args, **_kwargs: session) @@ -738,17 +739,19 @@ def step(self, action): env = Env() haptics = {"update": 0, "stop": 0} - def update(): - haptics["update"] += 1 + class HapticDriver: + def update(self): + haptics["update"] += 1 - def stop(): - haptics["stop"] += 1 + def stop(self): + haptics["stop"] += 1 + driver = HapticDriver() valid = torch.arange(36, dtype=torch.float32) - assert agent._apply_teleop_frame(env, valid, None, update, stop) - assert not agent._apply_teleop_frame(env, torch.zeros_like(valid), "missing left hand tracking", update, stop) - assert agent._apply_teleop_frame(env, valid, None, update, stop) + assert agent._apply_teleop_frame(env, valid, None, driver) + assert not agent._apply_teleop_frame(env, torch.zeros_like(valid), "missing left hand tracking", driver) + assert agent._apply_teleop_frame(env, valid, None, driver) assert env.sim.render_count == 1 assert len(env.actions) == 2 @@ -899,26 +902,6 @@ def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monk assert replay.args_cli.require_kit -@pytest.mark.parametrize( - "arguments,expected", - [ - (("--viz", "kit", "--disable_external_cameras"), True), - (("--disable_external_cameras",), False), - ((), True), - ], -) -def test_rtx_gate_reads_the_launcher_kit_visualizer(monkeypatch, arguments, expected): - """``setup_preset_cli`` consumes ``--viz``; the gate must read what the launcher resolved from it.""" - from isaaclab.app.sim_launcher import validate_launch - - agent = _load_teleop_agent(monkeypatch, "--task", _UNSET_PHYSICS_TASK, *arguments) - assert not hasattr(agent.args_cli, "visualizer") - env_cfg, _teleop_cfg, _use_isaac_teleop = agent._resolve_env_cfg() - # launch_simulation validates against the live namespace, which records kit_visualizer. - validate_launch(env_cfg, vars(agent.args_cli)) - assert agent._rtx_rendering_requested(agent.args_cli) is expected - - def test_rejected_ovxr_frame_does_not_render_on_the_control_thread(monkeypatch, run_live_workflow): """OVXR owns rendering, so a rejected frame holds the scene without a control-thread render or step.""" from isaaclab_teleop.entrypoints import teleop_se3_agent as agent diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py index 7123edf438d..4fedfd4ebfa 100644 --- a/source/isaaclab_teleop/test/test_teleop_failure_exit.py +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -22,9 +22,7 @@ def entry_namespace(monkeypatch): for name in ( "args_cli", "gym", - "_rtx_rendering_requested", "_resolve_cloudxr_env", - "_make_haptic_io", "has_kit", "create_teleop_device", "_install_ovxr_shutdown_handlers", @@ -51,9 +49,15 @@ def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, en reset=Mock(), close=Mock(), ) + env.unwrapped = env interface = MagicMock() interface.advance.side_effect = failure monkeypatch.setattr(isaaclab_teleop.XrCameraFeedSession, "prepare", Mock(return_value=camera)) + monkeypatch.setattr( + isaaclab_teleop, + "create_haptic_feedback_driver", + Mock(side_effect=failure if stage == "haptics" else None, return_value=None), + ) entry_namespace.update( { "args_cli": SimpleNamespace( @@ -70,9 +74,7 @@ def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, en side_effect=failure if stage == "environment" else None, return_value=SimpleNamespace(unwrapped=env) ) ), - "_rtx_rendering_requested": lambda _args: False, "_resolve_cloudxr_env": lambda *_args: None, - "_make_haptic_io": Mock(side_effect=failure if stage == "haptics" else None, return_value=(Mock(), Mock())), "has_kit": lambda: False, "create_teleop_device": Mock(side_effect=failure if stage == "device" else None, return_value=interface), "_install_ovxr_shutdown_handlers": Mock(return_value={}), From a5a6e23273c3276a741bc4cf9d09fb8fb559b6c1 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 13:59:21 -0500 Subject: [PATCH 094/145] Keep selecting the camera rendering experience for live teleoperation --- .../isaaclab_teleop/entrypoints/teleop_se3_agent.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index e62e10f9539..e35ebaf96b0 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -149,6 +149,12 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: sys.argv = [sys.argv[0], *argv] parser = _build_parser() args_cli, hydra_args = setup_preset_cli(parser) + + # Enable external camera rendering by default (``--disable_external_cameras`` turns it off). The + # ``--enable_cameras`` CLI flag was removed in Isaac Lab 3.0 (see #6656), so pass the intent to + # the launcher; this selects a camera-rendering experience that provides RTX/DLSS support. + # Everywhere else we read ``args_cli.disable_external_cameras`` directly. + args_cli.enable_cameras = not args_cli.disable_external_cameras if args_cli.xr and args_cli.using_ovxr: parser.error("--xr and --ovxr are mutually exclusive OpenXR frame-loop owners") if not math.isfinite(args_cli.step_hz) or args_cli.step_hz < 0.0: From 00de6325c40606efc1082f746d9b63991c85bf15 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:01:55 -0500 Subject: [PATCH 095/145] Strip cameras for disabled external cameras only under Kit XR again --- .../changelog.d/ovxr-live-replay.changed.rst | 2 -- .../entrypoints/teleop_se3_agent.py | 14 ++++++++++---- .../test/test_teleop_entrypoint_units.py | 7 +++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst index 046da91163e..2e57c229b32 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -8,8 +8,6 @@ * **Breaking:** Live teleoperation now exits non-zero when environment creation, device creation or a teleoperation step fails, instead of logging the error and exiting 0. Automation that inspected the log for failures should check the exit code. -* Changed ``--disable_external_cameras`` to remove camera sensors in every live teleoperation - mode, not only with ``--xr``. * Changed replay to launch Kit only when the resolved task configuration requires it (for example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index e35ebaf96b0..24d38269bb8 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -32,6 +32,7 @@ from isaaclab.app import add_launcher_args, launch_simulation, validate_launch from isaaclab.cli import load_external_tasks from isaaclab.devices import DeviceCfg +from isaaclab.devices.openxr import remove_camera_configs from isaaclab.envs import ManagerBasedRLEnvCfg from isaaclab.managers import TerminationTermCfg as DoneTerm from isaaclab.utils.string import string_to_callable @@ -294,11 +295,16 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg stage_mode = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") if stage_mode.strip() != "1": raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") - if args_cli.using_ovxr or args_cli.disable_external_cameras: - from .. import remove_camera_configs - - env_cfg = remove_camera_configs(env_cfg) + # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR + # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders + # normally, so gate on --xr alone. + if args_cli.xr: + # Keep camera configs when external cameras are enabled (defaulted on); otherwise + # strip them so the XR headset view is the sole render product. + if args_cli.disable_external_cameras: + env_cfg = remove_camera_configs(env_cfg) if args_cli.using_ovxr: + env_cfg = remove_camera_configs(env_cfg) require_kitless_physics(env_cfg, args_cli.task) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index a5289ada5b1..d37ad2dbc70 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -84,7 +84,6 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( """The task resolver preserves the process setting and rejects incompatible explicit modes before launch.""" from types import SimpleNamespace - import isaaclab_teleop from isaaclab_teleop import ( IsaacTeleopCfg, se3_gamepad_teleop_cfg, @@ -103,7 +102,6 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( monkeypatch.delenv(name, raising=False) else: monkeypatch.setenv(name, selection) - monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) preparations = [] from isaaclab_ov.physics import OvPhysxCfg @@ -130,6 +128,7 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( "sys": sys, "ManagerBasedRLEnvCfg": SimpleNamespace, "resolve_task_config": lambda *_args, **_kwargs: (config, None), + "remove_camera_configs": lambda cfg, **_kwargs: cfg, "validate_launch": lambda *_args: (scans.append(os.environ.get(name)) or ((), ())), "args_cli": SimpleNamespace( task="Test", @@ -137,6 +136,7 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( num_envs=1, teleop_device=None, sensitivity=1.0, + xr=False, using_ovxr=kitless, disable_external_cameras=False, physics=None, @@ -819,7 +819,7 @@ def validate_launch(selected_cfg, _args): monkeypatch.setattr(module, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) monkeypatch.setattr(module, "validate_launch", validate_launch) monkeypatch.setattr(module, "launch_simulation", launch) - monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) + monkeypatch.setattr(module, "remove_camera_configs", lambda cfg, **_kwargs: cfg) monkeypatch.setattr(SimulationContext, "instance", lambda: None) monkeypatch.setattr(ovxr_session, "SimulationContext", ContextAllocation) monkeypatch.setitem(sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _settings: None)) @@ -835,7 +835,6 @@ def validate_launch(selected_cfg, _args): ) else: monkeypatch.setattr(module, "hydra_overrides", None, raising=False) - monkeypatch.setattr(module, "remove_camera_configs", isaaclab_teleop.remove_camera_configs) arguments += ["--replay_file", "recording.mcap"] module._parse_args(arguments) From 672d80b24a59b3e1eaffb5d1dd44010389081be8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:04:00 -0500 Subject: [PATCH 096/145] Share the OVXR pre-launch checks between live and replay --- .../entrypoints/_ovxr_launch.py | 26 +++++++++++++++++++ .../entrypoints/teleop_replay_agent.py | 17 ++++++------ .../entrypoints/teleop_se3_agent.py | 12 ++------- .../test/test_teleop_entrypoint_units.py | 4 --- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 500fcf81939..f4f0eb9fb39 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -7,8 +7,34 @@ from __future__ import annotations +import os from typing import Any +from ..xr_cfg import remove_camera_configs + + +def prepare_ovxr_launch(env_cfg: Any, task: str, num_envs: int) -> Any: + """Check an OVXR launch before runtime startup and remove the task's camera sensors. + + Args: + env_cfg: Resolved environment configuration. + task: Task name, for the error message. + num_envs: Requested number of environments. + + Returns: + The configuration without camera sensors. + + Raises: + ValueError: If ``num_envs`` is not 1, ``ISAAC_LAB_OVRTX_USE_OVSTAGE`` is set to a value + other than ``1``, or the task needs Kit physics (see :func:`require_kitless_physics`). + """ + if num_envs != 1: + raise ValueError("--ovxr currently requires --num_envs 1") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") + require_kitless_physics(env_cfg, task) + return remove_camera_configs(env_cfg) + def require_kitless_physics(env_cfg: Any, task: str) -> None: """Reject a task whose physics selection needs the Kit runtime before an OVXR launch. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index c29732be782..1d9601163ef 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -295,7 +295,7 @@ poll_control_events, ) from ..action_policy import initial_hand_tracking_issue, prepare_teleop_action -from ._ovxr_launch import require_kitless_physics +from ._ovxr_launch import prepare_ovxr_launch logger = logging.getLogger(__name__) @@ -1117,15 +1117,14 @@ def _prepare_env_cfg( ) if hasattr(env_cfg.terminations, "time_out"): env_cfg.terminations.time_out = None - if args_cli.ovxr: - if args_cli.num_envs != 1 or args_cli.xr: - raise ValueError("--ovxr requires one environment and no Kit XR or local visualizer") - if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": - raise ValueError("--ovxr requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") - env_cfg = remove_camera_configs(env_cfg) - require_kitless_physics(env_cfg, task) - elif args_cli.disable_external_cameras: + # Keep camera configs when external cameras are enabled (defaulted on) so the replay + # renders them for production parity; otherwise strip them for a lighter headless replay. + if args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) + if args_cli.ovxr: + if args_cli.xr: + raise ValueError("--xr and --ovxr are mutually exclusive OpenXR frame-loop owners") + env_cfg = prepare_ovxr_launch(env_cfg, task, num_envs) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) if args_cli.ovxr: diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 24d38269bb8..7957a82bca5 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -19,7 +19,6 @@ import argparse import logging import math -import os import signal import sys import time @@ -56,7 +55,7 @@ ) from ..action_policy import prepare_teleop_action from ..device_selection import create_teleop_device, select_teleop_cfg -from ._ovxr_launch import require_kitless_physics +from ._ovxr_launch import prepare_ovxr_launch logger = logging.getLogger(__name__) @@ -289,12 +288,6 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg use_isaac_teleop = isinstance(teleop_cfg, IsaacTeleopCfg) if args_cli.using_ovxr and not use_isaac_teleop: raise ValueError("--ovxr requires a task configured with env_cfg.isaac_teleop") - if args_cli.using_ovxr and args_cli.num_envs != 1: - raise ValueError("--ovxr currently requires --num_envs 1") - if args_cli.using_ovxr: - stage_mode = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") - if stage_mode.strip() != "1": - raise ValueError("--ovxr requires OVStage (ISAAC_LAB_OVRTX_USE_OVSTAGE=1)") # XR-rendering setup (camera removal + DLSS) is only needed for the Kit XR # path. Without --xr, Isaac Capture runs standalone (I/O only) and renders # normally, so gate on --xr alone. @@ -304,8 +297,7 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg if args_cli.disable_external_cameras: env_cfg = remove_camera_configs(env_cfg) if args_cli.using_ovxr: - env_cfg = remove_camera_configs(env_cfg) - require_kitless_physics(env_cfg, args_cli.task) + env_cfg = prepare_ovxr_launch(env_cfg, args_cli.task, args_cli.num_envs) # A task without a physics selection keeps None: SimulationContext defaults it to PhysxCfg. configs, kit_sources = validate_launch(env_cfg, vars(args_cli).copy()) if args_cli.using_ovxr: diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index d37ad2dbc70..c453f206172 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -128,7 +128,6 @@ def test_teleop_ovxr_validates_external_stage_without_changing_environment( "sys": sys, "ManagerBasedRLEnvCfg": SimpleNamespace, "resolve_task_config": lambda *_args, **_kwargs: (config, None), - "remove_camera_configs": lambda cfg, **_kwargs: cfg, "validate_launch": lambda *_args: (scans.append(os.environ.get(name)) or ((), ())), "args_cli": SimpleNamespace( task="Test", @@ -448,8 +447,6 @@ def resolve(task, agent, *, overrides): monkeypatch.setattr(replay, "ManagerBasedRLEnvCfg", SimpleNamespace) monkeypatch.setattr(replay, "resolve_task_config", resolve) - monkeypatch.setattr(isaaclab_teleop, "remove_camera_configs", lambda cfg, **_kwargs: cfg) - monkeypatch.setattr(replay, "remove_camera_configs", isaaclab_teleop.remove_camera_configs) prepared, success = namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) assert resolutions == [("test-task", "", ["physics=ovphysx"])] assert prepared is cfg and success is not None @@ -819,7 +816,6 @@ def validate_launch(selected_cfg, _args): monkeypatch.setattr(module, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) monkeypatch.setattr(module, "validate_launch", validate_launch) monkeypatch.setattr(module, "launch_simulation", launch) - monkeypatch.setattr(module, "remove_camera_configs", lambda cfg, **_kwargs: cfg) monkeypatch.setattr(SimulationContext, "instance", lambda: None) monkeypatch.setattr(ovxr_session, "SimulationContext", ContextAllocation) monkeypatch.setitem(sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _settings: None)) From eb9cc4984620f3171366841864dc1a00b33bf9e8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:05:20 -0500 Subject: [PATCH 097/145] Log and return on live teleoperation failures as upstream does --- .../changelog.d/ovxr-live-replay.changed.rst | 3 --- .../entrypoints/teleop_se3_agent.py | 21 +++++++------------ .../test/test_teleop_failure_exit.py | 19 ++++++++++++----- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst index 2e57c229b32..1ef7f10e7f9 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -5,9 +5,6 @@ to the caller instead of silently suppressing them. Handle the reported failure; resources whose asynchronous users have not retired must remain alive until cleanup succeeds. -* **Breaking:** Live teleoperation now exits non-zero when environment creation, device - creation or a teleoperation step fails, instead of logging the error and exiting 0. - Automation that inspected the log for failures should check the exit code. * Changed replay to launch Kit only when the resolved task configuration requires it (for example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 7957a82bca5..febf20e915c 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -469,10 +469,9 @@ def run_teleoperation( # noqa: C901 f"The environment '{args_cli.task}' does not support gripper control. The device command will be" " ignored." ) - - except Exception: - logger.exception("Failed to create environment") - raise + except Exception as e: + logger.error(f"Failed to create environment: {e}") + return # Flags for controlling teleoperation flow should_reset_recording_instance = False @@ -535,13 +534,9 @@ def stop_teleoperation() -> None: ), haptic_cfg=getattr(env_cfg, "haptic_feedback", None), ) - except Exception: - logger.exception("Failed to create teleop device") - raise - - if teleop_interface is None: - logger.error("Failed to create teleop interface") - raise RuntimeError("Failed to create teleop interface") + except Exception as e: + logger.error(f"Failed to create teleop device: {e}") + return print(f"Using teleop device: {teleop_interface}") @@ -704,8 +699,8 @@ def run_loop(): ), ) except Exception as e: - logger.exception("Error during simulation step: %s", e) - raise + logger.error(f"Error during simulation step: {e}") + break # Install after environment/device initialization because embedded native # libraries may replace inherited signal dispositions during startup. diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py index 4fedfd4ebfa..88baec593a2 100644 --- a/source/isaaclab_teleop/test/test_teleop_failure_exit.py +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -6,6 +6,7 @@ """Execute the real public entry without native startup to prove failure/cleanup contracts.""" import contextlib +import logging import signal from types import SimpleNamespace from unittest.mock import MagicMock, Mock @@ -37,8 +38,10 @@ def entry_namespace(monkeypatch): @pytest.mark.parametrize( "stage,use_capture", [("environment", True), ("device", True), ("haptics", True), ("input", True), ("input", False)] ) -def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, entry_namespace, stage, use_capture): - """Every startup phase and the live loop must retain its failure after cleanup.""" +def test_live_entry_failure_closes_resources_and_reports_cause( + monkeypatch, caplog, entry_namespace, stage, use_capture +): + """Startup and live-loop failures are logged or propagated, and both release the resources.""" import isaaclab_teleop failure = RuntimeError(f"{stage} failed") @@ -91,9 +94,15 @@ def test_live_entry_failure_closes_resources_and_preserves_cause(monkeypatch, en ) if not use_capture: cfg = SimpleNamespace() # Native inputs must not read Capture-only action policy. - with pytest.raises(RuntimeError) as caught: - entry_namespace["run_teleoperation"](cfg, False, use_capture) - assert caught.value is failure + if stage == "haptics": + with pytest.raises(RuntimeError) as caught: + entry_namespace["run_teleoperation"](cfg, False, use_capture) + assert caught.value is failure + else: + # Environment, device and loop failures keep the upstream log-and-return exit. + with caplog.at_level(logging.ERROR): + entry_namespace["run_teleoperation"](cfg, False, use_capture) + assert f"{stage} failed" in caplog.text camera.close.assert_called_once_with() assert env.close.call_count == int(stage != "environment") From 30fdb76e6cdd1fdd60d36efcf0ab9a8a038c4979 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:05:44 -0500 Subject: [PATCH 098/145] Keep the CloudXR stop warning when the runtime outlives the session --- source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py | 4 ++-- source/isaaclab_teleop/test/test_cloudxr_lifecycle.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 6a645aa271e..2f16e842c52 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -730,8 +730,8 @@ def record_failure(cleanup_error: BaseException) -> None: if self._cloudxr_launcher is not None: try: self._cloudxr_launcher.stop() - except RuntimeError as cleanup_error: - record_failure(cleanup_error) + except RuntimeError: + logger.warning("CloudXR runtime process could not be terminated; handle retained for atexit cleanup") else: self._cloudxr_launcher = None logger.info("CloudXR runtime stopped") diff --git a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py index 16f4dd7deb8..5de1d6e42fa 100644 --- a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py +++ b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py @@ -572,8 +572,7 @@ def test_stop_retains_launcher_on_runtime_error(self): lifecycle, mock_launcher = self._make_started_lifecycle() mock_launcher.stop.side_effect = RuntimeError("process not found") - with pytest.raises(RuntimeError, match="process not found"): - lifecycle.stop() + lifecycle.stop() assert lifecycle._cloudxr_launcher is mock_launcher From d8f94b4685063d71a4f99f80e92c33ba193e8fb3 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:10:29 -0500 Subject: [PATCH 099/145] Suppress Isaac Capture session exit errors unless OVXR lends the handles --- .../changelog.d/ovxr-live-replay.changed.rst | 4 -- .../changelog.d/ovxr-live-replay.fixed.rst | 3 + .../isaaclab_teleop/session_lifecycle.py | 66 +++++++++++++------ .../test/test_cloudxr_lifecycle.py | 19 +++++- .../test/test_session_step_failure.py | 33 +++++++--- 5 files changed, 88 insertions(+), 37 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst index 1ef7f10e7f9..664e9837e58 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -1,10 +1,6 @@ * **Breaking:** Replay follows recorded START and RESET controls by default. Add ``--start_on_replay`` for recordings made without START events that should begin immediately. -* **Breaking:** Teleoperation context exit reports session or runtime cleanup failures - to the caller instead of silently suppressing them. Handle the reported failure; - resources whose asynchronous users have not retired must remain alive until - cleanup succeeds. * Changed replay to launch Kit only when the resolved task configuration requires it (for example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst index cab1837f07c..a65dcaa50f4 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst @@ -1,2 +1,5 @@ * Applied the same filtered tracking pose to separate render anchors in Kit and OVXR. * Deferred anchor USD imports until use so Kit can initialize its own USD provider. +* Fixed teleoperation cleanup stopping at the first failure: a retargeting UI or XR anchor + cleanup failure no longer skips the Isaac Capture session and CloudXR shutdown, and a failed + device entry stops a runtime it already launched. The first failure is still reported. diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 2f16e842c52..5281e06f8a4 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -660,12 +660,16 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: """Shut down the session and clean up resources. Closes the retargeting tuning UI and exits the ``TeleopSession`` - context manager before stopping CloudXR. Cleanup failures propagate - (or are attached to the caller's original exception). If native drain - fails, latch the error and prohibit reuse or restart. The retained session - reference does not prove native drain; the application must not release - borrowed resources or stop the runtime after that failure. Native owners - classify expected disconnects. + context manager. If the underlying OpenXR session was already torn + down externally (e.g. "Stop AR"), cleanup errors are suppressed. + + A retargeting UI failure does not skip session cleanup; it propagates + afterwards, or is attached to the caller's original exception. With an + external OVXR handle provider, a failed session exit is not suppressed: + it is reported the same way and latched, and the lifecycle prohibits + reuse or restart. The retained session reference does not prove native + drain; the application must not release borrowed resources or stop the + runtime after that failure. Args: exc_type: Exception type (for context manager protocol). @@ -675,7 +679,6 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: self._detach_keyboards() self._pressed_keys.clear() - # Close the retargeting tuning UI first error = exc_val def record_failure(cleanup_error: BaseException) -> None: @@ -685,6 +688,7 @@ def record_failure(cleanup_error: BaseException) -> None: else: error.add_note(f"Isaac Capture cleanup failed: {cleanup_error!r}") + # Close the retargeting tuning UI first # A UI failure must not prevent native/session cleanup. if self._retargeting_ui_ctx is not None: try: @@ -704,12 +708,21 @@ def record_failure(cleanup_error: BaseException) -> None: return try: self._session.__exit__(exc_type, exc_val, exc_tb) - except BaseException as cleanup_error: - self._session_cleanup_error = cleanup_error - record_failure(cleanup_error) - if exc_val is None: - raise error - return + except BaseException as e: + if self._external_oxr_handles_provider is not None: + # Borrowed OVXR handles must be proven drained before their owner releases them. + self._session_cleanup_error = e + record_failure(e) + if exc_val is None: + raise error + return + if not isinstance(e, Exception): + raise + # The OpenXR session may have already been torn down externally + # (e.g. user clicked "Stop AR"), so destroying spaces/action + # sets will fail with XR_ERROR_HANDLE_INVALID. This is + # expected and safe to suppress. + logger.debug(f"Suppressed error during Isaac Capture session cleanup: {e}") self._session = None # Always clear pipeline state (session may never have been created if @@ -1430,6 +1443,8 @@ def step( try: self._teardown_dead_session() except BaseException as cleanup_error: + if self._external_oxr_handles_provider is None: + raise # Failed drain retains the owner; never restart over live resources. self._session_cleanup_error = e e.add_note(f"Teleop session drain also failed: {cleanup_error!r}") @@ -1481,18 +1496,27 @@ def _teardown_dead_session(self) -> None: This is called when :meth:`step` detects that the XR session was destroyed externally (e.g. user clicked "Stop AR"). The - ``TeleopSession`` classifies expected native disconnects. Failed drain - propagates and latches a terminal error; only successful drain resets - state so deferred start can re-acquire handles when the user restarts AR. + ``TeleopSession`` is exited with error suppression (since its XR + resources are already gone), and the internal state is reset so that + the deferred-start logic in :meth:`step` can re-acquire handles if + the user restarts AR. + + With an external OVXR handle provider, a failed exit propagates instead + and latches a terminal error, so borrowed handles are never reused or + restarted before they are proven drained. """ if self._session is not None: try: self._session.__exit__(None, None, None) - except BaseException as error: - # Exit may already be terminal. Never infer successful drain from - # a later no-op __exit__, or resume/restart after this failure. - self._session_cleanup_error = error - raise + except BaseException as e: + if self._external_oxr_handles_provider is not None: + # Exit may already be terminal. Never infer successful drain from + # a later no-op __exit__, or resume/restart after this failure. + self._session_cleanup_error = e + raise + if not isinstance(e, Exception): + raise + logger.debug(f"Suppressed error tearing down dead session: {e}") self._session = None self._session_start_deferred_logged = False self._last_step_result = None diff --git a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py index 5de1d6e42fa..23d056dc535 100644 --- a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py +++ b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py @@ -374,10 +374,25 @@ def test_ui_cleanup_failure_still_drains_session_and_runtime(error_type): assert lifecycle._cloudxr_launcher is None +def test_session_cleanup_error_without_borrowed_handles_is_suppressed(): + """Kit "Stop AR" invalidates the handles, so a failed exit still releases the session and runtime.""" + lifecycle = _make_lifecycle() + session, launcher = MagicMock(), MagicMock() + session.__exit__.side_effect = RuntimeError("XR_ERROR_HANDLE_INVALID") + lifecycle._session, lifecycle._cloudxr_launcher = session, launcher + lifecycle.stop() + session.__exit__.assert_called_once() + assert lifecycle._session is None + assert not lifecycle.session_cleanup_failed + launcher.stop.assert_called_once() + + @pytest.mark.parametrize("body_error", [False, True]) @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) -def test_session_cleanup_failure_is_visible_and_retains_runtime(body_error, error_type): - lifecycle = _make_lifecycle() +def test_borrowed_session_cleanup_failure_is_visible_and_retains_runtime(body_error, error_type): + lifecycle = TeleopSessionLifecycle( + _make_cfg(), use_kit_xr_bridge=False, external_oxr_handles_provider=lambda: object() + ) session, launcher = MagicMock(), MagicMock() cleanup_error = error_type("native drain failed") primary = ValueError("body failed") if body_error else None diff --git a/source/isaaclab_teleop/test/test_session_step_failure.py b/source/isaaclab_teleop/test/test_session_step_failure.py index 4ffa0f2a00e..e07dde4793f 100644 --- a/source/isaaclab_teleop/test/test_session_step_failure.py +++ b/source/isaaclab_teleop/test/test_session_step_failure.py @@ -122,9 +122,13 @@ def _make_lifecycle(**kwargs) -> TeleopSessionLifecycle: return TeleopSessionLifecycle(cfg, **kwargs) -def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: +_BORROWED_OVXR_HANDLES = {"use_kit_xr_bridge": False, "external_oxr_handles_provider": lambda: object()} +"""Lifecycle options for an OVXR session whose handles must be proven drained.""" + + +def _make_failing_lifecycle(error: Exception, **kwargs) -> TeleopSessionLifecycle: """Lifecycle with a live session mock whose step raises *error*.""" - lifecycle = _make_lifecycle() + lifecycle = _make_lifecycle(**kwargs) lifecycle._pipeline = object() session = MagicMock() session.step.side_effect = error @@ -137,7 +141,7 @@ class TestStepFailureDiagnosis: def test_failed_drain_retains_owner_and_original_step_error(self, error_type): step_error = ValueError("retarget worker failed") drain_error = error_type("worker did not join") - lifecycle = _make_failing_lifecycle(step_error) + lifecycle = _make_failing_lifecycle(step_error, **_BORROWED_OVXR_HANDLES) session = lifecycle._session session.__exit__.side_effect = drain_error with ( @@ -168,17 +172,26 @@ def test_replay_step_failure_is_not_restarted_from_the_beginning(self): session.__exit__.assert_called_once() assert lifecycle._session is None - def test_xr_shutdown_drain_failure_remains_visible_and_owned(self): - lifecycle = _make_failing_lifecycle(RuntimeError("unused")) + def test_stop_ar_exit_error_without_borrowed_handles_allows_restart(self): + """Kit "Stop AR" invalidates the handles; a failed exit is suppressed so Start AR can restart.""" + lifecycle = _make_failing_lifecycle(RuntimeError("XR_ERROR_INSTANCE_LOST")) session = lifecycle._session - session.__exit__.side_effect = RuntimeError("native drain failed") - with pytest.raises(RuntimeError, match="native drain failed"): - lifecycle._on_pre_shutdown(None) - assert lifecycle._session is session + session.__exit__.side_effect = RuntimeError("XR_ERROR_HANDLE_INVALID") + with ( + patch.object(TeleopSessionLifecycle, "_kit_xr_session_is_active", return_value=False), + patch.object(lifecycle, "_build_external_inputs", return_value=None), + ): + assert lifecycle.step() is None + session.__exit__.assert_called_once() + assert lifecycle._session is None + assert not lifecycle.session_cleanup_failed + with patch.object(lifecycle, "_try_start_session", return_value=False) as try_start: + assert lifecycle.step() is None + try_start.assert_called_once() @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) def test_terminal_exit_failure_is_latched_across_callbacks_and_restart(self, error_type): - lifecycle = _make_lifecycle() + lifecycle = _make_lifecycle(**_BORROWED_OVXR_HANDLES) lifecycle._pipeline = object() error = error_type("terminal native cleanup failed") From 40d1cc0c0d2857805325492a66d47893ef9c0c78 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:15:24 -0500 Subject: [PATCH 100/145] Restore upstream replay agent text, task parsing and comments --- .../entrypoints/teleop_replay_agent.py | 110 +++++++++++------- .../test/test_teleop_entrypoint_units.py | 16 ++- .../test/test_teleop_replay_bootstrap.py | 2 + 3 files changed, 84 insertions(+), 44 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 1d9601163ef..9e73fe59599 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -7,7 +7,7 @@ This is the non-interactive counterpart to ``teleop_se3_agent.py``. It builds a teleop environment, attaches an :class:`~isaaclab_teleop.IsaacTeleopDevice` -configured in :class:`isaacteleop.teleop_session_manager.SessionMode.REPLAY`, +configured in :class:`isacteleop.teleop_session_manager.SessionMode.REPLAY`, and pumps the simulation loop until the recorded operator presses STOP (or ``--max_replay_duration_s`` elapses, or the simulator is closed). The user-journey teleop script remains ``teleop_se3_agent.py``. @@ -25,7 +25,7 @@ messages (head / hands / controllers) plus the ``_teleop_control`` ``MessageChannelTracker`` that captured the operator's START / STOP / RESET gestures. TeleopCore's - :class:`~isaacteleop.deviceio_session.ReplaySession` re-emits all of + :class:`~isacteleop.deviceio_session.ReplaySession` re-emits all of them on the same monotonic-time cadence they were recorded on, so :func:`~isaaclab_teleop.poll_control_events` returns the same edges here that ``record_demos.py``'s loop saw at recording time. @@ -73,28 +73,31 @@ Stats output: The wall-clock deltas between successive ``env.sim.render()`` calls - produced during the active window are recorded as a per-render-call + produced during the active window are recorded as a per-rendered-frame interval series (N gated renders produce N-1 samples). Pre-START render-only frames, warmup ticks, and post-quit render-only spin-down are excluded so the numbers reflect the steady-state replay workload rather than agent bookkeeping overhead. Per-run samples are summarised into mean / p50 / p90 / p95 / p99 / min / max / stddev (under ``cpu_frame_time_ms``) plus - derived call-rate metrics (under the legacy ``fps`` key). Both blocks measure - successive application ``env.sim.render()`` calls, not completed renderer - frames, encoded video, or client/headset presentation. The two + derived FPS metrics (under ``fps``). Both blocks are reported on + a **per-render** basis -- ``cpu_frame_time_ms`` is the wall time + between rendered frames and ``fps`` is the render rate, matching + what the simulator's HUD shows and what the headset wearer / + spectator actually perceives during real-time teleop. The two blocks stay self-consistent: ``fps.mean`` equals ``1000 / cpu_frame_time_ms.mean`` (harmonic mean of FPS = total frames / total step time). Each interval is the wall-clock delta between successive ``env.sim.render`` calls, so a run needs at least two - render calls during the active window to contribute. The agent + rendered frames during the active window to contribute. The agent asserts this per-run inside :func:`_run_single_replay`: if a run stepped the env but produced no usable intervals (0 or 1 renders -- typically the sim is not rendering, or the active window closed too quickly), it raises ``RuntimeError`` immediately, so the remaining replays never start and no stdout summary or JSON file - is written. OVXR owns rendering independently, so application render-call - timing statistics remain empty for OVXR replays. Partial reports are never emitted: CI relies on - "no JSON output" as an unambiguous failure signal. + is written. Partial reports are never emitted: CI relies on + "no JSON output" as an unambiguous failure signal. OVXR owns rendering + independently, so application render-call timing statistics remain empty + for OVXR replays. Each active iteration also emits one ``GpuStatsProvider.sample()`` call. The default :class:`NvmlGpuStatsProvider` snapshots GPU @@ -123,7 +126,7 @@ { "schema_version": 1, "task": "IsaacContrib-PickPlace-GR1T2-Abs", - "replay_file": "/absolute/path/to/session.mcap", + "replay_file": "/tmp/pickplace_gr1t2.mcap", "num_replays": 5, "requested_num_replays": 5, "batch_complete": true, @@ -151,13 +154,15 @@ "run_index": 0, "outcome": "success", "active_iterations": 322, + "rejected_action_frames": 0, + "last_action_rejection": null, "active_duration_s": 21.503, "success_step_count": 1, "cpu_frame_time_ms": { # per-render ms (wall time between sim.render calls) "mean": ..., "p50": ..., "p90": ..., "p95": ..., "p99": ..., "min": ..., "max": ..., "stddev": ..., "n": ... }, - "fps": { # application render-call rate, not headset presentation + "fps": { # per-render rate (matches simulator HUD) "mean": ..., "min_instantaneous": ..., "max_instantaneous": ... }, "gpu_stats": { @@ -284,7 +289,7 @@ from isaaclab.utils.version import has_kit import isaaclab_tasks # noqa: F401 -from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli +from isaaclab_tasks.utils import parse_env_cfg, setup_preset_cli from .. import ( CLOUDXR_AVP_ENV, @@ -391,9 +396,10 @@ def _build_parser() -> argparse.ArgumentParser: type=float, default=300.0, help=( - "Safety cap for initial stage readiness. Kit stage loading warns and continues" - " at this cap; OVXR render-worker bootstrap fails and drains on expiry." - " Default is 300s." + "Safety cap on how long to wait for omni.usd to finish loading the stage before" + " proceeding anyway. Hit only when something is misconfigured (missing asset, slow" + " Nucleus, etc.); a warning is logged and replay continues. Default is 300s." + " With --ovxr it also bounds render-worker startup, which fails and drains on expiry." ), ) parser.add_argument( @@ -438,12 +444,12 @@ def _build_parser() -> argparse.ArgumentParser: def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse workflow arguments, leaving task overrides in ``sys.argv``.""" + """Parse ``argv`` (default ``sys.argv[1:]``) into :data:`args_cli` and :data:`hydra_overrides`.""" global args_cli, hydra_overrides if argv is not None: sys.argv = [sys.argv[0], *argv] parser = _build_parser() - + # forward unrecognized args as Hydra-style task config overrides args_cli, hydra_overrides = setup_preset_cli(parser) sys.argv = [sys.argv[0]] + hydra_overrides @@ -485,7 +491,7 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: class _RunStats: """Per-replay performance + outcome record. - ``active_frame_times_ms`` is the per-render-call series: wall-clock + ``active_frame_times_ms`` is the per-rendered-frame series: wall-clock deltas between successive :meth:`SimulationContext.render` calls produced from inside ``env.step`` during the active window (post-START, pre-terminator). Pre-START render-only frames, warmup @@ -588,7 +594,7 @@ def _compute_frame_stats(samples_ms: list[float]) -> dict: def _compute_fps_stats(samples_ms: list[float]) -> dict: - """Compute render-call rate stats from application intervals (in ms). + """Compute FPS stats from per-rendered-frame intervals (in ms). All three fields are derived from the same ``samples_ms`` series that feeds ``cpu_frame_time_ms`` and stay self-consistent with it: @@ -599,7 +605,7 @@ def _compute_fps_stats(samples_ms: list[float]) -> dict: ``n / active_duration_s`` (dragged down by inter-step bookkeeping). ``min_instantaneous`` and ``max_instantaneous`` are derived from - the slowest / fastest individual render-call interval respectively. + the slowest / fastest individual rendered frame respectively. """ n = len(samples_ms) if n == 0: @@ -1058,7 +1064,10 @@ def _maybe_launch_cloudxr(cloudxr_env_path: str | None, auto_launch: bool): def _prepare_env_cfg( - task: str, num_envs: int, device: str, overrides: Sequence[str] = () + task: str, + num_envs: int, + device: str, + overrides: Sequence[str] = (), ) -> tuple[ManagerBasedRLEnvCfg, object | None]: """Build and tweak an env config suitable for non-interactive replay. @@ -1088,21 +1097,25 @@ def _prepare_env_cfg( cycle (sim reinit + teleop device reset) so Pink IK starts the next attempt with fresh articulation views. + It also checks the launch before any runtime starts, with two side + effects: it sets ``args_cli.require_kit`` to whether the resolved + configuration needs Kit, and it runs the task's + :attr:`~isaaclab_teleop.IsaacTeleopCfg.prepare_environment` hook, which may + modify the returned configuration. With ``--ovxr`` it also removes camera + sensors and rejects a launch that needs more than one environment, a stage + path other than OVStage, Kit or a local visualizer. + Args: task: Registered task name to load the env config for. num_envs: Number of parallel environments. device: Simulation device (e.g. ``"cuda:0"``). - overrides: Hydra-style ``key=value`` overrides passed to :func:`resolve_task_config`. + overrides: Hydra-style ``key=value`` overrides forwarded to :func:`parse_env_cfg`. Returns: Tuple ``(env_cfg, success_term)``. ``success_term`` is ``None`` when the env doesn't define a ``success`` termination term. """ - overrides = list(overrides) - env_cfg, _ = resolve_task_config(task, "", overrides=overrides) - if device is not None: - env_cfg.sim.device = device - env_cfg.scene.num_envs = num_envs + env_cfg = parse_env_cfg(task, device=device, num_envs=num_envs, overrides=overrides) env_cfg.env_name = task.split(":")[-1] if not isinstance(env_cfg, ManagerBasedRLEnvCfg): raise ValueError( @@ -1135,7 +1148,7 @@ def _prepare_env_cfg( args_cli.require_kit = bool(kit_sources) if env_cfg.isaac_teleop is not None and env_cfg.isaac_teleop.prepare_environment is not None: env_cfg.isaac_teleop.prepare_environment(env_cfg) - return (env_cfg, success_term) + return env_cfg, success_term def _replay_requires_hard_sim_reset(physics_backend: str | None) -> bool: @@ -1200,9 +1213,14 @@ def _process_success_condition( _RENDERER_SETTLE_FRAMES: int = 30 -"""Additional Kit render ticks after asynchronous USD asset loading. - -This legacy Kit warmup is not a kitless renderer-completion or FPS guarantee. +"""Number of additional render frames pumped after the USD stage finishes loading. + +The stage-load status flips to ``count_loading == 0`` as soon as every referenced asset +has been resolved, but the renderer pipeline (shader compilation, articulation-view +binding, material warm-up) typically needs a few more event-loop ticks to converge. Thirty +frames at the default render cadence is ~0.5 s on most machines and is deterministic +per-machine -- unlike a wall-clock delay it does not have to be tuned for hardware. +Used only when Kit is active. """ @@ -1267,9 +1285,15 @@ def close(self): def _warmup_replay(env, run_index: int, presentation) -> None: """Wait for Kit assets or pump the independently owned viewer before input.""" - # Only an active Kit application owns the asynchronous USD loader. - # Kitless environments finish their stage construction before this call. + # Deterministic warmup is only required on the first run -- once the + # stage has been streamed in and the renderer settled, subsequent + # runs share the same stage and only need the per-run reset in + # ``_run_single_replay``. Skipping it on later runs saves the + # renderer-settle frames at the cost of doing nothing measurable (the + # wait returns immediately on a fully-loaded stage anyway). if run_index == 0: + # Only an active Kit application owns the asynchronous USD loader. + # Kitless environments finish their stage construction before this call. if has_kit(): from isaaclab_physx.renderers.isaac_rtx_renderer_utils import wait_for_stage_load @@ -1369,7 +1393,7 @@ def _run_single_replay( teleop_interface = create_isaac_teleop_device( isaac_teleop_cfg, env_prim_path=env.scene.env_prim_paths[0], - sim_device=env.sim.device, + sim_device=args_cli.device, callbacks={}, cloudxr_env_file=None, auto_launch_cloudxr=False, @@ -1410,9 +1434,14 @@ def _patched_sim_render(): env.sim.render = _orig_sim_render with _patched_sim_render(), teleop_interface: - # PhysX backends need the hard reset that refreshes articulation views - # before Pink reads them (PR #5507). Newton's constructed solver must - # instead remain alive across the ordinary environment reset. + # Mirror the reset sequence used by ``record_demos.py``: ``sim.reset()`` + # does a hard physics reinit (re-binds articulation views, plays the + # timeline) that ``env.reset()`` alone does not perform. Pink IK reads + # ``data.joint_pos.torch`` every step to seed Pinocchio's configuration + # and to compute ``target = curr + delta``; if the articulation view is + # stale, every IK call produces zero-delta arm targets while the + # hand-finger path (which bypasses IK) keeps tracking. See PR #5507. + # Newton skips the hard reset: its constructed solver must stay alive. _reset_replay_environment( env, hard_sim_reset=_replay_requires_hard_sim_reset(env.scene.physics_backend), @@ -1546,7 +1575,8 @@ def _patched_sim_render(): # only renders fired from within it contribute # interval samples. iter_start_s = time.perf_counter() - active_start_s = iter_start_s if active_start_s is None else active_start_s + if active_start_s is None: + active_start_s = iter_start_s actions = action.repeat(env.num_envs, 1) sampling_active[0] = True try: @@ -1578,7 +1608,7 @@ def _patched_sim_render(): # meaningful replay. End the run with a failure # outcome so the batch's exit code reflects it. if bool(terminated.any().item()) or bool(truncated.any().item()): - print("Replay environment terminated; ending run.") + print("Replay failure: env terminated/truncated mid-trajectory; ending run.") stats.outcome = "failure" break diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index c453f206172..a7b6de34495 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -413,6 +413,8 @@ def test_teleop_replay_agent_resolves_typed_presets_without_reapplying_physics(m from isaaclab.managers import TerminationTermCfg from isaaclab.sim import SimulationCfg + from isaaclab_tasks.utils import parse_cfg + source_path = ROOT / "source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py" argv = [str(source_path), "--task", "Test", "--replay_file", "recording.mcap", *selector] if ovxr: @@ -446,9 +448,9 @@ def resolve(task, agent, *, overrides): return cfg, None monkeypatch.setattr(replay, "ManagerBasedRLEnvCfg", SimpleNamespace) - monkeypatch.setattr(replay, "resolve_task_config", resolve) + monkeypatch.setattr(parse_cfg, "resolve_task_config", resolve) prepared, success = namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) - assert resolutions == [("test-task", "", ["physics=ovphysx"])] + assert resolutions == [("test-task", None, ["physics=ovphysx"])] assert prepared is cfg and success is not None assert cfg.sim.physics is physics assert namespace["args_cli"].require_kit is False @@ -519,6 +521,7 @@ def last_control_events(self): replay, "args_cli", SimpleNamespace( + device="cpu", replay_file="recording.mcap", start_on_replay=True, enable_debug_visualization=False, @@ -769,6 +772,8 @@ def test_task_preparation_precedes_ovxr_simulation_context(monkeypatch, entrypoi from isaaclab.sim import SimulationCfg, SimulationContext + from isaaclab_tasks.utils import parse_cfg + calls = [] def prepare_environment(selected_cfg): @@ -813,7 +818,8 @@ def validate_launch(selected_cfg, _args): monkeypatch.setattr(sys, "argv", ["teleop-test"]) monkeypatch.setattr(module, "args_cli", None, raising=False) monkeypatch.setattr(module, "ManagerBasedRLEnvCfg", SimpleNamespace) - monkeypatch.setattr(module, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) + resolver_owner = teleop_se3_agent if entrypoint == "live" else parse_cfg + monkeypatch.setattr(resolver_owner, "resolve_task_config", lambda *_args, **_kwargs: (config, None)) monkeypatch.setattr(module, "validate_launch", validate_launch) monkeypatch.setattr(module, "launch_simulation", launch) monkeypatch.setattr(SimulationContext, "instance", lambda: None) @@ -879,6 +885,8 @@ def test_live_ovxr_rejects_unset_task_physics_before_launch(monkeypatch): def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monkeypatch, ovxr): from isaaclab_teleop.entrypoints import teleop_replay_agent as replay + from isaaclab_tasks.utils import parse_cfg + argv = ["teleop_replay_agent.py", "--task", _UNSET_PHYSICS_TASK, "--replay_file", "recording.mcap"] if ovxr: argv.append("--ovxr") @@ -887,7 +895,7 @@ def test_replay_unset_task_physics_keeps_simulation_default_or_rejects_ovxr(monk monkeypatch.setattr(replay, "args_cli", None, raising=False) monkeypatch.setattr(replay, "hydra_overrides", None, raising=False) replay._parse_args() - _unset_task_physics(monkeypatch, replay) + _unset_task_physics(monkeypatch, parse_cfg) if ovxr: with pytest.raises(ValueError, match=_UNSET_PHYSICS_OVXR_ERROR): replay._prepare_env_cfg(_UNSET_PHYSICS_TASK, 1, None, overrides=replay.hydra_overrides) diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index cbe5d1a7ecd..f9cf70af6e2 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -106,6 +106,7 @@ def create_device(selected, **kwargs): namespace = { "args_cli": SimpleNamespace( + device="cpu", replay_file="fixture.mcap", enable_debug_visualization=False, max_stage_load_wait_s=5, @@ -319,6 +320,7 @@ def step(_actions): replay, "args_cli", SimpleNamespace( + device="cpu", replay_file="recording.mcap", start_on_replay=start_on_replay, enable_debug_visualization=False, From 6591d50bc64968435649275bdfa595a5df0851a8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:17:48 -0500 Subject: [PATCH 101/145] Restore upstream live teleoperation text, comments and main entry --- .../entrypoints/teleop_se3_agent.py | 176 ++++++++++-------- .../test/test_teleop_failure_exit.py | 1 + 2 files changed, 103 insertions(+), 74 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index febf20e915c..695c2b54079 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -67,7 +67,7 @@ def _build_parser() -> argparse.ArgumentParser: - """Build workflow and simulation launcher arguments.""" + """Build the command-line parser, including the simulation launcher arguments.""" parser = argparse.ArgumentParser(description="Teleoperation for Isaac Lab environments.") parser.add_argument("--num_envs", type=int, default=1, help="Number of environments to simulate.") parser.add_argument( @@ -75,8 +75,9 @@ def _build_parser() -> argparse.ArgumentParser: type=str, default=None, help=( - "Teleop device name: a task device or keyboard, spacemouse, gamepad. When omitted, use the task's" - " Isaac Capture pipeline, otherwise keyboard. Built-in devices run through Isaac Capture." + "Teleop device name: an entry of env_cfg.teleop_devices.devices, or a built-in device (keyboard," + " spacemouse, gamepad). When omitted, the Isaac Capture pipeline is used if configured in the env," + " otherwise keyboard. Keyboard, gamepad and spacemouse run through Isaac Capture (the teleop extra)." ), ) parser.add_argument("--task", type=str, default=None, help="Name of the task.") @@ -143,7 +144,7 @@ def _build_parser() -> argparse.ArgumentParser: def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: - """Parse workflow arguments, leaving task overrides in ``sys.argv``.""" + """Parse ``argv`` (default ``sys.argv[1:]``) into :data:`args_cli`; Hydra task overrides stay in ``sys.argv``.""" global args_cli if argv is not None: sys.argv = [sys.argv[0], *argv] @@ -183,68 +184,26 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: return args_cli -def _pace_ovxr_control_loop( - using_ovxr: bool, - action_applied: bool, - *, - iteration_started_s: float, - action_period_s: float, - step_hz: float = 0.0, - wait_for_shutdown: Callable[[float], bool] | None = None, -) -> None: - """Limit host control independently of the XR frame clock. - - Credit elapsed input, physics and publication work; an overrun adds no wait - or catch-up steps. With no explicit limit, active control remains unpaced - and paused polling retains the environment's action interval. +def main() -> None: # noqa: C901 """ - if not using_ovxr or (action_applied and step_hz == 0.0): - return - period_s = 1.0 / step_hz if step_hz > 0.0 else action_period_s - remaining_s = period_s - (time.perf_counter() - iteration_started_s) - if remaining_s > 0.0: - if wait_for_shutdown is None: - time.sleep(remaining_s) - else: - wait_for_shutdown(remaining_s) + Run teleoperation with an Isaac Lab manipulation environment. + Creates the environment, sets up teleoperation interfaces and callbacks, + and runs the main simulation loop until the application is closed. -def _apply_teleop_frame(env, action, issue: str | None, haptic_driver, *, render: bool = True) -> bool: - """Step one valid action, or hold the scene while an unsafe tracker frame is rejected. - - ``render`` is false under OVXR, which owns rendering and republishes the unchanged scene. + Returns: + None """ - if issue is not None: - if render: - env.sim.render() - if haptic_driver is not None: - haptic_driver.stop() - return False - # process actions - actions = action.repeat(env.num_envs, 1) - # apply actions - env.step(actions) - # render controller haptics from post-step contact forces - if haptic_driver is not None: - haptic_driver.update() - return True - - -def _ovxr_publication_identity( - physics_step_id: int, *, reset_generation: int | None = None -) -> int | tuple[str, int, int]: - """Return a dedup identity that also represents a reset at an unchanged physics step.""" - if reset_generation is None: - return physics_step_id - return ("reset", reset_generation, physics_step_id) + env_cfg, teleop_cfg, use_isaac_teleop = _resolve_env_cfg() + with launch_simulation(env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): + run_teleoperation(env_cfg, teleop_cfg, use_isaac_teleop) def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool]: """Resolve task configuration and teleop ownership before runtime launch.""" - overrides = list(sys.argv[1:]) - env_cfg, _ = resolve_task_config(args_cli.task, "", overrides=overrides) - if args_cli.device is not None: - env_cfg.sim.device = args_cli.device + # Resolve the task configuration through Hydra so CLI presets are applied. + env_cfg, _ = resolve_task_config(args_cli.task, "") + env_cfg.sim.device = args_cli.device env_cfg.scene.num_envs = args_cli.num_envs env_cfg.env_name = args_cli.task if not isinstance(env_cfg, ManagerBasedRLEnvCfg): @@ -310,6 +269,62 @@ def _resolve_env_cfg() -> tuple[ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg return (env_cfg, teleop_cfg, use_isaac_teleop) +def _pace_ovxr_control_loop( + using_ovxr: bool, + action_applied: bool, + *, + iteration_started_s: float, + action_period_s: float, + step_hz: float = 0.0, + wait_for_shutdown: Callable[[float], bool] | None = None, +) -> None: + """Limit host control independently of the XR frame clock. + + Credit elapsed input, physics and publication work; an overrun adds no wait + or catch-up steps. With no explicit limit, active control remains unpaced + and paused polling retains the environment's action interval. + """ + if not using_ovxr or (action_applied and step_hz == 0.0): + return + period_s = 1.0 / step_hz if step_hz > 0.0 else action_period_s + remaining_s = period_s - (time.perf_counter() - iteration_started_s) + if remaining_s > 0.0: + if wait_for_shutdown is None: + time.sleep(remaining_s) + else: + wait_for_shutdown(remaining_s) + + +def _apply_teleop_frame(env, action, issue: str | None, haptic_driver, *, render: bool = True) -> bool: + """Step one valid action, or hold the scene while an unsafe tracker frame is rejected. + + ``render`` is false under OVXR, which owns rendering and republishes the unchanged scene. + """ + if issue is not None: + if render: + env.sim.render() + if haptic_driver is not None: + haptic_driver.stop() + return False + # process actions + actions = action.repeat(env.num_envs, 1) + # apply actions + env.step(actions) + # render controller haptics from post-step contact forces + if haptic_driver is not None: + haptic_driver.update() + return True + + +def _ovxr_publication_identity( + physics_step_id: int, *, reset_generation: int | None = None +) -> int | tuple[str, int, int]: + """Return a dedup identity that also represents a reset at an unchanged physics step.""" + if reset_generation is None: + return physics_step_id + return ("reset", reset_generation, physics_step_id) + + def _install_ovxr_shutdown_handlers(ov_libraries_xr_session) -> dict: """Make owner-script signals request the same orderly shutdown as OVUI.""" if ov_libraries_xr_session is None: @@ -410,11 +425,11 @@ def run_teleoperation( # noqa: C901 teleop_cfg: IsaacTeleopCfg | DeviceCfg, use_isaac_teleop: bool, ) -> None: - """Run teleoperation through the selected Kit or OVXR runtime.""" + """Create the environment and teleop device, then run the teleoperation loop until the app is closed.""" from isaaclab.sim import SimulationContext - from .. import XrCameraFeedSession + from .. import XrCameraFeedSession, poll_control_events previous_context = SimulationContext.instance() camera_feed_session = XrCameraFeedSession.prepare( @@ -480,19 +495,40 @@ def run_teleoperation( # noqa: C901 # Callback handlers def reset_recording_instance() -> None: - """Request an environment reset at the next simulation step.""" + """ + Reset the environment to its initial state. + + Sets a flag to reset the environment on the next simulation step. + + Returns: + None + """ nonlocal should_reset_recording_instance should_reset_recording_instance = True print("Reset triggered - Environment will reset on next step") def start_teleoperation() -> None: - """Resume applying teleoperation commands to the environment.""" + """ + Activate teleoperation control of the robot. + + Enables the application of teleoperation commands to the environment. + + Returns: + None + """ nonlocal teleoperation_active teleoperation_active = True print("Teleoperation activated") def stop_teleoperation() -> None: - """Pause applying teleoperation commands to the environment.""" + """ + Deactivate teleoperation control of the robot. + + Disables the application of teleoperation commands to the environment. + + Returns: + None + """ nonlocal teleoperation_active teleoperation_active = False print("Teleoperation deactivated") @@ -515,14 +551,11 @@ def stop_teleoperation() -> None: # Always active for other devices teleoperation_active = True - # Use the upstream device factory for Capture and native inputs alike. - from .. import poll_control_events - try: teleop_interface = create_teleop_device( teleop_cfg, env_prim_path=env.scene.env_prim_paths[0], - sim_device=env.sim.device, + sim_device=args_cli.device, callbacks=teleoperation_callbacks, visualizers=env.sim.visualizers, cloudxr_env_file=(None if args_cli.using_ovxr else args_cli.cloudxr_env), @@ -556,6 +589,7 @@ def stop_teleoperation() -> None: action_state: dict = {} if use_isaac_teleop: + # the device reads these from the focused visualizer window print("Isaac Capture control keys: [B] start/resume [P] pause [R] reset") def run_loop(): @@ -705,6 +739,7 @@ def run_loop(): # Install after environment/device initialization because embedded native # libraries may replace inherited signal dispositions during startup. previous_shutdown_handlers = _install_ovxr_shutdown_handlers(ov_libraries_xr_session) + # Run the teleoperation loop # Isaac Capture requires a context manager, native devices don't if use_isaac_teleop: if ov_libraries_xr_session is not None: @@ -737,13 +772,6 @@ def run_loop(): ) -def main() -> None: - """Resolve the task and run teleoperation with its selected runtime.""" - resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop = _resolve_env_cfg() - with launch_simulation(resolved_env_cfg, args_cli, cleanup_context=not args_cli.using_ovxr): - run_teleoperation(resolved_env_cfg, selected_teleop_cfg, uses_isaac_teleop) - - def run_teleop_se3_agent_cli(argv: list[str] | None = None) -> int: """Run interactive teleoperation from the command line. diff --git a/source/isaaclab_teleop/test/test_teleop_failure_exit.py b/source/isaaclab_teleop/test/test_teleop_failure_exit.py index 88baec593a2..eac6d229c0b 100644 --- a/source/isaaclab_teleop/test/test_teleop_failure_exit.py +++ b/source/isaaclab_teleop/test/test_teleop_failure_exit.py @@ -65,6 +65,7 @@ def test_live_entry_failure_closes_resources_and_reports_cause( { "args_cli": SimpleNamespace( task="test-task", + device="cpu", xr=False, using_ovxr=False, disable_external_cameras=True, From f749edebda49eed9deae361c30528a4af22558a4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:19:04 -0500 Subject: [PATCH 102/145] Restore upstream XR anchor docstrings, logging and optional carb import --- .../isaaclab_teleop/xr_anchor_manager.py | 18 +++---- .../isaaclab_teleop/xr_anchor_utils.py | 52 +++++++++++++++---- .../test/test_xr_anchor_manager_contract.py | 20 +------ 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py index 328c9aab8cb..b0648d0dc6c 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_manager.py @@ -18,6 +18,10 @@ from .xr_cfg import XrCfg # Import Kit components with fallback for sessions without Kit +carb = None +with contextlib.suppress(ModuleNotFoundError): + import carb + XRCore = None XRCoreEventType = None with contextlib.suppress(ModuleNotFoundError): @@ -172,15 +176,11 @@ def __init__(self, xr_cfg: XrCfg, *, ovxr_session: bool = False): self._xr_anchor_headset_path, self._xr_render_anchor_path ) - # Configure settings only when Kit owns a live XRCore. Kitless external- - # present sessions retain this anchor but have no carb settings interface. - if self._xr_core is not None and has_kit(): - import carb.settings # noqa: PLC0415 - - settings = carb.settings.get_settings() - settings.set_float("/persistent/xr/render/nearPlane", self._xr_cfg.near_plane) - settings.set_string("/persistent/xr/anchorMode", "custom anchor") - settings.set_string("/xrstage/customAnchor", self._xr_render_anchor_path) + # Configure carb settings for XR rendering; carb can import while Kit is not running + if carb is not None and has_kit(): + carb.settings.get_settings().set_float("/persistent/xr/render/nearPlane", self._xr_cfg.near_plane) + carb.settings.get_settings().set_string("/persistent/xr/anchorMode", "custom anchor") + carb.settings.get_settings().set_string("/xrstage/customAnchor", self._xr_render_anchor_path) self._anchor_sync: XrAnchorSynchronizer | None = None if self._xr_core is not None: diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py index a9ac5c47932..2143dcc66fa 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_anchor_utils.py @@ -40,12 +40,18 @@ def reset(self): self.__smoothed_anchor_quat = None self.__last_anchor_quat = None self.__anchor_rotation_enabled = True - self.__cached_world_pos = None - self.__cached_world_quat_xyzw = None + + # Cached anchor world transform (pos, quat_xyzw) set by update(). + # Reading back from the prim hierarchy is unreliable when the anchor is a child of a + # physics-driven prim (e.g. the pelvis) because Fabric computes the hierarchy world + # matrix using the physics-updated parent while the local xform was decomposed against + # the USD-level parent, which can diverge. + self.__cached_world_pos: np.ndarray | None = None + self.__cached_world_quat_xyzw: np.ndarray | None = None def toggle_anchor_rotation(self): self.__anchor_rotation_enabled = not self.__anchor_rotation_enabled - logger.info("XR: Toggling anchor rotation: %s", self.__anchor_rotation_enabled) + logger.info(f"XR: Toggling anchor rotation: {self.__anchor_rotation_enabled}") def get_world_transform(self): if self.__cached_world_pos is None: @@ -167,7 +173,7 @@ def __init__( self._policy = _XrAnchorPosePolicy(xr_cfg) stage = get_current_stage() anchor = stage.GetPrimAtPath(xr_anchor_headset_path) - self._layer_identifier = anchor.GetPrimStack()[0].layer.identifier + self.__anchor_headset_layer_identifier = anchor.GetPrimStack()[0].layer.identifier self._render_anchor_path = xr_render_anchor_path self._render_anchor_offset = None if xr_render_anchor_path is not None and xr_render_anchor_path != xr_anchor_headset_path: @@ -192,11 +198,37 @@ def reset(self): def toggle_anchor_rotation(self): self._policy.toggle_anchor_rotation() - def get_world_transform(self): + def get_world_transform(self) -> tuple[np.ndarray, np.ndarray] | None: + """Return the anchor world transform. + + Returns the cached world transform that was computed by the most recent + call to :meth:`sync_headset_to_anchor`. Using the cached value avoids + a Fabric/USD layer mismatch: when the XR anchor prim is a child of a + physics-driven prim (e.g. the robot pelvis), reading + ``GetFabricHierarchyWorldMatrixAttr`` would compose the Fabric-side + parent transform (updated by physics) with a local xform that was + decomposed against the USD-side parent (which can lag behind), + producing an incorrect world matrix that drifts as the robot moves. + + Returns: + A ``(position, quat_xyzw)`` tuple of numpy float64 arrays, + or ``None`` if :meth:`sync_headset_to_anchor` has not run yet. + """ return self._policy.get_world_transform() def sync_headset_to_anchor(self): - """Read the Fabric reference and update the Kit anchor once per pre-sync.""" + """Sync XR anchor pose in USD for both dynamic and static anchoring. + + For **dynamic** anchoring (``anchor_prim_path`` is set), the reference + prim's world position is read from Fabric and ``anchor_pos`` is added + as an offset. For **static** anchoring (no prim path), ``anchor_pos`` + is used directly as the world position. + + In both cases the function calls ``set_world_transform_matrix`` on the + XR core so that the rendering anchor and the pipeline's + ``world_T_anchor`` matrix are guaranteed to agree, and caches the + world transform for :meth:`get_world_transform`. + """ try: reference = None if self._reference_world_matrix is not None: @@ -207,11 +239,13 @@ def sync_headset_to_anchor(self): lambda: SimulationContext.instance().get_rendering_dt(), getattr(self, "_previous_headpose", None), ) - self._xr_core.set_world_transform_matrix(self._xr_anchor_headset_path, matrix, self._layer_identifier) + self._xr_core.set_world_transform_matrix( + self._xr_anchor_headset_path, matrix, self.__anchor_headset_layer_identifier + ) if self._render_anchor_offset is not None: # Gf uses row vectors: apply the view offset before the filtered world pose. self._xr_core.set_world_transform_matrix( self._render_anchor_path, self._render_anchor_offset * matrix, self._render_layer_identifier ) - except Exception as exc: - logger.warning("XR: Anchor sync failed: %s", exc) + except Exception as e: + logger.warning(f"XR: Anchor sync failed: {e}") diff --git a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py index 0ba5e45e65e..7ccd8494354 100644 --- a/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py +++ b/source/isaaclab_teleop/test/test_xr_anchor_manager_contract.py @@ -7,8 +7,7 @@ from __future__ import annotations -import sys -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace from unittest.mock import MagicMock import numpy as np @@ -116,23 +115,6 @@ def test_public_device_exposes_dynamic_anchor_only_for_ovxr_session(monkeypatch, lifecycle.assert_called_once() -def test_no_xr_core_does_not_acquire_carb_settings(monkeypatch): - settings = ModuleType("carb.settings") - settings.get_settings = MagicMock(side_effect=AssertionError("kitless anchor touched carb settings")) - carb = ModuleType("carb") - carb.settings = settings - monkeypatch.setitem(sys.modules, "carb", carb) - monkeypatch.setitem(sys.modules, "carb.settings", settings) - monkeypatch.setattr(xr_anchor_manager, "XRCore", None) - monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) - cfg = XrCfg(anchor_pos=(1.0, 2.0, 3.0), anchor_rot=(0.0, 0.0, 0.0, 1.0)) - - manager = xr_anchor_manager.XrAnchorManager(cfg) - - settings.get_settings.assert_not_called() - np.testing.assert_allclose(manager.get_world_matrix()[:3, 3], cfg.anchor_pos) - - def test_render_anchor_does_not_offset_tracked_device_poses(monkeypatch): created_prims = [] monkeypatch.setattr(xr_anchor_manager, "XRCore", None) From c4b9b1c7949bb6be6dd45cde81a2b6c6168caaa0 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:19:23 -0500 Subject: [PATCH 103/145] Cover empty CPU Warp conversion in its own proxy-array test --- source/isaaclab/test/utils/warp/test_proxy_array.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/test/utils/warp/test_proxy_array.py b/source/isaaclab/test/utils/warp/test_proxy_array.py index 8e96e56eb5e..8ff5b089373 100644 --- a/source/isaaclab/test/utils/warp/test_proxy_array.py +++ b/source/isaaclab/test/utils/warp/test_proxy_array.py @@ -461,7 +461,7 @@ def test_proxy_array_warning_is_one_shot(self): assert not deprecation, "shim must not re-warn after the first call" def test_requires_grad_forwarded_to_raw_wp_array(self): - """Native and empty CPU views preserve dtype, shape and gradient selection.""" + """``wp.to_torch(wp.array)`` still produces a torch view and forwards ``requires_grad``.""" import isaaclab.utils.warp # noqa: F401 # ensure shim is installed arr = wp.array([1.0, 2.0], dtype=wp.float32, device="cpu") @@ -470,6 +470,10 @@ def test_requires_grad_forwarded_to_raw_wp_array(self): assert t.shape == (2,) assert t.requires_grad is False + def test_empty_cpu_arrays_convert_through_dlpack(self): + """Empty CPU views preserve dtype, shape and gradient selection.""" + import isaaclab.utils.warp # noqa: F401 # ensure shim is installed + empty_ids = wp.from_torch(torch.arange(4, dtype=torch.int32)[:0]) ids = wp.to_torch(empty_ids) assert ids.shape == (0,) and ids.dtype == torch.int32 From a237fd4bc275ee698f4536306c3ea3c4bbb9a7c7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:20:05 -0500 Subject: [PATCH 104/145] Restore the upstream Capture stubs in the control-event and hand tests --- .../test/test_control_events.py | 75 ++++++++++++++++++- .../test/test_hand_debug_viz.py | 43 ++++++----- 2 files changed, 95 insertions(+), 23 deletions(-) diff --git a/source/isaaclab_teleop/test/test_control_events.py b/source/isaaclab_teleop/test/test_control_events.py index 1eb331bc824..556d0618521 100644 --- a/source/isaaclab_teleop/test/test_control_events.py +++ b/source/isaaclab_teleop/test/test_control_events.py @@ -18,14 +18,81 @@ import dataclasses import json -from types import SimpleNamespace +import sys +from types import ModuleType, SimpleNamespace from unittest.mock import MagicMock import pytest -# Capture's retargeting API is usable without starting a simulator. Use the real -# optional dependency so this module cannot replace packages seen by other tests. -pytest.importorskip("isaacteleop.retargeting_engine.interface") +# --------------------------------------------------------------------------- +# Stub out isaacteleop modules before any isaaclab_teleop imports so the +# tests can run in a plain Python environment without Omniverse. +# --------------------------------------------------------------------------- + +_MODULES_TO_STUB = [ + "isaacteleop", + "isaacteleop.deviceio", + "isaacteleop.deviceio_trackers", + "isaacteleop.retargeting_engine", + "isaacteleop.retargeting_engine.deviceio_source_nodes", + "isaacteleop.retargeting_engine.deviceio_source_nodes.deviceio_tensor_types", + "isaacteleop.retargeting_engine.interface", + "isaacteleop.retargeting_engine.interface.retargeter_core_types", + "isaacteleop.retargeting_engine.interface.tensor_group_type", + "isaacteleop.retargeting_engine_ui", + "isaacteleop.schema", + "isaacteleop.teleop_session_manager", + "isaacteleop.teleop_session_manager.teleop_state_manager_retargeter", + "isaacteleop.teleop_session_manager.teleop_state_manager_types", +] + +_stubs: dict[str, ModuleType | MagicMock] = {} + + +def _install_stubs(): + for name in _MODULES_TO_STUB: + if name not in sys.modules: + _stubs[name] = MagicMock() + sys.modules[name] = _stubs[name] + + from enum import Enum + + class ExecutionState(str, Enum): + UNKNOWN = "unknown" + STOPPED = "stopped" + PAUSED = "paused" + RUNNING = "running" + + @dataclasses.dataclass + class ExecutionEvents: + reset: bool = False + execution_state: ExecutionState = ExecutionState.UNKNOWN + + ee_mod = sys.modules["isaacteleop.retargeting_engine.interface.execution_events"] = ModuleType( + "isaacteleop.retargeting_engine.interface.execution_events" + ) + ee_mod.ExecutionState = ExecutionState # type: ignore[attr-defined] + ee_mod.ExecutionEvents = ExecutionEvents # type: ignore[attr-defined] + + iface = sys.modules["isaacteleop.retargeting_engine.interface"] + iface.ExecutionState = ExecutionState # type: ignore[attr-defined] + iface.ExecutionEvents = ExecutionEvents # type: ignore[attr-defined] + iface.RetargeterIOType = dict # type: ignore[attr-defined] + + class FakeBaseRetargeter: + def __init__(self, name: str) -> None: + self.name = name + + iface.BaseRetargeter = FakeBaseRetargeter # type: ignore[attr-defined] + + tsm_types = sys.modules["isaacteleop.teleop_session_manager.teleop_state_manager_types"] + tsm_types.bool_signal = MagicMock # type: ignore[attr-defined] + + dt_mod = sys.modules["isaacteleop.retargeting_engine.deviceio_source_nodes.deviceio_tensor_types"] + dt_mod.MessageChannelMessagesTrackedGroup = MagicMock # type: ignore[attr-defined] + + +_install_stubs() from isaaclab_teleop.control_events import ControlEvents, poll_control_events # noqa: E402 from isaaclab_teleop.session_lifecycle import TeleopSessionLifecycle # noqa: E402 diff --git a/source/isaaclab_teleop/test/test_hand_debug_viz.py b/source/isaaclab_teleop/test/test_hand_debug_viz.py index 9f056071af7..bf8c190c404 100644 --- a/source/isaaclab_teleop/test/test_hand_debug_viz.py +++ b/source/isaaclab_teleop/test/test_hand_debug_viz.py @@ -21,6 +21,7 @@ from __future__ import annotations import sys +from types import ModuleType from unittest.mock import MagicMock, patch import numpy as np @@ -63,7 +64,7 @@ "omni.kit.xr.system.openxr", ] -_stub_patch: pytest.MonkeyPatch | None = None +_stubs_installed: dict[str, ModuleType | MagicMock] = {} class _FakeHandsSource: @@ -121,37 +122,41 @@ def __init__(self, name: str) -> None: def _install_stubs(): - """Temporarily replace dependencies, including attributes on already loaded modules.""" - global _stub_patch - assert _stub_patch is None - _stub_patch = pytest.MonkeyPatch() + """Insert MagicMock modules for all heavy dependencies.""" for name in _MODULES_TO_STUB: if name not in sys.modules: - _stub_patch.setitem(sys.modules, name, MagicMock()) + sys.modules[name] = _stubs_installed.setdefault(name, MagicMock()) if "." in name: parent_name, child_name = name.rsplit(".", 1) - _stub_patch.setattr(sys.modules[parent_name], child_name, sys.modules[name], raising=False) + setattr(sys.modules[parent_name], child_name, sys.modules[name]) dsn_mod = sys.modules["isaacteleop.retargeting_engine.deviceio_source_nodes"] - _stub_patch.setattr(dsn_mod, "HandsSource", _FakeHandsSource, raising=False) - _stub_patch.setattr(dsn_mod, "ControllersSource", _FakeControllersSource, raising=False) + dsn_mod.HandsSource = _FakeHandsSource # type: ignore[attr-defined] + dsn_mod.ControllersSource = _FakeControllersSource # type: ignore[attr-defined] iface_mod = sys.modules["isaacteleop.retargeting_engine.interface"] - _stub_patch.setattr(iface_mod, "OutputCombiner", _FakeOutputCombiner, raising=False) - _stub_patch.setattr(iface_mod, "BaseRetargeter", _FakeBaseRetargeter, raising=False) - _stub_patch.setattr(iface_mod, "RetargeterIOType", dict, raising=False) + iface_mod.OutputCombiner = _FakeOutputCombiner # type: ignore[attr-defined] + iface_mod.BaseRetargeter = _FakeBaseRetargeter # type: ignore[attr-defined] + iface_mod.RetargeterIOType = dict # type: ignore[attr-defined] tt_mod = sys.modules["isaacteleop.retargeting_engine.tensor_types"] - _stub_patch.setattr(tt_mod, "HandInputIndex", _FakeHandInputIndex, raising=False) - _stub_patch.setattr(tt_mod, "ControllerInputIndex", _FakeControllerInputIndex, raising=False) + tt_mod.HandInputIndex = _FakeHandInputIndex # type: ignore[attr-defined] + tt_mod.ControllerInputIndex = _FakeControllerInputIndex # type: ignore[attr-defined] def _restore_stubs(): - """Restore both inserted modules and overwritten attributes exactly.""" - global _stub_patch - if _stub_patch is not None: - _stub_patch.undo() - _stub_patch = None + """Remove stubs installed for this test module from ``sys.modules``.""" + for name in reversed(_MODULES_TO_STUB): + stub = _stubs_installed.get(name) + if stub is None: + continue + if "." in name: + parent_name, child_name = name.rsplit(".", 1) + parent = sys.modules.get(parent_name) + if parent is not None and getattr(parent, child_name, None) is stub: + delattr(parent, child_name) + if sys.modules.get(name) is stub: + del sys.modules[name] _install_stubs() From 6f2376f8e1fd005bb91490118fd4afe2aa7d5d08 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:20:35 -0500 Subject: [PATCH 105/145] Keep the XR experience check in the upstream teleop smoke test --- .../test/test_teleop_entrypoint_units.py | 48 --------------- .../test/test_teleop_scripts_smoke.py | 61 +++++++++++++++---- 2 files changed, 50 insertions(+), 59 deletions(-) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index a7b6de34495..350fa8a761a 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -6,7 +6,6 @@ """In-process teleoperation entrypoint, configuration, and control-loop contracts.""" import os -import re import signal import sys import time @@ -19,53 +18,6 @@ # Repository root: this file lives at source/isaaclab_teleop/test/.py ROOT = Path(__file__).resolve().parents[3] -# XR experience file whose dependencies decide whether ``--xr`` can start offline. -_XR_EXPERIENCE = ROOT / "apps" / "isaaclab.python.xr.openxr.kit" - -# ``.kit`` files repeat section headers, so they cannot be parsed as TOML. -_SECTION_RE = re.compile(r"^\s*\[+(?P[^\[\]]+)\]+\s*$") -_DEPENDENCY_RE = re.compile(r'^\s*"(?P[^"]+)"\s*=') - - -def _kit_dependencies(experience: Path) -> set[str]: - """Collect the extension names declared in an experience file's dependency sections. - - Scoped to ``[dependencies]`` tables so quoted keys elsewhere in the file (settings tables - use them too) are not mistaken for extensions. - """ - dependencies: set[str] = set() - in_dependencies = False - - for line in experience.read_text(encoding="utf-8").splitlines(): - section = _SECTION_RE.match(line) - if section is not None: - in_dependencies = section.group("name").strip() == "dependencies" - continue - if in_dependencies: - dependency = _DEPENDENCY_RE.match(line) - if dependency is not None: - dependencies.add(dependency.group("name")) - - return dependencies - - -def test_xr_experience_declares_only_shipped_extensions(): - """The XR experience depends on XR extensions Isaac Sim ships, not on a registry-only bundle. - - ``omni.kit.xr.bundle.generic`` is not shipped and resolves only from the Kit SDK registry, so - depending on it aborted every ``--xr`` run that could not reach the registry. - - Static on purpose: a machine that reaches the registry caches the bundle on first use and - loads it locally forever after, so a launch test cannot catch this once that cache is warm. - """ - dependencies = _kit_dependencies(_XR_EXPERIENCE) - - assert "omni.kit.xr.system.openxr" in dependencies - assert "omni.kit.xr.ui.window.profile" in dependencies, "omni.kit.xr.profile.ar's Kit 110 successor is missing" - assert not [name for name in dependencies if name.startswith("omni.kit.xr.bundle.")], ( - "XR bundle meta-extensions are not shipped with Isaac Sim and resolve only from the Kit SDK registry" - ) - @pytest.mark.parametrize( ("kitless", "selection", "expected", "reject"), diff --git a/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py b/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py index 0773650c5dc..1796331576e 100644 --- a/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py +++ b/source/isaaclab_teleop/test/test_teleop_scripts_smoke.py @@ -27,6 +27,7 @@ """ import os +import re import signal import subprocess import sys @@ -35,9 +36,40 @@ import pytest +pytestmark = [pytest.mark.integration, pytest.mark.rendering] + # Repository root: this file lives at source/isaaclab_teleop/test/.py ROOT = Path(__file__).resolve().parents[3] +# XR experience file whose dependencies decide whether ``--xr`` can start offline. +_XR_EXPERIENCE = ROOT / "apps" / "isaaclab.python.xr.openxr.kit" + +# ``.kit`` files repeat section headers, so they cannot be parsed as TOML. +_SECTION_RE = re.compile(r"^\s*\[+(?P[^\[\]]+)\]+\s*$") +_DEPENDENCY_RE = re.compile(r'^\s*"(?P[^"]+)"\s*=') + + +def _kit_dependencies(experience: Path) -> set[str]: + """Collect the extension names declared in an experience file's dependency sections. + + Scoped to ``[dependencies]`` tables so quoted keys elsewhere in the file (settings tables + use them too) are not mistaken for extensions. + """ + dependencies: set[str] = set() + in_dependencies = False + + for line in experience.read_text(encoding="utf-8").splitlines(): + section = _SECTION_RE.match(line) + if section is not None: + in_dependencies = section.group("name").strip() == "dependencies" + continue + if in_dependencies: + dependency = _DEPENDENCY_RE.match(line) + if dependency is not None: + dependencies.add(dependency.group("name")) + + return dependencies + # Lightweight Franka task that configures an Isaac Capture pipeline (controllers), so omitting # ``--teleop_device`` drives the scripts through the Isaac Capture stack rather than the legacy @@ -135,8 +167,6 @@ def _assert_started_cleanly(output: str, markers: list[str], script: str) -> Non ) -@pytest.mark.integration -@pytest.mark.rendering def test_teleop_se3_agent_starts(tmp_path): """teleop_se3_agent.py builds the Isaac Capture device (past the #6656 RTX/camera regression). @@ -155,8 +185,6 @@ def test_teleop_se3_agent_starts(tmp_path): _assert_started_cleanly(output, [_ISAAC_TELEOP_MARKER], "teleop_se3_agent.py") -@pytest.mark.integration -@pytest.mark.rendering def test_record_demos_starts(tmp_path): """record_demos.py builds the Isaac Capture device without AttributeError-ing on enable_cameras. @@ -176,8 +204,6 @@ def test_record_demos_starts(tmp_path): _assert_started_cleanly(output, [_ISAAC_TELEOP_MARKER], "record_demos.py") -@pytest.mark.integration -@pytest.mark.rendering def test_record_demos_starts_when_reward_references_success(tmp_path): """record_demos.py preserves success-term references while disabling automatic resets.""" argv = [ @@ -190,14 +216,29 @@ def test_record_demos_starts_when_reward_references_success(tmp_path): str(tmp_path / "dataset.hdf5"), "physics=isaacsim_physx", "presets=diffik", - *_NO_CLOUDXR, ] output = _launch_until_marker(argv, [_ENV_READY_MARKER], tmp_path / "record_demos_success_reward.log") _assert_started_cleanly(output, [_ENV_READY_MARKER], "record_demos.py with a success reward") -@pytest.mark.integration -@pytest.mark.rendering +def test_xr_experience_declares_only_shipped_extensions(): + """The XR experience depends on XR extensions Isaac Sim ships, not on a registry-only bundle. + + ``omni.kit.xr.bundle.generic`` is not shipped and resolves only from the Kit SDK registry, so + depending on it aborted every ``--xr`` run that could not reach the registry. + + Static on purpose: a machine that reaches the registry caches the bundle on first use and + loads it locally forever after, so a launch test cannot catch this once that cache is warm. + """ + dependencies = _kit_dependencies(_XR_EXPERIENCE) + + assert "omni.kit.xr.system.openxr" in dependencies + assert "omni.kit.xr.ui.window.profile" in dependencies, "omni.kit.xr.profile.ar's Kit 110 successor is missing" + assert not [name for name in dependencies if name.startswith("omni.kit.xr.bundle.")], ( + "XR bundle meta-extensions are not shipped with Isaac Sim and resolve only from the Kit SDK registry" + ) + + def test_record_demos_starts_headless_xr(tmp_path): """record_demos.py starts under ``--xr`` with no ``--viz kit`` (the QA-reported failure). @@ -226,8 +267,6 @@ def test_record_demos_starts_headless_xr(tmp_path): _assert_started_cleanly(output, markers, "record_demos.py --xr") -@pytest.mark.integration -@pytest.mark.rendering def test_teleop_replay_agent_applies_rtx_without_replicator_crash(tmp_path): """teleop_replay_agent.py clears the RTX-apply site (create env) without the replicator crash. From f6d3c939fc7a71b599de2dda66f1088067252cf2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:21:30 -0500 Subject: [PATCH 106/145] Keep the upstream SO101 pipeline test and derive its full-pose home --- .../contrib/test_so101_teleop_pipeline.py | 112 ++++++------------ 1 file changed, 35 insertions(+), 77 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py index e3dec65bc14..a2bffd6544f 100644 --- a/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py +++ b/source/isaaclab_tasks/test/contrib/test_so101_teleop_pipeline.py @@ -3,31 +3,16 @@ # # SPDX-License-Identifier: BSD-3-Clause -"""Exercise the real SO-101 Capture graph after hardware controller conversion, without XR/GPU.""" +"""Run the SO-101 Capture graph with encoded controller snapshots and verify its 8D action contract.""" import numpy as np import pytest from scipy.spatial.transform import Rotation -# The test drives Capture's retargeting-engine cache execution; skip releases without the parts it uses. -_core_types = pytest.importorskip("isaacteleop.retargeting_engine.interface.retargeter_core_types") -if not ( - hasattr(_core_types, "ExecutionCache") - and hasattr(_core_types, "GraphTime") - and hasattr(getattr(_core_types, "GraphExecutable", None), "execute_pipeline_with_cache") -): - pytest.skip("Capture does not provide retargeting-engine cache execution", allow_module_level=True) +pytest.importorskip("isaacteleop") -from isaacteleop.retargeting_engine.deviceio_source_nodes import ControllersSource -from isaacteleop.retargeting_engine.interface import ( - ComputeContext, - ExecutionEvents, - ExecutionState, - OptionalTensorGroup, - TensorGroup, -) -from isaacteleop.retargeting_engine.interface.retargeter_core_types import ExecutionCache, GraphTime -from isaacteleop.retargeting_engine.tensor_types import ControllerInput, ControllerInputIndex, TransformMatrix +from isaacteleop.retargeting_engine.interface import ComputeContext, ExecutionEvents, ExecutionState, TensorGroup +from isaacteleop.schema import ControllerInputState, ControllerPose, ControllerSnapshot, Point, Pose, Quaternion from isaaclab_tasks.contrib.stack.config.so101.stack_ik_abs_env_cfg import ( _BASE_T_GRIPPER_HOME, @@ -35,65 +20,38 @@ ) -def test_so101_pipeline_full_pose_clutch_and_action_order(): - """Home, motion, re-clutch and reset survive the real transform/retarget/flatten graph.""" +def test_so101_pipeline_tracks_engage_relative_pose_and_gripper(): + """Pose/closedness occupy the declared slots, and clutching holds the pose across release and re-engage.""" pipeline = _build_so101_stack_pipeline() - controllers = next(node for node in pipeline.get_leaf_nodes() if isinstance(node, ControllersSource)) - anchor = TensorGroup(TransformMatrix()) - anchor[0] = np.eye(4, dtype=np.float32) - - def step(position, rotation, *, running=True, trigger=0.0, reset=False): - controller = TensorGroup(ControllerInput()) - # Populate the same complete typed input as ControllersSource, replacing hardware polling - # only. Cached source output still traverses the real frame transform and all retargeters. - for index in ControllerInputIndex: - if index in (ControllerInputIndex.GRIP_POSITION, ControllerInputIndex.AIM_POSITION): - controller[index] = np.asarray(position, dtype=np.float32) - elif index in (ControllerInputIndex.GRIP_ORIENTATION, ControllerInputIndex.AIM_ORIENTATION): - controller[index] = rotation.as_quat().astype(np.float32) - elif index in (ControllerInputIndex.GRIP_IS_VALID, ControllerInputIndex.AIM_IS_VALID): - controller[index] = True - else: - controller[index] = 0.0 - controller[ControllerInputIndex.SQUEEZE_VALUE] = float(running) - controller[ControllerInputIndex.TRIGGER_VALUE] = trigger - context = ComputeContext( - graph_time=GraphTime(sim_time_ns=0, real_time_ns=0), - execution_events=ExecutionEvents( - reset=reset, execution_state=ExecutionState.RUNNING if running else ExecutionState.STOPPED - ), + assert pipeline.output_types()["action"].types[0].shape == (8,) + inputs = { + leaf.name: {name: TensorGroup(group_type) for name, group_type in leaf.input_spec().items()} + for leaf in pipeline.get_leaf_nodes() + } + inputs["world_T_anchor"]["value"][0] = np.eye(4, dtype=np.float32) + inputs["controllers"]["deviceio_controller_left"][0] = None + + def step(position, orientation, squeeze=1.0, trigger=1.0, reset=False): + inputs["controllers"]["deviceio_controller_right"][0] = ControllerSnapshot( + grip_pose=ControllerPose(Pose(Point(*position), Quaternion(*orientation)), True), + inputs=ControllerInputState(False, False, False, False, 0.0, 0.0, squeeze, trigger), ) - cache = ExecutionCache({"world_T_anchor": {"value": anchor}}, context) - cache.cache( - id(controllers), - {ControllersSource.LEFT: OptionalTensorGroup(ControllerInput()), ControllersSource.RIGHT: controller}, - ) - action = np.from_dlpack(pipeline.execute_pipeline_with_cache(cache)["action"][0]).copy() - assert action.shape == (8,) - assert np.isfinite(action).all() - return action - - def check(action, position, rotation, gripper): - np.testing.assert_allclose(action[:3], position, atol=1e-6) - np.testing.assert_allclose(Rotation.from_quat(action[3:7]).as_matrix(), rotation.as_matrix(), atol=1e-6) - assert action[7] == pytest.approx(gripper) + context = ComputeContext(execution_events=ExecutionEvents(reset=reset, execution_state=ExecutionState.RUNNING)) + action = pipeline.execute_pipeline(inputs, context)["action"][0] + assert action.shape == (8,) and action.dtype == np.float32 + return action.copy() - home_position = _BASE_T_GRIPPER_HOME[:3, 3] home_rotation = Rotation.from_matrix(_BASE_T_GRIPPER_HOME[:3, :3]) - origin = Rotation.from_euler("xyz", [0.2, -0.4, 0.3]) - position = np.array([0.3, -0.1, 0.5]) - check(step(position, origin, running=False, reset=True), home_position, home_rotation, 0.0) - check(step(position, origin), home_position, home_rotation, 0.0) - delta = Rotation.from_rotvec([0.0, 0.0, 0.3]) - displacement = np.array([0.02, -0.03, 0.01]) - moved_position = home_position + displacement - moved_rotation = delta * home_rotation - check(step(position + displacement, delta * origin, trigger=1.0), moved_position, moved_rotation, 1.0) - other = Rotation.from_euler("xyz", [-0.5, 0.2, 0.8]) - check(step(-position, other, running=False), moved_position, moved_rotation, 0.0) - check(step(-position, other), moved_position, moved_rotation, 0.0) - # Re-engaging composes from the held full pose. - delta2 = Rotation.from_rotvec([0.2, 0.0, 0.0]) - check(step(-position + displacement, delta2 * other), moved_position + displacement, delta2 * moved_rotation, 0.0) - check(step(position, origin, running=False, reset=True), home_position, home_rotation, 0.0) - check(step(-position, other), home_position, home_rotation, 0.0) + home = [*_BASE_T_GRIPPER_HOME[:3, 3], *home_rotation.as_quat(), 1.0] + # The first squeeze emits the configured home even with a rotated controller. + np.testing.assert_allclose(step((0.2, 0.3, 0.4), (0.0, 0.0, 2**-0.5, 2**-0.5)), home, atol=1e-6) + # Relative to the initial Z quarter-turn, this controller quaternion gives [1, 2, 3, 4] / sqrt(30). + # The clutch applies that world-frame rotation and the controller displacement to the full home pose. + delta = Rotation.from_quat(np.array([1, 2, 3, 4]) / np.sqrt(30)) + moved = [*np.add(home[:3], (0.1, 0.2, 0.3)), *(delta * home_rotation).as_quat(), 1.0] + np.testing.assert_allclose(step((0.3, 0.5, 0.7), np.array([3, 1, 7, 1]) / np.sqrt(60)), moved, atol=1e-6) + # Release at another pose, then re-engage there: the commanded arm pose stays held. + held = [*moved[:7], 0.0] + np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), squeeze=0.0, trigger=0.0), held, atol=1e-6) + np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), trigger=0.0), held, atol=1e-6) + np.testing.assert_allclose(step((0.8, 0.9, 1.0), (0.0, 0.0, 0.0, 1.0), reset=True), home, atol=1e-6) From 8021f9ecb8c8c2927d60d27b8654d5941b0f5562 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:22:41 -0500 Subject: [PATCH 107/145] Leave the upstream OpenXR constructor test without a Kit launch The deprecated OpenXR device imports carb at module level, so the test cannot collect before Kit starts. That predates this stack (#8797 records it) and no kitless change needs the launch, so it is not part of this PR. --- .../isaaclab_teleop/test/test_openxr_device_constructors.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/isaaclab_teleop/test/test_openxr_device_constructors.py b/source/isaaclab_teleop/test/test_openxr_device_constructors.py index 6e7d59663b1..6f971b8ca64 100644 --- a/source/isaaclab_teleop/test/test_openxr_device_constructors.py +++ b/source/isaaclab_teleop/test/test_openxr_device_constructors.py @@ -7,11 +7,6 @@ from typing import cast import pytest - -from isaaclab.test.utils import launch_test_simulation - -launch_test_simulation() - from isaaclab_teleop.deprecated.openxr import OpenXRDevice, OpenXRDeviceCfg, XrCfg from isaaclab_teleop.deprecated.openxr.retargeters import ( GripperRetargeter, From 52d82bc2a31607815e6c42a69283a733494941c6 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:23:33 -0500 Subject: [PATCH 108/145] Narrow the OvPhysX contact-setting docs and quiet the hand-presence log --- docs/source/concepts/physics-backends/ovphysx.rst | 8 ++++---- .../isaaclab_ov/physics/ovphysx_manager_cfg.py | 12 +++++------- .../isaaclab_teleop/session_lifecycle.py | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/docs/source/concepts/physics-backends/ovphysx.rst b/docs/source/concepts/physics-backends/ovphysx.rst index 1f4e9c669b7..7ed7db97675 100644 --- a/docs/source/concepts/physics-backends/ovphysx.rst +++ b/docs/source/concepts/physics-backends/ovphysx.rst @@ -14,10 +14,10 @@ OvPhysX OvPhysX runs PhysX without Omniverse Kit. It reads scene-level solver settings from the USD ``PhysicsScene`` prim. :class:`~isaaclab_ov.physics.OvPhysxCfg` holds -GPU buffer settings and can also author selected contact settings onto that prim: -``bounce_threshold_velocity``, ``friction_correlation_distance`` and -``solve_articulation_contact_last``. Contact settings left at ``None`` keep the -stage value. +GPU buffer settings that USD does not represent. It can also author selected +contact settings onto that prim: ``bounce_threshold_velocity``, +``friction_correlation_distance`` and ``solve_articulation_contact_last``. +Contact settings left at ``None`` keep the stage value. OvPhysX is selected through :class:`~isaaclab_ov.physics.OvPhysxCfg`: diff --git a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py index a19d5e2683d..f5306f58e7a 100644 --- a/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py +++ b/source/isaaclab_ov/isaaclab_ov/physics/ovphysx_manager_cfg.py @@ -43,13 +43,11 @@ class OvPhysxBackendCfg(BackendCfg): class OvPhysxCfg(PhysicsCfg): """Configuration for the ovphysx physics manager. - PhysX scene-level parameters that this config does not set, such as solver iterations, - are read from the USD PhysicsScene prim. Before loading, the manager authors the scene - settings held here onto that prim: determinism, external forces, GPU buffer sizes (GPU - only) and the optional contact parameters :attr:`bounce_threshold_velocity`, - :attr:`friction_correlation_distance` and :attr:`solve_articulation_contact_last`. A - contact parameter left at ``None`` keeps the stage value, or the ``PhysxSceneAPI`` - schema default when the stage does not author it. + PhysX scene-level parameters (solver iterations, GPU buffer sizes, etc.) are + read from the USD PhysicsScene prim. Before loading, the manager authors the + settings held here onto that prim, including the optional contact parameters + :attr:`bounce_threshold_velocity`, :attr:`friction_correlation_distance` and + :attr:`solve_articulation_contact_last`, which keep the stage value when ``None``. """ class_type = "{DIR}.ovphysx_manager:OvPhysxManager" diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 5281e06f8a4..4e0083cb94d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -882,7 +882,7 @@ def _chain_hand_presence_outputs(user_pipeline, pipeline_outputs: dict) -> None: pipeline_outputs[TeleopSessionLifecycle.HAND_RIGHT_KEY] = leaf.output(HandsSource.RIGHT) logger.debug(f"Auto-discovered HandsSource '{leaf.name}'; chained raw hand-presence outputs") return - logger.info("No HandsSource found in the teleop pipeline; raw hand presence unavailable") + logger.debug("No HandsSource found in the teleop pipeline; raw hand presence unavailable") # ------------------------------------------------------------------ # Control pipeline construction From 31311bf871906f0f2b2650924f18273b89a79a2a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:24:23 -0500 Subject: [PATCH 109/145] Describe the replay controls and render anchors accurately in the changelog No teleop bullet is breaking any more, so the fragment raises a minor version for the new public API. --- .../isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst | 6 ++++-- .../changelog.d/ovxr-live-replay.changed.rst | 3 --- .../isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst | 2 +- .../{ovxr-live-replay.major => ovxr-live-replay.minor} | 0 4 files changed, 5 insertions(+), 6 deletions(-) rename source/isaaclab_teleop/changelog.d/{ovxr-live-replay.major => ovxr-live-replay.minor} (100%) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst index 73b3a24a1e6..0b8d26e926b 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.added.rst @@ -1,10 +1,12 @@ * Added kitless OVXR teleoperation and MCAP replay with independently paced rendering, backend-derived anchors, and ordered shutdown. -* Preserved recorded replay controls by default, with explicit ``--start_on_replay`` for recordings without start events. +* Added ``--start_on_replay`` to the MCAP replay agent. It starts playback at startup and after + each recorded RESET, for captures made without recorded START messages. * Added task hooks to :class:`~isaaclab_teleop.IsaacTeleopCfg`: ``prepare_environment``, ``action_conditioner``, ``action_validator`` and ``require_initial_bimanual_hand_tracking``. * Added ``render_anchor_pos`` and ``render_anchor_rot`` to :class:`~isaaclab_teleop.XrCfg` for a - render-only anchor offset. + render-only anchor offset. Kit and OVXR apply the same filtered tracking pose to the separate + render anchor. * Added external OVXR inputs to :class:`~isaaclab_teleop.IsaacTeleopDevice`, together with the ``target_frame_prim_path``, ``xr_render_anchor_path``, ``external_anchor_state``, ``required_openxr_extensions``, ``session_cleanup_failed`` and ``hand_tracking_presence`` diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst index 664e9837e58..f855d5239ef 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst @@ -1,6 +1,3 @@ -* **Breaking:** Replay follows recorded START and RESET controls by default. Add - ``--start_on_replay`` for recordings made without START events that should begin - immediately. * Changed replay to launch Kit only when the resolved task configuration requires it (for example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst index a65dcaa50f4..4f8bc4eb993 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst @@ -1,4 +1,4 @@ -* Applied the same filtered tracking pose to separate render anchors in Kit and OVXR. +* Fixed replay ending at a recorded RESET; it now resets and waits for the next recorded START. * Deferred anchor USD imports until use so Kit can initialize its own USD provider. * Fixed teleoperation cleanup stopping at the first failure: a retargeting UI or XR anchor cleanup failure no longer skips the Isaac Capture session and CloudXR shutdown, and a failed diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.major b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.minor similarity index 100% rename from source/isaaclab_teleop/changelog.d/ovxr-live-replay.major rename to source/isaaclab_teleop/changelog.d/ovxr-live-replay.minor From 49cecdf34020f69ebe1d35500ca990312bace91b Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:27:13 -0500 Subject: [PATCH 110/145] Leave upstream doc lines intact beside the kitless additions --- .../concepts/physics-backends/ovphysx.rst | 9 ++-- .../entrypoints/teleop_replay_agent.py | 52 +++++++++++-------- .../test/test_session_step_failure.py | 13 ++--- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/docs/source/concepts/physics-backends/ovphysx.rst b/docs/source/concepts/physics-backends/ovphysx.rst index 7ed7db97675..1d609549f8e 100644 --- a/docs/source/concepts/physics-backends/ovphysx.rst +++ b/docs/source/concepts/physics-backends/ovphysx.rst @@ -14,10 +14,11 @@ OvPhysX OvPhysX runs PhysX without Omniverse Kit. It reads scene-level solver settings from the USD ``PhysicsScene`` prim. :class:`~isaaclab_ov.physics.OvPhysxCfg` holds -GPU buffer settings that USD does not represent. It can also author selected -contact settings onto that prim: ``bounce_threshold_velocity``, -``friction_correlation_distance`` and ``solve_articulation_contact_last``. -Contact settings left at ``None`` keep the stage value. +GPU buffer settings that USD does not represent. +It can also author selected contact settings onto that prim: +``bounce_threshold_velocity``, ``friction_correlation_distance`` and +``solve_articulation_contact_last``. Contact settings left at ``None`` keep the +stage value. OvPhysX is selected through :class:`~isaaclab_ov.physics.OvPhysxCfg`: diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 9e73fe59599..9e212c1e383 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -36,10 +36,10 @@ :func:`~isaaclab_teleop.poll_control_events`, gates ``env.step()`` on ``ctrl.is_active`` (so pre-START operator setup frames are rendered without stepping), and handles mid-demo ``ctrl.should_reset`` events - by running the full sim/env/teleop reset cycle. Recorded controls govern - playback by default. For legacy captures started locally without recorded - START messages, pass ``--start_on_replay`` to start at replay startup and - resume after each recorded RESET. + by running the full sim/env/teleop reset cycle. + Recorded controls govern playback by default. For legacy captures started + locally without recorded START messages, pass ``--start_on_replay`` to start + at replay startup and resume after each recorded RESET. End-of-replay termination: Four distinct signals end the current run by breaking out of the @@ -95,9 +95,9 @@ too quickly), it raises ``RuntimeError`` immediately, so the remaining replays never start and no stdout summary or JSON file is written. Partial reports are never emitted: CI relies on - "no JSON output" as an unambiguous failure signal. OVXR owns rendering - independently, so application render-call timing statistics remain empty - for OVXR replays. + "no JSON output" as an unambiguous failure signal. + OVXR owns rendering independently, so application render-call timing + statistics remain empty for OVXR replays. Each active iteration also emits one ``GpuStatsProvider.sample()`` call. The default :class:`NvmlGpuStatsProvider` snapshots GPU @@ -1172,12 +1172,14 @@ def _handle_reset(env: gym.Env, teleop_interface: IsaacTeleopDevice, *, hard_sim Mirrors :func:`isaaclab_teleop.entrypoints.record_demos.handle_reset` (sans the instruction-display update, which the headless replay agent doesn't - own). The optional hard physics reset keeps Pink seeded against fresh - articulation views on PhysX backends. Newton instead retains the solver - constructed before replay. ``env.recorder_manager.reset()`` is a no-op - when no recorders are configured (the default for this script), but kept - for parity with record_demos.py so future recorder additions don't have - to re-derive the call sequence. + own). ``env.sim.reset()`` does the hard physics reinit that keeps Pink + IK seeded against fresh articulation views; see the initial-reset note + in :func:`main`. ``env.recorder_manager.reset()`` is a no-op when no + recorders are configured (the default for this script), but kept for + parity with record_demos.py so future recorder additions don't have to + re-derive the call sequence. + With ``hard_sim_reset=False`` (Newton), the hard reinit is skipped so the + solver constructed before replay stays alive. """ print("Resetting environment...") _reset_replay_environment(env, hard_sim_reset=hard_sim_reset, reset_recorder=True) @@ -1724,18 +1726,22 @@ def main() -> int: persisted to disk; a one-line-per-run stdout summary is always printed. See the module docstring for the full JSON schema. - Cleanup drains replay input before the optional OVXR owner releases its - borrowed environment. Without that owner the batch closes the environment - directly. Failed input drain retains borrowed resources and raises rather - than releasing a stage that a worker may still use. Repeated shutdown signals - request owner shutdown without interrupting its drain. + Resource cleanup is wrapped in a ``try/finally`` so that ``env.close()`` + always runs, even when device construction or any subsequent setup + raises -- otherwise the USD stage would leak across CI runs. + With ``--ovxr``, cleanup first drains replay input and the OVXR viewer, + which may own the environment. A failed input drain retains the borrowed + environment and raises rather than releasing a stage that a worker may + still use. Repeated shutdown signals request owner shutdown without + interrupting its drain. Returns: - ``1`` if the requested batch did not finish. For a complete batch, - return the worst-of-N outcome: ``0`` if every run met task success, - ``2`` if any run hit - ``--max_replay_duration_s``, otherwise ``1`` (any failure or incomplete - run). The report retains each completed run's outcome in either case. + The host process exit code, mapped from the worst-of-N outcome + across the multi-run batch: ``0`` if every run's + ``success_term`` fired, ``2`` if any run hit + ``--max_replay_duration_s``, otherwise ``1`` (any failure or + incomplete run). + A batch that stops before ``--num_replays`` runs complete returns ``1``. """ if args_cli.num_replays < 1: raise ValueError(f"--num_replays must be >= 1; got {args_cli.num_replays}") diff --git a/source/isaaclab_teleop/test/test_session_step_failure.py b/source/isaaclab_teleop/test/test_session_step_failure.py index e07dde4793f..f52b2b402a5 100644 --- a/source/isaaclab_teleop/test/test_session_step_failure.py +++ b/source/isaaclab_teleop/test/test_session_step_failure.py @@ -122,13 +122,9 @@ def _make_lifecycle(**kwargs) -> TeleopSessionLifecycle: return TeleopSessionLifecycle(cfg, **kwargs) -_BORROWED_OVXR_HANDLES = {"use_kit_xr_bridge": False, "external_oxr_handles_provider": lambda: object()} -"""Lifecycle options for an OVXR session whose handles must be proven drained.""" - - -def _make_failing_lifecycle(error: Exception, **kwargs) -> TeleopSessionLifecycle: +def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: """Lifecycle with a live session mock whose step raises *error*.""" - lifecycle = _make_lifecycle(**kwargs) + lifecycle = _make_lifecycle() lifecycle._pipeline = object() session = MagicMock() session.step.side_effect = error @@ -141,7 +137,8 @@ class TestStepFailureDiagnosis: def test_failed_drain_retains_owner_and_original_step_error(self, error_type): step_error = ValueError("retarget worker failed") drain_error = error_type("worker did not join") - lifecycle = _make_failing_lifecycle(step_error, **_BORROWED_OVXR_HANDLES) + lifecycle = _make_failing_lifecycle(step_error) + lifecycle._external_oxr_handles_provider = lambda: object() # Borrowed OVXR handles. session = lifecycle._session session.__exit__.side_effect = drain_error with ( @@ -191,7 +188,7 @@ def test_stop_ar_exit_error_without_borrowed_handles_allows_restart(self): @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) def test_terminal_exit_failure_is_latched_across_callbacks_and_restart(self, error_type): - lifecycle = _make_lifecycle(**_BORROWED_OVXR_HANDLES) + lifecycle = _make_lifecycle(use_kit_xr_bridge=False, external_oxr_handles_provider=lambda: object()) lifecycle._pipeline = object() error = error_type("terminal native cleanup failed") From 302e2444faae79679ef17033a2610339189c63f8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:27:43 -0500 Subject: [PATCH 111/145] Keep the upstream warmup and STOP-edge comment lines in replay --- .../entrypoints/teleop_replay_agent.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 9e212c1e383..f046378392d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -1523,8 +1523,9 @@ def _patched_sim_render(): # ``ctrl.is_active`` does not flip. Recordings # without an operator STOP are terminated instead by # the success / failure / wall-clock terminators - # below. RESET also pauses the control graph, but must - # reach the reset handler instead of ending the replay. + # below. + # RESET also pauses the control graph, but must reach + # the reset handler instead of ending the replay. if prev_active and not teleop_active and teleop_was_active and not ctrl.should_reset: print("Replay end observed (STOP edge); ending run.") break @@ -1674,11 +1675,11 @@ def main() -> int: multi-run batches start essentially instantly. Per-replay control flow (see :func:`_run_single_replay` for details): - * First-run Kit warmup: ``wait_for_stage_load`` polls + * Pre-loop warmup: ``wait_for_stage_load`` polls ``omni.usd.UsdContext.get_stage_loading_status`` until the simulator reports zero pending assets, then renders a fixed number of - settle frames only when Kit is active and ``run_index == 0``. - Kitless runs do not import Kit's asset loader. An optional + settle frames (only on ``run_index == 0`` and only when Kit is + active; kitless runs do not import Kit's asset loader). An optional ``--replay_start_delay_s`` buffer can be appended for hardware that needs more grace. ``advance()`` is not called during warmup so ``ReplaySession.update`` does not consume MCAP From 6740d3c6fd2a67008508c4d7b562ec3d60b95d50 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:43:46 -0500 Subject: [PATCH 112/145] Lead the kitless guide's live section with the task table and CPU OvPhysX GR1T2 --- source/isaaclab_teleop/docs/kitless_xr.md | 34 ++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 9ff0d48d807..83e0aee1e50 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -44,13 +44,8 @@ and restore the caller's arguments on return. The scripts under ## Live teleoperation -```bash -uv run --extra ovxr isaaclab teleop run \ - --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ - --device cuda:0 --physics newton_mjwarp --ovxr -``` - -The same registered-task selectors apply to live input and MCAP replay: +Start with CPU OvPhysX using the command below. The same registered-task +selectors apply to live input and MCAP replay: | Task | Physics selectors | | --- | --- | @@ -70,6 +65,19 @@ defines a `newton_mjwarp` preset, but the environment catalog (`tools/environ_docs.py`) disables that combination after the environment smoke matrix found runtime constraints, so it is not listed here. +For example, replace the task in this command with any row above: + +```bash +uv run --extra ovxr isaaclab teleop run \ + --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ + --device cpu --physics ovphysx --ovxr +``` + +`--ovxr` selects OVStage and starts CloudXR with the `cloudxrjs` profile by +default. Connect a supported CloudXR client, then use its Play and Reset +controls. Pause/resume is available only on clients that expose it; Reset is a +separate action. + The SO101 IK task resets its joints to the configured nominal pose without joint-position noise, giving the clutch a repeatable reset pose. Its configured home transform supplies the clutch calibration. This applies to its Kit and @@ -80,18 +88,6 @@ re-engagement, proportional trigger control and Reset separately from cube grasping. Object retention and cube stacking also depend on contact properties and grasp placement; validate them separately from the input controls. -For example, replace the task in this command with any row above: - -```bash -uv run --python 3.12 --extra ovxr --locked isaaclab teleop run \ - --task IsaacContrib-PickPlace-GR1T2-Abs --num_envs 1 \ - --device cpu --physics ovphysx --ovxr -``` - -`--ovxr` selects OVStage and starts CloudXR with the `cloudxrjs` profile by default. Connect a supported -CloudXR client, then use its Play and Reset controls. Pause/resume is available -only on clients that expose it; Reset is a separate action. - For desktop IWER, select **VR** and set **Reference Space** to **local-floor** under **Teleoperator adjustment** before connecting. This keeps the emulated head and controllers relative to the floor. Between trials, end the previous XR From 00d261de8110ff7da522587e703fdfe2f1220513 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:43:47 -0500 Subject: [PATCH 113/145] Replay the GR1T2 task in the kitless OVXR replay example --- docs/source/features/isaac_teleop.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index f36bc490850..6ac2878556d 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1355,14 +1355,14 @@ Capture with its OVXR extra and select the compatible rendering dependencies. uv run --extra ovxr python \ scripts/environments/teleoperation/teleop_replay_agent.py \ - --task IsaacContrib-Stack-Cube-SO101-IK-Abs-v0 --physics ovphysx \ + --task IsaacContrib-PickPlace-GR1T2-Abs --physics ovphysx \ --device cpu --num_envs 1 --ovxr \ --replay_file /absolute/path/to/session.mcap -MCAP supplies tracker and control input; the connected client supplies the view. -A recorded STOP, task success or failure, or the replay duration limit ends the -run. The separate ``isaaclab teleop replay`` command replays HDF5 demonstrations -through Kit. +Use ``--physics newton_mjwarp`` for Newton. MCAP supplies tracker and control +input; the connected client supplies the view. A recorded STOP, task success +or failure, or the replay duration limit ends the run. The separate +``isaaclab teleop replay`` command replays HDF5 demonstrations through Kit. .. _isaac-teleop-xr-anchor: From 956505d4391b089a1dffe26b0a376c16650726dc Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:43:47 -0500 Subject: [PATCH 114/145] Describe humanoid backend preparation in the task package README --- source/isaaclab_tasks/docs/README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 2ccb195fb5d..258250364a1 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -58,3 +58,15 @@ gym.register( > **Note:** As a practice, we specify all the environments in a single file to avoid name conflicts between different > tasks or environments. However, this practice is debatable and we are open to suggestions to deal with a large > scaling in the number of tasks or environments. + +## Standalone physics for GR1T2 and G1 teleoperation tasks + +For standalone OVPhysX, the fixed-base G1 task's backend preparation hook moves the +tabletop collision anchor onto its existing meshes, using the same preparation as GR1T2. +OVPhysX cooks the steering wheel's authored collision meshes. The bin's existing +primitive colliders and the object's drop termination remain enabled. +Registered GR1T2 and G1 tasks prepare the selected backend before scene construction, +including when created through `gym.make()` or `instantiate(cfg)` without the +teleoperation launcher. Newton uses its separate backend preparation. Resolve a +fresh task configuration when changing backends after preparation; copying a prepared +configuration preserves its selected backend and caller overrides. From 4e84b410f69aa3330e6fdfbab81fa060fe73aff8 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:49:22 -0500 Subject: [PATCH 115/145] Run the native OVRTX transfer harness on prepared GR1T2 with CPU OvPhysX --- source/isaaclab_teleop/docs/OVXR_ADAPTER.md | 7 +++++-- .../test/test_ovrtx_transfer_native_harness.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md index 306845f60bd..e858007932d 100644 --- a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -96,8 +96,11 @@ An opt-in harness runs the render-owner transfer and complete scene publication against the real libraries, on the manager-based `Isaac-Cartpole` task with its `ovphysx` physics preset and one environment on `cuda:0`. The transfer case also runs with CPU physics, which publishes host scene tensors to the GPU -renderer. It needs a CUDA GPU, OVRTX, OVStage, OvPhysX and an Isaac Capture -release with OVXR session support; it starts no XR session or CloudXR runtime. +renderer. Both cases also run `IsaacContrib-PickPlace-GR1T2-Abs` with CPU +OvPhysX, which its registered entry point prepares for kitless physics; its +cameras are removed as in an `--ovxr` launch. The harness needs a CUDA GPU, +OVRTX, OVStage, OvPhysX and an Isaac Capture release with OVXR session support; +it starts no XR session or CloudXR runtime. Each case runs in a child process: ```bash diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py index 70d8f94593d..201dae8ac90 100644 --- a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -30,6 +30,9 @@ _TRANSFER_DEVICES = ("cuda:0", "cpu") """Simulation devices for the transfer case; CPU physics publishes host scene tensors.""" +_GR1T2_TASK = "IsaacContrib-PickPlace-GR1T2-Abs" +"""Humanoid task whose registered entry point prepares it for kitless OvPhysX.""" + @dataclass class _CaseResult: @@ -45,6 +48,7 @@ def _make_env_and_bootstrap(task: str, device: str): import gymnasium as gym from isaaclab_ov.physics import OvPhysxCfg from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop import remove_camera_configs from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap import isaaclab_tasks # noqa: F401 -- registers the task @@ -56,6 +60,8 @@ def _make_env_and_bootstrap(task: str, device: str): assert isinstance(cfg.sim.physics, OvPhysxCfg) cfg.sim.device = device cfg.scene.num_envs = 1 + # Like an --ovxr launch, keep task cameras off the leased renderer. + cfg = remove_camera_configs(cfg) bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) try: return bootstrap, gym.make(task, cfg=cfg).unwrapped @@ -174,6 +180,12 @@ def test_cross_thread_preuse_owner_transfer_native(device): _run_native_subprocess(_TASK, device, True) +@pytest.mark.parametrize("transfer", [False, True], ids=["same-thread", "transfer"]) +def test_gr1t2_cpu_ovphysx_native(transfer): + """The prepared GR1T2 scene publishes from CPU OvPhysX, with and without the owner transfer.""" + _run_native_subprocess(_GR1T2_TASK, "cpu", transfer) + + if __name__ == "__main__": if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": raise RuntimeError("native transfer harness requires its explicit opt-in") From 5c27cbf2ef628f640b97cd4a7ec8a0cf3b66b726 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:49:35 -0500 Subject: [PATCH 116/145] Leave fixed-base G1's reset re-render count unchanged until it has image observations --- .../isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst | 2 -- .../locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py | 1 - 2 files changed, 3 deletions(-) diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst index 65f84d13c4b..6cac80fd578 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -8,8 +8,6 @@ * Changed the GR1T2 upper-body action term to :class:`~isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. With Isaac Sim PhysX it behaves as the previous Pink term. -* Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), - which adds render work per reset when cameras are rendered. * Changed the G1 hand contact-sensor paths to name the ``left_hand``/``right_hand`` parent of the hand links in the Isaac 6.2 ``g1.usd``. Newton matches whole body paths, so it needs that parent; Isaac Sim PhysX and OvPhysX resolve the same links by name as before. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index 93ce7b32468..4dda4674ef0 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -407,7 +407,6 @@ def __post_init__(self): # simulation settings self.sim.dt = 1 / 200 # 200Hz self.sim.render_interval = 2 - self.num_rerenders_on_reset = 3 # Set the URDF and mesh paths for the IK controller urdf_omniverse_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 From 9de9aad8953a1e08528e5cdcab5d73650cded93d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:50:29 -0500 Subject: [PATCH 117/145] Remove only the cameras each G1 task declares in the OvPhysX wheel test --- .../test/contrib/test_g1_kitless_ovphysx.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py index 55372fb44e1..436213e4554 100644 --- a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py @@ -61,13 +61,9 @@ def test_g1_wheel_remains_supported_on_ovphysx_cpu(cfg_type, task) -> None: env_cfg.sim.device = "cpu" env_cfg.sim.physics = OvPhysxCfg() env_cfg.scene.robot_pov_cam = None - env_cfg.observations.policy.robot_pov_cam = None - if isinstance(env_cfg, FixedBaseUpperBodyIKG1EnvCfg): - env_cfg.scene.left_wrist_cam = None - env_cfg.scene.right_wrist_cam = None - env_cfg.observations.policy.left_wrist_cam = None - env_cfg.observations.policy.right_wrist_cam = None - env_cfg.image_obs_list = [] + if isinstance(env_cfg, LocomanipulationG1EnvCfg): + env_cfg.observations.policy.robot_pov_cam = None + env_cfg.image_obs_list = [] env_cfg.stabilize_for_kitless_ovphysx() env = None From fa70f0db215b759e2519b23b33f3e3cb5ec05c96 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:50:58 -0500 Subject: [PATCH 118/145] Drop task-test assertions that cannot fail and name the G1 default-config check --- source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py | 3 +-- .../isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py index 436213e4554..f7fa1e6d75f 100644 --- a/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py +++ b/source/isaaclab_tasks/test/contrib/test_g1_kitless_ovphysx.py @@ -22,7 +22,7 @@ _TASK = "IsaacContrib-PickPlace-Locomanipulation-G1-Abs" -def test_fixed_base_g1_ovphysx_conditions_assets_without_changing_other_backends() -> None: +def test_fixed_base_g1_ovphysx_conditions_assets_without_mutating_default_configs() -> None: cfg = FixedBaseUpperBodyIKG1EnvCfg() default_cfg = FixedBaseUpperBodyIKG1EnvCfg() action_type = cfg.actions.upper_body_ik.class_type @@ -41,7 +41,6 @@ def test_fixed_base_g1_ovphysx_conditions_assets_without_changing_other_backends assert cfg.actions.upper_body_ik.class_type is action_type assert cfg.terminations.object_dropping.func is drop_term assert not isinstance(default_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) - assert not isinstance(default_cfg.scene.object.spawn, PackingTableUsdFileCfg) @pytest.mark.isaacsim_ci diff --git a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py index 0393148d945..3ee691aeb0d 100644 --- a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py +++ b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py @@ -365,4 +365,3 @@ def spawn_wheel(path, cfg): assert all(p.IsA(UsdGeom.Mesh) for p in colliders) assert all(UsdPhysics.CollisionAPI(p).GetCollisionEnabledAttr().Get() for p in colliders) assert all(UsdPhysics.MeshCollisionAPI(p).GetApproximationAttr().Get() == "convexDecomposition" for p in colliders) - assert not stage.GetPrimAtPath("/World/Wheel/Body/KitlessPrimitiveCollision") From 0b60caec4a3c12d9a4ccff088456f564d619badc Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:51:08 -0500 Subject: [PATCH 119/145] Keep the upstream camera-feed layout position in locomanipulation G1 --- .../contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py index e0528727008..b90993acad7 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/locomanipulation_g1_env_cfg.py @@ -499,6 +499,7 @@ def __post_init__(self): pipeline_builder=_build_g1_locomanipulation_pipeline, sim_device=self.sim.device, xr_cfg=self.xr, + xr_camera_feed_layout=XrCameraFeedLayoutCfg(placement="head_locked", use_scene_partition=True), xr_camera_feeds=[ XrCameraFeedCfg( camera_name="robot_pov_cam", @@ -508,7 +509,6 @@ def __post_init__(self): max_update_hz=0.0, ) ], - xr_camera_feed_layout=XrCameraFeedLayoutCfg(placement="head_locked", use_scene_partition=True), ) # Drive controller haptics from the per-hand contact sensors. From a65a3f8a476284a641fe564c2bcec0fd6e972c9d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:51:39 -0500 Subject: [PATCH 120/145] Keep the upstream waist-joint append in the WaistEnabled GR1T2 config --- .../pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py index cb152477b62..36510d1fa12 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_waist_enabled_env_cfg.py @@ -67,8 +67,7 @@ def __post_init__(self): # Add waist joint to pink_ik_cfg waist_joint_names = ["waist_yaw_joint", "waist_pitch_joint", "waist_roll_joint"] for joint_name in waist_joint_names: - if joint_name not in self.actions.upper_body_ik.pink_controlled_joint_names: - self.actions.upper_body_ik.pink_controlled_joint_names.append(joint_name) + self.actions.upper_body_ik.pink_controlled_joint_names.append(joint_name) # Defer USD→URDF conversion to controller initialization (requires Isaac Sim at runtime). self.actions.upper_body_ik.controller.usd_path = self.scene.robot.spawn.usd_path From 4166bdf77bdfd131aba8bc8d464a961c914fb5e2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:51:59 -0500 Subject: [PATCH 121/145] Restore the upstream GR1T2 import layout and pipeline builder text --- .../contrib/pick_place/pickplace_gr1t2_env_cfg.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index f517a3473fd..ab9e7f42999 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -30,6 +30,7 @@ from isaaclab.visualizers import VisualizerCfg from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE +from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions from isaaclab_tasks.contrib.steering_wheel import ( condition_newton_steering_wheel_detail_collisions, ) @@ -43,9 +44,7 @@ from .gr1t2_kitless_action_cfg import GR1T2KitlessPinkInverseKinematicsActionCfg from isaaclab_assets.robots.fourier import GR1T2_HIGH_PD_CFG # isort: skip - from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip -from isaaclab_tasks.contrib.packing_table import configure_packing_table_collisions # isort: skip def build_gr1t2_pickplace_pipeline(): @@ -275,7 +274,6 @@ def build_gr1t2_pickplace_pipeline(): } ) - # The session lifecycle discovers the raw HandsSource outputs for hand-presence safety. pipeline = OutputCombiner({"action": connected_reorderer.output("output")}) return pipeline, [left_dex, right_dex] From c1b11fa6554cd05994a696f709dda41d3d293aab Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:54:15 -0500 Subject: [PATCH 122/145] Check teleop-free humanoid preparation in its own test beside the upstream one --- .../core/test_env_cfg_no_forbidden_imports.py | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py index f2f299d2013..952510f65a2 100644 --- a/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py +++ b/source/isaaclab_tasks/test/core/test_env_cfg_no_forbidden_imports.py @@ -194,11 +194,10 @@ def test_config_load_does_not_import_backend_modules(task_name: str, all_cfg_che def test_teleop_configs_resolve_and_validate_without_teleop(): - """Optional teleop integrations must not block task validation or standalone physics preparation.""" + """Configs with optional Isaac Teleop integrations must resolve and validate without the package.""" script = textwrap.dedent("""\ import builtins import importlib.util - from pathlib import Path original_import = builtins.__import__ original_find_spec = importlib.util.find_spec @@ -253,11 +252,45 @@ def hide_teleop(name, *args, **kwargs): assert cfg.scene.left_hand_contact is not None assert cfg.scene.right_hand_contact is not None assert cfg.scene.robot.spawn.activate_contact_sensors + """) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, ( + f"Subprocess failed:\n--- stdout ---\n{result.stdout}\n--- stderr ---\n{result.stderr}" + ) - from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( - G1NewtonBoundedPinkInverseKinematicsAction, - ) + +def test_humanoid_standalone_preparation_without_teleop(): + """Registered humanoid tasks must prepare standalone physics without the Isaac Teleop package.""" + script = textwrap.dedent("""\ + import builtins + import importlib.util + + original_import = builtins.__import__ + original_find_spec = importlib.util.find_spec + + def block_teleop(name, *args, **kwargs): + if name == "isaaclab_teleop" or name.startswith("isaaclab_teleop."): + raise ModuleNotFoundError("No module named 'isaaclab_teleop'", name="isaaclab_teleop") + return original_import(name, *args, **kwargs) + + def hide_teleop(name, *args, **kwargs): + if name == "isaaclab_teleop" or name.startswith("isaaclab_teleop."): + return None + return original_find_spec(name, *args, **kwargs) + + builtins.__import__ = block_teleop + importlib.util.find_spec = hide_teleop + + import gymnasium + import isaaclab_tasks # noqa: F401 from isaaclab.envs import ManagerBasedRLEnv + from isaaclab.utils import validate + from isaaclab_tasks.utils import resolve_task_config class ConstructionReached(Exception): pass @@ -277,8 +310,6 @@ def stop_before_simulation(self, cfg, *args, **kwargs): ) for task_name, physics in standalone_tasks: cfg, _ = resolve_task_config(task_name, "", overrides=[f"physics={physics}"]) - assert cfg.isaac_teleop is None - controlled_joints = list(cfg.actions.upper_body_ik.pink_controlled_joint_names) try: gymnasium.make(task_name, cfg=cfg) except ConstructionReached: @@ -286,18 +317,8 @@ def stop_before_simulation(self, cfg, *args, **kwargs): else: raise AssertionError("Registered constructor did not reach the simulation boundary") validate(cfg) - assert cfg.xr is None assert cfg.isaac_teleop is None - action = cfg.actions.upper_body_ik - assert action.pink_controlled_joint_names == controlled_joints - if "GR1T2" in task_name: - assert action.kitless_target_conditioning_enabled - assert Path(action.controller.urdf_path).is_file() - else: - assert action.class_type is G1NewtonBoundedPinkInverseKinematicsAction - if physics == "newton_mjwarp": - assert cfg.sim.physics.solver_cfg.nconmax == 4096 - assert cfg.sim.physics.solver_cfg.njmax == 1536 + assert cfg._prepared_kitless_backend == physics """) result = subprocess.run( [sys.executable, "-c", script], From 4ec9ac379d2c8fb090eb80106a40208df364e050 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:54:51 -0500 Subject: [PATCH 123/145] Keep the launcher physics-preservation test and check the humanoid hook separately --- .../test/test_teleop_entrypoint_units.py | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 1cfc22c10ed..596abbd2850 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -554,6 +554,26 @@ def test_product_xr_default_resolves_cloudxrjs_and_autolaunch(monkeypatch): assert agent.args_cli.auto_launch_cloudxr is True +def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch): + """Launch validation must not replace task-specific settings on the selected backend.""" + agent = _load_teleop_agent(monkeypatch) + from isaaclab_newton.physics import NewtonCfg + + from isaaclab.app.sim_launcher import validate_launch + + physics_cfg = NewtonCfg() + physics_cfg.solver_cfg.nconmax = 4096 + physics_cfg.solver_cfg.njmax = 1536 + env_cfg = SimpleNamespace(sim=SimpleNamespace(physics=physics_cfg)) + configs, _ = validate_launch(env_cfg, vars(agent.args_cli).copy()) + + assert not hasattr(agent.args_cli, "physics") + assert env_cfg.sim.physics is physics_cfg + assert any(cfg is physics_cfg for cfg in configs) + assert physics_cfg.solver_cfg.nconmax == 4096 + assert physics_cfg.solver_cfg.njmax == 1536 + + @pytest.mark.parametrize( "task_name,physics", [ @@ -566,25 +586,13 @@ def test_product_xr_default_resolves_cloudxrjs_and_autolaunch(monkeypatch): ("IsaacContrib-PickPlace-FixedBaseUpperBodyIK-G1-Abs", "newton_mjwarp"), ], ) -def test_teleop_launcher_preserves_the_resolved_physics_configuration(monkeypatch, task_name, physics): - """The entrypoint validates the launch, then prepares the task, keeping its backend settings.""" +def test_teleop_launcher_prepares_humanoid_backends(monkeypatch, task_name, physics): + """The entrypoint runs the task's preparation hook for the selected standalone backend.""" # Standalone backends run under --ovxr, which removes the task cameras before launch validation. ovxr = () if physics == "isaacsim_physx" else ("--ovxr",) agent = _load_teleop_agent(monkeypatch, "--task", task_name, "--physics", physics, *ovxr) env_cfg, _teleop_cfg, _use_isaac_teleop = agent._resolve_env_cfg() - physics_cfg = env_cfg.sim.physics - assert not hasattr(agent.args_cli, "physics") - if physics == "newton_mjwarp": - if "Locomanipulation-G1" in task_name: - assert physics_cfg.num_substeps == 2 - assert physics_cfg.solver_cfg.integrator == "implicitfast" - assert physics_cfg.solver_cfg.nconmax >= 4096 - assert physics_cfg.solver_cfg.njmax >= 1536 - else: - from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg - - assert isinstance(env_cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) == (physics == "ovphysx") - assert getattr(env_cfg, "_prepared_kitless_backend", None) == (None if physics == "isaacsim_physx" else physics) + assert env_cfg._prepared_kitless_backend == (None if physics == "isaacsim_physx" else physics) def test_task_action_hook_validates_before_and_after_conditioning(monkeypatch): From fb146d9082a078bec8e798201a4f1dd97bf518b6 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 14:55:46 -0500 Subject: [PATCH 124/145] Assert only preparation state in the registered-construction preset test --- .../test/core/test_preset_cli.py | 51 +++++-------------- 1 file changed, 13 insertions(+), 38 deletions(-) diff --git a/source/isaaclab_tasks/test/core/test_preset_cli.py b/source/isaaclab_tasks/test/core/test_preset_cli.py index ae10a560332..80c32dc235c 100644 --- a/source/isaaclab_tasks/test/core/test_preset_cli.py +++ b/source/isaaclab_tasks/test/core/test_preset_cli.py @@ -12,6 +12,7 @@ import argparse import sys +from copy import deepcopy from pathlib import Path import pytest @@ -283,7 +284,10 @@ def load(task, entry): ) @pytest.mark.parametrize("physics", [None, "isaacsim_physx", "ovphysx", "newton_mjwarp"]) def test_teleop_tasks_resolve_public_physics_selectors(monkeypatch, task_name, physics): - """Registered construction prepares the selected backend before simulation starts.""" + """Resolve registered tasks through the public flag parser before backend preparation. + + Registered humanoid construction then prepares a standalone backend and leaves Kit physics unprepared. + """ import gymnasium as gym from isaaclab_newton.physics import MJWarpSolverCfg, NewtonCfg from isaaclab_ov.physics import OvPhysxCfg @@ -315,51 +319,22 @@ def test_teleop_tasks_resolve_public_physics_selectors(monkeypatch, task_name, p for name in ("bounce_threshold_velocity", "friction_correlation_distance", "solve_articulation_contact_last"): assert getattr(cfg.sim.physics, name) == getattr(physx_cfg, name), name + if "SO101" in task_name: + return + class ConstructionReached(Exception): pass - from isaaclab_tasks.contrib.packing_table import PackingTableUsdFileCfg - from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import GR1T2KitlessUsdFileCfg - - original_action_type = getattr(getattr(cfg.actions, "upper_body_ik", None), "class_type", None) - def inspect_before_simulation(self, cfg, *args, **kwargs): self._is_closed = True - standalone = physics in ("ovphysx", "newton_mjwarp") - if "GR1T2" in task_name: - action = cfg.actions.upper_body_ik - assert action.kitless_target_conditioning_enabled is standalone - if standalone: - assert Path(action.controller.urdf_path).is_file() - assert isinstance(cfg.scene.robot.spawn, GR1T2KitlessUsdFileCfg) - else: - assert action.controller.urdf_path is None - expected_waist = "WaistEnabled" in task_name - assert ("waist_yaw_joint" in action.pink_controlled_joint_names) is expected_waist - if "SO101" not in task_name: - assert isinstance(cfg.scene.packing_table.spawn, PackingTableUsdFileCfg) is standalone - if physics == "newton_mjwarp": - if "Locomanipulation" in task_name: - assert cfg.sim.physics.solver_cfg.integrator == "implicitfast" - assert cfg.sim.physics.num_substeps == 2 - else: - assert cfg.sim.physics.solver_cfg.nconmax == 4096 - assert cfg.sim.physics.solver_cfg.njmax == 1536 - if "FixedBaseUpperBodyIK" in task_name: - from isaaclab_tasks.contrib.locomanip_pick_place.g1_kitless_newton import ( - G1NewtonBoundedPinkInverseKinematicsAction, - ) - - assert cfg.actions.upper_body_ik.class_type is G1NewtonBoundedPinkInverseKinematicsAction - elif not standalone: - assert cfg.actions.upper_body_ik.class_type == original_action_type + assert cfg._prepared_kitless_backend == (physics if physics in ("ovphysx", "newton_mjwarp") else None) raise ConstructionReached monkeypatch.setattr(ManagerBasedRLEnv, "__init__", inspect_before_simulation) - with pytest.raises(ConstructionReached): - gym.make(task_name, cfg=cfg) - with pytest.raises(ConstructionReached): - instantiate(cfg) + # Each constructor receives its own unprepared copy, so each must prepare it. + for construct in (lambda env_cfg: gym.make(task_name, cfg=env_cfg), instantiate): + with pytest.raises(ConstructionReached): + construct(deepcopy(cfg)) @pytest.mark.parametrize( From afecad7c118a1fdbe494be2717bfb383e122a8f4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:01:53 -0500 Subject: [PATCH 125/145] Swap in the GR1T2 kitless action term during backend preparation The task keeps the upstream Pink action term, so Kit and Mimic users run the base term. Kitless OvPhysX and Newton preparation replace it with the kitless term, which now always conditions targets; the term type replaces the kitless_target_conditioning_enabled flag. --- .../changelog.d/kitless-robot-tasks.added.rst | 5 ++- .../kitless-robot-tasks.changed.rst | 3 -- .../pick_place/gr1t2_kitless_action.py | 31 ++++---------- .../pick_place/gr1t2_kitless_action_cfg.py | 12 +++--- .../pick_place/pickplace_gr1t2_env_cfg.py | 42 ++++++++++++------- .../test/contrib/test_gr1t2_kitless_action.py | 9 ++-- 6 files changed, 49 insertions(+), 53 deletions(-) diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst index 5407015deaa..0b8bc9cb415 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.added.rst @@ -1,5 +1,8 @@ * Added G1 and GR1T2 kitless teleoperation setup, including portable GR1T2 kinematics, - backend-specific actuators, collision preparation, and bounded action targets. + backend-specific actuators, collision preparation, and bounded action targets. Kitless preparation + replaces the GR1T2 Pink action term with + :class:`~isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`; + Isaac Sim PhysX keeps the Pink term. * Added public OvPhysX and Newton selection while retaining Kit defaults and supporting standalone backend preparation without optional teleoperation configuration. Registered construction prepares the selected backend and preserves prepared configuration copies diff --git a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst index 6cac80fd578..6b14bcf7ecf 100644 --- a/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst +++ b/source/isaaclab_tasks/changelog.d/kitless-robot-tasks.changed.rst @@ -5,9 +5,6 @@ backend before construction. Code that constructs the environment without that entry point should first call :func:`~isaaclab_tasks.contrib.teleop_physics_cfg.prepare_humanoid_teleop_environment`, which also applies the preset defaults to a configuration built without task resolution. -* Changed the GR1T2 upper-body action term to - :class:`~isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action_cfg.GR1T2KitlessPinkInverseKinematicsActionCfg`. - With Isaac Sim PhysX it behaves as the previous Pink term. * Changed the G1 hand contact-sensor paths to name the ``left_hand``/``right_hand`` parent of the hand links in the Isaac 6.2 ``g1.usd``. Newton matches whole body paths, so it needs that parent; Isaac Sim PhysX and OvPhysX resolve the same links by name as before. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py index bcb7358ffe2..56a6ff06586 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py @@ -78,20 +78,13 @@ class GR1T2KitlessPinkInverseKinematicsAction(PinkInverseKinematicsAction): this term reproduces their target relation in software and bounds hand targets. OVPhysX widens physical hand limits; Newton retains them. It also bounds cold-start IK demands against measured state to avoid a first-step - drive impulse. With the feature disabled, behavior is the base Pink action. + drive impulse. """ cfg: GR1T2KitlessPinkInverseKinematicsActionCfg def __init__(self, cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, env: ManagerBasedEnv) -> None: super().__init__(cfg, env) - self._kitless_target_conditioning_enabled = cfg.kitless_target_conditioning_enabled - self._mimic_pairs: tuple[tuple[int, int, float, float], ...] = () - self._hand_lower: torch.Tensor | None = None - self._hand_upper: torch.Tensor | None = None - self._hand_target_slew: _HandTargetSlewState | None = None - if not self._kitless_target_conditioning_enabled: - return from .pickplace_gr1t2_env_cfg import GR1T2KitlessUsdFileCfg @@ -151,27 +144,21 @@ def __init__(self, cfg: GR1T2KitlessPinkInverseKinematicsActionCfg, env: Manager def process_actions(self, actions: torch.Tensor) -> None: """Process input without mutating the action-manager input during coupling.""" super().process_actions(actions) - if self._kitless_target_conditioning_enabled: - assert self._hand_lower is not None and self._hand_upper is not None - assert self._hand_target_slew is not None - constrained = constrain_hand_joint_targets( - self._target_hand_joint_positions, - self._mimic_pairs, - self._hand_lower, - self._hand_upper, - ) - self._target_hand_joint_positions = self._hand_target_slew.condition(constrained) + constrained = constrain_hand_joint_targets( + self._target_hand_joint_positions, + self._mimic_pairs, + self._hand_lower, + self._hand_upper, + ) + self._target_hand_joint_positions = self._hand_target_slew.condition(constrained) def reset(self, env_ids: Sequence[int] | None = None) -> None: """Clear commanded-target history when the owning session/environment resets.""" super().reset(env_ids) - if self._hand_target_slew is not None: - self._hand_target_slew.reset(env_ids) + self._hand_target_slew.reset(env_ids) def _compute_ik_solutions(self) -> torch.Tensor: targets = super()._compute_ik_solutions() - if not self._kitless_target_conditioning_enabled: - return targets current = self._asset.data.joint_pos.torch[:, self._isaaclab_controlled_joint_ids] return bound_ik_joint_targets( targets, diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py index 6da9e4abaf6..e453f80dda6 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action_cfg.py @@ -19,20 +19,18 @@ @configclass class GR1T2KitlessPinkInverseKinematicsActionCfg(PinkInverseKinematicsActionCfg): - """Pink action with opt-in, backend-specific target conditioning for GR1T2. + """Pink action with backend-specific target conditioning for GR1T2. - The feature is disabled by default, preserving the regular Kit/PhysX action - path. The task enables it for kitless OVPhysX or Newton and fills the - hand metadata while conditioning the spawned USD before physics parses it. + Kitless OVPhysX and Newton preparation replace the task's Pink action term + with this one; the regular Kit/PhysX path keeps the base term. Preparation + fills the hand metadata while conditioning the spawned USD before physics + parses it. """ class_type: type[GR1T2KitlessPinkInverseKinematicsAction] | str = ( "{DIR}.gr1t2_kitless_action:GR1T2KitlessPinkInverseKinematicsAction" ) - kitless_target_conditioning_enabled: bool = False - """Enable the shared raw-GR1T2 joint/target policy for standalone OVPhysX and Newton.""" - max_ik_joint_delta_rad: float = 0.2 """Maximum IK target offset from measured joint position on each IK application [rad]. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index ab9e7f42999..92ec6f22c05 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -16,6 +16,7 @@ from isaaclab.assets import ArticulationCfg, AssetBaseCfg, RigidObjectCfg from isaaclab.controllers.pink_ik import DampingTaskCfg, FrameTaskCfg, NullSpacePostureTaskCfg, PinkIKControllerCfg from isaaclab.envs import ManagerBasedRLEnvCfg +from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg from isaaclab.managers import EventTermCfg as EventTerm from isaaclab.managers import ObservationGroupCfg as ObsGroup from isaaclab.managers import ObservationTermCfg as ObsTerm @@ -398,7 +399,7 @@ class PickPlaceGR1T2SceneCfg(ObjectTableSceneCfg): class ActionsCfg: """Action specifications for the MDP.""" - upper_body_ik = GR1T2KitlessPinkInverseKinematicsActionCfg( + upper_body_ik = PinkInverseKinematicsActionCfg( pink_controlled_joint_names=[ "left_shoulder_pitch_joint", "left_shoulder_roll_joint", @@ -726,10 +727,11 @@ def stabilize_for_kitless_newton_mjwarp(self) -> None: def prepare_gr1t2_for_kitless_ovphysx(env_cfg: ManagerBasedRLEnvCfg) -> None: """Configure a GR1T2 task's asset and action boundaries for kitless OVPhysX. - Selects the vendored kinematics URDF for Pink, enables target conditioning, requires initial - bimanual hand tracking, restores actuator coverage, conditions the robot spawn, and authors - PhysX contact offsets on the packing table. The configured controlled joints (with or without - the waist) are kept. A prepared configuration only re-attaches its spawn conditioning. + Replaces the Pink action term with the kitless GR1T2 term and selects the vendored kinematics + URDF for it, requires initial bimanual hand tracking, restores actuator coverage, conditions the + robot spawn, and authors PhysX contact offsets on the packing table. The configured controlled + joints (with or without the waist) are kept. A prepared configuration only re-attaches its spawn + conditioning. Args: env_cfg: GR1T2 pick-place task configuration. @@ -740,11 +742,7 @@ def prepare_gr1t2_for_kitless_ovphysx(env_cfg: ManagerBasedRLEnvCfg) -> None: if check_kitless_backend_preparation(env_cfg, "ovphysx"): condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik) return - action_cfg = env_cfg.actions.upper_body_ik - action_cfg.controller.urdf_path = os.path.join( - os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" - ) - action_cfg.kitless_target_conditioning_enabled = True + action_cfg = _use_kitless_action_term(env_cfg) # Do not apply retargeter defaults before hand tracking starts. if env_cfg.isaac_teleop is not None: env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True @@ -758,7 +756,7 @@ def prepare_gr1t2_for_kitless_ovphysx(env_cfg: ManagerBasedRLEnvCfg) -> None: def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: """Configure a GR1T2 task's actuators, contacts and action boundaries for kitless Newton. - Shares the URDF and target conditioning with :func:`prepare_gr1t2_for_kitless_ovphysx`, + Shares the kitless action term and its URDF with :func:`prepare_gr1t2_for_kitless_ovphysx`, applies Newton actuator gains and MJWarp contact budgets, and keeps the configured controlled joints. Hand joint limits are not widened on Newton. @@ -771,11 +769,7 @@ def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> No if check_kitless_backend_preparation(env_cfg, "newton_mjwarp"): condition_kitless_gr1t2_stage(env_cfg.scene.robot, env_cfg.actions.upper_body_ik, widen_hand_limits=False) return - action_cfg = env_cfg.actions.upper_body_ik - action_cfg.controller.urdf_path = os.path.join( - os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" - ) - action_cfg.kitless_target_conditioning_enabled = True + action_cfg = _use_kitless_action_term(env_cfg) if env_cfg.isaac_teleop is not None: env_cfg.isaac_teleop.require_initial_bimanual_hand_tracking = True _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg) @@ -784,6 +778,22 @@ def prepare_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> No env_cfg._prepared_kitless_backend = "newton_mjwarp" +def _use_kitless_action_term(env_cfg: ManagerBasedRLEnvCfg) -> GR1T2KitlessPinkInverseKinematicsActionCfg: + """Replace the Pink action term with the kitless GR1T2 term and select the vendored URDF. + + The kitless term keeps the task's Pink settings, including caller changes made before preparation. + """ + action_cfg = env_cfg.actions.upper_body_ik + if not isinstance(action_cfg, GR1T2KitlessPinkInverseKinematicsActionCfg): + settings = {name: value for name, value in vars(action_cfg).items() if name != "class_type"} + action_cfg = GR1T2KitlessPinkInverseKinematicsActionCfg(**settings) + env_cfg.actions.upper_body_ik = action_cfg + action_cfg.controller.urdf_path = os.path.join( + os.path.dirname(__file__), "assets", "GR1T2_fourier_hand_6dof_kinematics.urdf" + ) + return action_cfg + + def _stabilize_gr1t2_for_kitless_newton_mjwarp(env_cfg: ManagerBasedRLEnvCfg) -> None: """Apply Newton actuator, allocation-budget, and wheel-collision preparation.""" from isaaclab_newton.sim.schemas import NewtonArticulationCfg diff --git a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py index 3ee691aeb0d..0c628d1f99a 100644 --- a/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py +++ b/source/isaaclab_tasks/test/contrib/test_gr1t2_kitless_action.py @@ -16,6 +16,8 @@ from pxr import Usd, UsdGeom, UsdPhysics +from isaaclab.envs.mdp.actions.pink_actions_cfg import PinkInverseKinematicsActionCfg + from isaaclab_tasks.contrib.pick_place import pickplace_gr1t2_env_cfg as gr1t2_env_cfg from isaaclab_tasks.contrib.pick_place.gr1t2_kitless_action import ( GR1T2KitlessPinkInverseKinematicsAction, @@ -67,8 +69,8 @@ def test_newton_contract_is_backend_scoped_and_complete(config_type): assert newton_cfg.sim.physics.solver_cfg.nconmax == 4096 assert newton_cfg.sim.physics.solver_cfg.njmax == 1536 assert newton_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is True - assert newton_cfg.actions.upper_body_ik.kitless_target_conditioning_enabled is True - assert default_cfg.actions.upper_body_ik.kitless_target_conditioning_enabled is False + assert isinstance(newton_cfg.actions.upper_body_ik, GR1T2KitlessPinkInverseKinematicsActionCfg) + assert type(default_cfg.actions.upper_body_ik) is PinkInverseKinematicsActionCfg assert isinstance(newton_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) assert not isinstance(default_cfg.scene.robot.spawn, gr1t2_env_cfg.GR1T2KitlessUsdFileCfg) assert default_cfg.isaac_teleop.require_initial_bimanual_hand_tracking is False @@ -240,13 +242,12 @@ def test_kitless_kinematics_model_loads_without_the_kit_exporter(config_type, ba np.testing.assert_allclose(world_axis, [0.0, 0.0, 1.0], atol=1e-6) -def test_kitless_conditioning_is_opt_in_and_metadata_is_not_shared(): +def test_kitless_action_metadata_is_not_shared(): first = GR1T2KitlessPinkInverseKinematicsActionCfg() second = GR1T2KitlessPinkInverseKinematicsActionCfg() first.hand_joint_limits["joint"] = (-1.0, 1.0) - assert not first.kitless_target_conditioning_enabled assert second.hand_joint_limits == {} From ce2a331b661aa30039e394e51cb0dab3b76c9dbc Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:02:20 -0500 Subject: [PATCH 126/145] Keep the upstream Pink hand-joint comment and document the hand target order --- source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst | 3 ++- .../isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py | 5 ++++- .../isaaclab/envs/mdp/actions/pink_task_space_actions.py | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst index ca76d3f2e2e..d8d9c86fb40 100644 --- a/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst +++ b/source/isaaclab/changelog.d/pink-hand-joint-order.fixed.rst @@ -1 +1,2 @@ -* Preserved configured hand joint ordering when applying Pink action targets. +* Fixed Pink hand targets being applied to the wrong joints when the articulation's joint order differs + from ``hand_joint_names`` (for example, Newton's G1 order). diff --git a/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py b/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py index b5910b64d0f..f17a11253ee 100644 --- a/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py +++ b/source/isaaclab/isaaclab/envs/mdp/actions/pink_actions_cfg.py @@ -29,7 +29,10 @@ class PinkInverseKinematicsActionCfg(ActionTermCfg): """List of joint names or regular expression patterns that specify the joints controlled by pink IK.""" hand_joint_names: list[str] = MISSING - """List of joint names or regular expression patterns that specify the joints controlled by hand retargeting.""" + """List of joint names or regular expression patterns that specify the joints controlled by hand retargeting. + + Hand targets in the action are applied in this order. + """ controller: PinkIKControllerCfg = MISSING """Configuration for the Pink IK controller that will be used to solve the inverse kinematics.""" diff --git a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py index 454c965562f..56b73acfbc3 100644 --- a/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py +++ b/source/isaaclab/isaaclab/envs/mdp/actions/pink_task_space_actions.py @@ -72,6 +72,7 @@ def _initialize_joint_info(self) -> None: self._isaaclab_all_joint_ids = list(range(len(self._asset.data.joint_names))) self.cfg.controller.all_joint_names = self._asset.data.joint_names + # Resolve hand joints # Hand targets occupy the action tensor in configuration order. Newton's # articulation order groups these joints differently, so preserve the # declared order when resolving ids instead of silently permuting targets. From 2da5c5adcd1fa8b8e0c92c20da2bf93bae3a5b97 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:40:05 -0500 Subject: [PATCH 127/145] Render leased OVRTX camera panels from a CPU simulation device CPU PiP is the first user of OVRTX cameras on a CPU simulation device. Camera render products leave native GPU selection automatic instead of authoring Warp's CPU ordinal, camera pose and intrinsics writes run without a CUDA stream, and render vars map to host memory for CPU consumers. The device-selection test now uses Warp's real CPU ordinal, so it fails if the product pins device -1. --- .../leased-camera-products.added.rst | 3 + .../isaaclab_ov/renderers/ovrtx_renderer.py | 23 +++-- .../isaaclab_ov/renderers/ovrtx_usd.py | 7 +- .../isaaclab_ov/test/test_ovrtx_clone_plan.py | 3 +- .../test/test_ovrtx_renderer_contract.py | 97 +++++++++++++++++-- 5 files changed, 117 insertions(+), 16 deletions(-) diff --git a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst index 6dd14498977..9584406eff2 100644 --- a/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst +++ b/source/isaaclab_ov/changelog.d/leased-camera-products.added.rst @@ -1 +1,4 @@ * Added leased camera render products and independent camera-panel scheduling to external consumers. +* Added CPU simulation-device support to OVRTX cameras. Render products leave GPU selection automatic, + and camera poses, intrinsics and image reads use host memory without a CUDA stream, so kitless OVXR + camera panels can run with CPU physics. diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index 1e564681173..b827ad427d1 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -656,7 +656,10 @@ def _register_camera(self, spec: CameraRenderSpec, render_data: OVRTXCameraRende ] scope, product_path = render_data.render_scope_name, render_data.render_product_path usd = build_render_product_as_string( - spec, render_data, device_id=self._warp_device.ordinal, enable_shadows=self.cfg.enable_shadows + spec, + render_data, + device_id=self._warp_device.ordinal if self._warp_device.is_cuda else None, + enable_shadows=self.cfg.enable_shadows, ) try: # Registration mutates the shared scene; finish renders using its existing products first. @@ -820,7 +823,7 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Wait for rendering on the consuming stream, or on the host on Linux unless :data:`_DISABLE_LINUX_CUDA_CPU_SYNC_ENV` is ``1``. Consume the zero-copy view inside the context. Unmapping records the consuming stream; native release waits for its queued - reads and the last view to be dropped. + reads and the last view to be dropped. CPU consumers wait on the host and release without a CUDA stream. Args: render_var: OVRTX ``RenderVarOutput`` to map (looked up from ``frame.render_vars``). @@ -828,6 +831,14 @@ def _map_render_var_to_dlpack(self, render_var: RenderVarOutput) -> Iterator[wp. Yields: The render var's contents as a Warp array, valid for the duration of the context. """ + if not self._warp_device.is_cuda: + mapping = render_var.map(device=Device.CPU) + try: + mapping.wait() + yield wp.from_dlpack(mapping) + finally: + mapping.unmap() + return gpu_side_sync = _gpu_side_render_var_sync_enabled() # OVRTX uses 0 for no synchronization and 1 for Torch's legacy default stream (CUDA handle 0). stream = self._warp_device.stream.cuda_stream or 1 @@ -1193,7 +1204,7 @@ def _update_camera( binding = render_data.camera_xform_binding if binding is None: return - stream = self._warp_device.stream + stream = self._warp_device.stream if self._warp_device.is_cuda else None matrices = render_data.camera_writes.acquire(stream) wp.launch( create_camera_transforms_kernel, len(matrices), [positions, orientations, matrices], device=self._device @@ -1206,7 +1217,7 @@ def _update_camera( tensors=xform_tensor_from_warp(matrices), is_array=False, semantic=ovstage.AttributeSemantic.MATRIX, - cuda_stream=stream.cuda_stream or 1, + cuda_stream=(stream.cuda_stream or 1) if stream is not None else None, ).wait() else: operation = render_data.camera_writes.submit(binding, matrices, stream) @@ -1214,14 +1225,14 @@ def _update_camera( operation.wait() def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsics: wp.array, parameters: wp.array): - """Publish calibration columns from GPU memory into the renderer-owned scene.""" + """Publish calibration columns from their producing device into the renderer-owned scene.""" # A runtime calibration change is a scene write for this camera's bindings, so its # in-flight renders must finish first. Other cameras keep their pipelines. self._render_lease.reject_ordinary_operation("update ordinary camera intrinsics") errors = self.drain_pending_renders((render_data,)) if errors: raise ExceptionGroup("OVRTX renders failed before calibration update", errors) - stream = wp.get_stream(parameters.device).cuda_stream or 1 + stream = (wp.get_stream(parameters.device).cuda_stream or 1) if parameters.device.is_cuda else None if self._use_ovstage: self.scene.stage.write_attributes( render_data.camera_xform_binding, diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py index 85032bcbd42..30094ae61f2 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_usd.py @@ -144,7 +144,7 @@ def build_render_product_as_string( spec: CameraRenderSpec, render_data: OVRTXCameraRenderData, *, - device_id: int, + device_id: int | None = None, enable_shadows: bool = False, ) -> str: """Build a complete render product USD layer as a string. @@ -160,6 +160,7 @@ def build_render_product_as_string( render_data: Camera's render scope and product identity. device_id: CUDA device index the render product is pinned to, so its render var buffers are allocated on the same device as the Warp kernels that read them. + When None, leave native GPU selection automatic for CPU consumers. enable_shadows: Whether lights cast shadows. Defaults to False. Only honored for the ``simple_shading_*`` data types, which are the ones that select RTX Minimal mode. @@ -210,6 +211,8 @@ def build_render_product_as_string( for _, name, source in render_var_configs ) + device_ids_line = "" if device_id is None else f"uint[] deviceIds = [{device_id}]" + layer = f'''#usda 1.0 (defaultPrim = "{render_data.render_scope_name}") @@ -219,7 +222,7 @@ def RenderProduct "RenderProduct" ( prepend apiSchemas = ["OmniRtxSettingsCommonAdvancedAPI_1"] ) {{ rel camera = [<{render_data.camera_paths[0]}>] - uint[] deviceIds = [{device_id}] + {device_ids_line} {bg_type_line} {render_mode_block} token[] omni:rtx:waitForEvents = ["AllLoadingFinished", "OnlyOnFirstRequest"] diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index e82e0768d4f..1d85cb0d570 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -105,7 +105,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: write_array_attribute=lambda *args, **kwargs: None, write_attribute=lambda *args, **kwargs: None, ) - renderer._warp_device = SimpleNamespace(ordinal=0) + renderer._warp_device = SimpleNamespace(ordinal=0, is_cuda=True) renderer.scene.next_camera_id = 0 return renderer @@ -285,6 +285,7 @@ def test_create_render_data_writes_combined_stage_on_the_simulation_device(tmp_p class Device: ordinal = 1 + is_cuda = True def __str__(self): return "cuda:1" diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 8e43c9e4f1f..b168e516ad9 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -1042,7 +1042,7 @@ def test_ovrtx_render_var_sync_rejects_non_boolean_values(monkeypatch, value): @pytest.mark.parametrize( ("gpu_side", "cuda_stream", "expected_stream", "consumer_fails"), - [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False)], + [(True, 0, 1, False), (True, 99, 99, True), (False, 0, 1, False), (False, None, None, True)], ) def test_ovrtx_map_render_var_orders_the_read_against_render_completion( monkeypatch, gpu_side, cuda_stream, expected_stream, consumer_fails @@ -1050,15 +1050,17 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( """Order reads after rendering and release after reads, including when the consumer raises.""" events = [] mapping = types.SimpleNamespace( - wait=lambda: events.append("wait"), unmap=lambda *, stream: events.append(("release", stream)) + wait=lambda: events.append("wait"), unmap=lambda *, stream=None: events.append(("release", stream)) ) render_var = types.SimpleNamespace(map=MagicMock(return_value=mapping)) sentinel = object() monkeypatch.setattr(ovrtx_renderer_module, "_gpu_side_render_var_sync_enabled", lambda: gpu_side) - monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: sentinel) + monkeypatch.setattr(ovrtx_renderer_module.wp, "from_dlpack", lambda mapping: events.append("wrap") or sentinel) renderer = _make_ovrtx_renderer_without_backend() - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=cuda_stream)) + renderer._warp_device = types.SimpleNamespace( + is_cuda=cuda_stream is not None, stream=types.SimpleNamespace(cuda_stream=cuda_stream) + ) with pytest.raises(ValueError, match="consumer failed") if consumer_fails else contextlib.nullcontext(): with renderer._map_render_var_to_dlpack(render_var) as array: assert array is sentinel @@ -1066,8 +1068,11 @@ def test_ovrtx_map_render_var_orders_the_read_against_render_completion( if consumer_fails: raise ValueError("consumer failed") - assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) - assert events == ([] if gpu_side else ["wait"]) + ["consume", ("release", expected_stream)] + if cuda_stream is None: + assert render_var.map.call_args.kwargs == {"device": ovrtx_renderer_module.Device.CPU} + else: + assert render_var.map.call_args.kwargs["sync_stream"] == (expected_stream if gpu_side else 0) + assert events == ([] if gpu_side else ["wait"]) + ["wrap", "consume", ("release", expected_stream)] @pytest.mark.parametrize( @@ -1189,7 +1194,7 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs assert not vars(renderer).keys() & {"_camera_xform_binding", "_camera_xform_query", "_camera_paths_list"} monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(wp, "synchronize_stream", lambda stream: None) - renderer._warp_device = types.SimpleNamespace(stream=types.SimpleNamespace(cuda_stream=99)) + renderer._warp_device = types.SimpleNamespace(is_cuda=True, stream=types.SimpleNamespace(cuda_stream=99)) monkeypatch.setattr(ovrtx_renderer_module, "xform_tensor_from_warp", lambda value: value) positions = wp.array([[1, 2, 3], [4, 5, 6]], dtype=wp.vec3f, device="cpu") rotations = np.array([[1, 2, 3, 4], [0, 0, 0, 1]], dtype=np.float32) @@ -1213,6 +1218,7 @@ def test_pose_and_intrinsic_updates_target_the_given_camera(monkeypatch, use_ovs renderer.update_camera_intrinsics(cameras[1], wp.zeros(2, dtype=wp.mat33f, device="cpu"), parameters) if use_ovstage: assert renderer.scene.stage.write_attributes.call_args.args[0] is cameras[1].camera_xform_binding + assert all(desc.cuda_stream is None for desc in renderer.scene.stage.write_attributes.call_args.args[1]) bound_paths = [call.args[0] for call in renderer.scene.paths.create_path_list_from_strings.call_args_list] assert bound_paths == [ entry @@ -1907,6 +1913,83 @@ def get_points(*, output=None, offsets=None): renderer._geometry_writes.close() +@pytest.mark.parametrize("use_ovstage", [True, False]) +def test_cpu_camera_pose_updates_use_host_tensors_without_a_cuda_stream(use_ovstage): + renderer = _make_ovrtx_renderer_without_backend() + renderer._use_ovstage = use_ovstage + renderer.cfg = OVRTXRendererCfg() + renderer._warp_device = wp.get_device("cpu") + renderer._device = "cpu" + stage = MagicMock() + renderer.scene = types.SimpleNamespace(stage=stage, ordinal=11) + camera = _make_ovrtx_camera_render_data() + camera.camera_xform_binding = MagicMock() + camera.camera_writes = _AsyncWriteBuffers([wp.empty(2, dtype=wp.mat44d, device="cpu")]) + positions = wp.array([[1.0, 2.0, 3.0]] * 2, dtype=wp.vec3f, device="cpu") + orientations = wp.array([[0.0, 0.0, 0.0, 1.0]] * 2, dtype=wp.quatf, device="cpu") + update = renderer.update_camera + update(camera, positions, orientations, None) + call = stage.write_attribute.call_args if use_ovstage else camera.camera_xform_binding.write_async.call_args + assert call.kwargs["cuda_stream"] is None + if use_ovstage: + assert call.args == (camera.camera_xform_binding, "omni:xform") + assert call.kwargs["ordinal"] == 11 + else: + # Reusing the camera ring waits for previous writes without a CUDA stream. + update(camera, positions, orientations, None) + update(camera, positions, orientations, None) + camera.camera_writes.close() + + +@pytest.mark.parametrize("cuda_ordinal", [None, 2]) +def test_camera_registration_authors_valid_render_device_selection(monkeypatch, cuda_ordinal): + """CPU camera consumers leave GPU selection automatic; CUDA consumers retain their device.""" + from pxr import Usd + + class AuthoredProduct(Exception): + pass + + renderer = _make_ovrtx_renderer_without_backend() + renderer.cfg = OVRTXRendererCfg() + + class Device: + is_cuda = cuda_ordinal is not None + ordinal = -1 if cuda_ordinal is None else cuda_ordinal # Warp reports ordinal -1 for the CPU. + + def __str__(self): + return f"cuda:{cuda_ordinal}" if self.is_cuda else "cpu" + + renderer._warp_device = Device() + renderer._device = str(renderer._warp_device) + monkeypatch.setattr(ovrtx_renderer_module.wp, "get_device", lambda _device: renderer._warp_device) + renderer.scene.clone_env_paths = [] + renderer.drain_pending_renders = lambda: [] + spec = types.SimpleNamespace( + cfg=_make_camera_cfg(["rgb"]), + device=renderer._device, + num_instances=1, + camera_prim_paths=["/World/envs/env_0/Camera"], + render_settings={}, + ) + build_product = ovrtx_renderer_module.build_render_product_as_string + + def consume_product(spec, render_data, **kwargs): + stage = Usd.Stage.CreateInMemory() + assert stage.GetRootLayer().ImportFromString(build_product(spec, render_data, **kwargs)) + product = stage.GetPrimAtPath(render_data.render_product_path) + devices = product.GetAttribute("deviceIds") + if cuda_ordinal is None: + assert not devices.HasAuthoredValueOpinion() + else: + assert list(devices.Get()) == [cuda_ordinal] + # Native population is outside this parser/selection regression. + raise AuthoredProduct + + monkeypatch.setattr(ovrtx_renderer_module, "build_render_product_as_string", consume_product) + with pytest.raises(AuthoredProduct): + renderer.create_render_data(spec) + + def test_leased_camera_cleanup_waits_for_lease_release(leased_renderer, monkeypatch): """Releasing a leased camera would remove its product from the borrowed Stage; it waits for release.""" renderer = leased_renderer From df13e355a28736f89a4d34132081002ceb4a481a Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:40:26 -0500 Subject: [PATCH 128/145] Re-render fixed-base G1 three times after reset for its camera images --- .../changelog.d/ovxr-robot-camera-observations.changed.rst | 3 +++ .../fixed_base_upper_body_ik_g1_env_cfg.py | 1 + 2 files changed, 4 insertions(+) diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst index 1920a658d8b..772d5f8b153 100644 --- a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst @@ -2,3 +2,6 @@ and head-panel uploads are capped at 20 Hz (wrist panels at 10 Hz). * Changed GR1T2 Kit ``--xr`` camera panels to a world-fixed layout. OVXR presents head-locked panels only; pass ``--ovxr_pip_head_locked`` there. +* Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), so + its head and wrist images are not stale after a reset. This adds render work per reset when cameras + are rendered. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index 0c72f7b5584..08ec7863bf8 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -451,6 +451,7 @@ def __post_init__(self): # simulation settings self.sim.dt = 1 / 200 # 200Hz self.sim.render_interval = 2 + self.num_rerenders_on_reset = 3 # Set the URDF and mesh paths for the IK controller urdf_omniverse_path = f"{ISAACLAB_NUCLEUS_DIR}/Controllers/LocomanipulationAssets/unitree_g1_kinematics_asset/g1_29dof_with_hand_only_kinematics.urdf" # noqa: E501 From 7f5d4739911771a76583f7f39ddcba4f0ccf2d24 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:40:40 -0500 Subject: [PATCH 129/145] Point a blank main view with a working camera panel at the client origin --- source/isaaclab_teleop/docs/kitless_xr.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 11f4a0b9639..9a5aa600a29 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -92,6 +92,8 @@ For desktop IWER, select **VR** and set **Reference Space** to **local-floor** under **Teleoperator adjustment** before connecting. This keeps the emulated head and controllers relative to the floor. Between trials, end the previous XR session and reload the client page before reconnecting; recheck both settings. +A blank or below-scene main view with a working camera panel warrants checking +this client origin before changing the renderer. `--ovxr` and Kit's `--xr` select separate XR runtimes and are mutually exclusive. An explicit `ISAAC_LAB_OVRTX_USE_OVSTAGE=0` is incompatible with OVXR. From a939b77e5a91541d2fe122051764ca194ab8004e Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:41:06 -0500 Subject: [PATCH 130/145] Keep the GR1T2 Kit camera-panel layout unchanged --- .../ovxr-robot-camera-observations.changed.rst | 2 -- .../contrib/pick_place/pickplace_gr1t2_env_cfg.py | 13 +------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst index 772d5f8b153..07d4286ff83 100644 --- a/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst +++ b/source/isaaclab_tasks/changelog.d/ovxr-robot-camera-observations.changed.rst @@ -1,7 +1,5 @@ * Changed Kit ``--xr`` camera panels for the fixed-base G1 task: it shows head and wrist panels, and head-panel uploads are capped at 20 Hz (wrist panels at 10 Hz). -* Changed GR1T2 Kit ``--xr`` camera panels to a world-fixed layout. OVXR presents head-locked - panels only; pass ``--ovxr_pip_head_locked`` there. * Changed fixed-base G1 to re-render three times after each reset (``num_rerenders_on_reset = 3``), so its head and wrist images are not stale after a reset. This adds render work per reset when cameras are rendered. diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py index 0aec50e63e0..7ea3e1e18f1 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/pickplace_gr1t2_env_cfg.py @@ -683,13 +683,7 @@ def __post_init__(self): self.actions.upper_body_ik.controller.urdf_output_dir = self.temp_urdf_dir if ISAACLAB_TELEOP_AVAILABLE: - from isaaclab_teleop import ( - GloveHapticFeedbackCfg, - IsaacTeleopCfg, - XrCameraFeedCfg, - XrCameraFeedLayoutCfg, - XrCfg, - ) + from isaaclab_teleop import GloveHapticFeedbackCfg, IsaacTeleopCfg, XrCameraFeedCfg, XrCfg # Isaac Capture-based teleoperation pipeline. self.xr = XrCfg( @@ -713,11 +707,6 @@ def __post_init__(self): max_update_hz=0.0, ) ], - xr_camera_feed_layout=XrCameraFeedLayoutCfg( - placement="world", - world_position_m=(0.0, 0.8, 1.6), - world_orientation_xyzw=(0.70710678, 0.0, 0.0, 0.70710678), - ), ) # Per-finger haptic glove feedback: vibrate each finger of the operator's From 430a5979b8fb8b3b61e16f92c88a69e3187acbb9 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:42:48 -0500 Subject: [PATCH 131/145] Drop camera-panel tests that only parse arguments or pin other layers The live and replay session-construction tests already prove that the PiP and mirror options reach the OVXR session configuration, so the four tests that only read the parsed namespace go. So do an assertion on a Capture binding attribute that Lab never sets, and the GR1T2 anchor test, which pins earlier layers' anchor and configuration behaviour; the real-asset head-exclusion test covers the GR1T2 partition patterns. --- .../test/cli/test_teleop_entrypoints.py | 29 ------------------- .../test/core/test_xr_camera_feed_presets.py | 28 ------------------ .../test/test_ov_libraries_xr.py | 1 - .../test/test_teleop_entrypoint_units.py | 27 ----------------- .../test/test_teleop_replay_bootstrap.py | 15 ---------- 5 files changed, 100 deletions(-) diff --git a/source/isaaclab/test/cli/test_teleop_entrypoints.py b/source/isaaclab/test/cli/test_teleop_entrypoints.py index 4011b808a5b..fffe3ebc165 100644 --- a/source/isaaclab/test/cli/test_teleop_entrypoints.py +++ b/source/isaaclab/test/cli/test_teleop_entrypoints.py @@ -90,35 +90,6 @@ def parse(arguments): assert parse(["--ovxr", "--cloudxr_env", "none"]).cloudxr_env is None -def test_ovxr_output_options_reach_existing_runtime_fields(source_checkout_root, monkeypatch): - """Camera options configure panels, the desktop mirror and XR startup.""" - options = [ - "--ovxr", - "--ovxr_session_mode", - "vr", - "--ovxr_desktop_mirror", - "--ovxr_desktop_mirror_fps", - "20", - "--ovxr_pip_camera", - "robot_pov_cam", - "--ovxr_pip_fps", - "10", - "--ovxr_pip_head_locked", - "--ovxr_pip_view_position", - "0", - "0.2", - "-0.5", - ] - script = source_checkout_root / "scripts/environments/teleoperation/teleop_se3_agent.py" - monkeypatch.setattr(sys, "argv", [str(script), *options]) - parsed = importlib.import_module("isaaclab_teleop.entrypoints.teleop_se3_agent")._parse_args() - assert parsed.using_ovxr and parsed.ovxr_session_mode == "vr" - assert parsed.ovxr_desktop_mirror - assert parsed.ovxr_desktop_mirror_fps == 20 - assert parsed.ovxr_pip_camera == ["robot_pov_cam"] and parsed.ovxr_pip_fps == 10 - assert parsed.ovxr_pip_head_locked and parsed.ovxr_pip_view_position == [0, 0.2, -0.5] - - @pytest.mark.parametrize(("command", "script_parts"), TELEOP_WORKFLOWS.items()) def test_teleop_dispatches_to_its_workflow(source_checkout_root: Path, command, script_parts, monkeypatch): """``isaaclab teleop`` runs each workflow's entrypoint in process with the remaining arguments.""" diff --git a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py index 4eed3f74f1c..745b9d76f7c 100644 --- a/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py +++ b/source/isaaclab_tasks/test/core/test_xr_camera_feed_presets.py @@ -3,20 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause -import re - -import numpy as np import pytest from isaaclab_ov.renderers import OVRTXRendererCfg from isaaclab_physx.renderers import IsaacRtxRendererCfg -from isaaclab_teleop import xr_anchor_manager from isaaclab_tasks.contrib.locomanip_pick_place.fixed_base_upper_body_ik_g1_env_cfg import ( FixedBaseUpperBodyIKG1EnvCfg, ) from isaaclab_tasks.contrib.locomanip_pick_place.locomanipulation_g1_env_cfg import LocomanipulationG1EnvCfg from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_env_cfg import PickPlaceGR1T2EnvCfg -from isaaclab_tasks.contrib.pick_place.pickplace_gr1t2_waist_enabled_env_cfg import PickPlaceGR1T2WaistEnabledEnvCfg from isaaclab_tasks.utils.hydra import resolve_presets @@ -91,29 +86,6 @@ def test_locomanipulation_g1_retains_recorded_camera(): assert cfg.image_obs_list == ["robot_pov_cam"] -@pytest.mark.parametrize("env_cfg_type", [PickPlaceGR1T2EnvCfg, PickPlaceGR1T2WaistEnabledEnvCfg]) -def test_gr1t2_xr_anchor_preserves_operator_view_and_head_partition(monkeypatch, env_cfg_type): - """The operator faces the table; head exclusion remains camera-selective.""" - cfg = env_cfg_type() - assert cfg.xr.anchor_pos == (0.0, 0.0, 0.0) - assert cfg.xr.render_anchor_pos is None - assert cfg.xr.render_anchor_rot is None - assert any( - re.fullmatch(pattern, "/World/envs/env_0/Robot/head_yaw_link/visuals/head_yaw_link/mesh") - for pattern in cfg.xr.excluded_visual_meshes - ) - monkeypatch.setattr(xr_anchor_manager, "XRCore", None) - monkeypatch.setattr(xr_anchor_manager, "_xr_anchor_prim_exists", lambda _path: True) - anchor = xr_anchor_manager.XrAnchorManager(cfg.xr) - assert anchor.render_anchor_path == anchor.anchor_headset_path - transform = anchor.get_world_matrix() - eye = transform @ np.array([0.0, 1.6, 0.0, 1.0]) - forward = transform @ np.array([0.0, 0.0, -1.0, 0.0]) - np.testing.assert_allclose(eye, (0.0, 0.0, 1.6, 1.0)) - np.testing.assert_allclose(forward, (0.0, 1.0, 0.0, 0.0)) - assert np.dot(np.asarray(cfg.scene.packing_table.init_state.pos) - eye[:3], forward[:3]) > 0.0 - - def test_xr_camera_reference_renderer_resolves_for_supported_backends(): """Reference cameras retain Isaac RTX defaults and OVRTX compatibility.""" camera_renderer_cfg = PickPlaceGR1T2EnvCfg().scene.robot_pov_cam.renderer_cfg diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 87efd82e743..da78ceef84e 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -440,7 +440,6 @@ def test_host_lends_the_selected_auxiliary_product_without_rendering_it(xr_modul binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) assert binding.frame_consumers[0].render_product_paths == bootstrap.lease.auxiliary_render_product_paths - assert not hasattr(binding, "auxiliary_frame_source") def test_host_rejects_a_missing_selected_auxiliary_camera(xr_module, monkeypatch): diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 4eb3094af22..46dea02c9c7 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -739,33 +739,6 @@ def stop(self): assert haptics == {"update": 2, "stop": 1} -def test_desktop_mirror_cli_is_explicit_and_bounded(monkeypatch): - agent = _load_teleop_agent( - monkeypatch, - "--ovxr", - "--ovxr_desktop_mirror", - "--ovxr_desktop_mirror_fps", - "20", - ) - - assert agent.args_cli.ovxr_desktop_mirror - assert agent.args_cli.ovxr_desktop_mirror_fps == 20.0 - - -def test_auxiliary_camera_pip_cli_is_explicit_and_bounded(monkeypatch): - agent = _load_teleop_agent( - monkeypatch, - "--ovxr", - "--ovxr_pip_camera", - "robot_pov_cam", - "--ovxr_pip_fps", - "20", - ) - - assert agent.args_cli.ovxr_pip_camera == ["robot_pov_cam"] - assert agent.args_cli.ovxr_pip_fps == 20.0 - - @pytest.mark.parametrize("entrypoint", ["live", "replay"]) @pytest.mark.parametrize( "flags, expected", diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index 9d6d930e39f..6921e6e0a3a 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -456,18 +456,3 @@ def test_replay_and_live_share_ovxr_presentation_validation(monkeypatch, capsys, with pytest.raises(SystemExit): module._parse_args() assert message in capsys.readouterr().err - - -def test_replay_accepts_live_ovxr_rate_and_position_options(monkeypatch): - from isaaclab_teleop.entrypoints import teleop_replay_agent - - arguments = ["--task", "Test", "--replay_file", "recording.mcap", "--ovxr", "--ovxr_pip_camera", "head"] - arguments += ["--ovxr_pip_head_locked", "--ovxr_pip_view_position", "0", "-0.1", "-0.8"] - arguments += ["--ovxr_pip_fps", "15", "--ovxr_desktop_mirror_fps", "20", "--headless"] - monkeypatch.setattr(sys, "argv", ["teleop_replay_agent.py", *arguments]) - monkeypatch.setattr(teleop_replay_agent, "args_cli", None, raising=False) - args = teleop_replay_agent._parse_args() - assert args.ovxr_pip_fps == 15.0 - assert args.ovxr_desktop_mirror_fps == 20.0 - assert args.ovxr_pip_view_position == [0.0, -0.1, -0.8] - assert args.ovxr_desktop_mirror is False From 56687389fd5c129a85dbabfac07dc3a084ca12a2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:43:56 -0500 Subject: [PATCH 132/145] Let --headless alone turn off the OVXR desktop mirror --ovxr already enables the mirror and --headless disables it, so --ovxr_desktop_mirror changed nothing. Remove the flag, its no-op validation and the guide sentences about it. The session reads the mirror selection from --headless directly, and the tests no longer pin the flag. --- source/isaaclab/test/cli/test_teleop_entrypoints.py | 1 - source/isaaclab_teleop/docs/kitless_xr.md | 4 +--- .../isaaclab_teleop/entrypoints/_ovxr_launch.py | 13 ++----------- .../entrypoints/teleop_replay_agent.py | 2 +- .../isaaclab_teleop/entrypoints/teleop_se3_agent.py | 2 +- .../test/test_teleop_entrypoint_units.py | 4 ---- .../test/test_teleop_replay_bootstrap.py | 2 +- 7 files changed, 6 insertions(+), 22 deletions(-) diff --git a/source/isaaclab/test/cli/test_teleop_entrypoints.py b/source/isaaclab/test/cli/test_teleop_entrypoints.py index fffe3ebc165..e80d41b9bc1 100644 --- a/source/isaaclab/test/cli/test_teleop_entrypoints.py +++ b/source/isaaclab/test/cli/test_teleop_entrypoints.py @@ -60,7 +60,6 @@ def test_teleop_workflow_help_exposes_task_preset_selectors(source_checkout_root if command == "run": assert re.search(r"^\s+--ovxr(?:\s|$)", result.stdout, re.MULTILINE) assert "--ovxr_pip_camera" in result.stdout - assert "--ovxr_desktop_mirror" in result.stdout def test_ovxr_omitted_launch_options_match_documented_defaults(source_checkout_root, monkeypatch): diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 9a5aa600a29..b61ca8323f1 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -164,9 +164,7 @@ GR1T2 and G1 locomanipulation provide `robot_pov_cam`; fixed-base G1 also provid its scene. Use the desktop mirror without `--ovxr_pip_camera` for that task. The default desktop mirror shows completed eye or camera images locally. -`--headless` disables it and takes precedence over `--ovxr_desktop_mirror`. -The explicit mirror flag is also accepted, but is unnecessary for the default. -`--ovxr_desktop_mirror_fps` limits its update rate. The mirror requires the +`--headless` disables it. `--ovxr_desktop_mirror_fps` limits its update rate. The mirror requires the `desktop-mirror` extra, Python 3.12 or newer, and a usable display. On Linux, use a GPU-backed desktop session for the mirror. OVUI's default Vulkan backend requires no `OMNIUI_BACKEND` setting. Lab and Capture preserve the application's diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 78316d871f4..118bf044fa1 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -67,12 +67,6 @@ def require_kitless_physics(env_cfg: Any, task: str) -> None: def add_ovxr_presentation_args(parser: argparse.ArgumentParser) -> None: """Add the OVXR camera-panel and desktop-mirror options shared by live teleoperation and replay.""" - parser.add_argument( - "--ovxr_desktop_mirror", - action="store_true", - default=False, - help="Show the OVXR desktop mirror (enabled by default; --headless disables it).", - ) parser.add_argument("--headless", action="store_true", help="Disable the OVXR desktop mirror.") parser.add_argument( "--ovxr_desktop_mirror_fps", @@ -110,20 +104,17 @@ def add_ovxr_presentation_args(parser: argparse.ArgumentParser) -> None: def validate_ovxr_presentation_args( parser: argparse.ArgumentParser, args: argparse.Namespace, *, using_ovxr: bool ) -> None: - """Validate the shared OVXR presentation options and resolve the desktop-mirror default. + """Validate the shared OVXR presentation options. Args: parser: Parser used to report errors. - args: Parsed namespace; ``ovxr_desktop_mirror`` is set to the resolved selection. + args: Parsed namespace. using_ovxr: Whether the command selected ``--ovxr``. """ - if args.ovxr_desktop_mirror and not using_ovxr: - parser.error("--ovxr_desktop_mirror requires --ovxr") if args.ovxr_pip_camera is not None and not using_ovxr: parser.error("--ovxr_pip_camera requires --ovxr") if (args.ovxr_pip_head_locked or args.ovxr_pip_view_position is not None) and not using_ovxr: parser.error("OVXR presentation options require --ovxr") - args.ovxr_desktop_mirror = using_ovxr and not args.headless if args.ovxr_pip_camera is not None and getattr(args, "disable_external_cameras", False): parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") if not math.isfinite(args.ovxr_pip_fps) or args.ovxr_pip_fps <= 0.0: diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index c92b73c065b..34f12291f5b 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -1261,7 +1261,7 @@ def __init__(self, env_cfg, selected_cameras: tuple[OvLibrariesXrCameraConfig, . session_mode=args_cli.ovxr_session_mode, cloudxr_env_file=args_cli.cloudxr_env, auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, + desktop_mirror_enabled=not args_cli.headless, desktop_mirror_max_fps=args_cli.ovxr_desktop_mirror_fps, auxiliary_cameras=selected_cameras, ), diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index 0d866ddb57e..d06e00b6b25 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -491,7 +491,7 @@ def run_teleoperation( # noqa: C901 session_mode=args_cli.ovxr_session_mode, cloudxr_env_file=args_cli.cloudxr_env, auto_launch_cloudxr=args_cli.auto_launch_cloudxr, - desktop_mirror_enabled=args_cli.ovxr_desktop_mirror, + desktop_mirror_enabled=not args_cli.headless, desktop_mirror_max_fps=args_cli.ovxr_desktop_mirror_fps, auxiliary_cameras=selected_cameras, ), diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 46dea02c9c7..7c12af5e2c2 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -745,9 +745,6 @@ def stop(self): [ ([], True), (["--headless"], False), - (["--ovxr_desktop_mirror"], True), - (["--headless", "--ovxr_desktop_mirror"], False), - (["--ovxr_desktop_mirror", "--headless"], False), ], ) def test_ovxr_mirror_cli_reaches_session_construction(monkeypatch, entrypoint, flags, expected): @@ -817,7 +814,6 @@ def test_non_ovxr_cli_keeps_mirror_disabled(monkeypatch, entrypoint, flags): argv += ["--replay_file", "recording.mcap"] monkeypatch.setattr(module, "hydra_overrides", None, raising=False) args = module._parse_args(argv) - assert args.ovxr_desktop_mirror is False assert args.headless is ("--headless" in flags) diff --git a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py index 6921e6e0a3a..b36e2b6e746 100644 --- a/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py +++ b/source/isaaclab_teleop/test/test_teleop_replay_bootstrap.py @@ -79,7 +79,7 @@ def test_replay_presentation_preserves_camera_layout(monkeypatch, replay_present ovxr_session_mode="ar", cloudxr_env="none", auto_launch_cloudxr=False, - ovxr_desktop_mirror=True, + headless=False, ovxr_desktop_mirror_fps=20.0, ) monkeypatch.setitem(replay_presentation_type.__init__.__globals__, "args_cli", args) From cff7b8c393cd121c9bafa25cb8aa023356fd9286 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:45:10 -0500 Subject: [PATCH 133/145] Reject --headless without --ovxr Kit removed --headless and runs without a local viewport unless --visualizer is given, so a Kit teleoperation run accepted the option and ignored it. Like the other OVXR presentation options, --headless now requires --ovxr, and the parser says why. --- .../entrypoints/_ovxr_launch.py | 2 ++ .../test/test_teleop_entrypoint_units.py | 20 ++++++------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 118bf044fa1..c9fa7c90f81 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -115,6 +115,8 @@ def validate_ovxr_presentation_args( parser.error("--ovxr_pip_camera requires --ovxr") if (args.ovxr_pip_head_locked or args.ovxr_pip_view_position is not None) and not using_ovxr: parser.error("OVXR presentation options require --ovxr") + if args.headless and not using_ovxr: + parser.error("--headless requires --ovxr; Kit runs headless unless --visualizer is given") if args.ovxr_pip_camera is not None and getattr(args, "disable_external_cameras", False): parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") if not math.isfinite(args.ovxr_pip_fps) or args.ovxr_pip_fps <= 0.0: diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 7c12af5e2c2..51813b0869a 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -792,18 +792,9 @@ def prepare(_sim, xr_cfg, *_args, **_kwargs): assert isaaclab_teleop.OvLibrariesXrCfg().desktop_mirror_enabled is False -@pytest.mark.parametrize( - "entrypoint, flags", - [ - ("live", []), - ("live", ["--headless"]), - ("live", ["--xr"]), - ("live", ["--xr", "--headless"]), - ("replay", []), - ("replay", ["--headless"]), - ], -) -def test_non_ovxr_cli_keeps_mirror_disabled(monkeypatch, entrypoint, flags): +@pytest.mark.parametrize("entrypoint, flags", [("live", ["--xr", "--headless"]), ("replay", ["--headless"])]) +def test_headless_requires_ovxr(monkeypatch, capsys, entrypoint, flags): + """Kit runs headless by default, so only the OVXR desktop mirror accepts ``--headless``.""" from isaaclab_teleop.entrypoints import teleop_replay_agent, teleop_se3_agent module = teleop_se3_agent if entrypoint == "live" else teleop_replay_agent @@ -813,8 +804,9 @@ def test_non_ovxr_cli_keeps_mirror_disabled(monkeypatch, entrypoint, flags): if entrypoint == "replay": argv += ["--replay_file", "recording.mcap"] monkeypatch.setattr(module, "hydra_overrides", None, raising=False) - args = module._parse_args(argv) - assert args.headless is ("--headless" in flags) + with pytest.raises(SystemExit): + module._parse_args(argv) + assert "--headless requires --ovxr" in capsys.readouterr().err def _run_ovxr_pip_workflow(monkeypatch, entrypoint, layout, flags, calls): From 7bbaa10af7817e05ec91a40cc10f7b34a18e60c2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:46:03 -0500 Subject: [PATCH 134/145] Rename OvLibrariesXrCameraConfig to OvLibrariesXrCameraCfg --- .../source/api/lab_teleop/isaaclab_teleop.rst | 4 +-- .../changelog.d/ovxr-camera-panels.added.rst | 2 +- .../isaaclab_teleop/__init__.pyi | 4 +-- .../isaaclab_teleop/camera_feed.py | 8 ++--- .../entrypoints/_ovxr_launch.py | 4 +-- .../entrypoints/teleop_replay_agent.py | 8 ++--- .../entrypoints/teleop_se3_agent.py | 6 ++-- .../isaaclab_teleop/ov_libraries_xr.py | 4 +-- .../isaaclab_teleop/ovxr_cfg.py | 4 +-- .../test/test_ov_libraries_xr.py | 30 +++++++++---------- 10 files changed, 36 insertions(+), 38 deletions(-) diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 42148db8654..ca5d0b3ba06 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -23,7 +23,7 @@ isaaclab_teleop XrAnchorRotationMode XrAnchorSynchronizer OvLibrariesXrCfg - OvLibrariesXrCameraConfig + OvLibrariesXrCameraCfg OvLibrariesXrSession .. rubric:: Functions @@ -70,7 +70,7 @@ Kitless OVXR Session .. autoclass:: OvLibrariesXrCfg :members: -.. autoclass:: OvLibrariesXrCameraConfig +.. autoclass:: OvLibrariesXrCameraCfg :members: .. autoclass:: OvLibrariesXrSession diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst index 8dabe03c359..df8acda7090 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -1,6 +1,6 @@ * Added optional OVXR camera panels and a desktop mirror as application-configured Isaac Capture frame consumers using completed render products, with OVUI's default - Vulkan backend for the mirror. :class:`~isaaclab_teleop.OvLibrariesXrCameraConfig` selects a panel. + Vulkan backend for the mirror. :class:`~isaaclab_teleop.OvLibrariesXrCameraCfg` selects a panel. * Enabled the mirror by default in OVXR live and MCAP replay entrypoints; ``--headless`` disables the local window while preserving XR and camera-panel rendering. * Supported task-configured automatic layouts for head-locked panels diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index 148160376e0..4e94d8cbdaa 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,7 +15,7 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", - "OvLibrariesXrCameraConfig", + "OvLibrariesXrCameraCfg", "OvLibrariesXrCfg", "OvLibrariesXrSession", "SupportsControlEvents", @@ -70,7 +70,7 @@ from .isaac_teleop_cfg import ( ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device from .ov_libraries_xr import OvLibrariesXrSession -from .ovxr_cfg import OvLibrariesXrCameraConfig, OvLibrariesXrCfg +from .ovxr_cfg import OvLibrariesXrCameraCfg, OvLibrariesXrCfg from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index e2ac1091127..50d25bfb06a 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -32,7 +32,7 @@ from isaaclab.sensors import Camera, CameraCfg - from .ovxr_cfg import OvLibrariesXrCameraConfig + from .ovxr_cfg import OvLibrariesXrCameraCfg logger = logging.getLogger(__name__) @@ -71,13 +71,13 @@ def resolve_ovxr_camera_configs( *, max_fps: float, position_override: tuple[float, float, float] | None = None, -) -> tuple[OvLibrariesXrCameraConfig, ...]: +) -> tuple[OvLibrariesXrCameraCfg, ...]: """Resolve selected feeds into OVXR panels without creating native resources. A feed's positive update cap limits the caller's rate; zero leaves that rate unchanged. A position override applies only to a single selected panel. """ - from .ovxr_cfg import OvLibrariesXrCameraConfig + from .ovxr_cfg import OvLibrariesXrCameraCfg _validate_layout_cfg(layout) if selected and layout.placement != "head_locked": @@ -94,7 +94,7 @@ def resolve_ovxr_camera_configs( cameras = [] for (name, camera, _feed), feed in zip(selected, feeds, strict=True): cameras.append( - OvLibrariesXrCameraConfig( + OvLibrariesXrCameraCfg( name=name, camera_cfg=camera, max_fps=min(max_fps, feed.max_update_hz) if feed.max_update_hz > 0.0 else max_fps, diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index c9fa7c90f81..2a705db4f87 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -15,7 +15,7 @@ from ..xr_cfg import remove_camera_configs if TYPE_CHECKING: - from ..ovxr_cfg import OvLibrariesXrCameraConfig + from ..ovxr_cfg import OvLibrariesXrCameraCfg def prepare_ovxr_launch(env_cfg: Any, task: str, num_envs: int, *, keep: tuple[str, ...] = ()) -> Any: @@ -134,7 +134,7 @@ def validate_ovxr_presentation_args( def resolve_ovxr_pip_cameras( env_cfg: Any, args: argparse.Namespace, renderer_cfg: Any -) -> tuple[OvLibrariesXrCameraConfig, ...]: +) -> tuple[OvLibrariesXrCameraCfg, ...]: """Select the requested PiP cameras and resolve their OVXR panels before launch. Selection assigns ``renderer_cfg`` to each selected camera, so the OVXR session and its panels diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index 34f12291f5b..f2cd45c1aef 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -299,7 +299,7 @@ CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopDevice, - OvLibrariesXrCameraConfig, + OvLibrariesXrCameraCfg, create_isaac_teleop_device, poll_control_events, ) @@ -1079,7 +1079,7 @@ def _prepare_env_cfg( num_envs: int, device: str, overrides: Sequence[str] = (), -) -> tuple[ManagerBasedRLEnvCfg, object | None, tuple[OvLibrariesXrCameraConfig, ...]]: +) -> tuple[ManagerBasedRLEnvCfg, object | None, tuple[OvLibrariesXrCameraCfg, ...]]: """Build and tweak an env config suitable for non-interactive replay. Mirrors the env-config mutations performed by ``record_demos.py``'s @@ -1249,7 +1249,7 @@ def _process_success_condition( class _ReplayPresentation: """Own a batch's viewer independently of each recorded-input session.""" - def __init__(self, env_cfg, selected_cameras: tuple[OvLibrariesXrCameraConfig, ...]): + def __init__(self, env_cfg, selected_cameras: tuple[OvLibrariesXrCameraCfg, ...]): from isaaclab_ov.renderers import OVRTXRendererCfg from .. import OvLibrariesXrCfg, OvLibrariesXrSession @@ -1794,7 +1794,7 @@ def main() -> int: def _run_replay_batch( env_cfg: ManagerBasedRLEnvCfg, success_term: object | None, - selected_cameras: tuple[OvLibrariesXrCameraConfig, ...], + selected_cameras: tuple[OvLibrariesXrCameraCfg, ...], ) -> int: """Build the env, run the ``--num_replays`` replays, and report the stats; see :func:`main`. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py index d06e00b6b25..997dbd261a4 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_se3_agent.py @@ -46,7 +46,7 @@ CLOUDXR_JS_ENV, CLOUDXR_STANDALONE_ENV, IsaacTeleopCfg, - OvLibrariesXrCameraConfig, + OvLibrariesXrCameraCfg, se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, @@ -208,7 +208,7 @@ def main() -> None: # noqa: C901 def _resolve_env_cfg() -> tuple[ - ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, tuple[OvLibrariesXrCameraConfig, ...] + ManagerBasedRLEnvCfg, IsaacTeleopCfg | DeviceCfg, bool, tuple[OvLibrariesXrCameraCfg, ...] ]: """Resolve task configuration and teleop ownership before runtime launch.""" # Resolve the task configuration through Hydra so CLI presets are applied. @@ -443,7 +443,7 @@ def run_teleoperation( # noqa: C901 env_cfg: ManagerBasedRLEnvCfg, teleop_cfg: IsaacTeleopCfg | DeviceCfg, use_isaac_teleop: bool, - selected_cameras: tuple[OvLibrariesXrCameraConfig, ...], + selected_cameras: tuple[OvLibrariesXrCameraCfg, ...], ) -> None: """Create the environment and teleop device, then run the teleoperation loop until the app is closed.""" diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index f7d471e5dd1..72f0d7b5b8c 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -21,7 +21,7 @@ from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger from isaaclab_ov.renderers import OVRTXRendererCfg -from .ovxr_cfg import OvLibrariesXrCameraConfig, OvLibrariesXrCfg +from .ovxr_cfg import OvLibrariesXrCameraCfg, OvLibrariesXrCfg from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation if TYPE_CHECKING: @@ -30,7 +30,7 @@ from isaaclab.sim import SimulationCfg -__all__ = ["OvLibrariesXrCameraConfig", "OvLibrariesXrCfg", "OvLibrariesXrSession"] +__all__ = ["OvLibrariesXrCameraCfg", "OvLibrariesXrCfg", "OvLibrariesXrSession"] logger = logging.getLogger(__name__) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py index ae280a5527d..347d6538a84 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -25,7 +25,7 @@ @dataclass(frozen=True) -class OvLibrariesXrCameraConfig: +class OvLibrariesXrCameraCfg: """One application-selected sensor and its camera-panel policy.""" name: str @@ -96,7 +96,7 @@ class OvLibrariesXrCfg: """Whether to display the owner-managed desktop mirror.""" desktop_mirror_max_fps: float = 30.0 """Finite positive desktop mirror refresh cap [Hz].""" - auxiliary_cameras: tuple[OvLibrariesXrCameraConfig, ...] = () + auxiliary_cameras: tuple[OvLibrariesXrCameraCfg, ...] = () """Up to three uniquely named camera panels; an empty selection disables panels.""" extra_openxr_extensions: tuple[str, ...] = () """Additional OpenXR extension names requested when creating the session.""" diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index da78ceef84e..080a3c0ac35 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -432,7 +432,7 @@ def test_host_lends_the_selected_auxiliary_product_without_rendering_it(xr_modul env, xr_module.OvLibrariesXrCfg( auxiliary_cameras=( - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), ), ), ) @@ -451,7 +451,7 @@ def test_host_rejects_a_missing_selected_auxiliary_camera(xr_module, monkeypatch env, xr_module.OvLibrariesXrCfg( auxiliary_cameras=( - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), ), ), ) @@ -495,7 +495,7 @@ def data(self): cameras = {name: PoseOnlyCamera() for name in (*names, "unselected")} env.scene = SimpleNamespace(stage=None, sensors=cameras) host._auxiliary_cameras = tuple( - xr_module.OvLibrariesXrCameraConfig(name, SimpleNamespace(width=720, height=450)) for name in names + xr_module.OvLibrariesXrCameraCfg(name, SimpleNamespace(width=720, height=450)) for name in names ) bootstrap.lease.auxiliary_render_product_paths = tuple(f"/Render/{name}" for name in names) binding = host.attach_on_render_thread(host.offer_render_owner_transfer()) @@ -743,7 +743,7 @@ def set_status_text(self, text): desktop_mirror_enabled=True, desktop_mirror_max_fps=20.0, auxiliary_cameras=( - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=720, height=450)), + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=720, height=450)), ) if camera_enabled else (), @@ -883,16 +883,16 @@ def test_desktop_mirror_rate_must_be_positive_and_finite(xr_module): def test_auxiliary_camera_config_is_preconfigured(xr_module, monkeypatch): with pytest.raises(ValueError, match="name and configuration"): - xr_module.OvLibrariesXrCameraConfig("", object()) + xr_module.OvLibrariesXrCameraCfg("", object()) with pytest.raises(ValueError, match="name and configuration"): - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", None) + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", None) with pytest.raises(ValueError, match="max_fps"): - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", object(), max_fps=0.0) + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", object(), max_fps=0.0) bootstrap = _FakeBootstrap() camera_cfg = SimpleNamespace(width=720, height=450) monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_args: bootstrap) - camera = xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", camera_cfg) + camera = xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", camera_cfg) session = xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,))) assert isinstance(session, xr_module.OvLibrariesXrSession) assert bootstrap.auxiliary_camera_cfgs == [camera_cfg] @@ -1001,7 +1001,7 @@ def test_three_auxiliary_camera_configs_prepare_the_same_bootstrap(xr_module, mo bootstrap = _FakeBootstrap() monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) cameras = tuple( - xr_module.OvLibrariesXrCameraConfig( + xr_module.OvLibrariesXrCameraCfg( name=name, camera_cfg=SimpleNamespace(width=640, height=480), max_fps=hz, space="head_locked" ) for name, hz in zip(("robot_pov_cam", "left_wrist_cam", "right_wrist_cam"), (20, 10, 10), strict=True) @@ -1009,7 +1009,7 @@ def test_three_auxiliary_camera_configs_prepare_the_same_bootstrap(xr_module, mo session = xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=cameras)) assert bootstrap.auxiliary_camera_cfgs == [camera.camera_cfg for camera in cameras] assert session is not None - extra = xr_module.OvLibrariesXrCameraConfig("overview", SimpleNamespace(width=640, height=480)) + extra = xr_module.OvLibrariesXrCameraCfg("overview", SimpleNamespace(width=640, height=480)) with pytest.raises(ValueError, match="at most three"): xr_module.OvLibrariesXrCfg(auxiliary_cameras=(*cameras, extra)) @@ -1018,7 +1018,7 @@ def test_prepare_snapshots_camera_selection_without_mutating_caller(xr_module, m bootstrap = _FakeBootstrap() monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) camera_cfg = SimpleNamespace(width=640, height=480) - camera = xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", camera_cfg) + camera = xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", camera_cfg) cfg = xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,)) session = xr_module.OvLibrariesXrSession.prepare(object(), cfg) cfg.auxiliary_cameras = () @@ -1052,9 +1052,7 @@ def abort(): monkeypatch.setattr(xr_module.OvLibrariesXrSession, "__init__", fail) cfg = xr_module.OvLibrariesXrCfg( - auxiliary_cameras=( - xr_module.OvLibrariesXrCameraConfig("robot_pov_cam", SimpleNamespace(width=640, height=480)), - ), + auxiliary_cameras=(xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", SimpleNamespace(width=640, height=480)),), ) with pytest.raises(RuntimeError) as caught: xr_module.OvLibrariesXrSession.prepare(object(), cfg) @@ -1076,7 +1074,7 @@ def test_prepare_rejects_unsupported_quad_placement_before_native_preparation(xr ) monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated before validation")) with pytest.raises(ValueError, match="head_locked"): - camera = xr_module.OvLibrariesXrCameraConfig( + camera = xr_module.OvLibrariesXrCameraCfg( name="robot_pov_cam", camera_cfg=SimpleNamespace(width=640, height=480), space=space ) xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,))) @@ -1086,7 +1084,7 @@ def test_public_xr_exports_resolve_to_implemented_types(xr_module): import isaaclab_teleop assert isaaclab_teleop.OvLibrariesXrCfg is xr_module.OvLibrariesXrCfg - assert isaaclab_teleop.OvLibrariesXrCameraConfig is xr_module.OvLibrariesXrCameraConfig + assert isaaclab_teleop.OvLibrariesXrCameraCfg is xr_module.OvLibrariesXrCameraCfg assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession From b733a37d582dc763c7c23a3130f85e5728988bdf Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:48:42 -0500 Subject: [PATCH 135/145] Restore upstream texts that the camera panels do not change Keep the fixed-base G1 observation docstrings and group comment, the camera-feed constructor and exit signatures, the feed validator and the image binding lookup, the moved native-cloning comment, the Kit headless note and CloudXR packaging note, and the teleop README's Kit feedback and reference lines. Prepare-stage population now calls the lease's populate_stage in place of the upstream line, since it already falls back to plain population without a lease. The kitless additions follow the upstream lines. --- docs/source/features/isaac_teleop.rst | 5 ++- docs/source/how-to/cloudxr_teleoperation.rst | 7 ++-- .../isaaclab_ov/renderers/ovrtx_renderer.py | 8 +--- .../isaaclab_ov/renderers/render_lease.py | 2 +- .../fixed_base_upper_body_ik_g1_env_cfg.py | 8 ++-- source/isaaclab_teleop/docs/README.md | 13 ++++--- .../isaaclab_teleop/camera_feed.py | 39 ++++++++----------- 7 files changed, 38 insertions(+), 44 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 5a4020809fb..6ae1e8e4e12 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -2060,8 +2060,9 @@ Optimize XR Performance .. note:: - For Kit XR, omit ``--visualizer`` to run without the local viewport. - For kitless ``--ovxr``, pass ``--headless`` to disable its default desktop mirror. + The ``--headless`` CLI flag was removed in Isaac Lab 3.0. Headless is now the **default** + (omit ``--visualizer``), or set ``HEADLESS=1`` in the environment. + Kitless ``--ovxr`` accepts ``--headless`` to disable its default desktop mirror. .. dropdown:: Configure retargeting execution :open: diff --git a/docs/source/how-to/cloudxr_teleoperation.rst b/docs/source/how-to/cloudxr_teleoperation.rst index 48192e53e53..2a41ddd0fe8 100644 --- a/docs/source/how-to/cloudxr_teleoperation.rst +++ b/docs/source/how-to/cloudxr_teleoperation.rst @@ -146,10 +146,9 @@ to capture demonstrations, and ``replay`` to play a dataset back. :ref:`The next .. note:: - For XR teleoperation without Isaac Sim, follow the - :ref:`kitless XR guide `. It explains how to select a public - Isaac Capture wheel and its optional OVXR dependencies. The Kit setup below - uses the published ``teleop`` extra. + ``teleop`` cannot be combined with ``ov`` or ``ovphysx`` in a single ``uv run``: the + bundled Isaac Sim pins ``packaging==26.0`` while those runtimes require ``<24``. Install + the OV runtimes separately when you need them. For XR teleoperation without Isaac Sim, follow the :ref:`kitless XR guide `. It explains how to select a public Isaac Capture wheel and its optional OVXR dependencies. The Kit diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index b827ad427d1..d4f9bc8b1b7 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -431,7 +431,6 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" if not self._render_lease.prepare_stage(num_envs): return - lease = self._render_lease.active logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -461,10 +460,7 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: if self.cfg.temp_usd_dir is not None: _write_file(Path(self.cfg.temp_usd_dir), "ovrtx_renderer_stage.usda", usd) if self._use_ovstage: - if lease is not None: - self._render_lease.populate_stage(usd) - else: - self.scene.populate(usd) + self._render_lease.populate_stage(usd) else: self.backend.renderer.open_usd_from_string(usd) ovrtx_replicate(self.backend.renderer, self.scene.clone_copies, env_paths, self.scene.clone_positions) @@ -631,7 +627,7 @@ def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: env_partitions = {Sdf.Path(path): f"env_{i}" for i, path in enumerate(self.scene.clone_env_paths)} source_root = next((root for root in Sdf.Path(camera_paths[0]).GetPrefixes() if root in env_partitions), None) if source_root is not None and len(camera_paths) < spec.num_instances: - # Native cloning may leave only the source camera in USD. Preserve complete path row order. + # Native cloning may leave only the source camera in USD. Complete paths retain their row order. suffix = camera_paths[0][len(str(source_root)) :] camera_paths = [path + suffix for path in self.scene.clone_env_paths] data = OVRTXCameraRenderData(spec, f"RenderCamera_{self.scene.next_camera_id}", camera_paths) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 6a047edf6c1..545741a60e7 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -749,7 +749,7 @@ def bind_stage(self) -> None: lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) def populate_stage(self, usd: str) -> None: - """Populate the borrowed scene with the authorized camera schedule.""" + """Populate the renderer's scene, with the camera schedule an active lease authorizes.""" lease = self.active if lease is not None and lease._auxiliary_cameras: usd = self._author_camera_schedules(usd, lease) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index 08ec7863bf8..16b2e6b4409 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -329,11 +329,13 @@ class ActionsCfg: @configclass class ObservationsCfg: - """State observations, with opt-in head and wrist RGB images (``presets=rgb``).""" + """Observation specifications for the MDP. + This class is required by the environment configuration but not used in this implementation + """ @configclass class PolicyCfg(ObsGroup): - """Policy observations with state values.""" + """Observations for policy group with state values.""" actions = ObsTerm(func=manip_mdp.last_action) robot_joint_pos = ObsTerm( @@ -389,7 +391,7 @@ class RgbPolicyCfg(PolicyCfg): }, ) - # observation groups; the XR camera panels do not need the image terms + # observation groups policy: PolicyCfg = preset(default=PolicyCfg(), rgb=RgbPolicyCfg()) diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index d2fe870445c..9af820fc836 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -19,9 +19,10 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. modes for controlling how the anchor orientation tracks the reference prim. - **Retargeting tuning UI** -- optional ImGui window for real-time adjustment of retargeter parameters when `retargeters_to_tune` is provided. -- **XR camera feedback** -- selected scene cameras use Kit Scene UI or the optional - [OVXR path](kitless_xr.md)'s OpenXR quad layers. The latter also - supports a passive desktop eye-image mirror; separate quad layers are not in that image. +- **XR camera feedback** -- selected scene cameras can be shown through Kit Scene UI using a + feed-owned CUDA view of their existing RTX render product, with camera-buffer fallback. + The optional [OVXR path](kitless_xr.md) shows them as OpenXR quad layers; its desktop mirror + shows eye images without those layers. ## Architecture @@ -116,7 +117,7 @@ For kitless teleoperation, installation, MCAP replay, camera panels and the optional desktop mirror, see [Kitless XR teleoperation](kitless_xr.md). See [XR Camera Feedback](../../../docs/source/features/isaac_teleop.rst#xr-camera-feedback) for -reference tasks, camera selection, layout, placement, renderer, disable, and OVXR behavior. +reference tasks, camera selection, layout, placement, renderer, disable, and kitless behavior. ### 4. Programmatic Usage @@ -198,8 +199,8 @@ scenes; they are not lossless recorders or rolling-shutter playback. ## Utilities - **`remove_camera_configs(env_cfg, keep=())`** -- strips camera sensors and their associated policy - observation terms. Cameras named in `keep` stay in the scene, but their observation terms are - still removed. + observation terms. + Cameras named in `keep` stay in the scene, but their observation terms are still removed. ## Run with Docker diff --git a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py index 50d25bfb06a..2e362089fdf 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py +++ b/source/isaaclab_teleop/isaaclab_teleop/camera_feed.py @@ -26,8 +26,6 @@ from .isaac_teleop_cfg import XrCameraFeedCfg, XrCameraFeedLayoutCfg if TYPE_CHECKING: - from types import TracebackType - from isaaclab_ov.renderers import OVRTXRendererCfg from isaaclab.sensors import Camera, CameraCfg @@ -85,7 +83,7 @@ def resolve_ovxr_camera_configs( if position_override is not None and len(selected) > 1: raise ValueError("A PiP position override requires one camera; multiple panels use task layout") for _name, _camera, feed in selected: - _validate_feed_cfg(feed, layout.placement) + _XrCameraFeedManager._validate_cfg(feed, layout.placement) feeds = _layout_feed_cfgs( [feed for _name, _camera, feed in selected], [(camera.width, camera.height) for _name, camera, _feed in selected], @@ -234,7 +232,7 @@ def __init__( presenter: Any | None, *, requires_responsive_denoising: bool, - ) -> None: + ): self._cfgs = cfgs self._layout_cfg = layout_cfg self._presenter = presenter @@ -420,24 +418,10 @@ def __enter__(self) -> XrCameraFeedSession: raise RuntimeError("Call bind(env) before entering an XR camera feed session.") return self - def __exit__( - self, exc_type: type[BaseException] | None, exc_value: BaseException | None, traceback: TracebackType | None - ) -> None: + def __exit__(self, exc_type, exc_value, traceback) -> None: self.close() -def _validate_feed_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: - """Validate original feed values before layout or backend conversion.""" - if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: - raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") - if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): - raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") - if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): - raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") - if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: - raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") - - def _panel_size_m(cfg: XrCameraFeedCfg, image_size: tuple[int, int]) -> tuple[float, float]: width, height = image_size if width <= 0 or height <= 0: @@ -563,7 +547,7 @@ def __init__( cfgs: list[XrCameraFeedCfg], layout_cfg: XrCameraFeedLayoutCfg | None, presenter: Any, - ) -> None: + ): self._env = env self._feeds: list[_ActiveFeed] = [] self._presenter = presenter @@ -573,7 +557,7 @@ def __init__( try: bound_feeds = [] for cfg in cfgs: - _validate_feed_cfg(cfg, self._layout_cfg.placement) + self._validate_cfg(cfg, self._layout_cfg.placement) camera, image = self._bind_image(cfg) bound_feeds.append((camera, image)) image_sizes = [(int(image.shape[1]), int(image.shape[0])) for _, image in bound_feeds] @@ -605,8 +589,19 @@ def __init__( def _on_frame(self, _event: Any) -> None: self.update() + @staticmethod + def _validate_cfg(cfg: XrCameraFeedCfg, placement: str) -> None: + if not math.isfinite(cfg.panel_width_m) or cfg.panel_width_m <= 0.0: + raise ValueError(f"panel_width_m for XR camera feed {cfg.camera_name!r} must be positive.") + if len(cfg.offset_m) != 2 or not all(math.isfinite(value) for value in cfg.offset_m): + raise ValueError(f"offset_m for XR camera feed {cfg.camera_name!r} must contain two finite values.") + if placement != "world" and (not math.isfinite(cfg.distance_m) or cfg.distance_m <= 0.0): + raise ValueError(f"distance_m for XR camera feed {cfg.camera_name!r} must be positive.") + if not math.isfinite(cfg.max_update_hz) or cfg.max_update_hz < 0.0: + raise ValueError(f"max_update_hz for XR camera feed {cfg.camera_name!r} must be finite and non-negative.") + def _bind_image(self, cfg: XrCameraFeedCfg) -> tuple[Camera, torch.Tensor]: - sensors = self._env.scene.sensors + sensors = getattr(getattr(self._env, "scene", None), "sensors", {}) if cfg.camera_name not in sensors: raise ValueError( f"XR camera feed {cfg.camera_name!r} is not present in the interactive scene. " From 743345dd7a150fcf24fa548267a82b921219c4bd Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:49:24 -0500 Subject: [PATCH 136/145] Leave panel placement checks to the camera configuration OvLibrariesXrCameraCfg already rejects every space other than head_locked when it is constructed, so the same check in OvLibrariesXrSession.prepare could not run, and its test passed on the constructor's error. Drop the check and assert the constructor's rejection in the camera configuration test. --- .../isaaclab_teleop/ov_libraries_xr.py | 8 ++------ .../test/test_ov_libraries_xr.py | 17 ++--------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py index 72f0d7b5b8c..0b59e57b7bf 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -561,11 +561,7 @@ def prepare( use one configured session per process. No supported readback exists here to restore a previous renderer's settings on close. """ - cameras = tuple(cfg.auxiliary_cameras) - for camera in cameras: - if camera.space != "head_locked": - raise ValueError("Kitless XR camera quads currently require head_locked placement") - cfg = replace(cfg, auxiliary_cameras=cameras) + cfg = replace(cfg, auxiliary_cameras=tuple(cfg.auxiliary_cameras)) validate_ovxr_session_preparation() import ovxr @@ -573,7 +569,7 @@ def prepare( ovxr.apply_renderer_settings(cfg.renderer_settings.strip()) bootstrap = OvxrSessionBootstrap(sim_cfg, renderer_cfg) try: - for camera in cameras: + for camera in cfg.auxiliary_cameras: bootstrap.configure_auxiliary_camera(camera.camera_cfg, max_fps=camera.max_fps) return cls(bootstrap, cfg) except BaseException as preparation_error: diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py index 080a3c0ac35..5fd2c90eda3 100644 --- a/source/isaaclab_teleop/test/test_ov_libraries_xr.py +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -888,6 +888,8 @@ def test_auxiliary_camera_config_is_preconfigured(xr_module, monkeypatch): xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", None) with pytest.raises(ValueError, match="max_fps"): xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", object(), max_fps=0.0) + with pytest.raises(ValueError, match="head_locked"): + xr_module.OvLibrariesXrCameraCfg("robot_pov_cam", object(), space="world") bootstrap = _FakeBootstrap() camera_cfg = SimpleNamespace(width=720, height=450) @@ -1065,21 +1067,6 @@ def abort(): ) -@pytest.mark.parametrize("space", ["viewer_start", "world"]) -def test_prepare_rejects_unsupported_quad_placement_before_native_preparation(xr_module, monkeypatch, space): - monkeypatch.setitem( - sys.modules, - "ovxr", - SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("renderer settings changed before validation")), - ) - monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated before validation")) - with pytest.raises(ValueError, match="head_locked"): - camera = xr_module.OvLibrariesXrCameraCfg( - name="robot_pov_cam", camera_cfg=SimpleNamespace(width=640, height=480), space=space - ) - xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(auxiliary_cameras=(camera,))) - - def test_public_xr_exports_resolve_to_implemented_types(xr_module): import isaaclab_teleop From 10496d9e147790c424e14591e6527e4cc54aa9ad Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:51:36 -0500 Subject: [PATCH 137/145] Test native camera schedules through the lease's Stage population The schedule test built the population description itself, from a source that already held the rates, so it checked OVStage's selectors rather than RenderLeaseController.populate_stage. It now configures two scheduled cameras on a real lease and populates a real OVStage through populate_stage from a source without rates, which also covers the schedule authoring that the export-only test checked. Changing the selected paths or property names, skipping the authoring or dropping the description each fails it. --- .../test/test_ovrtx_renderer_contract.py | 125 ++++++------------ 1 file changed, 42 insertions(+), 83 deletions(-) diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index b168e516ad9..3cbd71b40a9 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -2020,29 +2020,6 @@ def test_leased_camera_cleanup_waits_for_lease_release(leased_renderer, monkeypa assert data not in renderer._camera_render_data -def test_external_camera_native_rate_is_authored_only_on_export(leased_renderer): - from pxr import Usd, UsdGeom - - renderer = leased_renderer - lease = renderer.acquire_ov_libraries_render_lease() - cfg = _make_camera_cfg(["rgb"]) - cfg.prim_path = "/World/envs/env_0/Camera" - lease.configure_auxiliary_camera(cfg, max_fps=20.0) - lease._auxiliary_cameras[0].spec = types.SimpleNamespace(camera_prim_paths=[cfg.prim_path]) - stage = Usd.Stage.CreateInMemory() - camera = UsdGeom.Camera.Define(stage, cfg.prim_path).GetPrim() - exported = Usd.Stage.CreateInMemory() - assert exported.GetRootLayer().ImportFromString( - renderer._render_lease._author_camera_schedules(stage.ExportToString(), lease) - ) - assert not camera.HasAttribute("omni:sensor:frameRate") - assert not camera.HasAttribute("omni:sensor:Core:partialOutputs") - prim = exported.GetPrimAtPath(cfg.prim_path) - assert tuple(prim.GetAttribute("omni:sensor:frameRate").Get()) == (20.0, 1.0) - assert prim.GetAttribute("omni:sensor:Core:partialOutputs").Get() is False - lease.release() - - def test_selected_external_camera_order_is_independent_of_sensor_preparation(leased_renderer): renderer = leased_renderer lease = renderer.acquire_ov_libraries_render_lease() @@ -2082,24 +2059,22 @@ def test_external_camera_partition_validation_is_not_stereo_specific(leased_rend lease.release() -def test_external_camera_schedules_reach_native_stage(monkeypatch): - """The renderer's real population path retains only selected camera scheduling properties.""" +def test_external_camera_schedules_reach_native_stage(): + """Leased population authors the selected cameras' native rates and retains only those properties.""" import ovrtx import ovstage + from isaaclab_ov.stage import OvstageBackend, OvstageBackendCfg ovrtx.register_schema_paths() camera_paths = [f"/World/envs/env_0/{name}" for name in ("Head", "Wrist", "Unselected")] + # The application export has no camera schedules; the unselected camera's own rate must not populate. source = """#usda 1.0 def Xform "World" { def Xform "envs" { def Xform "env_0" { def Camera "Head" { - double2 omni:sensor:frameRate = (20, 1) - bool omni:sensor:Core:partialOutputs = false } def Camera "Wrist" { - double2 omni:sensor:frameRate = (10, 1) - bool omni:sensor:Core:partialOutputs = false } def Camera "Unselected" { double2 omni:sensor:frameRate = (999, 1) @@ -2108,63 +2083,47 @@ def Camera "Unselected" { } } """ - cfg = _make_camera_cfg(["rgb"]) - cfg.prim_path = camera_paths[0] - spec = types.SimpleNamespace(cfg=cfg, num_instances=1, camera_prim_paths=[camera_paths[0]], render_settings={}) - data = OVRTXCameraRenderData(spec, "RenderCamera_0", list(spec.camera_prim_paths)) - from isaaclab_ov.stage import OvstageBackend, OvstageBackendCfg - - renderer = _make_ovrtx_renderer_without_backend() - renderer._use_ovstage = True - renderer._render_lease.stage_deferred = False - renderer._render_lease.active = None - renderer.backend.renderer = MagicMock() + renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) + lease = renderer.acquire_ov_libraries_render_lease() with contextlib.ExitStack() as resources: + resources.callback(lease.release) + for path, max_fps in zip(camera_paths[:2], (20.0, 10.0), strict=True): + cfg = _make_camera_cfg(["rgb"]) + cfg.prim_path = path + lease.configure_auxiliary_camera(cfg, max_fps=max_fps) + for camera in lease._auxiliary_cameras: + camera.spec = types.SimpleNamespace(camera_prim_paths=[camera.cfg.prim_path]) renderer.scene = OvstageBackend(OvstageBackendCfg()) resources.callback(renderer.scene.close) + renderer._render_lease.populate_stage(source) + stage, paths = renderer.scene.stage, renderer.scene.paths - population = ovstage.population - description = population.Desc( - domains=ovstage.PopulationDomain.RENDERING, - selectors=[ - population.Selector( - population.PrimPredicate.has_path(*camera_paths[:2]), - population.PropertyPredicate.has_name("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs"), - ) - ], - ) - renderer.scene.populate(source, population_desc=description) + names = ("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs") + resolved = {} + plist = paths.create_path_list_from_strings(camera_paths) try: - renderer._register_camera(spec, data) - renderer.scene.commit() - resolved = {} - plist = paths.create_path_list_from_strings(camera_paths) - try: - with stage.query_from_path_list(plist) as query: - query.wait() - names = ("omni:sensor:frameRate", "omni:sensor:Core:partialOutputs") - with stage.read_attributes( - query, - [paths.intern_token(name) for name in names], - ovstage.OrdinalRange.latest(renderer.scene.ordinal - 1), - ) as read: - read.wait() - for group in read.groups(): - try: - values = np.asarray(group.array(0).copy()).reshape(group.data_count, -1) - group_paths = paths.get_path_strings(group.prim_list) - attribute = paths.token_to_string(group.attribute) - for index in range(min(group.prim_count, group.data_count)): - path = group_paths[group.prim_index(index)] - resolved[path, attribute] = values[group.data_row_index(index)].tolist() - finally: - stage.release_group(group) - finally: - paths.destroy_path_list(plist) - assert resolved.get((camera_paths[0], names[0])) == [20.0, 1.0] - assert resolved.get((camera_paths[1], names[0])) == [10.0, 1.0] - assert resolved.get((camera_paths[0], names[1])) == [False] - assert resolved.get((camera_paths[1], names[1])) == [False] - assert (camera_paths[2], names[0]) not in resolved + with stage.query_from_path_list(plist) as query: + query.wait() + with stage.read_attributes( + query, + [paths.intern_token(name) for name in names], + ovstage.OrdinalRange.latest(renderer.scene.ordinal - 1), + ) as read: + read.wait() + for group in read.groups(): + try: + values = np.asarray(group.array(0).copy()).reshape(group.data_count, -1) + group_paths = paths.get_path_strings(group.prim_list) + attribute = paths.token_to_string(group.attribute) + for index in range(min(group.prim_count, group.data_count)): + path = group_paths[group.prim_index(index)] + resolved[path, attribute] = values[group.data_row_index(index)].tolist() + finally: + stage.release_group(group) finally: - data.resources.close() + paths.destroy_path_list(plist) + assert resolved.get((camera_paths[0], names[0])) == [20.0, 1.0] + assert resolved.get((camera_paths[1], names[0])) == [10.0, 1.0] + assert resolved.get((camera_paths[0], names[1])) == [False] + assert resolved.get((camera_paths[1], names[1])) == [False] + assert (camera_paths[2], names[0]) not in resolved From 0e237ac52288704eab9bb87e1f296929f0c813f4 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:52:56 -0500 Subject: [PATCH 138/145] Describe camera-panel rates, labels and head exclusion as OVXR applies them The head exclusion hides meshes from the OVXR eye cameras and camera panels; OVXR removes the observation terms of kept cameras and HDF5 recording is Kit-only, so no recorded image is affected. A camera authorized without a rate renders with every frame of its product. In OVXR each panel renders at the lower of --ovxr_pip_fps and its feed cap, not at the sensor's update_period, and --disable_external_cameras only applies with --xr. Capture uses a panel label only in the desktop mirror, so the guide says feed labels are Kit-only. --- source/isaaclab_tasks/docs/README.md | 16 ++++++++-------- .../changelog.d/ovxr-camera-panels.added.rst | 3 +-- source/isaaclab_teleop/docs/kitless_xr.md | 4 +++- .../isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py | 2 +- .../isaaclab_teleop/ovxr_session.py | 2 +- source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 2864f20014e..33955ad2cf4 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -93,13 +93,13 @@ only the names present in the policy observations. The same names identify the policy image observations and the task's XR camera feeds, with the head panel centered and wrist panels to its left and right. -Sensor `update_period` controls image sampling in simulation time; feed -`max_update_hz` caps uploads. Actual delivery also depends on simulation and -renderer throughput. Camera sampling rates are configuration targets, not -measured headset frame rates. - -The teleoperation launcher's `--disable_external_cameras` option removes these -cameras and their image observations. The kitless XR launcher accepts up to +In Kit, sensor `update_period` controls image sampling in simulation time; feed +`max_update_hz` caps uploads. In OVXR, each panel renders natively at the lower of +`--ovxr_pip_fps` and its feed's nonzero `max_update_hz`; `update_period` does not +apply. Actual delivery also depends on simulation and renderer throughput. Camera +sampling rates are configuration targets, not measured headset frame rates. + +With `--xr`, the teleoperation launcher's `--disable_external_cameras` option removes +these cameras and their image observations. The kitless XR launcher accepts up to three camera names with `--ovxr_pip_camera`. Camera feeds are opt-in: without this option, live OVXR and MCAP replay omit task cameras and PiP layers. -Selected feeds keep their own sampling cadence. diff --git a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst index df8acda7090..04fd6d18ab7 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-camera-panels.added.rst @@ -8,7 +8,6 @@ not head-locked in the OpenXR VIEW space. Replay accepts the live ``--ovxr_pip_fps``, ``--ovxr_pip_view_position`` and ``--ovxr_desktop_mirror_fps`` options. * Added :attr:`~isaaclab_teleop.XrCfg.excluded_visual_meshes`, which hides matching visual meshes - from the OVXR eye cameras and from environment cameras selecting the ``env_0`` partition, such - as camera panels and recorded camera images. + from the OVXR eye cameras and camera panels without changing the source scene's visibility. * Added the ``keep`` argument to :func:`~isaaclab_teleop.remove_camera_configs` to retain selected cameras while removing their observation terms. diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index b61ca8323f1..90a2d87e207 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -157,7 +157,9 @@ supported by the OVXR presenter: `--ovxr_pip_head_locked` places such a layout's panels in the OpenXR VIEW space, and without it the command fails before launch. The session snapshots its camera selection at preparation; create a new session to change it. Panels use the same renderer as -the headset and do not require a desktop display. +the headset and do not require a desktop display. OVXR panels show no label; a +feed's `label` applies to Kit panels only, and the desktop mirror lists each camera +by its sensor name. GR1T2 and G1 locomanipulation provide `robot_pov_cam`; fixed-base G1 also provides `left_wrist_cam` and `right_wrist_cam`. SO101 does not declare a PiP camera in diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py index 347d6538a84..fe40d6b0d7c 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -29,7 +29,7 @@ class OvLibrariesXrCameraCfg: """One application-selected sensor and its camera-panel policy.""" name: str - """Scene sensor name used for camera selection and panel labeling.""" + """Scene sensor name used for camera selection and as the desktop mirror's camera label.""" camera_cfg: CameraCfg = field(repr=False) """Selected sensor configuration, assigned to the session's shared renderer.""" max_fps: float = 30.0 diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py index 4539d6f8c8b..48edf65693e 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -112,7 +112,7 @@ def capture_scene_publisher(self) -> OvLibrariesScenePublisher: return self._lease.capture_scene_publisher() def configure_auxiliary_camera(self, camera_cfg: CameraCfg, *, max_fps: float | None = None) -> None: - """Authorize a camera with an update cap [Hz]; None retains its configured cadence.""" + """Authorize a camera with an update cap [Hz]; ``None`` renders it with every frame of its product.""" self._require_open() if self._bound or self._lease is None: raise RuntimeError("Auxiliary camera must be configured before OVXR Stage binding") diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py index e4218f57081..f77d6a319e9 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_cfg.py @@ -69,7 +69,7 @@ class XrCfg: """ excluded_visual_meshes: list[str] = [] - """Mesh-path regexes excluded from kitless environment cameras by scene partition. + """Mesh-path regexes excluded from the OVXR eye cameras and camera panels by scene partition. This leaves USD visibility and the source scene unchanged. Only matching visual meshes in the single-environment render export enter the From 8fa1caf67560a79290402e4b5eadf5c3590c5088 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:53:56 -0500 Subject: [PATCH 139/145] Type the visual-partition helper and tidy camera-panel conventions Annotate partition_visual_meshes and document its arguments, yield and errors. Read --disable_external_cameras directly, since both entrypoints define it. Import both camera factories in one statement, fix the "locomanipulation" spelling, and add the replay's camera-panel sentences after the earlier OVXR viewer text instead of reflowing it. --- .../fixed_base_upper_body_ik_g1_env_cfg.py | 5 ++--- .../entrypoints/_ovxr_launch.py | 2 +- .../entrypoints/teleop_replay_agent.py | 14 +++++++------ .../isaaclab_teleop/xr_scene_partition.py | 21 ++++++++++++++++++- 4 files changed, 31 insertions(+), 11 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py index 16b2e6b4409..cab0eeb5175 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/fixed_base_upper_body_ik_g1_env_cfg.py @@ -21,7 +21,7 @@ from isaaclab_tasks.contrib import ISAACLAB_TELEOP_AVAILABLE from isaaclab_tasks.contrib.locomanip_pick_place import mdp as locomanip_mdp from isaaclab_tasks.contrib.pick_place import mdp as manip_mdp -from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg +from isaaclab_tasks.contrib.robot_pov_camera_cfg import g1_robot_pov_camera_cfg, robot_pov_camera_cfg from isaaclab_tasks.contrib.teleop_physics_cfg import ( TeleopPhysicsCfg, check_kitless_backend_preparation, @@ -34,7 +34,6 @@ from isaaclab_tasks.contrib.locomanip_pick_place.configs.pink_controller_cfg import ( # isort: skip G1_UPPER_BODY_IK_ACTION_CFG, ) -from isaaclab_tasks.contrib.robot_pov_camera_cfg import robot_pov_camera_cfg # isort: skip def _build_g1_upper_body_pipeline(): @@ -269,7 +268,7 @@ class FixedBaseUpperBodyIKG1SceneCfg(InteractiveSceneCfg): # Unitree G1 Humanoid robot - fixed base configuration robot: ArticulationCfg = replace(G1_29DOF_CFG, prim_path="{ENV_REGEX_NS}/Robot") - # Share the calibrated head view with the locomomanipulation G1 task. + # Share the calibrated head view with the locomanipulation G1 task. robot_pov_cam = g1_robot_pov_camera_cfg().replace(update_period=1.0 / 20.0) # Mount above each hand, looking along wrist-local +X with +Z up. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py index 2a705db4f87..0db8f903452 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/_ovxr_launch.py @@ -117,7 +117,7 @@ def validate_ovxr_presentation_args( parser.error("OVXR presentation options require --ovxr") if args.headless and not using_ovxr: parser.error("--headless requires --ovxr; Kit runs headless unless --visualizer is given") - if args.ovxr_pip_camera is not None and getattr(args, "disable_external_cameras", False): + if args.ovxr_pip_camera is not None and args.disable_external_cameras: parser.error("--ovxr_pip_camera cannot be combined with --disable_external_cameras") if not math.isfinite(args.ovxr_pip_fps) or args.ovxr_pip_fps <= 0.0: parser.error("--ovxr_pip_fps must be finite and positive") diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index f2cd45c1aef..b32c2c7e45e 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -14,12 +14,14 @@ OVXR viewer: ``--ovxr`` presents replay through an independently owned OVXR session while - tracker and control input remain sourced from MCAP. Optional camera quads use - ``--ovxr_pip_camera NAME`` with the same rate, placement and position options as live - teleoperation (``--ovxr_pip_head_locked`` overrides a task layout that is not head-locked). - The desktop mirror is enabled by default; ``--headless`` disables it. See - ``source/isaaclab_teleop/docs/kitless_xr.md`` - for installation and launch instructions. + tracker and control input remain sourced from MCAP. See + ``source/isaaclab_teleop/docs/kitless_xr.md`` for installation and launch + instructions. + + Optional camera quads use ``--ovxr_pip_camera NAME`` with the same rate, placement + and position options as live teleoperation (``--ovxr_pip_head_locked`` overrides a + task layout that is not head-locked). The desktop mirror is enabled by default; + ``--headless`` disables it. Inputs: ``--replay_file`` is an MCAP capture produced by Isaac Capture's diff --git a/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py index 79eff4aba1d..6cc9ef8ac39 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py +++ b/source/isaaclab_teleop/isaaclab_teleop/xr_scene_partition.py @@ -5,17 +5,36 @@ """Camera-selective visual exclusions for the single-environment XR export.""" +from __future__ import annotations + import re +from collections.abc import Iterator, Sequence from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from pxr import Usd @contextmanager -def partition_visual_meshes(stage, patterns): +def partition_visual_meshes(stage: Usd.Stage, patterns: Sequence[str]) -> Iterator[tuple[str, ...]]: """Exclude matching visual meshes from environment cameras during export. A temporary session sublayer is flattened into the borrowed render Stage. The application's source layers, visibility, physics and cameras are not changed. Patterns match mesh paths, including instance proxies, not links. + + Args: + stage: Application Stage with exactly one environment, ``/World/envs/env_0``. + patterns: Regular expressions that must each fully match at least one visual mesh path. + + Yields: + The excluded mesh paths, or an empty tuple without patterns. + + Raises: + ValueError: If the Stage has other environments, a pattern matches no visual mesh, + or a match is collision geometry. + RuntimeError: If a stronger source opinion prevents the partition or deinstancing. """ if not patterns: yield () From b395bd59cbee3bfa78c98d42c0a12047c166eff2 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:54:40 -0500 Subject: [PATCH 140/145] Keep the deferred-Stage materializer at the lease controller's original position --- .../isaaclab_ov/renderers/render_lease.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py index 545741a60e7..2febeb116b8 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -630,21 +630,6 @@ def accept_camera_spec(self, spec: CameraRenderSpec) -> bool: raise RuntimeError("Auxiliary camera specification changed after authorization") return True - def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: - """Materialize a deferred Stage, with any selected cameras, after render-thread ownership transfer.""" - self.require_owner_thread("materialize a deferred Stage") - # Unclaimed, a fallback prepare_stage would only defer again; reject even if nothing is deferred yet. - if (self.defer_until_transfer or self.stage_deferred) and not self._transfer_claimed: - raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") - if not self.stage_deferred: - return False - if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: - raise RuntimeError("Deferred Stage is not eligible for materialization") - if any(camera.spec is None or camera.render_data is None for camera in lease._auxiliary_cameras): - raise RuntimeError("Deferred auxiliary camera Stage is not eligible for materialization") - self._renderer.prepare_stage(stage, num_envs) - return True - def product_paths(self, lease: OvLibrariesRenderLease) -> tuple[str, ...]: """Return products in application selection order, independently of sensor initialization order.""" self.validate_live(lease, lease.ovrtx_renderer, lease.ovstage, lease._generation) @@ -739,6 +724,21 @@ def prepare_stage(self, num_envs: int) -> bool: raise RuntimeError("The render lease requires its reserved private OVStage renderer") return True + def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: + """Materialize a deferred Stage, with any selected cameras, after render-thread ownership transfer.""" + self.require_owner_thread("materialize a deferred Stage") + # Unclaimed, a fallback prepare_stage would only defer again; reject even if nothing is deferred yet. + if (self.defer_until_transfer or self.stage_deferred) and not self._transfer_claimed: + raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") + if not self.stage_deferred: + return False + if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Deferred Stage is not eligible for materialization") + if any(camera.spec is None or camera.render_data is None for camera in lease._auxiliary_cameras): + raise RuntimeError("Deferred auxiliary camera Stage is not eligible for materialization") + self._renderer.prepare_stage(stage, num_envs) + return True + def bind_stage(self) -> None: """Bind the prepared native pair after renderer initialization succeeds.""" lease = self.active From 3d022d7ba597e6f992d4747dc37c8507f219ca74 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 15:58:05 -0500 Subject: [PATCH 141/145] Keep two more upstream lines beside the camera-panel docs The Scene UI note now changes only the line about a future kitless entry point, and the replay preparation's return description keeps its upstream success-term line and adds the camera sentence after it. --- docs/source/features/isaac_teleop.rst | 6 ++---- .../isaaclab_teleop/entrypoints/teleop_replay_agent.py | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/docs/source/features/isaac_teleop.rst b/docs/source/features/isaac_teleop.rst index 6ae1e8e4e12..abc7b75f0bf 100644 --- a/docs/source/features/isaac_teleop.rst +++ b/docs/source/features/isaac_teleop.rst @@ -1623,10 +1623,8 @@ Kit Scene UI presentation PiP presentation uses Kit Scene UI and ``SpatialSource`` placement. Kit imports are deferred until an enabled feed is requested. If the Scene UI modules cannot be imported, the scripts log a warning and continue without PiP; task-owned cameras and recording observations remain unchanged. This keeps -the camera selection configuration usable without Scene UI. The -:ref:`kitless XR guide ` uses separate OpenXR quad -layers and does not import Scene UI; those layers are visible in the headset/client, not in the -passive eye-image desktop mirror. Render-product tuning is also best-effort: backends without a compatible render product +the camera selection configuration usable by the :ref:`kitless XR path `, whose OpenXR quad layers need no +Scene UI. Render-product tuning is also best-effort: backends without a compatible render product keep the Camera-buffer path, and schema-authoring failures warn while PiP continues. Other configuration, camera-buffer, and panel-initialization errors still fail during startup. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index b32c2c7e45e..6437a9f111d 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -1127,8 +1127,8 @@ def _prepare_env_cfg( Returns: Tuple ``(env_cfg, success_term, selected_cameras)``. ``success_term`` is ``None`` when - the env doesn't define a ``success`` termination term. ``selected_cameras`` holds the - resolved OVXR PiP panels, empty without ``--ovxr_pip_camera``. + the env doesn't define a ``success`` termination term. + ``selected_cameras`` holds the resolved OVXR PiP panels, empty without ``--ovxr_pip_camera``. """ env_cfg = parse_env_cfg(task, device=device, num_envs=num_envs, overrides=overrides) env_cfg.env_name = task.split(":")[-1] From dec2d9dbf55309a24523683e05e1a216251006ce Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 20:24:07 -0500 Subject: [PATCH 142/145] Start Kit for every replay without --ovxr Replay chose Kit from the resolved task configuration, so an ordinary replay of a kitless-capable task ran without Kit and then always failed its render timing check (no render calls), writing no report. Restore the upstream Kit requirement for replay and drop it only for --ovxr, whose viewer replaces Kit. A standalone physics selection without --ovxr is rejected by launch validation, and the changelog no longer claims the broader mode. --- .../changelog.d/ovxr-live-replay.changed.rst | 3 --- .../isaaclab_teleop/entrypoints/teleop_replay_agent.py | 9 +++------ .../isaaclab_teleop/test/test_teleop_entrypoint_units.py | 7 +++++++ 3 files changed, 10 insertions(+), 9 deletions(-) delete mode 100644 source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst deleted file mode 100644 index f855d5239ef..00000000000 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.changed.rst +++ /dev/null @@ -1,3 +0,0 @@ -* Changed replay to launch Kit only when the resolved task configuration requires it (for - example Isaac Sim PhysX, ``--xr`` or a Kit visualizer) instead of always. Select a Kit - physics preset to keep the previous runtime for a kitless task. diff --git a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py index f046378392d..a3838d05fdf 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py +++ b/source/isaaclab_teleop/isaaclab_teleop/entrypoints/teleop_replay_agent.py @@ -459,8 +459,8 @@ def _parse_args(argv: list[str] | None = None) -> argparse.Namespace: # the launcher; this selects a camera-rendering experience that provides RTX/DLSS. # Everywhere else we read ``args_cli.disable_external_cameras`` directly. args_cli.enable_cameras = not args_cli.disable_external_cameras - # Let the resolved task/preset select Kit; only Kit paths use its stage-load APIs. - args_cli.require_kit = False + # the stage-load wait and the RTX settings use Kit APIs directly; the OVXR viewer replaces Kit + args_cli.require_kit = not args_cli.ovxr # Resolve CLI profile names once; runtime construction receives a path or None. if args_cli.ovxr and args_cli.cloudxr_env is None: @@ -1097,9 +1097,7 @@ def _prepare_env_cfg( cycle (sim reinit + teleop device reset) so Pink IK starts the next attempt with fresh articulation views. - It also checks the launch before any runtime starts, with two side - effects: it sets ``args_cli.require_kit`` to whether the resolved - configuration needs Kit, and it runs the task's + It also checks the launch before any runtime starts and runs the task's :attr:`~isaaclab_teleop.IsaacTeleopCfg.prepare_environment` hook, which may modify the returned configuration. With ``--ovxr`` it also removes camera sensors and rejects a launch that needs more than one environment, a stage @@ -1145,7 +1143,6 @@ def _prepare_env_cfg( if kit_sources or any(isinstance(cfg, VisualizerBackendCfg) for cfg in configs): raise ValueError("--ovxr requires no Kit runtime or local visualizer") - args_cli.require_kit = bool(kit_sources) if env_cfg.isaac_teleop is not None and env_cfg.isaac_teleop.prepare_environment is not None: env_cfg.isaac_teleop.prepare_environment(env_cfg) return env_cfg, success_term diff --git a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py index 350fa8a761a..60d22024818 100644 --- a/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py +++ b/source/isaaclab_teleop/test/test_teleop_entrypoint_units.py @@ -401,6 +401,13 @@ def resolve(task, agent, *, overrides): monkeypatch.setattr(replay, "ManagerBasedRLEnvCfg", SimpleNamespace) monkeypatch.setattr(parse_cfg, "resolve_task_config", resolve) + if not ovxr: + # Replay without --ovxr starts Kit, which cannot host standalone OvPhysX. + with pytest.raises(ValueError, match="Invalid backend combination"): + namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) + assert resolutions == [("test-task", None, ["physics=ovphysx"])] + assert not prepared_configs + return prepared, success = namespace["_prepare_env_cfg"]("test-task", 1, None, overrides=namespace["hydra_overrides"]) assert resolutions == [("test-task", None, ["physics=ovphysx"])] assert prepared is cfg and success is not None From 3047331be4dbe02d78c3f6d1af213e08c5656e1d Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 21:50:18 -0500 Subject: [PATCH 143/145] Preserve replay failures during session cleanup --- .../changelog.d/ovxr-live-replay.fixed.rst | 1 + .../isaaclab_teleop/session_lifecycle.py | 18 +++++++-------- .../test/test_cloudxr_lifecycle.py | 21 ++++++++++++++---- .../test/test_session_step_failure.py | 22 ++++++++++++------- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst index 4f8bc4eb993..448fe7b0bb6 100644 --- a/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst +++ b/source/isaaclab_teleop/changelog.d/ovxr-live-replay.fixed.rst @@ -3,3 +3,4 @@ * Fixed teleoperation cleanup stopping at the first failure: a retargeting UI or XR anchor cleanup failure no longer skips the Isaac Capture session and CloudXR shutdown, and a failed device entry stops a runtime it already launched. The first failure is still reported. +* Reported replay session-exit failures and retained their resources instead of suppressing them as Kit XR teardown. diff --git a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py index 4e0083cb94d..760d0a5945f 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py +++ b/source/isaaclab_teleop/isaaclab_teleop/session_lifecycle.py @@ -664,8 +664,8 @@ def stop(self, exc_type=None, exc_val=None, exc_tb=None) -> None: down externally (e.g. "Stop AR"), cleanup errors are suppressed. A retargeting UI failure does not skip session cleanup; it propagates - afterwards, or is attached to the caller's original exception. With an - external OVXR handle provider, a failed session exit is not suppressed: + afterwards, or is attached to the caller's original exception. For replay + or an external OVXR handle provider, a failed session exit is not suppressed: it is reported the same way and latched, and the lifecycle prohibits reuse or restart. The retained session reference does not prove native drain; the application must not release borrowed resources or stop the @@ -709,8 +709,8 @@ def record_failure(cleanup_error: BaseException) -> None: try: self._session.__exit__(exc_type, exc_val, exc_tb) except BaseException as e: - if self._external_oxr_handles_provider is not None: - # Borrowed OVXR handles must be proven drained before their owner releases them. + if self._is_replay or self._external_oxr_handles_provider is not None: + # Replay failures are terminal; borrowed resources also require a proven drain. self._session_cleanup_error = e record_failure(e) if exc_val is None: @@ -1443,7 +1443,7 @@ def step( try: self._teardown_dead_session() except BaseException as cleanup_error: - if self._external_oxr_handles_provider is None: + if not self._is_replay and self._external_oxr_handles_provider is None: raise # Failed drain retains the owner; never restart over live resources. self._session_cleanup_error = e @@ -1501,15 +1501,15 @@ def _teardown_dead_session(self) -> None: the deferred-start logic in :meth:`step` can re-acquire handles if the user restarts AR. - With an external OVXR handle provider, a failed exit propagates instead - and latches a terminal error, so borrowed handles are never reused or - restarted before they are proven drained. + For replay or an external OVXR handle provider, a failed exit propagates + instead and latches a terminal error. Replay cannot recover by restarting + from the beginning, and borrowed resources require a proven drain. """ if self._session is not None: try: self._session.__exit__(None, None, None) except BaseException as e: - if self._external_oxr_handles_provider is not None: + if self._is_replay or self._external_oxr_handles_provider is not None: # Exit may already be terminal. Never infer successful drain from # a later no-op __exit__, or resume/restart after this failure. self._session_cleanup_error = e diff --git a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py index 23d056dc535..d425d851287 100644 --- a/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py +++ b/source/isaaclab_teleop/test/test_cloudxr_lifecycle.py @@ -387,11 +387,24 @@ def test_session_cleanup_error_without_borrowed_handles_is_suppressed(): launcher.stop.assert_called_once() -@pytest.mark.parametrize("body_error", [False, True]) -@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) -def test_borrowed_session_cleanup_failure_is_visible_and_retains_runtime(body_error, error_type): +@pytest.mark.parametrize( + "body_error,error_type,replay", + [ + (False, RuntimeError, False), + (True, RuntimeError, False), + (False, KeyboardInterrupt, False), + (True, KeyboardInterrupt, False), + (False, ValueError, True), + (True, RuntimeError, True), + ], +) +def test_session_exit_failure_is_visible_and_retains_resources(body_error, error_type, replay): + """Replay exit failures stay visible even though input borrows no OpenXR handles.""" lifecycle = TeleopSessionLifecycle( - _make_cfg(), use_kit_xr_bridge=False, external_oxr_handles_provider=lambda: object() + _make_cfg(), + use_kit_xr_bridge=False, + external_oxr_handles_provider=None if replay else lambda: object(), + mcap_replay_path="recording.mcap" if replay else None, ) session, launcher = MagicMock(), MagicMock() cleanup_error = error_type("native drain failed") diff --git a/source/isaaclab_teleop/test/test_session_step_failure.py b/source/isaaclab_teleop/test/test_session_step_failure.py index f52b2b402a5..39e771ba06d 100644 --- a/source/isaaclab_teleop/test/test_session_step_failure.py +++ b/source/isaaclab_teleop/test/test_session_step_failure.py @@ -122,9 +122,9 @@ def _make_lifecycle(**kwargs) -> TeleopSessionLifecycle: return TeleopSessionLifecycle(cfg, **kwargs) -def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: +def _make_failing_lifecycle(error: Exception, **kwargs) -> TeleopSessionLifecycle: """Lifecycle with a live session mock whose step raises *error*.""" - lifecycle = _make_lifecycle() + lifecycle = _make_lifecycle(**kwargs) lifecycle._pipeline = object() session = MagicMock() session.step.side_effect = error @@ -133,12 +133,18 @@ def _make_failing_lifecycle(error: Exception) -> TeleopSessionLifecycle: class TestStepFailureDiagnosis: - @pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) - def test_failed_drain_retains_owner_and_original_step_error(self, error_type): + @pytest.mark.parametrize( + "error_type,replay", [(RuntimeError, False), (KeyboardInterrupt, False), (RuntimeError, True)] + ) + def test_failed_drain_retains_owner_and_original_step_error(self, error_type, replay): step_error = ValueError("retarget worker failed") - drain_error = error_type("worker did not join") - lifecycle = _make_failing_lifecycle(step_error) - lifecycle._external_oxr_handles_provider = lambda: object() # Borrowed OVXR handles. + drain_error = error_type("input cleanup failed") + lifecycle = _make_failing_lifecycle( + step_error, + use_kit_xr_bridge=False, + external_oxr_handles_provider=None if replay else lambda: object(), + mcap_replay_path="recording.mcap" if replay else None, + ) session = lifecycle._session session.__exit__.side_effect = drain_error with ( @@ -150,7 +156,7 @@ def test_failed_drain_retains_owner_and_original_step_error(self, error_type): assert raised.value is step_error assert raised.value.__cause__ is drain_error assert lifecycle._session is session - assert any("worker did not join" in note for note in step_error.__notes__) + assert any("input cleanup failed" in note for note in step_error.__notes__) notes = tuple(step_error.__notes__) lifecycle.stop(type(step_error), step_error, step_error.__traceback__) assert tuple(step_error.__notes__) == notes From 01c1e164820ab0758ef303556c97f2c2216dada7 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 21:50:52 -0500 Subject: [PATCH 144/145] Clarify humanoid joint target contracts --- .../locomanip_pick_place/g1_kitless_newton.py | 6 ++-- .../pick_place/gr1t2_kitless_action.py | 32 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py index 72348cc3412..4b847b032e0 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/locomanip_pick_place/g1_kitless_newton.py @@ -90,8 +90,10 @@ def bound_pink_joint_targets( ) -> torch.Tensor: """Keep one Pink update finite, inside limits, and close to current state. - Each target moves at most :data:`JOINT_TARGET_MAX_STEP_RAD` from ``current`` and stays - :data:`JOINT_LIMIT_MARGIN_RAD` inside ``[lower, upper]``. + Targets are first limited to :data:`JOINT_TARGET_MAX_STEP_RAD` from ``current``, then + clamped :data:`JOINT_LIMIT_MARGIN_RAD` inside ``[lower, upper]``. The joint limits take + precedence: when ``current`` lies outside those limits, the final change can exceed + the step bound. Args: current: Current joint positions [rad]. Shape is (num_envs, num_joints), in the Pink action's diff --git a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py index 56a6ff06586..a8d5f8e030f 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py +++ b/source/isaaclab_tasks/isaaclab_tasks/contrib/pick_place/gr1t2_kitless_action.py @@ -27,7 +27,21 @@ def bound_ik_joint_targets( max_delta_rad: float, max_magnitude_rad: float, ) -> torch.Tensor: - """Hold invalid rows and bound finite IK targets relative to measured state.""" + """Bound finite IK targets and hold rows containing an invalid target. + + Args: + targets: Proposed joint positions [rad], shape (num_envs, num_joints), in the + controlled-joint order of the Pink action. + current: Measured joint positions [rad], with the same shape, joint order, dtype and device. + max_delta_rad: Maximum requested change from ``current`` [rad], before the magnitude clamp. + max_magnitude_rad: Maximum absolute target position [rad]. This clamp takes precedence + over the step bound if ``current`` lies outside the magnitude limit. + + Returns: + New targets [rad] with the same shape, joint order, dtype and device. A row containing + any non-finite target holds its entire ``current`` row without clamping it. Inputs are + not mutated; callers must supply valid measured positions and nonnegative bounds. + """ finite_rows = torch.isfinite(targets).all(dim=1, keepdim=True) safe_targets = torch.where(finite_rows, targets, current) delta = torch.clamp(safe_targets - current, -max_delta_rad, max_delta_rad) @@ -41,7 +55,21 @@ def constrain_hand_joint_targets( lower: torch.Tensor, upper: torch.Tensor, ) -> torch.Tensor: - """Apply software mimic relations and anatomical bounds to hand targets.""" + """Apply mimic relations in tuple order, then clamp the hand targets. + + Args: + targets: Hand joint positions [rad], shape (num_envs, num_hand_joints), in the action's + controlled hand-joint order. + mimic_pairs: Tuples of (mimic_index, reference_index, factor, bias). Indices address + columns of ``targets``; the factor is dimensionless and the bias is in radians. + Each relation reads the targets updated by preceding relations. + lower: Lower positions [rad], broadcastable to ``targets`` in the same joint order, dtype and device. + upper: Upper positions [rad], broadcastable to ``targets`` in the same joint order, dtype and device. + + Returns: + Clamped targets [rad] with the same shape, joint order, dtype and device. The input + targets are cloned before applying mimic relations; none of the inputs are mutated. + """ constrained = targets.clone() for mimic_index, reference_index, factor, bias in mimic_pairs: constrained[:, mimic_index] = factor * constrained[:, reference_index] + bias From e5dc42a889d4af8c7294bdb52715ad43904c0fc0 Mon Sep 17 00:00:00 2001 From: Hougant Chen Date: Sun, 1 Nov 2026 21:50:53 -0500 Subject: [PATCH 145/145] Clarify OVXR camera observation boundaries --- source/isaaclab_tasks/docs/README.md | 5 ++++- source/isaaclab_teleop/docs/kitless_xr.md | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/source/isaaclab_tasks/docs/README.md b/source/isaaclab_tasks/docs/README.md index 33955ad2cf4..85751c59464 100644 --- a/source/isaaclab_tasks/docs/README.md +++ b/source/isaaclab_tasks/docs/README.md @@ -102,4 +102,7 @@ sampling rates are configuration targets, not measured headset frame rates. With `--xr`, the teleoperation launcher's `--disable_external_cameras` option removes these cameras and their image observations. The kitless XR launcher accepts up to three camera names with `--ovxr_pip_camera`. Camera feeds are opt-in: without -this option, live OVXR and MCAP replay omit task cameras and PiP layers. +this option, live OVXR and MCAP replay omit task cameras and PiP layers. Selected +OVXR cameras feed the panels only: their policy image observations are removed even +with `presets=rgb`, so the application does not request a second camera render. +The RGB recording example above uses Kit `--xr`. diff --git a/source/isaaclab_teleop/docs/kitless_xr.md b/source/isaaclab_teleop/docs/kitless_xr.md index 90a2d87e207..8a19bc54f24 100644 --- a/source/isaaclab_teleop/docs/kitless_xr.md +++ b/source/isaaclab_teleop/docs/kitless_xr.md @@ -165,6 +165,11 @@ GR1T2 and G1 locomanipulation provide `robot_pov_cam`; fixed-base G1 also provid `left_wrist_cam` and `right_wrist_cam`. SO101 does not declare a PiP camera in its scene. Use the desktop mirror without `--ovxr_pip_camera` for that task. +Selected OVXR cameras feed the panels only. Their policy image observations are +removed even with `presets=rgb`, so the application does not request a second +camera render while OVXR owns rendering. The `presets=rgb` policy-image selection +applies to Kit workflows. + The default desktop mirror shows completed eye or camera images locally. `--headless` disables it. `--ovxr_desktop_mirror_fps` limits its update rate. The mirror requires the `desktop-mirror` extra, Python 3.12 or newer, and a usable display. On Linux,