diff --git a/docs/conf.py b/docs/conf.py index bd62975a2e54..da4c74c9eea0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -294,6 +294,7 @@ def _read_pinned_versions() -> dict: "ipywidgets", "mpl_toolkits", "isaacteleop", + "isaaccapture", "scipy", "hydra", "hydra.core", diff --git a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst index de123825d152..84518c38e1d6 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.cloner.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.cloner.rst @@ -23,6 +23,10 @@ both OVPhysX and Newton. OVPhysX also retains native physics cloning. ``0`` selects native OVRTX cloning. Set the flag before creating the simulation. Reset retains the selected route. This flag alone never enables physics/rendering stage sharing. +An active :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` makes OVStage rendering the +unset default for every OVRTX renderer in that simulation, and rejects +``ISAAC_LAB_OVRTX_USE_OVSTAGE=0`` and ``ISAAC_LAB_SHARE_OVSTAGE=1``. + To share one OVStage with OVPhysX and OVRTX, set ``ISAAC_LAB_SHARE_OVSTAGE=1``. Sharing requires OVPhysX and selects OVStage rendering regardless of ``ISAAC_LAB_OVRTX_USE_OVSTAGE``. diff --git a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst index 4909893f624c..4c779b1a74f6 100644 --- a/docs/source/api/lab_ov/isaaclab_ov.renderers.rst +++ b/docs/source/api/lab_ov/isaaclab_ov.renderers.rst @@ -16,9 +16,12 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. OVRTXRenderer OVRTXRendererCfg OVRTXBackendCfg + OvLibrariesRenderLease + OvLibrariesScenePublisher .. autoclass:: OVRTXRenderer :show-inheritance: + :members: acquire_ov_libraries_render_lease .. autoclass:: OVRTXRendererCfg :show-inheritance: @@ -26,6 +29,23 @@ The following classes are part of the public :mod:`isaaclab_ov.renderers` API. .. autoclass:: OVRTXBackendCfg :show-inheritance: +Renderer leases +--------------- + +An external consumer, such as the OVXR teleoperation adapter, can borrow one exact OVRTX +renderer and its private OVStage with :meth:`OVRTXRenderer.acquire_ov_libraries_render_lease` +before clone preparation. While the lease is active, ordinary camera rendering, scene writes and +renderer/stage destruction are rejected; eager scene updates are left to the lease's scene +publisher, which :meth:`OvLibrariesRenderLease.capture_scene_publisher` returns. A failed leased +publication poisons publication only; the borrower can still retire the pair, after which the +renderer must be recreated. + +.. autoclass:: OvLibrariesRenderLease + :members: + +.. autoclass:: OvLibrariesScenePublisher + :members: + Simulation-owned stages ----------------------- diff --git a/docs/source/api/lab_teleop/isaaclab_teleop.rst b/docs/source/api/lab_teleop/isaaclab_teleop.rst index 26ceeec6f5b9..3400a2dc7499 100644 --- a/docs/source/api/lab_teleop/isaaclab_teleop.rst +++ b/docs/source/api/lab_teleop/isaaclab_teleop.rst @@ -22,6 +22,8 @@ isaaclab_teleop XrCfg XrAnchorRotationMode XrAnchorSynchronizer + OvLibrariesXrCfg + OvLibrariesXrSession .. rubric:: Functions @@ -61,6 +63,15 @@ XR Camera Feedback .. autoclass:: XrCameraFeedSession :members: +Kitless OVXR Session +-------------------- + +.. autoclass:: OvLibrariesXrCfg + :members: + +.. autoclass:: OvLibrariesXrSession + :members: + Device ------ diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.added.rst b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst new file mode 100644 index 000000000000..7478cad04aa5 --- /dev/null +++ b/source/isaaclab_ov/changelog.d/renderer-lease.added.rst @@ -0,0 +1,6 @@ +* Added :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` and + :meth:`~isaaclab_ov.renderers.OVRTXRenderer.acquire_ov_libraries_render_lease` for application-owned + OVRTX/OVStage borrowing by external consumers, with private-stage selection, capability-checked + render-thread ownership transfer, deferred Stage materialization, scene publication through + :class:`~isaaclab_ov.renderers.OvLibrariesScenePublisher`, and ordered cleanup. +* Published dirty visual materials with leased scene updates, including material-only changes. diff --git a/source/isaaclab_ov/changelog.d/renderer-lease.minor b/source/isaaclab_ov/changelog.d/renderer-lease.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py index f350bd6b2626..387d1531621b 100644 --- a/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py +++ b/source/isaaclab_ov/isaaclab_ov/cloner/replicate.py @@ -177,6 +177,7 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderers = sim.get_backends(OVRTXRendererCfg) if not renderers: return + leased = [renderer for renderer in renderers if renderer._render_lease.active is not None] initialized = next((renderer for renderer in renderers if renderer.backend is not None), None) if initialized is not None: use_ovstage = initialized._use_ovstage @@ -185,13 +186,16 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], # Default to native OVRTX cloning to preserve camera-task throughput on both physics backends. # Keep sharing opt-in until OVPhysX cold binding and shared-stage update performance are fixed. with_ovphysx = sim.backend_manager.clone_context_type is OvPhysxReplicateContext - value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0").strip() + # An explicit lease selects its private OVStage without changing process globals. + value = os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1" if leased else "0").strip() if value not in {"0", "1"}: raise ValueError(f"Invalid ISAAC_LAB_OVRTX_USE_OVSTAGE: {value!r}. Expected 0 or 1.") share_ovstage = os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1" use_ovstage = share_ovstage or value == "1" if share_ovstage and not with_ovphysx: raise ValueError("Shared OVStage cloning requires OVPhysX.") + if leased and (not use_ovstage or share_ovstage): + raise RuntimeError("The render lease requires a private rendering OVStage") if share_ovstage and sim.backend_manager.backend is not None and sim.backend_manager.backend.owns_scene: raise RuntimeError("Configure OVRTX cameras before the first OVPhysX reset to share its OVStage.") if use_ovstage: @@ -230,6 +234,8 @@ def prepare(sim: SimulationContext, routing: dict[type[cloner.ReplicateContext], renderer.backend = sim.get_or_create_backend(backend_cfg) renderer.scene = scene if use_ovstage else renderer.backend renderer._use_ovstage = use_ovstage + if renderer in leased: + renderer._render_lease.reserve_pair() if renderer.cfg.async_rendering and use_ovstage: logger.warning("Asynchronous OVRTX rendering is unsupported with ovstage; rendering synchronously.") diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi index d430a333224a..71a097ec2cbd 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi +++ b/source/isaaclab_ov/isaaclab_ov/renderers/__init__.pyi @@ -7,9 +7,12 @@ __all__ = [ "OVRTXRenderer", "OVRTXRendererCfg", "OVRTXBackendCfg", + "OvLibrariesRenderLease", + "OvLibrariesScenePublisher", "Renderer", ] from .ovrtx_renderer import OVRTXRenderer from .ovrtx_renderer import OVRTXRenderer as Renderer from .ovrtx_renderer_cfg import OVRTXBackendCfg, OVRTXRendererCfg +from .render_lease import OvLibrariesRenderLease, OvLibrariesScenePublisher diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py index a9c7152d2753..4093c4d430d7 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/ovrtx_renderer.py @@ -14,6 +14,8 @@ its own pending ``step_async`` render operation and publishes it at the next ``read_output``. The ovrtx binding write operations retain their input buffers until ovrtx has consumed them. +- **render_lease.py**: External borrowing, ownership transfer and scene publication ordering. + - **ovrtx_renderer_kernels.py**: Warp GPU kernels for OVRTX rendering pipeline. - **ovrtx_usd.py**: USD helpers for OVRTX: render var config, camera injection, etc. @@ -107,6 +109,8 @@ xform_tensor_from_warp, ) +from .render_lease import OvLibrariesRenderLease, RenderLeaseController + if TYPE_CHECKING: from isaaclab_ppisp import PpispPipeline from ovrtx import AttributeBinding, FrameOutput, Operation, PendingFetch, RenderVarOutput @@ -213,6 +217,7 @@ class OVRTXBackend: def __init__(self, cfg: OVRTXBackendCfg): self.cfg = cfg self.attached = False + self._render_lease_owner: OVRTXRenderer | None = None self.render_products: set[str] = set() # Resolve the wheel's native dependency here, including callers without a viewer. dependency = Path(ovstage.__file__).parent / "bin/plugins/libosdCPU.so.3.6.0" @@ -242,13 +247,25 @@ def __init__(self, cfg: OVRTXBackendCfg): self.renderer = Renderer(native_cfg) def detach(self) -> None: - """Release the borrowed stage before it is rebuilt or destroyed.""" + """Release the borrowed stage before it is rebuilt or destroyed. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot detach a backend with an active render lease") if self.attached: self.renderer.detach_ovstage() self.attached = False def close(self) -> None: - """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward.""" + """Destroy the engine; the simulation's backend registry releases any borrowed stage afterward. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy a backend with an active render lease") if self.renderer is not None: # The SDK destroys bindings and detaches only if attached, including partial initialization. self.renderer.destroy() @@ -334,6 +351,8 @@ def supported_output_types(self) -> dict[RenderBufferKind, RenderBufferSpec]: def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): super().__init__(cfg, sim=sim) + self._closed = False + self._render_lease = RenderLeaseController(self) self._warp_device = wp.get_device(self._sim.device) self._device = str(self._warp_device) self._camera_render_data: list[OVRTXCameraRenderData] = [] @@ -366,17 +385,48 @@ def __init__(self, cfg: OVRTXRendererCfg, *, sim: SimulationContext): @property def visual_material_writer(self): - """Return a writer factory once this renderer has a scene consumer.""" - return self._create_visual_material_writer if self._initialized_scene else None + """Return a writer factory once this renderer has a scene consumer. + + A leased pair registers its writer before the borrower binds the Stage, because + environment reset finalizes material consumers first; addresses compile on the + first publication after population. + """ + return ( + self._create_visual_material_writer + if self._initialized_scene or self._render_lease.active is not None + else None + ) def _create_visual_material_writer(self, batches: tuple[VisualMaterialBatch, ...]) -> OVRTXVisualMaterialWriter: - """Create the detached-scene material writer after scene population.""" + """Retain material buffers; deferred scenes compile addresses after population.""" writer = OVRTXVisualMaterialWriter(self, batches) self._visual_material_writer_ref = weakref.ref(writer) return writer + @contextlib.contextmanager + def _publish_visual_materials(self) -> Iterator[None]: + """Publish dirty materials and complete their reads before buffers may change.""" + material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None + try: + if material_writer is not None: + material_writer.publish() + yield + finally: + if material_writer is not None: + # When another exception is already propagating, log the drain failure instead of + # replacing it. Losing it silently would hide a failed material write. + primary_active = sys.exc_info()[0] is not None + try: + material_writer.drain() + except Exception as e: + if not primary_active: + raise + logger.warning("Error draining material writes after a failed scene update: %s", e, exc_info=True) + def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: """Load, clone, and bind the shared scene before any camera products are registered.""" + if not self._render_lease.prepare_stage(num_envs): + return logger.info("Preparing stage (%d envs)...", num_envs) env_paths = self.scene.clone_env_paths if self.cfg.temp_usd_dir is not None or not self._use_ovstage or not self.scene.ordinal: @@ -447,6 +497,11 @@ def prepare_stage(self, stage: Usd.Stage, num_envs: int) -> None: self.scene.resources.callback(self.close) self._initialized_scene = True self._needs_transform_initialization = not self._use_ovstage and self.backend.cfg.read_gpu_transforms + self._render_lease.bind_stage() + + def acquire_ov_libraries_render_lease(self) -> OvLibrariesRenderLease: + """Reserve the native renderer/scene pair before stage preparation.""" + return self._render_lease.acquire() def _setup_xform_bindings(self, stage: Usd.Stage) -> None: """Bind SDP body paths and preserve their authored scales when writing world poses.""" @@ -556,6 +611,7 @@ def _setup_geometry_bindings(self) -> None: def create_render_data(self, spec: CameraRenderSpec) -> OVRTXCameraRenderData: """Register a camera's product and bindings in the prepared scene.""" + self._render_lease.reject_ordinary_operation("create ordinary camera render data") if str(wp.get_device(spec.device)) != self._device: raise ValueError("Cameras sharing an OVRTX renderer must use the simulation device.") camera_paths = list(spec.camera_prim_paths) @@ -982,6 +1038,17 @@ def _process_render_products( ) def update_transforms(self) -> None: + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes, so eager scene + updates (for example ``lazy_sensor_update=False``) return without writing. + """ + if self._render_lease.owns_scene_updates(): + return + self._render_lease.reject_ordinary_operation("update ordinary transforms") + self._update_transforms() + + def _update_transforms(self) -> None: """Write changed SDP transforms to OVRTX.""" binding = self._object_xform_binding if binding is None: @@ -1014,6 +1081,16 @@ def update_transforms(self) -> None: self._transforms_timestamp = timestamp def update_geometries(self) -> None: + """Publish ordinary scene data when the native pair is not leased. + + While a lease is active its scene publisher owns these writes. + """ + if self._render_lease.owns_scene_updates(): + return + self._render_lease.reject_ordinary_operation("update ordinary geometries") + self._update_geometries() + + def _update_geometries(self) -> None: """Write changed SDP geometry to OVRTX.""" for callback in self.geometry_callbacks.values(): callback(self) @@ -1069,6 +1146,7 @@ def update_camera( intrinsics: ProxyArray, ) -> None: """Update camera poses using the camera's reusable conversion buffers.""" + self._render_lease.reject_ordinary_operation("update ordinary cameras") binding = render_data.camera_xform_binding if binding is None: return @@ -1096,6 +1174,7 @@ def update_camera_intrinsics(self, render_data: OVRTXCameraRenderData, intrinsic """Publish calibration columns from GPU memory into the renderer-owned scene.""" # A runtime calibration change is a scene write for this camera's bindings, so its # in-flight renders must finish first. Other cameras keep their pipelines. + self._render_lease.reject_ordinary_operation("update ordinary camera intrinsics") errors = self.drain_pending_renders((render_data,)) if errors: raise ExceptionGroup("OVRTX renders failed before calibration update", errors) @@ -1126,6 +1205,7 @@ def render(self, render_data: OVRTXCameraRenderData) -> None: def log_mesh(self, name: str, *args, **kwargs) -> None: """Publish a runtime mesh through the same scene used by every camera.""" + self._render_lease.reject_ordinary_operation("log runtime meshes") if name not in self.meshes: self.meshes[name] = OVRTXMesh(self, f"/IsaacLabMeshes/{name.lstrip('/')}") self.meshes[name].update(*args, **kwargs) @@ -1147,15 +1227,13 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: """ if not render_data: return + self._render_lease.reject_ordinary_operation("render an ordinary camera batch") if not self._initialized_scene: raise RuntimeError("Scene not initialized. Call initialize() first.") if self.backend.renderer is None or not self._camera_render_data: return products = self.backend.render_products.copy() - material_writer = self._visual_material_writer_ref() if self._visual_material_writer_ref is not None else None - try: - if material_writer is not None: - material_writer.publish() + with self._publish_visual_materials(): if self._needs_transform_initialization: # OVRTX 0.6 initializes GPU world transforms on its first step using # authored poses. Flush that initialization before capturing runtime @@ -1177,18 +1255,6 @@ def render_batch(self, render_data: Sequence[OVRTXCameraRenderData]) -> None: else: result = self.backend.renderer.step(render_products=products, delta_time=_RENDER_DELTA_TIME) self._process_render_products(render_data, result) - finally: - if material_writer is not None: - # When another exception is already propagating, log the drain failure instead of - # replacing it. Losing it silently would hide a failed material write. - primary_active = sys.exc_info()[0] is not None - try: - material_writer.drain() - except Exception as e: - if not primary_active: - raise - logger.warning("Error draining material writes after a failed render: %s", e, exc_info=True) - if self._use_ovstage: # Stage writes and material reads finish before rendering. Consume only after advancing # the ordinal so a failed readback cannot leave later writes at the sealed floor. @@ -1220,7 +1286,12 @@ def _remove_usd_reference(self, reference: int) -> None: ovstage.population.apply_usd_changes(self.scene.stage, ordinal=self.scene.ordinal) def close(self) -> None: - """Complete borrowed-buffer reads and release bindings; the registry owns the native engine.""" + """Complete borrowed-buffer reads and release bindings; the registry owns the native engine. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + self._render_lease.reject_ordinary_operation("close the renderer", retiring=True) try: with contextlib.ExitStack() as resources: resources.callback(self._bindings.close) @@ -1245,6 +1316,7 @@ def close(self) -> None: self.meshes.clear() self.markers.clear() self.geometry_callbacks.clear() + self._closed = True class OVRTXMesh: diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py new file mode 100644 index 000000000000..46904f164afc --- /dev/null +++ b/source/isaaclab_ov/isaaclab_ov/renderers/render_lease.py @@ -0,0 +1,578 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + + +"""Exclusive borrowing, thread handoff and publication ordering for OVRTX/OVStage.""" + +from __future__ import annotations + +import contextlib +import os +import threading +from collections.abc import Hashable, Iterator +from enum import Enum, auto +from typing import TYPE_CHECKING, Any, cast + +if TYPE_CHECKING: + from ovrtx import Renderer + from ovstage import Stage as OvStage + + from .ovrtx_renderer import OVRTXRenderer +_NO_PHYSICS_STEP_ID = object() + + +class _OvLibrariesRenderLeaseState(Enum): + """Internal lifecycle state for a renderer/scene pair lease.""" + + UNBOUND = auto() + BOUND = auto() + RELEASED = auto() + + +class OvLibrariesRenderLease: + """Exclusive non-owning lease over one exact attached OVRTX/OVStage pair. + + The lease is acquired before stage preparation. Stage preparation then + materializes the renderer-owned scene without requiring a camera sensor and + binds this object to the exact live Python ``Renderer`` and ``Stage`` + instances. External consumers must release the lease before the renderer is + closed; they must never detach or destroy either borrowed object. + """ + + _controller: RenderLeaseController + """Renderer-owned controller that created this lease.""" + + def __init__(self) -> None: + raise RuntimeError("OvLibrariesRenderLease must be acquired from OVRTXRenderer") + + def __enter__(self) -> OvLibrariesRenderLease: + self._validate_active() + return self + + def __exit__(self, *_exc: object) -> None: + self.release() + + @property + def ovrtx_renderer(self) -> Renderer: + """Return the exact retained application renderer.""" + self.validate_live() + return cast("Renderer", self._renderer) + + @property + def ovstage(self) -> OvStage: + """Return the exact retained renderer-attached OVStage.""" + self.validate_live() + return cast("OvStage", self._stage) + + def offer_owner_transfer(self) -> object: + """Freeze one unbound lease and return its one-shot owner capability.""" + owner = self._controller + return owner.offer_owner_transfer(self) + + def claim_owner_transfer(self, capability: object) -> None: + """Make the calling thread the render owner using the offered capability. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If the capability is stale, already claimed, or the lease is no longer unbound. + """ + self._controller.claim_owner_transfer(self, capability) + + def cancel_owner_transfer(self, capability: object) -> None: + """Withdraw an unclaimed offer on the original owner thread. + + Args: + capability: The object returned by :meth:`offer_owner_transfer`. + + Raises: + RuntimeError: If called off the original owner thread or with a stale capability. + """ + self._controller.cancel_owner_transfer(self, capability) + + def defer_stage_until_owner_transfer(self) -> None: + """Defer Stage materialization requested before the owner transfer is claimed. + + Camera initialization calls ``prepare_stage`` for every registered renderer on the + application thread. With deferral, that call only records the request; the claimed + render owner completes it with :meth:`materialize_deferred_stage`. + """ + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Stage deferral must be selected before Stage materialization") + self._controller.defer_until_transfer = True + + def materialize_deferred_stage(self, stage: Any, num_envs: int) -> bool: + """Complete a deferred Stage preparation on the claimed render owner. + + Args: + stage: The environment USD stage. + num_envs: Number of environments; the lease supports exactly one. + + Returns: + Whether a deferred preparation was completed. ``False`` means nothing was deferred. + + Raises: + RuntimeError: If called off the render-owner thread, or deferral is selected and + the owner transfer has not been claimed. + """ + return self._controller.materialize_deferred_stage(self, stage, num_envs) + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application-thread scene publisher before render ownership transfers. + + Call this on the owner thread while the lease is unbound and before + :meth:`offer_owner_transfer`. The publisher stays bound to the calling thread; see + :class:`~isaaclab_ov.renderers.OvLibrariesScenePublisher` for the publication + protocol. + + Returns: + The lease's only scene publisher. + + Raises: + RuntimeError: If called off the owner thread or while a transfer is offered, after the + Stage is bound, or when the lease already has a publisher. + """ + return self._controller.capture_scene_publisher(self) + + @contextlib.contextmanager + def ovstage_owner_lane(self) -> Iterator[None]: + """Guard one non-reentrant operation on the render-owner thread. + + The application's publication guard must separately exclude concurrent + producer writes while this operation uses the borrowed stage. A failed + producer write poisons publication only: the lane stays available so the + borrower can destroy its consumers and retire the pair. + """ + self._validate_live(allow_poisoned=True) + owner = self._controller + with owner.owner_lane("external renderer operation"): + yield + + def release(self) -> None: + """Release the borrowing intent without destroying either pair member.""" + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + return + owner = self._controller + owner.release(self) + self._renderer = None + self._stage = None + self._state = _OvLibrariesRenderLeaseState.RELEASED + + def validate_live(self) -> None: + """Verify that this lease still refers to its live, unpoisoned renderer and stage.""" + self._validate_live(allow_poisoned=False) + + def _validate_live(self, *, allow_poisoned: bool) -> None: + self._validate_active() + if self._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OvLibrariesRenderLease is not bound to a prepared Stage") + owner = self._controller + owner.validate_live( + self, + cast("Renderer", self._renderer), + cast("OvStage", self._stage), + self._generation, + allow_poisoned=allow_poisoned, + ) + + @classmethod + def _create_unbound(cls, owner: RenderLeaseController) -> OvLibrariesRenderLease: + lease = object.__new__(cls) + lease._controller = owner + lease._state = _OvLibrariesRenderLeaseState.UNBOUND + lease._renderer = None + lease._stage = None + lease._generation = 0 + return lease + + def _bind(self, renderer: Renderer, stage: OvStage, generation: int) -> None: + if self._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("OvLibrariesRenderLease can only bind once") + self._renderer = renderer + self._stage = stage + self._generation = generation + self._state = _OvLibrariesRenderLeaseState.BOUND + + def _validate_active(self) -> None: + if self._state is _OvLibrariesRenderLeaseState.RELEASED: + raise RuntimeError("OvLibrariesRenderLease is released") + owner = self._controller + owner.validate_active(self) + + +class OvLibrariesScenePublisher: + """Application-thread writer for complete provider-derived OVStage batches. + + Obtain it with :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.capture_scene_publisher`. + While the lease is active, the renderer skips its own eager transform, geometry and material + writes, and the borrower publishes each complete physics state through this object instead: + + 1. On the application thread, call :meth:`publication_required` with the identity of the + current physics state. + 2. If it returns ``True``, reserve a nonzero Stage ordinal and call + :meth:`write_complete_scene` with it. The write does not advance the Stage write floor: + the borrower seals that ordinal afterwards, or abandons it if the write fails. + 3. Hold one application publication guard around both calls and around every operation the + render owner performs in :meth:`~isaaclab_ov.renderers.OvLibrariesRenderLease.ovstage_owner_lane`, so that scene + writes and native frame operations never interleave. + + A failed write poisons the pair: later publication and ordinary renderer use are rejected, + but the borrower can still retire its consumers and release the lease. The renderer must then + be recreated. :meth:`close` stops publication; releasing the lease also closes the publisher. + The OVXR teleoperation adapter, ``isaaclab_teleop.OvLibrariesXrSession``, follows this protocol. + """ + + def __init__(self, owner: RenderLeaseController, lease: OvLibrariesRenderLease) -> None: + self._controller = owner + self._lease = lease + self._application_thread = threading.current_thread() + self._lock = threading.Lock() + self._closed = False + + def publication_required(self, physics_step_id: Hashable) -> bool: + """Return whether a new physics step or dirty material requires a scene write. + + Args: + physics_step_id: Hashable identity of the complete physics state to publish. Only the + last written identity is remembered, so the identity must change whenever the + state does, including after a reset that does not advance the physics step count. + + Returns: + Whether ``physics_step_id`` differs from the last written batch or a visual material + has pending updates. + + Raises: + RuntimeError: If called off the captured application thread, the publisher is closed + or inactive, the lease is not bound, or a failed write poisoned the pair. + TypeError: If ``physics_step_id`` is not hashable. + """ + self._require_application_thread() + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + return self._controller.producer_publication_required(self._lease, physics_step_id) + + def write_complete_scene(self, ordinal: int, physics_step_id: Hashable) -> None: + """Write, but do not publish, one complete batch at ``ordinal``. + + Writes the current transforms, geometries and pending visual materials at ``ordinal`` and + records ``physics_step_id`` as written. The caller seals the ordinal afterwards. + + Args: + ordinal: Nonzero Stage ordinal reserved by the caller for this batch. + physics_step_id: The identity passed to the preceding :meth:`publication_required` call. + + Raises: + ValueError: If ``ordinal`` is not positive. + RuntimeError: If called off the captured application thread, the publisher is closed or + inactive, the state needs no publication, or the write failed. A failed write + poisons the pair. + """ + self._require_application_thread() + with self._lock: + if self._closed: + raise RuntimeError("OVStage scene publisher is closed") + self._controller.write_producer_scene(self._lease, ordinal, physics_step_id) + + def close(self) -> None: + """Stop scene publication after any in-flight batch finishes. Safe to call repeatedly.""" + with self._lock: + self._closed = True + + def _require_application_thread(self) -> None: + if threading.current_thread() is not self._application_thread: + raise RuntimeError("OVStage scene publisher must run on its captured application thread") + + +class RenderLeaseController: + """Coordinate external borrowing without owning the renderer's native resources. + + Lease state, thread transfer and publication ordering live here. The renderer + retains scene/camera storage and supplies the component-write operations. + """ + + def __init__(self, renderer: OVRTXRenderer) -> None: + self._renderer = renderer + self._owner_thread = threading.current_thread() + self._transfer_lock = threading.Lock() + self._transfer_capability: object | None = None + self._transfer_claimed = False + self.active: OvLibrariesRenderLease | None = None + self._publisher: OvLibrariesScenePublisher | None = None + self._generation = 0 + self._lane_operation: str | None = None + self._last_step_id: Hashable = _NO_PHYSICS_STEP_ID + self._poisoned = False + self.defer_until_transfer = False + self.stage_deferred = False + + def acquire(self) -> OvLibrariesRenderLease: + """Reserve the exact OVRTX/OVStage pair before stage preparation.""" + self.require_owner_thread("acquire a render lease") + if os.environ.get("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1").strip() != "1": + raise RuntimeError("OvLibrariesRenderLease requires ISAAC_LAB_OVRTX_USE_OVSTAGE=1") + if os.environ.get("ISAAC_LAB_SHARE_OVSTAGE") == "1": + raise RuntimeError("The render lease requires a private rendering OVStage") + if self._renderer._closed: + raise RuntimeError("Cannot acquire OvLibrariesRenderLease from a closed renderer") + if self.active is not None: + raise RuntimeError("An OvLibrariesRenderLease is already active") + if self._renderer.backend is not None or self._renderer._initialized_scene or self._generation != 0: + raise RuntimeError("OvLibrariesRenderLease must be acquired before clone preparation") + lease = OvLibrariesRenderLease._create_unbound(self) + self.active = lease + return lease + + def require_owner_thread(self, operation: str) -> None: + """Reject lease operations outside the renderer construction thread.""" + if self._transfer_capability is not None: + raise RuntimeError("OVRTXRenderer owner transfer is pending") + if threading.current_thread() is not self._owner_thread: + raise RuntimeError(f"OVRTXRenderer owner thread must {operation}") + + def offer_owner_transfer(self, lease: OvLibrariesRenderLease) -> object: + self.require_owner_thread("offer an owner transfer") + with self._transfer_lock: + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + or self._transfer_capability is not None + or self._transfer_claimed + ): + raise RuntimeError("OVRTXRenderer is not eligible for a pre-use owner transfer") + capability = object() + self._transfer_capability = capability + return capability + + def claim_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + if ( + capability is None + or capability is not self._transfer_capability + or self._transfer_claimed + or self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._renderer._initialized_scene + or (self._renderer.backend is not None and self._renderer.backend.attached) + or self._generation != 0 + or self._lane_operation is not None + ): + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._owner_thread = threading.current_thread() + self._transfer_claimed = True + self._transfer_capability = None + + def cancel_owner_transfer(self, lease: OvLibrariesRenderLease, capability: object) -> None: + with self._transfer_lock: + # Cancellation is the one original-owner operation allowed while an offer is pending. + if threading.current_thread() is not self._owner_thread: + raise RuntimeError("OVRTXRenderer owner thread must cancel an owner transfer") + if capability is None or capability is not self._transfer_capability or self.active is not lease: + raise RuntimeError("OVRTXRenderer owner transfer capability is invalid") + self._transfer_capability = None + + def capture_scene_publisher(self, lease: OvLibrariesRenderLease) -> OvLibrariesScenePublisher: + """Create the scene publisher before render ownership transfers.""" + self.require_owner_thread("capture an OVStage scene publisher") + if ( + self.active is not lease + or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND + or self._publisher is not None + ): + raise RuntimeError("OVRTXRenderer is not eligible to capture an OVStage scene publisher") + publisher = OvLibrariesScenePublisher(self, lease) + self._publisher = publisher + return publisher + + @contextlib.contextmanager + def owner_lane(self, operation: str) -> Iterator[None]: + """Reject nested operations and access outside the render-owner thread. + + This lane is not a cross-thread lock. The external consumer must share + its application publication guard with the captured scene publisher. + """ + self.require_owner_thread(f"enter the OVStage owner lane for {operation}") + if self._lane_operation is not None: + raise RuntimeError( + f"OVStage owner lane is already active for {self._lane_operation}; nested writers are forbidden" + ) + self._lane_operation = operation + try: + yield + finally: + self._lane_operation = None + + def validate_active(self, lease: OvLibrariesRenderLease) -> None: + self.require_owner_thread("access a render lease") + if self.active is not lease: + raise RuntimeError("OvLibrariesRenderLease is not active for this renderer") + if self._renderer._closed: + raise RuntimeError("OvLibrariesRenderLease owner is closed or no longer uses OVStage") + + def validate_live( + self, + lease: OvLibrariesRenderLease, + ovrtx_renderer: Renderer, + ovstage_instance: OvStage, + generation: int, + *, + allow_poisoned: bool = False, + ) -> None: + """Validate retained Python-object identity for the live attached pair. + + ``allow_poisoned`` admits retirement work after a failed publication. + """ + self.validate_active(lease) + if self._poisoned and not allow_poisoned: + raise RuntimeError("OvLibrariesRenderLease owner pair is poisoned") + if generation != self._generation: + raise RuntimeError("OvLibrariesRenderLease has a stale Stage generation") + if ovrtx_renderer is not self._renderer.backend.renderer or ovstage_instance is not self._renderer.scene.stage: + raise RuntimeError("OvLibrariesRenderLease no longer matches its exact renderer-owned pair") + if not self._renderer._initialized_scene or not self._renderer.backend.attached: + raise RuntimeError("OvLibrariesRenderLease pair is no longer initialized and attached") + + def release(self, lease: OvLibrariesRenderLease) -> None: + """Retire the lease after borrower destruction and rebase the dormant ordinal.""" + self.validate_active(lease) + if self._lane_operation is not None: + raise RuntimeError("Cannot release OvLibrariesRenderLease while the owner lane is active") + if lease._state is _OvLibrariesRenderLeaseState.BOUND and (not self._poisoned): + with self.owner_lane("lease-release ordinal rebase"): + self._renderer.scene.ordinal = self._next_ordinal() + self._last_step_id = _NO_PHYSICS_STEP_ID + if self._publisher is not None: + self._publisher.close() + self._publisher = None + self._transfer_claimed = False + self.defer_until_transfer = False + self.stage_deferred = False + self.active = None + if self._renderer.backend is not None: + self._renderer.backend._render_lease_owner = None + if self._renderer.scene is not None: + self._renderer.scene._render_lease_owner = None + + def reject_ordinary_operation(self, operation: str, *, retiring: bool = False) -> None: + """Reject ordinary OVRTX operations while an external lease is active. + + A pair poisoned by a failed leased write keeps rejecting ordinary use after + release, because its ordinal is not rebased; only ``retiring`` operations + (closing the renderer) are admitted so the pair can be recreated. + """ + if self.active is not None or ( + self._renderer.backend is not None and self._renderer.backend._render_lease_owner is not None + ): + raise RuntimeError(f"Cannot {operation} while an OvLibrariesRenderLease is active") + if self._poisoned and not retiring: + raise RuntimeError( + f"Cannot {operation}: a failed leased publication poisoned this OVRTX/OVStage pair; recreate it" + ) + + def owns_scene_updates(self) -> bool: + """Whether an active lease's scene publisher owns transform and geometry writes.""" + return self.active is not None + + def producer_publication_required(self, lease: OvLibrariesRenderLease, physics_step_id: Hashable) -> bool: + """Check publication on the application thread without entering the render thread.""" + if self.active is not lease or self._publisher is None: + raise RuntimeError("OVStage scene publisher is not active for this lease") + if self._poisoned or lease._state is not _OvLibrariesRenderLeaseState.BOUND: + raise RuntimeError("OVStage scene publisher pair is unavailable") + try: + hash(physics_step_id) + except TypeError as exc: + raise TypeError("physics_step_id must be hashable") from exc + writer_ref = self._renderer._visual_material_writer_ref + writer = writer_ref() if writer_ref is not None else None + return self._last_step_id != physics_step_id or (writer is not None and writer.has_pending_updates) + + def write_producer_scene(self, lease: OvLibrariesRenderLease, ordinal: int, physics_step_id: Hashable) -> None: + """Write a complete scene batch without advancing its reserved ordinal.""" + if ordinal <= 0: + raise ValueError("OVStage publication ordinal must be nonzero") + if not self.producer_publication_required(lease, physics_step_id): + raise RuntimeError("OVStage scene publication is not required for this complete state") + self._renderer.scene.ordinal = ordinal + try: + with self._renderer._publish_visual_materials(): + self._renderer._update_transforms() + self._renderer._update_geometries() + except Exception as exc: + self._poisoned = True + raise RuntimeError("Producer scene publication failed; recreate the exact pair") from exc + self._last_step_id = physics_step_id + + def prepare_stage(self, num_envs: int) -> bool: + """Validate or defer leased preparation; ordinary rendering is immediately allowed. + + Returns: + Whether the renderer should materialize the Stage now. + """ + lease = self.active + if lease is not None: + if self.defer_until_transfer and not self._transfer_claimed: + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + self.stage_deferred = True + return False + self.require_owner_thread("prepare the leased Stage") + if lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Cannot replace the Stage of a bound renderer/scene pair") + if num_envs != 1: + raise RuntimeError("The render lease requires exactly one environment") + if not self._renderer._use_ovstage or self._renderer.backend._render_lease_owner is not self._renderer: + raise RuntimeError("The render lease requires its reserved private OVStage renderer") + return True + + def materialize_deferred_stage(self, lease: OvLibrariesRenderLease, stage: Any, num_envs: int) -> bool: + """Materialize a deferred Stage after render-thread ownership transfer.""" + self.require_owner_thread("materialize a deferred Stage") + # Unclaimed, a fallback prepare_stage would only defer again; reject even if nothing is deferred yet. + if self.defer_until_transfer and not self._transfer_claimed: + raise RuntimeError("Deferred Stage materialization requires a claimed owner transfer") + if not self.stage_deferred: + return False + if self.active is not lease or lease._state is not _OvLibrariesRenderLeaseState.UNBOUND: + raise RuntimeError("Deferred Stage is not eligible for materialization") + self._renderer.prepare_stage(stage, num_envs) + return True + + def bind_stage(self) -> None: + """Bind the prepared native pair after renderer initialization succeeds.""" + lease = self.active + if lease is not None: + self.stage_deferred = False + self._generation += 1 + lease._bind(self._renderer.backend.renderer, self._renderer.scene.stage, self._generation) + + def reserve_pair(self) -> None: + """Reserve a newly acquired private native pair during clone preparation.""" + renderer = self._renderer + if renderer.backend._render_lease_owner is not None or renderer.scene._render_lease_owner is not None: + raise RuntimeError("The native pair already has a render lease") + with renderer.scene.stage.get_attribute_write_floor() as floor: + floor.wait() + if int(floor.fetch()) != 0: + raise RuntimeError("The render lease requires an unused private OVStage") + renderer.backend._render_lease_owner = renderer + renderer.scene._render_lease_owner = renderer + + def _next_ordinal(self) -> int: + """Return ``floor + 1`` from the exact renderer-owned Stage.""" + if self._renderer.scene.stage is None: + raise RuntimeError("Cannot query an OVStage floor before scene materialization") + with self._renderer.scene.stage.get_attribute_write_floor() as floor_query: + floor_query.wait() + floor = int(floor_query.fetch()) + if floor >= (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + return floor + 1 diff --git a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py index 3138cf964e8a..921b640bc1a0 100644 --- a/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py +++ b/source/isaaclab_ov/isaaclab_ov/renderers/visual_materials.py @@ -62,8 +62,23 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, start = end self._device = str(device) self._event = wp.Event(device=self._device) + self._pending_groups = groups + self._resources = contextlib.ExitStack() + if renderer._initialized_scene: + self._compile_addresses(renderer) + + @property + def has_pending_updates(self) -> bool: + """Whether material changes require a scene publication even without a physics step.""" + return bool(self._dirty_channels) + + def _compile_addresses(self, renderer: OVRTXRenderer) -> None: + """Bind addresses only after the leased or ordinary scene has been populated.""" + if not renderer._initialized_scene: + raise RuntimeError("Visual materials require a populated renderer scene before publication.") + addresses = [] with contextlib.ExitStack() as resources: - for channel, attribute_name, shader_paths, rows, dtype, shape in groups: + for channel, attribute_name, shader_paths, rows, dtype, shape in self._pending_groups: if renderer._use_ovstage: address = resources.enter_context(renderer.scene.query(shader_paths)) else: @@ -76,8 +91,10 @@ def __init__(self, renderer: OVRTXRenderer, batches: tuple[VisualMaterialBatch, flags=BindingFlag.OPTIMIZE, ) resources.callback(address.unbind) - self._addresses.append((channel, address, attribute_name, rows)) + addresses.append((channel, address, attribute_name, rows)) self._resources = resources.pop_all() + self._addresses = addresses + self._pending_groups = [] def __call__(self, material_offsets: dict[str, Any] | None = None, env_ids: Any | None = None) -> None: """Mark channels dirty; OVRTX currently copies each dirty channel's full device buffer.""" @@ -92,6 +109,8 @@ def publish(self) -> None: if not channels: return renderer = self._renderer_ref() + if self._pending_groups: + self._compile_addresses(renderer) operations = [] try: for channel, address, attribute_name, rows in self._addresses: @@ -152,5 +171,6 @@ def close(self) -> None: logger.warning("Error draining in-flight render before material release: %s", error) finally: self._addresses.clear() + self._pending_groups.clear() self._dirty_channels.clear() self._buffers.clear() diff --git a/source/isaaclab_ov/isaaclab_ov/stage.py b/source/isaaclab_ov/isaaclab_ov/stage.py index e9f9f67a0d57..36a438b47207 100644 --- a/source/isaaclab_ov/isaaclab_ov/stage.py +++ b/source/isaaclab_ov/isaaclab_ov/stage.py @@ -175,6 +175,7 @@ def __init__(self, cfg: OvstageBackendCfg): self.clone_env_paths: list[str] = [] self.population_env_paths: list[str] = [] self.clone_positions: np.ndarray | None = None + self._render_lease_owner = None self.resources = contextlib.ExitStack() """Release borrowers before paths and stage, in reverse registration order.""" self.reset() @@ -224,12 +225,22 @@ def commit(self) -> int: return ordinal def close(self) -> None: - """Release borrowers, then the path dictionary and stage.""" + """Release borrowers, then the path dictionary and stage. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ + if self._render_lease_owner is not None: + raise RuntimeError("Cannot destroy or reset a stage with an active render lease") self.resources.close() self.stage = self.paths = None def reset(self) -> None: - """Recreate the native stage after all consumers detach, retaining the prepared clone operations.""" + """Recreate the native stage after all consumers detach, retaining the prepared clone operations. + + Raises: + RuntimeError: If an :class:`~isaaclab_ov.renderers.OvLibrariesRenderLease` is active. + """ self.close() self.ordinal = self.next_camera_id = 0 with contextlib.ExitStack() as resources: diff --git a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py index 91d0bbc7d634..e82e0768d4fa 100644 --- a/source/isaaclab_ov/test/test_ovrtx_clone_plan.py +++ b/source/isaaclab_ov/test/test_ovrtx_clone_plan.py @@ -94,6 +94,7 @@ def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: clone_positions=None, render_products=set(), cfg=SimpleNamespace(read_gpu_transforms=False), + _render_lease_owner=None, ) renderer.scene = renderer.backend renderer.backend.renderer = SimpleNamespace( diff --git a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py index 529ab0a469b5..1dad8fc7c555 100644 --- a/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py +++ b/source/isaaclab_ov/test/test_ovrtx_deformable_bindings.py @@ -42,7 +42,7 @@ def _simulation(monkeypatch): def _make_renderer_without_backend() -> tuple[OVRTXRenderer, MagicMock]: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) - renderer.backend = SimpleNamespace(renderer=MagicMock()) + renderer.backend = SimpleNamespace(renderer=MagicMock(), _render_lease_owner=None) renderer.scene = renderer.backend renderer.scene.query = OvstageBackend.query.__get__(renderer.scene) renderer._device = "cpu" diff --git a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py index 9c7b174f0a9f..bb24bf98f0b6 100644 --- a/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py +++ b/source/isaaclab_ov/test/test_ovrtx_renderer_contract.py @@ -37,7 +37,7 @@ ] if not _MISSING_MODULES: - from isaaclab_ov.renderers import OVRTXBackendCfg, OVRTXRendererCfg # noqa: E402 + from isaaclab_ov.renderers import OvLibrariesScenePublisher, OVRTXBackendCfg, OVRTXRendererCfg # noqa: E402 from isaaclab_ov.renderers import ovrtx_renderer as ovrtx_renderer_module # noqa: E402 from isaaclab_ov.renderers.ovrtx_renderer import ( # noqa: E402 _DISABLE_LINUX_CUDA_CPU_SYNC_ENV, @@ -81,6 +81,7 @@ def _make_ovrtx_camera_render_data() -> OVRTXCameraRenderData: def _make_ovrtx_renderer_without_backend() -> OVRTXRenderer: renderer = OVRTXRenderer(OVRTXRendererCfg(), sim=SimulationContext.instance()) renderer.backend = OVRTXBackend.__new__(OVRTXBackend) + renderer.backend._render_lease_owner = None renderer.scene = renderer.backend from isaaclab_ov.stage import OvstageBackend @@ -1270,6 +1271,7 @@ def query(paths): events.append(f"release_query:{paths}") renderer.scene = OvstageBackend.__new__(OvstageBackend) + renderer.scene._render_lease_owner = None SimulationContext.instance()._backend_registry.insert( 0, (OvstageBackendCfg(scene_key=renderer.cfg), renderer.scene) ) @@ -1412,3 +1414,405 @@ def attach(native_stage): assert events.count("disable-camera-reads") == (needs_fallback and stage_mode != "legacy") if stage_mode != "legacy": assert events[-2:] == ["commit", "attach"] + + +@pytest.fixture +def leased_renderer(monkeypatch): + """Exercise real deferred resource acquisition with native SDK endpoints doubled.""" + import isaaclab_ov.stage as stage_module + + monkeypatch.delenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", raising=False) + native, stage, paths = MagicMock(), MagicMock(), MagicMock() + floor = stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 0 + + def advance(*, ordinal, **kwargs): + return types.SimpleNamespace(wait=lambda: setattr(floor.fetch, "return_value", ordinal)) + + stage.advance_write_floor.side_effect = advance + monkeypatch.setattr(ovrtx_renderer_module, "Renderer", lambda _cfg: native) + monkeypatch.setattr(ovrtx_renderer_module, "redirect_shader_cache", lambda _cfg: None) + monkeypatch.setattr(stage_module, "create_ovstage", lambda _name: contextlib.nullcontext(stage)) + monkeypatch.setattr(stage_module.ovstage, "PathDictionary", lambda _: contextlib.nullcontext(paths)) + monkeypatch.setattr(stage_module.ovstage.population, "open_usd_from_string", lambda *args, **kwargs: None) + renderer = SimulationContext.instance().get_or_create_backend(OVRTXRendererCfg()) + assert renderer.backend is renderer.scene is None + return renderer + + +def _prepare_leased_renderer(renderer, *, bind=True): + from isaaclab_ov.cloner import OvrtxReplicateContext + + OvrtxReplicateContext.prepare(renderer._sim, {}) + if bind: + renderer.prepare_stage(renderer._sim.stage, 1) + + +def test_render_lease_borrows_registry_pair_and_blocks_shared_users(leased_renderer, monkeypatch): + renderer = leased_renderer + shared = SimulationContext.instance().get_or_create_backend(renderer.cfg) + assert shared is renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="already active"): + shared.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + with pytest.raises(RuntimeError, match="Lease"): + shared.render_batch((object(),)) + shared.render_batch(()) + with monkeypatch.context() as patch: + patch.setattr(ovrtx_renderer_module, "OVRTXMesh", lambda *_: pytest.fail("constructed a mesh while leased")) + with pytest.raises(RuntimeError, match="log runtime meshes"): + shared.log_mesh("blocked") + assert "blocked" not in shared.meshes + with pytest.raises(RuntimeError, match="active render lease"): + SimulationContext.instance().close_backend(renderer.backend) + with pytest.raises(RuntimeError, match="active render lease"): + renderer.scene.reset() + native = lease.ovrtx_renderer + assert native is shared.backend.renderer + assert lease.ovstage is shared.scene.stage + lease.release() + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene._render_lease_owner is None + native.destroy.assert_not_called() + SimulationContext.instance().close_backend(renderer.backend) + native.destroy.assert_called_once_with() + + +def test_render_lease_rejects_acquisition_after_clone_preparation(leased_renderer): + renderer = leased_renderer + _prepare_leased_renderer(renderer, bind=False) + with pytest.raises(RuntimeError, match="before clone preparation"): + renderer.acquire_ov_libraries_render_lease() + assert renderer.backend._render_lease_owner is None + + +def test_render_lease_rejects_calibration_before_scene_access(leased_renderer, monkeypatch): + """An ordinary calibration write cannot bypass the external owner's publication lane.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + camera = _make_ovrtx_camera_render_data() + intrinsics = wp.zeros(2, dtype=wp.mat33f, device="cpu") + parameters = wp.zeros((5, 2), dtype=wp.float32, device="cpu") + drain = MagicMock(return_value=[]) + monkeypatch.setattr(renderer, "drain_pending_renders", drain) + monkeypatch.setattr(wp, "get_stream", lambda device: types.SimpleNamespace(cuda_stream=99)) + try: + with pytest.raises(RuntimeError, match="Lease"): + renderer.update_camera_intrinsics(camera, intrinsics, parameters) + drain.assert_not_called() + assert renderer.backend is renderer.scene is None + finally: + lease.release() + + +def test_render_lease_owner_transfer_is_single_use(leased_renderer): + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(None) + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + with pytest.raises(RuntimeError, match="capability is invalid"): + lease.claim_owner_transfer(token) + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._render_lease._owner_thread is thread + assert renderer.backend is renderer.scene is None + + +@pytest.fixture +def complete_scene_publication(leased_renderer, monkeypatch): + """Exercise the application publisher with native component endpoints doubled.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + stage = renderer.scene.stage + stage.get_attribute_write_floor.reset_mock() + stage.advance_write_floor.reset_mock() + state = types.SimpleNamespace( + renderer=renderer, + lease=lease, + publisher=publisher, + ordinal=23, + events=[], + failure=None, + cause=ValueError("component endpoint failed"), + initial_id=renderer._render_lease._last_step_id, + ) + + def component(name): + state.events.append((name, renderer.scene.ordinal)) + if state.failure == name: + raise state.cause + + writer = types.SimpleNamespace( + has_pending_updates=False, + publish=lambda: component("material"), + drain=lambda: component("material-drain"), + ) + monkeypatch.setattr(renderer, "_visual_material_writer_ref", lambda: writer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: component("transform")) + monkeypatch.setattr(renderer, "_update_geometries", lambda: component("geometry")) + state.publish = lambda step_id="step": publisher.write_complete_scene(state.ordinal, step_id) + try: + yield state + finally: + lease.release() + + +def test_render_lease_complete_scene_components_share_the_supplied_ordinal(complete_scene_publication): + state = complete_scene_publication + renderer = state.renderer + assert state.publisher.publication_required("step") + assert state.publish() is None + names = ["material", "transform", "geometry", "material-drain"] + assert state.events == [(name, state.ordinal) for name in names] + assert renderer._render_lease._last_step_id == "step" + assert renderer.scene.ordinal == state.ordinal + assert not state.publisher.publication_required("step") + with pytest.raises(RuntimeError, match="publication is not required"): + state.publish() + assert state.events == [(name, state.ordinal) for name in names] + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + renderer.scene.stage.advance_write_floor.assert_not_called() + assert not renderer._render_lease._poisoned + + +@pytest.mark.parametrize("failure", ["material", "transform", "geometry", "material-drain"]) +def test_render_lease_complete_scene_failure_preserves_commit_boundary(complete_scene_publication, failure): + state = complete_scene_publication + renderer, lease = state.renderer, state.lease + state.failure = failure + with pytest.raises(RuntimeError, match="Producer scene publication failed") as caught: + state.publish() + assert caught.value.__cause__ is state.cause + order = ["material", "transform", "geometry", "material-drain"] + expected = order[: order.index(failure) + 1] + if failure != "material-drain": + expected.append("material-drain") + assert state.events == [(name, state.ordinal) for name in expected] + assert renderer._render_lease._last_step_id == state.initial_id + assert renderer.scene.ordinal == state.ordinal + assert renderer._render_lease._poisoned + with pytest.raises(RuntimeError, match="poisoned"): + _ = lease.ovstage + previous_events = list(state.events) + with pytest.raises(RuntimeError, match="unavailable|poisoned"): + state.publish("next-step") + assert state.events == previous_events + renderer.scene.stage.advance_write_floor.assert_not_called() + renderer.scene.stage.get_attribute_write_floor.assert_not_called() + + +def test_render_lease_complete_scene_public_authority_precedes_components(complete_scene_publication): + import threading + + state = complete_scene_publication + errors = [] + + def foreign_thread(): + try: + state.publish() + except BaseException as exc: + errors.append(exc) + + thread = threading.Thread(target=foreign_thread) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert len(errors) == 1 and isinstance(errors[0], RuntimeError) + assert "thread" in str(errors[0]) + with pytest.raises(TypeError, match="physics_step_id must be hashable"): + state.publish([]) + with pytest.raises(ValueError, match="ordinal must be nonzero"): + state.publisher.write_complete_scene(0, "step") + state.publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + state.publish() + assert not state.events + state.renderer.scene.stage.get_attribute_write_floor.assert_not_called() + state.renderer.scene.stage.advance_write_floor.assert_not_called() + assert not state.renderer._render_lease._poisoned + + +def test_render_lease_closed_publisher_reports_no_pending_publication(leased_renderer, monkeypatch): + """A closed publisher must not report a write that :meth:`write_complete_scene` would then reject.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + assert isinstance(publisher, OvLibrariesScenePublisher) + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", lambda: None) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + assert publisher.publication_required("step-1") + publisher.close() + publisher.close() + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.publication_required("step-1") + with pytest.raises(RuntimeError, match="publisher is closed"): + publisher.write_complete_scene(18, "step-1") + lease.release() + + +def test_render_lease_poison_blocks_publication_but_not_retirement(leased_renderer, monkeypatch): + """A failed producer write must not prevent the borrower from retiring the pair (Capture's close order).""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + _prepare_leased_renderer(renderer) + monkeypatch.setattr(renderer, "_update_transforms", MagicMock(side_effect=ValueError("write failed"))) + monkeypatch.setattr(renderer, "_update_geometries", lambda: None) + with pytest.raises(RuntimeError, match="scene publication failed"): + publisher.write_complete_scene(19, "step-1") + with pytest.raises(RuntimeError, match="unavailable"): + publisher.publication_required("step-2") + with pytest.raises(RuntimeError, match="poisoned"): + lease.validate_live() + # OvxrFramePump.close: close publication, destroy the OVXR context inside the owner lane, release the lease. + publisher.close() + with lease.ovstage_owner_lane(): + pass + ordinal = renderer.scene.ordinal + lease.release() + assert renderer.backend._render_lease_owner is None + assert renderer.scene.ordinal == ordinal # The abandoned ordinal is not rebased onto a sealed floor. + with pytest.raises(RuntimeError, match="poisoned this OVRTX/OVStage pair; recreate it"): + renderer.render_batch((object(),)) + with pytest.raises(RuntimeError, match="recreate it"): + renderer.update_transforms() + renderer.close() + + +def test_render_lease_release_rebases_ordinal_and_restores_ordinary_use(leased_renderer): + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + floor = renderer.scene.stage.get_attribute_write_floor.return_value.__enter__.return_value + floor.fetch.return_value = 41 + lease.release() + assert renderer.scene.ordinal == 42 + renderer.update_transforms() + renderer.update_geometries() + + +@pytest.mark.parametrize( + ("variable", "value"), [("ISAAC_LAB_OVRTX_USE_OVSTAGE", "0"), ("ISAAC_LAB_SHARE_OVSTAGE", "1")] +) +def test_render_lease_rejects_explicit_stage_opt_out(leased_renderer, monkeypatch, variable, value): + monkeypatch.setenv(variable, value) + with pytest.raises(RuntimeError, match="OVSTAGE=1|private rendering OVStage"): + leased_renderer.acquire_ov_libraries_render_lease() + + +def test_render_lease_defers_application_thread_stage_until_owner_transfer(leased_renderer): + """Camera initialization prepares every renderer on the application thread before XR transfers ownership.""" + import threading + + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + lease.defer_stage_until_owner_transfer() + _prepare_leased_renderer(renderer, bind=False) + context = renderer._sim.render_context + context.ensure_prepare_stage(renderer._sim.stage, 1) + assert not renderer._initialized_scene + assert renderer._render_lease.stage_deferred + # The deferred call must leave the pair eligible for the XR worker handoff. + publisher = lease.capture_scene_publisher() + token = lease.offer_owner_transfer() + errors = [] + + def owner(): + try: + lease.claim_owner_transfer(token) + assert lease.materialize_deferred_stage(renderer._sim.stage, 1) + assert not lease.materialize_deferred_stage(renderer._sim.stage, 1) + lease.validate_live() + publisher.close() + lease.release() + except BaseException as error: + errors.append(error) + + thread = threading.Thread(target=owner) + thread.start() + thread.join(timeout=5) + assert not thread.is_alive() + assert not errors + assert renderer._initialized_scene + + +def test_render_lease_eager_scene_updates_defer_to_the_publisher(leased_renderer, monkeypatch): + """``lazy_sensor_update=False`` refreshes every renderer; the leased renderer's publisher owns those writes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + _prepare_leased_renderer(renderer) + transforms, geometries = MagicMock(), MagicMock() + monkeypatch.setattr(renderer, "_update_transforms", transforms) + monkeypatch.setattr(renderer, "_update_geometries", geometries) + renderer._sim.render_context.update_scene_state(1) + transforms.assert_not_called() + geometries.assert_not_called() + lease.release() + renderer._sim.render_context.update_scene_state(2) + transforms.assert_called_once_with() + geometries.assert_called_once_with() + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA material buffers") +def test_leased_material_updates_reach_the_reserved_scene(leased_renderer, monkeypatch): + """Real material composition and the lease publisher must reach native writes, including before physics changes.""" + renderer = leased_renderer + lease = renderer.acquire_ov_libraries_render_lease() + publisher = lease.capture_scene_publisher() + context = renderer._sim.render_context + # Real lifecycle: clone preparation, then environment reset finalizes material consumers, + # then the XR borrower binds the Stage. + _prepare_leased_renderer(renderer, bind=False) + material = types.SimpleNamespace( + channels=("color",), + is_per_env=False, + num_instances=1, + _material_paths=("/Looks/Feedback",), + _shader_paths=("/Looks/Feedback/Shader",), + _input_names={"color": "diffuseColor"}, + _offsets={}, + _values={"color": torch.tensor([[1.0, 0.0, 0.0]], device="cuda")}, + ) + context.register_visual_material(material) + context.finalize_consumers([]) + assert renderer._visual_material_writer_ref is not None + renderer.prepare_stage(renderer._sim.stage, 1) + stage = renderer.scene.stage + try: + for ordinal, color in ((23, [0.0, 1.0, 0.0]), (24, [0.0, 0.0, 1.0])): + context.write_visual_materials([material], {"color": torch.tensor([color], device="cuda")}) + assert publisher.publication_required("unchanged-physics") + stage.write_attribute.reset_mock() + completion = MagicMock() + stage.write_attribute.return_value = completion + publisher.write_complete_scene(ordinal, "unchanged-physics") + writes = [c for c in stage.write_attribute.call_args_list if c.args[1] == "inputs:diffuseColor"] + assert len(writes) == 1 + assert writes[0].kwargs["ordinal"] == ordinal + torch.testing.assert_close(writes[0].kwargs["tensors"].cpu(), torch.tensor([color])) + assert completion.wait.called + assert not publisher.publication_required("unchanged-physics") + finally: + lease.release() + for writer in context._visual_material_writers: + writer.close() diff --git a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py index 6c7b0c9efc29..ca892860c012 100644 --- a/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py +++ b/source/isaaclab_ov/test/test_ovrtx_visual_material_sync.py @@ -23,6 +23,7 @@ if not _MISSING_MODULES: from isaaclab_ov.renderers.ovrtx_renderer import OVRTXRenderer from isaaclab_ov.renderers.ovrtx_renderer_cfg import OVRTXRendererCfg + from isaaclab_ov.renderers.render_lease import RenderLeaseController from ovrtx import DataAccess from isaaclab.renderers.base_renderer import VisualMaterialBatch @@ -121,7 +122,8 @@ def destroy_path_list(self, path_list): def _renderer(*, use_ovstage: bool = False): events: list[str] = [] renderer = OVRTXRenderer.__new__(OVRTXRenderer) - renderer.backend = SimpleNamespace(render_products=set()) + renderer._render_lease = RenderLeaseController(renderer) + renderer.backend = SimpleNamespace(render_products=set(), _render_lease_owner=None) renderer.scene = renderer.backend renderer._initialized_scene = True renderer._needs_transform_initialization = False diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst new file mode 100644 index 000000000000..0565d24bd9f3 --- /dev/null +++ b/source/isaaclab_teleop/changelog.d/kitless-ovxr.added.rst @@ -0,0 +1,6 @@ +* Added :class:`~isaaclab_teleop.OvLibrariesXrSession`, an OVXR application adapter and session + bootstrap that borrow the simulation-owned OVRTX renderer and OVStage, publish scene updates, + and coordinate render-thread ownership and shutdown with Isaac Capture. + Publication failures close admission, and failed lease retirement retains borrowed resources. +* Added :class:`~isaaclab_teleop.OvLibrariesXrCfg` in :mod:`isaaclab_teleop.ovxr_cfg`, which + imports without the native OVXR packages. diff --git a/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor b/source/isaaclab_teleop/changelog.d/kitless-ovxr.minor new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/source/isaaclab_teleop/docs/OVXR_ADAPTER.md b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md new file mode 100644 index 000000000000..2e7c91fbf4f3 --- /dev/null +++ b/source/isaaclab_teleop/docs/OVXR_ADAPTER.md @@ -0,0 +1,102 @@ +# OVXR application adapter + +`isaaclab_teleop.OvLibrariesXrSession` connects an Isaac Lab environment to +Isaac Capture's OVXR session owner. It uses the simulation's existing OVRTX +renderer and OVStage through `isaaclab_ov.renderers.OvLibrariesRenderLease`. + +## Ownership + +- `isaaclab_ov` owns the renderer, stage, scene publication and their lifetime. + Its lease validates access and enforces render-thread ownership. The adapter's + publication guard serializes application writes with Capture's frame operations. + It does not create an XR session or select teleoperation behavior. +- `isaaclab_teleop.ovxr_session.OvxrSessionBootstrap` creates the simulation + context and reserves the lease before environment construction. It binds the + completed environment and reads current target-frame transforms. +- `isaaclab_teleop.ov_libraries_xr.OvLibrariesXrSession` supplies the application + adapter to Isaac Capture. It declares XR render products, publishes scene and + anchor updates, transfers render-thread ownership and coordinates shutdown. +- Isaac Capture owns the OVXR session and render worker. Its borrowed OpenXR + handles are available to the teleoperation input session. + +## Lifecycle + +Prepare the adapter before constructing the environment so that both use the +same simulation context and native renderer/scene pair. Start the render worker +after the environment exists. Publish simulation updates through the session; +the worker owns rendering. Stop input consumers before closing the XR session, +and retire the lease before destroying the simulation's renderer or stage. +If startup fails before the worker takes ownership, the caller closes the +environment after releasing the lease. If input retirement fails, request +shutdown and retain the borrowed resources until their retirement is proved. + +The lease currently requires one environment and a private rendering OVStage. +Ownership transfer is single-use. Failed retirement must preserve resources +for the owning thread to finish cleanup; a failed drain is not successful +shutdown. Importing `OvLibrariesXrSession` requires a Capture release with OVXR +session support; see [Installing kitless XR](kitless_xr_source.md). The +`OvLibrariesXrCfg` dataclass lives in `isaaclab_teleop.ovxr_cfg` and imports +without those packages. + +A failed scene write or seal stops further publication. Recreate the session +after retiring the failed pair; retrying the same generation cannot return an +older scene as if the failed update succeeded. The failure poisons producer +publication only: the render-owner lane stays available so Capture can destroy +its OVXR context and the adapter can release the lease and close the +environment. After a poisoned lease is released, the renderer keeps rejecting +ordinary use (except closing it) because its write ordinal was not rebased. +During shutdown, publication storage drains before lease guards are removed. +The adapter retains the environment if lease retirement cannot establish completion. + +## Environment requirements + +- `OvLibrariesXrSession.prepare()` creates the `SimulationContext` from the + `sim_cfg` it receives, and the environment reuses that context. Pass the + environment's own `env_cfg.sim`; before the worker claims the lease, + `start_render_worker()` rejects an environment whose simulation device or + time step differs, or whose `num_envs` is not 1. +- Only environments that accept a pre-created simulation context are + supported. Manager-based environments do; `DirectRLEnv` always creates its own + context and fails with "Simulation context already exists". +- Camera sensors on other renderers are supported: their initialization + prepares every registered renderer on the application thread, and the leased + renderer defers its Stage until the XR worker claims ownership. A camera that + resolves to the leased renderer itself (for example an OVRTX camera with the + default `OVRTXRendererCfg()`) is rejected while the lease is active. +- Eager scene updates (`lazy_sensor_update=False`) skip the leased renderer; its + scene publisher writes the same physics state at the reserved ordinal. +- Dynamic anchoring takes any object implementing + `OvLibrariesXrSession.ExternalAnchorState`; `start_render_worker()` rejects + other objects before the worker claims the lease. + +## Backend implementation + +The ordinary renderer remains in `isaaclab_ov.renderers.ovrtx_renderer`. +`render_lease.RenderLeaseController` owns the lease state, thread-transfer +capability and per-owner operation guards; it borrows the renderer's native resources +and delegates component writes back to that renderer. Public renderer and lease +imports remain available from `isaaclab_ov.renderers`. + +Acquiring a lease selects a private rendering OVStage without changing process +environment variables. An explicit OVStage disable or shared-stage selection is +rejected. Without an active lease, the controller permits +ordinary renderer operations and does not start an XR session. While a consumer +holds the lease, ordinary camera rendering, scene writes and renderer destruction +are rejected to prevent concurrent access to the borrowed resources. The same +generic borrowing interface is available to applications outside teleoperation. + +## Native validation + +The adapter and lease unit tests double the native OVRTX and OVStage endpoints. +An opt-in harness runs the render-owner transfer and complete scene publication +against the real libraries, on the manager-based `Isaac-Cartpole` task with its +`ovphysx` physics preset and one environment on `cuda:0`. It needs a CUDA GPU, +OVRTX, OVStage, OvPhysX and an Isaac Capture release with OVXR session support; +it starts no XR session or CloudXR runtime. Each case runs in a child process: + +```bash +ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1 uv run python -m pytest \ + source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py +``` + +Without `ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER=1`, both cases are skipped. diff --git a/source/isaaclab_teleop/docs/README.md b/source/isaaclab_teleop/docs/README.md index 266f5470dda9..05e7848dcba8 100644 --- a/source/isaaclab_teleop/docs/README.md +++ b/source/isaaclab_teleop/docs/README.md @@ -24,6 +24,10 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. ## Architecture +The [OVXR application adapter](OVXR_ADAPTER.md) connects a simulation-owned +renderer to Isaac Capture. XR session integration belongs to `isaaclab_teleop`; +`isaaclab_ov` supplies the generic renderer/scene lease and lifetime checks. + `IsaacTeleopDevice` composes three focused collaborators: | Component | Responsibility | @@ -32,6 +36,11 @@ XR anchor synchronization, retargeting pipelines, and action-tensor generation. | `TeleopSessionLifecycle` | Pipeline building, OpenXR handle acquisition, session create/destroy, action-tensor extraction | | `CommandHandler` | Callback registration and XR message-bus command dispatch | +For kitless XR, `isaaclab_teleop` owns session preparation and tracking anchors. +It borrows the application renderer through `isaaclab_ov`'s `OvLibrariesRenderLease` +interface. The renderer retains ownership of native resources, thread synchronization +and GPU scene publication. + ## Usage ### 1. Configure Your Environment @@ -180,5 +189,7 @@ run automatically. ## Dependencies - **`isaaclab`** -- core Isaac Lab framework +- **`isaaclab_ov`** -- required by the optional OVXR session adapter; the standalone `isaaclab_teleop` package does not install this backend automatically - **`isaacteleop`** -- Isaac Capture retargeting engine, device I/O, and session management - **`isaacsim`** -- Isaac Sim runtime (provides the Kit XR bridge for OpenXR handle acquisition) +- **OVXR / OVRTX / OVStage** -- matched native runtime and stock peers for OVXR diff --git a/source/isaaclab_teleop/docs/kitless_xr_source.md b/source/isaaclab_teleop/docs/kitless_xr_source.md new file mode 100644 index 000000000000..389c1119ecd3 --- /dev/null +++ b/source/isaaclab_teleop/docs/kitless_xr_source.md @@ -0,0 +1,63 @@ +# Installing kitless XR + +Use Python 3.12 on Linux x86-64. The Isaac Capture wheel includes the +OVXR session integration. + +Complete the CloudXR, client, network and license prerequisites before +starting a session. + +From the Isaac Lab root, select the Capture version in an `ovxr` project +extra. Lab already configures NVIDIA's public Python index and allows prereleases: + +```bash +uv add --optional ovxr --frozen \ + "isaaccapture[ovxr,retargeters,ui,cloudxr]==1.7.93a1 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +uv add --optional ovxr --frozen \ + "isaaclab-dev[ov] ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-teleop ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "isaaclab-mimic ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "ipywidgets>=8.1.5 ; platform_system == 'Linux' and platform_machine == 'x86_64'" \ + "dex-retargeting==0.5.0 ; platform_system == 'Linux' and platform_machine == 'x86_64'" +``` + +The `ov` extra supplies the compatible OVRTX, OVStage, OVPhysX and PhysX USD +schemas. Keep its provider requirements and the workspace package metadata unchanged. + +Add these mutually exclusive selections to `tool.uv.conflicts`, preserving existing entries: + +```toml +[tool.uv] +conflicts = [ + [{ extra = "ovxr" }, { extra = "teleop" }], + [{ extra = "ovxr" }, { extra = "isaacsim" }], + [{ extra = "ovxr" }, { extra = "viser" }], + [{ extra = "ovxr" }, { extra = "all" }], +] +``` + +The `teleop` extra selects Capture packages for Kit and non-Kit input +workflows. Use the separate `ovxr` selection for the XR commands here. +Isaac Sim and Viser require older WebSocket versions than this Capture +release, so these selections cannot be combined. The `all` extra +includes Viser. + +Resolve the selected environment and check its dependencies: + +```bash +uv lock --refresh-package isaaccapture --upgrade-package isaaccapture \ + --upgrade-package numpy==2.3.5 --upgrade-package packaging==23.2 +uv sync --python 3.12 --locked --extra ovxr +uv run --extra ovxr python -c 'import importlib.metadata as m; print(m.version("isaaccapture"), m.version("ovxr"))' +uv pip check +``` + +The lock selects compatible NumPy 2.3.5 and packaging 23.2 versions without +changing project-wide overrides. OVPhysX declares `packaging<24`; the explicit +selection above satisfies its metadata in this non-Kit selection. Stop applications before changing selections. +The Kit and OVXR provider selections are different; preserve the source metadata +and lockfile when changing between them, and check the resulting dependency graph. +Repeat this selection if a later lock upgrade replaces it. Isaac Capture's +`ovxr` extra selects OVXR and its compatible OVRTX/OVStage dependencies; see its +OVXR guide for application version overrides. + +Continue with the [OVXR application adapter](OVXR_ADAPTER.md). diff --git a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi index d810345978e6..79656e7f0876 100644 --- a/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi +++ b/source/isaaclab_teleop/isaaclab_teleop/__init__.pyi @@ -15,6 +15,8 @@ __all__ = [ "HapticFeedbackReceiver", "IsaacTeleopCfg", "IsaacTeleopDevice", + "OvLibrariesXrCfg", + "OvLibrariesXrSession", "SupportsControlEvents", "SystemCheckItem", "SystemCheckResult", @@ -42,8 +44,12 @@ __all__ = [ from .camera_feed import XrCameraFeedSession from .control_events import TELEOP_CONTROL_CHANNEL_UUID, ControlEvents, SupportsControlEvents, poll_control_events from .device_pipelines import ( - se2_gamepad_teleop_cfg, se2_keyboard_teleop_cfg, se2_spacemouse_teleop_cfg, - se3_gamepad_teleop_cfg, se3_keyboard_teleop_cfg, se3_spacemouse_teleop_cfg, + se2_gamepad_teleop_cfg, + se2_keyboard_teleop_cfg, + se2_spacemouse_teleop_cfg, + se3_gamepad_teleop_cfg, + se3_keyboard_teleop_cfg, + se3_spacemouse_teleop_cfg, ) from .haptic_feedback import ( ControllerHapticFeedbackCfg, @@ -62,6 +68,8 @@ from .isaac_teleop_cfg import ( XrCameraFeedLayoutCfg, ) from .isaac_teleop_device import IsaacTeleopDevice, create_isaac_teleop_device +from .ov_libraries_xr import OvLibrariesXrSession +from .ovxr_cfg import OvLibrariesXrCfg from .session_lifecycle import cloudxr_eula_accepted from .system_check import SystemCheckItem, SystemCheckResult, check_system_requirements from .xr_anchor_utils import XrAnchorSynchronizer diff --git a/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py new file mode 100644 index 000000000000..9c0ca2d83890 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ov_libraries_xr.py @@ -0,0 +1,649 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Isaac Lab host adapter for Isaac Capture-owned OVXR presentation.""" + +from __future__ import annotations + +import contextlib +import logging +import threading +import time +from collections.abc import Callable, Hashable, Iterable +from dataclasses import replace +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +import numpy as np +from isaaccapture.ovxr_session_manager import OvxrApplicationInterface, StageOrdinalLedger +from isaaclab_ov.renderers import OVRTXRendererCfg + +from .ovxr_cfg import OvLibrariesXrCfg +from .ovxr_session import OvxrSessionBootstrap, validate_ovxr_session_preparation + +if TYPE_CHECKING: + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + from isaacteleop.oxr import OpenXRSessionHandles + + from isaaclab.sim import SimulationCfg + +__all__ = ["OvLibrariesXrCfg", "OvLibrariesXrSession"] + +logger = logging.getLogger(__name__) + +_DEFAULT_OVXR_RENDER_PRODUCT_PATHS = ( + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", + "/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", +) +_DEFAULT_OVXR_RENDER_VAR_PATHS = ( + "/Render/Vars/LdrColor", + "/Render/Vars/XRDepth", +) +_DEFAULT_OVXR_RENDER_SETTINGS_PATH = "/Render/OmniverseGlobalRenderSettings" + + +def _declare_ovxr_runtime_prims(env: Any, cfg: OvLibrariesXrCfg) -> None: + """Declare type-only OVXR prims for the application Stage's lifetime.""" + if not cfg.create_render_products: + return + + unwrapped = getattr(env, "unwrapped", env) + stage = unwrapped.scene.stage + if stage is None: + raise RuntimeError("OVXR runtime prim declaration requires an environment Stage") + + render_products = cfg.render_products or _DEFAULT_OVXR_RENDER_PRODUCT_PATHS + declarations = ( + *((path, "Camera") for path in cfg.camera_paths), + *((path, "RenderProduct") for path in render_products), + *((path, "RenderVar") for path in _DEFAULT_OVXR_RENDER_VAR_PATHS), + (_DEFAULT_OVXR_RENDER_SETTINGS_PATH, "RenderSettings"), + ) + expected_types: dict[str, str] = {} + for path, type_name in declarations: + if not path.startswith("/") or path == "/": + raise ValueError(f"OVXR runtime prim path must be absolute and non-root: {path!r}") + previous_type = expected_types.setdefault(path, type_name) + if previous_type != type_name: + raise ValueError(f"OVXR runtime prim path {path!r} has conflicting requested types") + + for path, type_name in expected_types.items(): + existing = stage.GetPrimAtPath(path) + existing_type = str(existing.GetTypeName()) if existing and existing.IsValid() else "" + if existing_type and existing_type != type_name: + raise RuntimeError( + f"OVXR runtime prim {path!r} already has incompatible type {existing_type!r}; expected {type_name!r}" + ) + stage.DefinePrim(path, type_name) + + +class _StageOrdinalLedger(StageOrdinalLedger): + """Reserve explicit OVStage ordinals for the application publication lane.""" + + def __init__(self, stage: Any) -> None: + self._stage = stage + self._state_lock = threading.Lock() + self._active_ordinal: int | None = None + self._closed = False + self._floor = self._read_floor() + + def _read_floor(self) -> int: + with self._stage.get_attribute_write_floor() as query: + query.wait() + floor = int(query.fetch()) + if floor < 0 or floor > (1 << 64) - 1: + raise RuntimeError(f"OVStage returned an invalid write floor: {floor}") + return floor + + def _require_active(self, ordinal: int) -> None: + if self._active_ordinal != ordinal: + raise RuntimeError( + f"OVStage ordinal {ordinal} is not the active application reservation ({self._active_ordinal=})" + ) + + def reserve(self) -> int: + """Reserve exactly ``floor + 1`` while the application publication guard is held.""" + with self._state_lock: + if self._closed: + raise RuntimeError("OVStage ordinal ledger is closed") + if self._active_ordinal is not None: + raise RuntimeError(f"OVStage ordinal {self._active_ordinal} is still unresolved") + if self._floor == (1 << 64) - 1: + raise RuntimeError("OVStage ordinal range is exhausted") + self._active_ordinal = self._floor + 1 + return self._active_ordinal + + def publish(self, ordinal: int) -> None: + """Seal one application-written reservation and mark it resolved.""" + with self._state_lock: + self._require_active(ordinal) + self._stage.advance_write_floor(ordinal=ordinal).wait() + self._floor = ordinal + self._active_ordinal = None + + def resolve_external(self, ordinal: int, *, expected_published: bool | None) -> bool: + """Resolve an external write and verify its documented publication result.""" + with self._state_lock: + self._require_active(ordinal) + if expected_published is True: + published = True + else: + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + published = floor == ordinal + # Record native side effects even when the response contradicts the + # expected result. A reported error cannot make a sealed ordinal reusable. + if published: + self._floor = ordinal + if expected_published is not None and published != expected_published: + self._active_ordinal = None + state = "sealed" if published else "unsealed" + expected = "sealed" if expected_published else "unsealed" + raise RuntimeError(f"OVStage ordinal {ordinal} is {state} after an external write; expected {expected}") + self._active_ordinal = None + return published + + def abandon(self, ordinal: int) -> None: + """Reconcile a failed writer before releasing its reservation.""" + with self._state_lock: + self._require_active(ordinal) + # A publication wait can fail after the native Stage has sealed it. + # Query only on the failure/abandon path; an unreadable or unrelated + # floor leaves the reservation unresolved so it cannot be reused. + floor = self._read_floor() + if floor < self._floor or floor > ordinal: + raise RuntimeError( + f"OVStage write floor {floor} is outside the active ledger range [{self._floor}, {ordinal}]" + ) + self._floor = floor + self._active_ordinal = None + + def destroy(self) -> None: + """Close the ledger after every application and external write is resolved.""" + with self._state_lock: + if self._active_ordinal is not None: + raise RuntimeError(f"Cannot close with unresolved OVStage ordinal {self._active_ordinal}") + self._closed = True + + +class _IsaacLabOvxrApplicationInterface(OvxrApplicationInterface): + """Supply only the renderer/Stage/coordinator and simulation publication.""" + + def __init__( + self, + bootstrap: OvxrSessionBootstrap, + env: Any, + cfg: OvLibrariesXrCfg, + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, + ) -> None: + self._bootstrap = bootstrap + self._env = env + self._cfg = cfg + self._publication_lock = threading.Lock() + self._scene_publisher = None + self._lease = None + self._ordinal_ledger = None + self._last_published_ordinal: int | None = None + self._publication_closed = False + self._external_anchor = external_anchor_state + self._anchor_snapshot: tuple[int, np.ndarray] | None = None + self._last_anchor_frame_time: float | None = None + self._anchor_resources = contextlib.ExitStack() + self._anchor_query = None + self._anchor_stage = None + self._anchor_retirement_failed = False + + def offer_render_owner_transfer(self) -> object: + self._scene_publisher = self._bootstrap.capture_scene_publisher() + return self._bootstrap.offer_render_owner_transfer() + + def attach_on_render_thread(self, capability: object) -> OvxrSessionBinding: + from isaaccapture.ovxr_session_manager.ovxr_session import OvxrSessionBinding + + self._bootstrap.claim_render_owner_transfer(capability) + _declare_ovxr_runtime_prims(self._env, self._cfg) + self._lease = self._bootstrap.bind_environment(self._env) + with self._lease.ovstage_owner_lane(): + self._ordinal_ledger = _StageOrdinalLedger(self._lease.ovstage) + if self._external_anchor is not None: + import ovstage + + self._anchor_stage = self._lease.ovstage + try: + paths = self._anchor_resources.enter_context(ovstage.PathDictionary(self._anchor_stage)) + path_list = paths.create_path_list_from_strings(self._external_anchor.external_anchor_paths) + self._anchor_resources.callback(paths.destroy_path_list, path_list) + self._anchor_query = self._anchor_resources.enter_context( + self._anchor_stage.query_from_path_list(path_list) + ) + self._anchor_query.wait() + except BaseException as setup_error: + try: + self._anchor_resources.close() + except BaseException as cleanup_error: + self._anchor_retirement_failed = True + setup_error.add_note(f"Anchor query cleanup also failed: {cleanup_error!r}") + raise + return OvxrSessionBinding( + renderer=self._lease.ovrtx_renderer, + stage=self._lease.ovstage, + ordinal_ledger=self._ordinal_ledger, + prepare_frame_scene=self.prepare_frame_scene if self._external_anchor is not None else None, + ) + + def publication_guard(self) -> contextlib.AbstractContextManager[None]: + return self._publication_lock + + def owner_lane(self) -> contextlib.AbstractContextManager[None]: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + return self._lease.ovstage_owner_lane() + + def publish_complete_scene(self, publication_id: Hashable) -> int: + if self._publication_closed or self._scene_publisher is None or self._ordinal_ledger is None: + raise RuntimeError("IsaacLab scene publication is not available") + anchor_epoch = None + if self._external_anchor is not None: + # A failed seed can follow the publisher recording its identity. + # Check the terminal policy before dedup may return an older seal. + anchor_epoch = self._external_anchor.external_epoch + if not self._scene_publisher.publication_required(publication_id): + if self._last_published_ordinal is None: + raise RuntimeError("OVStage publication deduplicated before a sealed ordinal exists") + return self._last_published_ordinal + snapshot = None + seed = False + if self._external_anchor is not None: + epoch, seed = anchor_epoch + reference = self._bootstrap.get_world_transform(self._external_anchor.external_reference_path) + snapshot = (epoch, reference) + ordinal = self._ordinal_ledger.reserve() + try: + self._scene_publisher.write_complete_scene(ordinal, publication_id) + except BaseException as write_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + write_error.add_note(f"Abandon after failed scene write also failed: {abandon_error!r}") + raise + try: + if seed: + epoch, reference = snapshot + + def commit(matrices: np.ndarray) -> None: + self._write_anchor_matrices(matrices, ordinal) + self._ordinal_ledger.publish(ordinal) + + # Reset seeding must be sealed with this scene before input resumes. + # The first pose initializes the filter without an elapsed interval. + if not self._external_anchor.commit_external_pose(reference, 0.0, epoch, commit): + raise RuntimeError("XR anchor reset changed during complete scene publication") + self._last_anchor_frame_time = None + else: + self._ordinal_ledger.publish(ordinal) + except BaseException as publish_error: + # Component writes already consumed this generation. Do not let a + # retry deduplicate it to an older seal after publication failed. + self.close_publication() + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publish_error.add_note(f"Abandon after failed scene publication also failed: {abandon_error!r}") + raise + self._last_published_ordinal = ordinal + self._anchor_snapshot = snapshot + return ordinal + + def _write_anchor_matrices(self, matrices: np.ndarray, ordinal: int) -> None: + import ovstage + from isaaclab_ov.stage import xform_tensor_from_numpy + + # SDP uses column vectors; the Stage's omni:xform stores USD row matrices. + rows = np.ascontiguousarray(matrices.transpose(0, 2, 1), dtype=np.float64) + tensor = xform_tensor_from_numpy(rows) + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:xform", + ordinal=ordinal, + tensors=tensor, + semantic=ovstage.AttributeSemantic.MATRIX, + is_array=False, + ).wait() + self._anchor_stage.write_attribute( + self._anchor_query, + "omni:resetXformStack", + ordinal=ordinal, + tensors=np.ones(len(rows), dtype=np.bool_), + is_array=False, + ).wait() + + def prepare_frame_scene(self) -> None: + """Advance the committed reference with elapsed XR-frame time [s]. + + The first begun frame after reset establishes the monotonic clock with + zero elapsed time. Scene publication never advances this frame clock. + """ + if self._anchor_snapshot is None: + return + epoch, reference = self._anchor_snapshot + now = time.monotonic() + dt = 0.0 if self._last_anchor_frame_time is None else now - self._last_anchor_frame_time + + def commit(matrices: np.ndarray) -> None: + ordinal = self._ordinal_ledger.reserve() + try: + self._write_anchor_matrices(matrices, ordinal) + # OVXR reads sealed anchor state at bind; bind reserves the next ordinal. + self._ordinal_ledger.publish(ordinal) + except BaseException as publication_error: + try: + self._ordinal_ledger.abandon(ordinal) + except BaseException as abandon_error: + publication_error.add_note( + f"Abandon after failed anchor publication also failed: {abandon_error!r}" + ) + raise + + if self._external_anchor.commit_external_pose(reference, dt, epoch, commit): + self._last_anchor_frame_time = now + + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: + """Replace a replay device's policy while preserving native query ownership.""" + if self._external_anchor is not None: + # A new replay manager must not revive a failed borrowed native pair. + _ = self._external_anchor.external_epoch + if (state is None) != (self._external_anchor is None) or ( + state is not None and state.external_anchor_paths != self._external_anchor.external_anchor_paths + ): + raise ValueError("A reused XR environment must preserve its anchor paths") + self._external_anchor = state + self._anchor_snapshot = None + self._last_anchor_frame_time = None + + def validate_first_published_scene(self, ordinal: int) -> None: + if self._lease is None: + raise RuntimeError("OVRTX/OVStage lease is not attached") + with self._lease.ovstage_owner_lane(): + self._lease.validate_live() + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + return self._bootstrap.get_target_frame_T_world(prim_path) + + def close_publication(self) -> None: + if self._publication_closed: + return + self._publication_closed = True + if self._scene_publisher is not None: + self._scene_publisher.close() + + def close_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + # Unresolved writes may still use the anchor queries and Stage. + # Retain their resources; retirement requires proven completion. + self._ordinal_ledger.destroy() + self._ordinal_ledger = None + try: + self._anchor_resources.close() + except BaseException: + self._anchor_retirement_failed = True + raise + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its borrowed Stage") + self._bootstrap.close() + + def close_environment_on_render_thread(self) -> None: + if self._ordinal_ledger is not None: + raise RuntimeError("OVStage write retirement is unproved; retaining its environment") + if self._anchor_retirement_failed: + raise RuntimeError("Anchor query retirement is unproved; retaining its environment") + if not self._bootstrap.closed: + raise RuntimeError("Render lease retirement is unproved; retaining its environment") + self._env.close() + + def abort(self) -> None: + # The caller owns an environment even when worker startup failed. + # Release the lease before it closes that environment and its context. + self._bootstrap.close() + + +class OvLibrariesXrSession: + """Thin Isaac Lab adapter around Isaac Capture's OVXR session.""" + + @runtime_checkable + class ExternalAnchorState(Protocol): + """Dynamic XR-anchor policy sealed together with complete scene publications.""" + + @property + def external_anchor_paths(self) -> tuple[str, ...]: + """Anchor Xform paths written together at each committed pose.""" + ... + + @property + def external_reference_path(self) -> str: + """Absolute path of the physics prim whose world transform drives the anchor.""" + ... + + @property + def external_epoch(self) -> tuple[int, bool]: + """Reset identity and whether that epoch still needs its seeding publication. + + Raises: + RuntimeError: If a previous anchor publication failed. + """ + ... + + def commit_external_pose( + self, + reference: np.ndarray, + rendering_dt: float, + epoch: int, + commit: Callable[[np.ndarray], None], + ) -> bool: + """Advance the policy from ``reference`` and seal its matrices through ``commit``. + + Args: + reference: World transform of :attr:`external_reference_path` [4, 4]. + rendering_dt: Elapsed XR-frame time since the last committed pose [s]. + epoch: Reset identity read from :attr:`external_epoch`. + commit: Callable that writes and seals the anchor matrices [N, 4, 4]. + + Returns: + ``False`` when ``epoch`` is stale and nothing was committed. + """ + ... + + def __init__(self, bootstrap: OvxrSessionBootstrap, cfg: OvLibrariesXrCfg) -> None: + self._bootstrap = bootstrap + self._cfg = cfg + self._owner = None + self._host = None + self._closed = False + + @classmethod + def prepare( + cls, + sim_cfg: SimulationCfg, + cfg: OvLibrariesXrCfg, + renderer_cfg: OVRTXRendererCfg | None = None, + ) -> OvLibrariesXrSession: + """Snapshot session selection, create simulation and reserve its OVRTX lease. + + Renderer settings are process-global on the stock renderer interface; + use one configured session per process. No supported readback exists + here to restore a previous renderer's settings on close. + """ + cfg = replace(cfg) + validate_ovxr_session_preparation() + import ovxr + + # Stock settings are applied before Renderer construction. Preserve explicit caller choices. + ovxr.apply_renderer_settings(cfg.renderer_settings.strip()) + bootstrap = OvxrSessionBootstrap(sim_cfg, renderer_cfg) + try: + return cls(bootstrap, cfg) + except BaseException as preparation_error: + try: + bootstrap.abort() + except BaseException as abort_error: + # Cleanup follows the failure; retain its original cause. + preparation_error.add_note(f"Cleanup after failed XR preparation also failed: {abort_error!r}") + raise + + def start_render_worker( + self, + env: Any, + *, + tracking_anchor_path: str | None = None, + required_openxr_extensions: Iterable[str] = (), + max_blank_frames: int = 1, + timeout_s: float = 30.0, + external_anchor_state: OvLibrariesXrSession.ExternalAnchorState | None = None, + ) -> bool: + """Give Isaac Capture the application pair and report whether XR started. + + Args: + env: Constructed environment sharing the prepared simulation context. + tracking_anchor_path: Override the configured USD tracking-anchor path. + required_openxr_extensions: Extensions required by the input pipeline. + max_blank_frames: Nonnegative integer bootstrap-frame count; zero skips bootstrap frames. + timeout_s: Maximum worker startup wait [s]. + external_anchor_state: Dynamic anchor policy seeded from committed scene publications. + + Raises: + RuntimeError: If the session is closed or already started, or ``env`` does not reuse the + prepared simulation context, has no completed stage or not exactly one environment, + or selects a different simulation device or physics time step. + TypeError: If ``external_anchor_state`` does not implement :class:`ExternalAnchorState`. + ValueError: If ``max_blank_frames`` is not a nonnegative integer. + """ + self._require_open() + if self._owner is not None: + raise RuntimeError("OVXR async render worker is already started") + if isinstance(max_blank_frames, bool) or not isinstance(max_blank_frames, int) or max_blank_frames < 0: + raise ValueError("max_blank_frames must be a non-negative integer") + _require_external_anchor_state(external_anchor_state) + # Reject an inconsistent environment before the worker claims the lease. + self._bootstrap.validate_environment(env) + from isaaccapture.ovxr_session_manager.ovxr_session import ( + OvxrSession, + OvxrSessionConfig, + ) + + from .session_lifecycle import cloudxr_eula_accepted + + anchor_path = self._cfg.tracking_anchor_path if tracking_anchor_path is None else tracking_anchor_path + host = _IsaacLabOvxrApplicationInterface(self._bootstrap, env, self._cfg, external_anchor_state) + self._host = host + self._owner = OvxrSession( + host, + OvxrSessionConfig( + application_name=self._cfg.application_name, + session_mode=self._cfg.session_mode, + cloudxr_env_file=self._cfg.cloudxr_env_file, + auto_launch_cloudxr=self._cfg.auto_launch_cloudxr, + accept_eula=cloudxr_eula_accepted(), + render_products=self._cfg.render_products, + camera_paths=self._cfg.camera_paths, + create_render_products=self._cfg.create_render_products, + meters_per_unit=self._cfg.meters_per_unit, + depth_near_z=self._cfg.depth_near_z, + depth_far_z=self._cfg.depth_far_z, + tracking_anchor_path=anchor_path, + floor_aligned_anchor=False, + host_wait_render_output=self._cfg.host_wait_render_output, + extra_openxr_extensions=self._cfg.extra_openxr_extensions, + ), + required_openxr_extensions=required_openxr_extensions, + max_blank_frames=max_blank_frames, + ) + return self._owner.start(timeout_s=timeout_s) + + def external_oxr_handles(self) -> OpenXRSessionHandles | None: + """Return Isaac Capture's typed external handles once its owner is ready.""" + self._require_open() + return None if self._owner is None else self._owner.external_oxr_handles() + + def request_render(self, physics_step_id: Hashable, *, force: bool = False) -> int | None: + """Offer one complete application generation for Isaac Capture to render. + + Return the admitted ordinal, or ``None`` when render demand is already + satisfied. Retry the latest dirty generation until admitted, including + while paused. ``force`` preserves reset/lifecycle discontinuities. + """ + self._require_started() + return self._owner.request_render(physics_step_id, force=force) + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + self._require_started() + return self._owner.get_target_frame_T_world(prim_path) + + def close(self) -> None: + """Close Isaac Capture's owner before releasing the application environment.""" + if self._closed: + return + if self._owner is None: + self._bootstrap.close() + else: + self._owner.close() + self._closed = True + + @property + def async_worker_owns_environment(self) -> bool: + """Whether Isaac Capture accepted environment teardown ownership.""" + return self._owner is not None and self._owner.owns_environment + + @property + def shutdown_requested(self) -> bool: + """Whether the OVXR session requested application shutdown.""" + return self._owner is not None and self._owner.shutdown_requested + + def request_shutdown(self) -> None: + """Request orderly OVXR shutdown.""" + if self._owner is not None: + self._owner.request_shutdown() + + def set_external_anchor_state(self, state: OvLibrariesXrSession.ExternalAnchorState | None) -> None: + """Adopt the current replay device's anchor policy between runs.""" + self._require_started() + _require_external_anchor_state(state) + with self._host.publication_guard(): + self._host.set_external_anchor_state(state) + + def wait_for_shutdown(self, timeout_s: float) -> bool: + """Wait up to timeout_s [s]; return whether application cancellation was requested.""" + self._require_started() + return self._owner.wait_for_shutdown(timeout_s) + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR session is closed") + + def _require_started(self) -> None: + self._require_open() + if self._owner is None: + raise RuntimeError("OVXR session is not started") + + def __enter__(self) -> OvLibrariesXrSession: + self._require_open() + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +def _require_external_anchor_state(state: object) -> None: + """Reject anchor policies that lack the external-publication protocol on the caller's thread.""" + if state is None: + return + if not isinstance(state, OvLibrariesXrSession.ExternalAnchorState): + raise TypeError( + f"external_anchor_state must implement OvLibrariesXrSession.ExternalAnchorState; got {type(state).__name__}" + ) + reference_path = state.external_reference_path + if not isinstance(reference_path, str) or not reference_path.startswith("/"): + raise ValueError(f"external_reference_path must be an absolute prim path; got {reference_path!r}") diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py new file mode 100644 index 000000000000..1773a4969b76 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_cfg.py @@ -0,0 +1,71 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Configuration for an Isaac Capture-owned OVXR session. + +This module has no runtime dependencies, so launchers can build and validate the +configuration without the native OVXR, OVRTX or OVStage packages. +""" + +from __future__ import annotations + +import math +from typing import Literal + +from isaaclab.utils import configclass + +_DEFAULT_OVXR_CAMERA_PATHS = ( + "/OVXRGeneratedCameras/OVXR_LeftEyeCamera", + "/OVXRGeneratedCameras/OVXR_RightEyeCamera", +) + + +@configclass +class OvLibrariesXrCfg: + """Isaac Lab configuration for an Isaac Capture-owned OVXR session.""" + + application_name: str = "isaaclab-kitless-teleop" + """Application identifier passed to the OpenXR runtime.""" + session_mode: Literal["ar", "vr"] = "ar" + """OpenXR presentation mode: augmented or virtual reality.""" + renderer_settings: str = "--/app/hydra/aperture/conform=stretch" + """Process-global OVXR renderer settings applied before renderer construction.""" + cloudxr_env_file: str | None = None + """CloudXR environment-file path; None leaves runtime launch to the caller.""" + auto_launch_cloudxr: bool = True + """Whether the session owner launches the CloudXR runtime automatically.""" + render_products: tuple[str, ...] = () + """Stereo RenderProduct paths; an empty selection uses OVXR's default paths.""" + camera_paths: tuple[str, ...] = _DEFAULT_OVXR_CAMERA_PATHS + """Exactly two non-empty USD camera paths, ordered left eye then right eye.""" + create_render_products: bool = True + """Whether to declare and configure the selected stereo rendering prims.""" + meters_per_unit: float = 1.0 + """Finite positive scene scale [m per USD unit].""" + depth_near_z: float = 0.0 + """Near depth distance [m]; both depth distances zero select provider defaults.""" + depth_far_z: float = 0.0 + """Far depth distance [m]; explicit distances must satisfy 0 < near < far.""" + tracking_anchor_path: str = "" + """USD tracking-anchor path; an empty path uses the stage origin.""" + host_wait_render_output: bool = True + """Whether OVXR waits for render output before returning from frame rendering.""" + extra_openxr_extensions: tuple[str, ...] = () + """Additional OpenXR extension names requested when creating the session.""" + + def __post_init__(self) -> None: + if self.session_mode not in ("ar", "vr"): + raise ValueError("session_mode must be 'ar' or 'vr'") + if not math.isfinite(self.meters_per_unit) or self.meters_per_unit <= 0: + raise ValueError("meters_per_unit must be finite and positive") + if (self.depth_near_z, self.depth_far_z) != (0.0, 0.0) and ( + not math.isfinite(self.depth_near_z) + or not math.isfinite(self.depth_far_z) + or self.depth_near_z <= 0.0 + or self.depth_far_z <= self.depth_near_z + ): + raise ValueError("depth_near_z and depth_far_z must both be zero or finite with 0 < near < far") + if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): + raise ValueError("Kitless stereo requires exactly two non-empty OVXR camera paths") diff --git a/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py new file mode 100644 index 000000000000..ef0086299af7 --- /dev/null +++ b/source/isaaclab_teleop/isaaclab_teleop/ovxr_session.py @@ -0,0 +1,292 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Lifecycle bootstrap for an application-owned OVRTX OVXR pair.""" + +from __future__ import annotations + +import builtins +from typing import TYPE_CHECKING, Any + +import numpy as np +import warp as wp +from isaaclab_ov.renderers import OvLibrariesRenderLease, OVRTXRenderer, OVRTXRendererCfg + +from isaaclab.app import get_settings_manager +from isaaclab.scene_data import SceneDataFormat +from isaaclab.sim import SimulationContext + +_TERMINAL_MODE_SENTINEL = "ISAAC_LAUNCHED_FROM_TERMINAL" + +if TYPE_CHECKING: + from isaaclab_ov.renderers import OvLibrariesScenePublisher + + from isaaclab.sim import SimulationCfg + + +def validate_ovxr_session_preparation() -> None: + """Reject ineligible preparation before the caller changes renderer globals.""" + if SimulationContext.instance() is not None: + raise RuntimeError("OVXR bootstrap must create the SimulationContext immediately before the environment") + if hasattr(builtins, _TERMINAL_MODE_SENTINEL) and getattr(builtins, _TERMINAL_MODE_SENTINEL) is not True: + raise RuntimeError("OVXR bootstrap cannot reuse a context in extension mode") + + +def resolve_cuda_device(device: str) -> str: + """Resolve a bare ``"cuda"`` device to the GPU that the simulation selects for it. + + :class:`~isaaclab.sim.SimulationContext` and :func:`~isaaclab.app.launch_simulation` read + the ``/physics/cudaDevice`` setting the same way. + + Args: + device: Simulation device, for example ``"cpu"``, ``"cuda"`` or ``"cuda:1"``. + + Returns: + ``"cuda:"`` for a bare ``"cuda"``; any other device unchanged. + """ + if device != "cuda": + return device + cuda_device = get_settings_manager().get("/physics/cudaDevice") + return f"cuda:{max(0, int(cuda_device) if cuda_device is not None else 0)}" + + +class OvxrSessionBootstrap: + """Reserve an OVRTX/OVStage lease before a real environment is constructed. + + The bootstrap creates the :class:`~isaaclab.sim.SimulationContext` from the ``sim_cfg`` + given here, and the environment reuses it. Only environments that accept a pre-created + context are supported (manager-based environments; :class:`~isaaclab.envs.DirectRLEnv` + always creates its own). :meth:`validate_environment` requires ``num_envs == 1`` and + rejects a simulation device or time step that disagrees with the prepared context. + """ + + def __init__(self, sim_cfg: SimulationCfg, renderer_cfg: OVRTXRendererCfg | None = None) -> None: + validate_ovxr_session_preparation() + + self._sim: SimulationContext | None = None + self._lease: OvLibrariesRenderLease | None = None + self._owner_transfer_capability: object | None = None + self._bound = False + self._closed = False + self._installed_terminal_mode_sentinel = False + self._target_frame_readers: dict[str, tuple[Any, Any, Any]] = {} + + if not hasattr(builtins, _TERMINAL_MODE_SENTINEL): + setattr(builtins, _TERMINAL_MODE_SENTINEL, True) + self._installed_terminal_mode_sentinel = True + + try: + sim = SimulationContext(sim_cfg) + # The simulation registry owns the renderer, including cloning and + # cleanup; the XR session only borrows its exact native pair. + renderer = sim.get_or_create_backend(renderer_cfg or OVRTXRendererCfg()) + if not isinstance(renderer, OVRTXRenderer): + raise TypeError(f"OVRTXRendererCfg resolved to unexpected renderer {type(renderer).__name__}") + lease = renderer.acquire_ov_libraries_render_lease() + # Camera sensors prepare every registered renderer on the application thread + # during environment construction. Only the claimed render owner materializes + # this pair, so that call is recorded and completed in bind_environment(). + lease.defer_stage_until_owner_transfer() + except BaseException as preparation_error: + sim = SimulationContext.instance() + try: + if sim is not None: + sim.clear_instance() + except BaseException as cleanup_error: + preparation_error.add_note(f"Cleanup after failed XR construction also failed: {cleanup_error!r}") + finally: + self._restore_terminal_mode_sentinel() + raise + + self._sim = sim + self._lease = lease + + def capture_scene_publisher(self) -> OvLibrariesScenePublisher: + """Capture the application scene publisher before render ownership transfers.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Scene publisher is available only before OVXR Stage binding") + return self._lease.capture_scene_publisher() + + @property + def simulation_context(self) -> SimulationContext: + """Return the exact pre-created simulation context.""" + self._require_open() + assert self._sim is not None + return self._sim + + def validate_environment(self, env: Any) -> None: + """Check that a constructed environment uses the prepared simulation context. + + Args: + env: Constructed environment, possibly wrapped. + + Raises: + RuntimeError: If the environment did not reuse the prepared context, has no + completed stage or not exactly one environment, or its simulation + configuration selects a different device or physics time step [s]. + """ + self._require_open() + unwrapped = getattr(env, "unwrapped", env) + if unwrapped.sim is not self._sim: + raise RuntimeError("Environment did not reuse the exact pre-created SimulationContext") + if unwrapped.scene.stage is None: + raise RuntimeError("Environment has no completed InteractiveScene Stage") + if unwrapped.num_envs != 1: + raise RuntimeError( + f"The OVXR render lease requires exactly one environment; got num_envs={unwrapped.num_envs!r}" + ) + env_sim_cfg = unwrapped.cfg.sim + prepared_cfg = self._sim.cfg + if env_sim_cfg is not prepared_cfg: + # SimulationContext resolved the prepared configuration's bare CUDA selection the same way. + env_device = resolve_cuda_device(env_sim_cfg.device) + for field, env_value, prepared_value in ( + ("device", env_device, prepared_cfg.device), + ("dt", env_sim_cfg.dt, prepared_cfg.dt), + ): + if env_value != prepared_value: + raise RuntimeError( + f"Environment sim.{field}={getattr(env_sim_cfg, field)!r} differs from the OVXR" + f" prepared simulation context ({prepared_value!r}); prepare the" + " session with the environment's sim configuration" + ) + + def bind_environment(self, env: Any) -> OvLibrariesRenderLease: + """Prepare and bind the completed environment Stage to the reserved pair. + + The bootstrap defers Stage materialization to the render owner, so call this on the + worker after :meth:`claim_render_owner_transfer`. + + Args: + env: Constructed environment, possibly wrapped. + + Returns: + The bound lease. + + Raises: + RuntimeError: If the bootstrap is closed or already bound, the owner transfer has not + been claimed, or :meth:`validate_environment` rejects the environment. + """ + self._require_open() + if self._bound: + raise RuntimeError("OVXR bootstrap is already bound") + try: + self.validate_environment(env) + unwrapped = getattr(env, "unwrapped", env) + scene = unwrapped.scene + num_envs = unwrapped.num_envs + assert self._sim is not None and self._lease is not None + if not self._lease.materialize_deferred_stage(scene.stage, num_envs): + self._sim.render_context.ensure_prepare_stage(scene.stage, num_envs) + self._lease.validate_live() + self._bound = True + return self._lease + finally: + self._restore_terminal_mode_sentinel() + + def offer_render_owner_transfer(self) -> object: + """Offer the exact unbound OVXR lease before Stage binding.""" + self._require_open() + if self._bound or self._lease is None: + raise RuntimeError("Owner transfer is available only before OVXR Stage binding") + capability = self._lease.offer_owner_transfer() + self._owner_transfer_capability = capability + return capability + + def claim_render_owner_transfer(self, capability: object) -> OvLibrariesRenderLease: + """Claim the offered lease on the worker that will bind, use, and close it.""" + self._require_open() + assert self._lease is not None + self._lease.claim_owner_transfer(capability) + self._owner_transfer_capability = None + return self._lease + + def get_world_transform(self, prim_path: str) -> np.ndarray: + """Read target-to-world transform [4, 4], float64, with translation in scene units.""" + self._require_open() + if not self._bound or self._sim is None: + raise RuntimeError("Target-frame transforms require a bound OVXR environment") + if not isinstance(prim_path, str) or not prim_path.startswith("/"): + raise ValueError(f"Target-frame prim path must be absolute: {prim_path!r}") + + provider = self._sim.get_scene_data_provider() + cached = self._target_frame_readers.get(prim_path) + if cached is None or cached[0] is not provider: + transform_paths = list(provider.backend.transform_paths) + match_count = transform_paths.count(prim_path) + if match_count != 1: + raise RuntimeError( + f"Target-frame prim path {prim_path!r} matched {match_count} live backend transforms" + ) + mapping = provider.create_mapping([prim_path]) + output = SceneDataFormat.Matrix44() + cached = (provider, mapping, output) + self._target_frame_readers[prim_path] = cached + + provider, mapping, output = cached + if not provider.get_transforms(output, mapping=mapping, allow_passthrough=False, count=1): + raise RuntimeError(f"No SceneData transform conversion is available for target frame {prim_path!r}") + if output.matrices is None: + raise RuntimeError(f"SceneData returned no matrix buffer for target frame {prim_path!r}") + + # SceneData converts on the device's current stream. Wait only for that stream so a + # publication does not serialize with unrelated work (such as in-flight XR rendering). + if output.matrices.device.is_cuda: + wp.synchronize_stream(wp.get_stream(output.matrices.device)) + world_T_target = np.asarray(output.matrices.numpy()[0], dtype=np.float64) + if world_T_target.shape != (4, 4) or not np.all(np.isfinite(world_T_target)): + raise RuntimeError(f"Target-frame transform {prim_path!r} is missing or non-finite") + if not np.allclose(world_T_target[3], (0.0, 0.0, 0.0, 1.0), atol=1e-5, rtol=0.0): + raise RuntimeError(f"Target-frame transform {prim_path!r} is not homogeneous") + return world_T_target.copy() + + def get_target_frame_T_world(self, prim_path: str) -> np.ndarray: + """Read world-to-target transform [4, 4], float32, with translation in scene units.""" + world_T_target = self.get_world_transform(prim_path) + try: + target_T_world = np.linalg.inv(world_T_target) + except np.linalg.LinAlgError as exc: + raise RuntimeError(f"Target-frame transform {prim_path!r} is singular") from exc + if not np.all(np.isfinite(target_T_world)): + raise RuntimeError(f"Inverse target-frame transform {prim_path!r} is non-finite") + return target_T_world.astype(np.float32) + + @property + def closed(self) -> bool: + """Whether the lease and its retained publication resources retired successfully.""" + return self._closed + + def close(self) -> None: + """Release after every borrower is destroyed; retain ownership for retry on failure.""" + if self._closed: + return + # Keep ownership and the bootstrap retryable until retirement succeeds. + if self._lease is not None: + if self._owner_transfer_capability is not None: + self._lease.cancel_owner_transfer(self._owner_transfer_capability) + self._owner_transfer_capability = None + self._lease.release() + self._target_frame_readers.clear() + self._restore_terminal_mode_sentinel() + self._closed = True + + def abort(self) -> None: + """Release a failed construction; retain its context if lease retirement fails.""" + if self._closed: + return + self.close() + if not self._bound and self._sim is not None and SimulationContext.instance() is self._sim: + self._sim.clear_instance() + + def _require_open(self) -> None: + if self._closed: + raise RuntimeError("OVXR bootstrap is closed") + + def _restore_terminal_mode_sentinel(self) -> None: + if self._installed_terminal_mode_sentinel: + if hasattr(builtins, _TERMINAL_MODE_SENTINEL): + delattr(builtins, _TERMINAL_MODE_SENTINEL) + self._installed_terminal_mode_sentinel = False diff --git a/source/isaaclab_teleop/test/test_ov_libraries_xr.py b/source/isaaclab_teleop/test/test_ov_libraries_xr.py new file mode 100644 index 000000000000..8eff1d710679 --- /dev/null +++ b/source/isaaclab_teleop/test/test_ov_libraries_xr.py @@ -0,0 +1,936 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Focused tests for the Isaac Lab host side of external OVXR presentation.""" + +from __future__ import annotations + +import builtins +import contextlib +import importlib +import subprocess +import sys +import threading +from contextlib import contextmanager +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest +from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg + + +class _FakeLease: + def __init__(self): + self.ovrtx_renderer = object() + self.ovstage = object() + self.owner_lane_entries = 0 + + @contextmanager + def ovstage_owner_lane(self): + self.owner_lane_entries += 1 + yield + + def validate_live(self): + pass + + +class _FakePublisher: + def __init__(self): + self.calls = [] + self.fail = None + self.required = True + self.closed = False + + def publication_required(self, _publication_id): + return self.required + + def write_complete_scene(self, ordinal, publication_id): + self.calls.append((ordinal, publication_id, threading.get_native_id())) + if self.fail is not None: + raise self.fail + + def close(self): + self.closed = True + + +class _FakeOrdinalLedger: + def __init__(self, stage): + self.stage = stage + self.next_ordinal = 11 + self.reserved = [] + self.published = [] + self.abandoned = [] + self.publish_error = None + self.destroyed = False + + def reserve(self): + ordinal = self.next_ordinal + self.next_ordinal += 1 + self.reserved.append(ordinal) + return ordinal + + def publish(self, ordinal): + if self.publish_error is not None: + raise self.publish_error + self.published.append(ordinal) + + def abandon(self, ordinal): + self.abandoned.append(ordinal) + + def destroy(self): + self.destroyed = True + + +class _FakeBootstrap: + def __init__(self): + self.lease = _FakeLease() + self.publisher = _FakePublisher() + self.events = [] + self.closed = False + self.aborted = False + + def capture_scene_publisher(self): + self.events.append("capture") + return self.publisher + + def offer_render_owner_transfer(self): + self.events.append("offer") + return "capability" + + def claim_render_owner_transfer(self, capability): + self.events.append(("claim", capability)) + + def bind_environment(self, env): + self.events.append(("bind", env)) + return self.lease + + def validate_environment(self, env): + pass + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + self.closed = True + + def abort(self): + self.aborted = True + + +@pytest.fixture +def real_lease_type(): + from isaaclab_ov.renderers.render_lease import OvLibrariesRenderLease + + return OvLibrariesRenderLease + + +@pytest.fixture +def xr_module(monkeypatch, real_lease_type): + """Import the adapter with native renderer dependencies stubbed. + + The stub-bound adapter is removed from ``sys.modules`` and its package on teardown so + later tests import the real module. + """ + # The adapter needs a Capture release with OVXR session support. The Kit ``teleop`` extra + # selects an older release without it; skip rather than error in that environment. + pytest.importorskip("isaaccapture.ovxr_session_manager") + import isaaclab_teleop + + isaaclab_ov = ModuleType("isaaclab_ov") + isaaclab_ov.__path__ = [] + renderers = ModuleType("isaaclab_ov.renderers") + renderers.OVRTXRendererCfg = object + renderers.OVRTXRenderer = object + renderers.OvLibrariesRenderLease = object + monkeypatch.setitem(sys.modules, "isaaclab_ov", isaaclab_ov) + monkeypatch.setitem(sys.modules, "isaaclab_ov.renderers", renderers) + source = Path(__file__).parents[1] / "isaaclab_teleop/ovxr_session.py" + spec = importlib.util.spec_from_file_location("isaaclab_teleop.ovxr_session", source) + ovxr_session = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, ovxr_session) + spec.loader.exec_module(ovxr_session) + name = "isaaclab_teleop.ov_libraries_xr" + saved_module = sys.modules.pop(name, None) + saved_attribute = isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + try: + module = importlib.import_module(name) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + yield module + finally: + sys.modules.pop(name, None) + isaaclab_teleop.__dict__.pop("ov_libraries_xr", None) + if saved_module is not None: + sys.modules[name] = saved_module + if saved_attribute is not None: + isaaclab_teleop.ov_libraries_xr = saved_attribute + + +def _fake_ovxr(): + return SimpleNamespace(apply_renderer_settings=lambda _settings: None) + + +def test_stage_ordinal_ledger_tracks_one_application_reservation(xr_module): + class ImmediateQuery: + def __init__(self, fetch): + self._fetch = fetch + + def __enter__(self): + return self + + def __exit__(self, *_exc): + pass + + def wait(self): + pass + + def fetch(self): + return self._fetch() + + class ImmediateAdvance: + def wait(self): + pass + + class Stage: + def __init__(self): + self.floor = 10 + self.advances = [] + self.queries = 0 + + def get_attribute_write_floor(self): + self.queries += 1 + return ImmediateQuery(lambda: self.floor) + + def advance_write_floor(self, *, ordinal): + self.advances.append(ordinal) + self.floor = ordinal + return ImmediateAdvance() + + stage = Stage() + ledger = xr_module._StageOrdinalLedger(stage) + assert stage.queries == 1 + + assert ledger.reserve() == 11 + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + ledger.abandon(11) + + assert ledger.reserve() == 11 + ledger.publish(11) + assert stage.advances == [11] + assert stage.queries == 2 # Only the earlier abandon needs reconciliation. + + assert ledger.reserve() == 12 + stage.floor = 12 # The explicit-ordinal OVXR call sealed its own write. + assert ledger.resolve_external(12, expected_published=True) + assert stage.queries == 2 + + assert ledger.reserve() == 13 + assert not ledger.resolve_external(13, expected_published=None) + assert stage.queries == 3 + assert ledger.reserve() == 13 + stage.floor = 13 + with pytest.raises(RuntimeError, match="expected unsealed"): + ledger.resolve_external(13, expected_published=False) + assert stage.queries == 4 + # Even a contradicted response must not permit reuse of an observed sealed ordinal. + assert ledger.reserve() == 14 + ledger.abandon(14) + ledger.destroy() + with pytest.raises(RuntimeError, match="closed"): + ledger.reserve() + + +def test_runtime_render_prims_have_the_declared_paths_and_types(xr_module): + class FakePrim: + def __init__(self, type_name=""): + self.type_name = type_name + + def __bool__(self): + return bool(self.type_name) + + def IsValid(self): + return bool(self.type_name) + + def GetTypeName(self): + return self.type_name + + class FakeStage: + def __init__(self): + self.prims = {} + self.definitions = [] + + def GetPrimAtPath(self, path): + return self.prims.get(path, FakePrim()) + + def DefinePrim(self, path, type_name): + self.definitions.append((path, type_name)) + self.prims[path] = FakePrim(type_name) + + stage = FakeStage() + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) + + assert stage.definitions == [ + ("/OVXRGeneratedCameras/OVXR_LeftEyeCamera", "Camera"), + ("/OVXRGeneratedCameras/OVXR_RightEyeCamera", "Camera"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Left", "RenderProduct"), + ("/Render/OmniverseKit/HydraTextures/omni_kit_widget_viewport_ViewportTexture_0_OVXR_Right", "RenderProduct"), + ("/Render/Vars/LdrColor", "RenderVar"), + ("/Render/Vars/XRDepth", "RenderVar"), + ("/Render/OmniverseGlobalRenderSettings", "RenderSettings"), + ] + + +def test_runtime_render_prim_declaration_preserves_disabled_and_fails_on_conflict(xr_module): + class ExistingPrim: + def IsValid(self): + return True + + def GetTypeName(self): + return "Xform" + + stage = SimpleNamespace(GetPrimAtPath=lambda _path: ExistingPrim(), DefinePrim=lambda *_args: None) + env = SimpleNamespace(unwrapped=SimpleNamespace(scene=SimpleNamespace(stage=stage))) + + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg(create_render_products=False)) + with pytest.raises(RuntimeError, match="incompatible type"): + xr_module._declare_ovxr_runtime_prims(env, xr_module.OvLibrariesXrCfg()) + + +def _make_host(xr_module, monkeypatch): + bootstrap = _FakeBootstrap() + env = SimpleNamespace(scene=SimpleNamespace(stage=None), close=lambda: bootstrap.events.append("environment-close")) + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "_StageOrdinalLedger", _FakeOrdinalLedger) + monkeypatch.setattr(xr_module, "_declare_ovxr_runtime_prims", lambda _env, _cfg: bootstrap.events.append("declare")) + host = xr_module._IsaacLabOvxrApplicationInterface(bootstrap, env, xr_module.OvLibrariesXrCfg()) + return host, bootstrap, env + + +def test_host_lends_exact_pair_and_keeps_application_publication(xr_module, monkeypatch): + host, bootstrap, env = _make_host(xr_module, monkeypatch) + + capability = host.offer_render_owner_transfer() + binding = host.attach_on_render_thread(capability) + with host.publication_guard(): + ordinal = host.publish_complete_scene("step-1") + + assert binding.renderer is bootstrap.lease.ovrtx_renderer + assert binding.stage is bootstrap.lease.ovstage + assert binding.ordinal_ledger.stage is bootstrap.lease.ovstage + assert bootstrap.events == ["capture", "offer", ("claim", "capability"), "declare", ("bind", env)] + assert ordinal == 11 + assert bootstrap.publisher.calls[0][:2] == (11, "step-1") + host.validate_first_published_scene(ordinal) + with host.owner_lane(): + pass + assert bootstrap.lease.owner_lane_entries == 3 + assert host.get_target_frame_T_world("/World/target") == ("target", "/World/target") + + host.close_publication() + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.publisher.closed and bootstrap.closed + assert binding.ordinal_ledger.destroyed + assert bootstrap.events[-1] == "environment-close" + + +def test_host_abandons_failed_write_and_failed_publish(xr_module, monkeypatch): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + bootstrap.publisher.fail = RuntimeError("write failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="write failed"): + host.publish_complete_scene("step-1") + assert host._ordinal_ledger.abandoned == [11] + + bootstrap.publisher.fail = None + host._ordinal_ledger.publish_error = RuntimeError("publish failed") + with host.publication_guard(), pytest.raises(RuntimeError, match="publish failed"): + host.publish_complete_scene("step-2") + assert host._ordinal_ledger.abandoned == [11, 12] + + +@pytest.mark.parametrize("sealed", [False, True]) +def test_host_reconciles_failed_publication_without_reusing_generation(xr_module, monkeypatch, real_lease_type, sealed): + ledger_type = xr_module._StageOrdinalLedger + host, _bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + failure = RuntimeError("native publication wait failed") + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + # Use the real publication identity owner; a permanently-dirty fake cannot + # detect a retry returning the older ordinal after a failed seal. + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _update_transforms=lambda: None, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + host._scene_publisher = lease.capture_scene_publisher() + renderer.scene = SimpleNamespace(stage=stage, ordinal=10) + lease._bind(object(), stage, 1) + stage.advance_write_floor = lambda *, ordinal: SimpleNamespace(wait=lambda: setattr(stage, "floor", ordinal)) + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(): + assert host.publish_complete_scene("previous") == 11 + + def failed_advance(*, ordinal): + if sealed: + stage.floor = ordinal + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("changed") + assert caught.value is failure + with host.publication_guard(), pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("changed") + next_ordinal = ledger.reserve() + assert next_ordinal == (13 if sealed else 12) + ledger.abandon(next_ordinal) + host.close_on_render_thread() + lease.release() + + +def test_failed_bootstrap_retirement_retains_environment(xr_module, monkeypatch): + """Capture's independent environment callback cannot bypass failed lease retirement.""" + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + + def fail_close(): + raise RuntimeError("producer drain failed") + + monkeypatch.setattr(bootstrap, "close", fail_close) + with pytest.raises(RuntimeError, match="producer drain failed"): + host.close_on_render_thread() + with pytest.raises(RuntimeError, match="lease retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +def test_failed_publication_with_unknown_floor_retains_reservation_and_first_error(xr_module, monkeypatch): + ledger_type = xr_module._StageOrdinalLedger + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + stage = SimpleNamespace(unknown=False) + failure = RuntimeError("native publication wait failed") + + def fetch(): + if stage.unknown: + raise RuntimeError("floor read unavailable") + return 10 + + stage.get_attribute_write_floor = lambda: contextlib.nullcontext(SimpleNamespace(wait=lambda: None, fetch=fetch)) + + def failed_advance(*, ordinal): + stage.unknown = True + + def wait(): + raise failure + + return SimpleNamespace(wait=wait) + + stage.advance_write_floor = failed_advance + ledger = ledger_type(stage) + host._ordinal_ledger = ledger + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is failure + assert any("floor read unavailable" in note for note in getattr(caught.value, "__notes__", [])) + with pytest.raises(RuntimeError, match="still unresolved"): + ledger.reserve() + + # Even repeated cleanup cannot retire queries/Stage while the write is unresolved. + anchor_resources_closed = [] + host._anchor_resources.callback(lambda: anchor_resources_closed.append(True)) + for _ in range(2): + with pytest.raises(RuntimeError, match="unresolved OVStage ordinal"): + host.close_on_render_thread() + assert not bootstrap.closed + assert not anchor_resources_closed + with pytest.raises(RuntimeError, match="write retirement is unproved"): + host.close_environment_on_render_thread() + assert "environment-close" not in bootstrap.events + + +@pytest.mark.parametrize("failure", ["write", "publish"]) +def test_publication_retains_primary_cause_when_abandon_fails(xr_module, monkeypatch, failure): + host, bootstrap, _env = _make_host(xr_module, monkeypatch) + host.attach_on_render_thread(host.offer_render_owner_transfer()) + original_cause = ValueError("original publication cause") + first_error = RuntimeError("publication failed") + first_error.__cause__ = original_cause + if failure == "write": + bootstrap.publisher.fail = first_error + else: + host._ordinal_ledger.publish_error = first_error + + def fail_abandon(_ordinal): + raise RuntimeError("abandon failed") + + monkeypatch.setattr(host._ordinal_ledger, "abandon", fail_abandon) + with host.publication_guard(), pytest.raises(RuntimeError) as caught: + host.publish_complete_scene("step-1") + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert any("abandon failed" in note for note in caught.value.__notes__) + + +def test_session_delegates_lifecycle_to_isaacteleop_owner(xr_module, monkeypatch): + calls = [] + + class FakeOwner: + owns_environment = True + shutdown_requested = False + + def __init__(self, host, config, **kwargs): + calls.append(("init", host, config, kwargs)) + + def start(self, timeout_s): + calls.append(("start", timeout_s)) + return True + + def external_oxr_handles(self): + return "handles" + + def request_render(self, identity, *, force=False): + calls.append(("request", identity, force)) + return 17 + + def request_shutdown(self): + self.shutdown_requested = True + calls.append(("request_shutdown",)) + + def get_target_frame_T_world(self, path): + return ("target", path) + + def close(self): + calls.append(("close",)) + + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", FakeOwner) + bootstrap = _FakeBootstrap() + assert xr_module.OvLibrariesXrCfg().extra_openxr_extensions == () + session = xr_module.OvLibrariesXrSession( + bootstrap, xr_module.OvLibrariesXrCfg(extra_openxr_extensions=("XR_EXT_debug_utils",)) + ) + env = SimpleNamespace() + assert session.start_render_worker( + env, required_openxr_extensions=("XR_EXT_hand_tracking",), max_blank_frames=0, timeout_s=4.0 + ) + assert calls[0][0] == "init" + assert calls[0][2].session_mode == "ar" + assert calls[0][3]["required_openxr_extensions"] == ("XR_EXT_hand_tracking",) + assert calls[0][3]["max_blank_frames"] == 0 + assert calls[0][2].extra_openxr_extensions == ("XR_EXT_debug_utils",) + assert calls[0][2].host_wait_render_output is True + assert calls[1] == ("start", 4.0) + assert session.external_oxr_handles() == "handles" + assert session.request_render("step") == 17 + assert session.request_render("reset", force=True) == 17 + assert calls[-2:] == [("request", "step", False), ("request", "reset", True)] + assert session.get_target_frame_T_world("/target") == ("target", "/target") + assert session.async_worker_owns_environment + assert not session.shutdown_requested + session.request_shutdown() + assert calls[-1] == ("request_shutdown",) + assert session.shutdown_requested + session._owner.shutdown_requested = True + assert session.shutdown_requested + session.close() + assert calls[-1] == ("close",) + + +@pytest.fixture +def bootstrap_context(xr_module, monkeypatch): + """Use the real bootstrap with only simulation/native construction replaced.""" + module = sys.modules["isaaclab_teleop.ovxr_session"] + events = [] + + class Context: + current = None + failure = None + cleanup_failure = None + + def __init__(self, _cfg): + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + events.append("context-close") + cls.current = None + if cls.cleanup_failure is not None: + raise cls.cleanup_failure + + def get_or_create_backend(self, _cfg): + if self.failure is not None: + raise self.failure + lease = SimpleNamespace( + release=lambda: events.append("lease-release"), defer_stage_until_owner_transfer=lambda: None + ) + return SimpleNamespace(acquire_ov_libraries_render_lease=lambda: lease) + + monkeypatch.setattr(module, "SimulationContext", Context) + monkeypatch.delattr(builtins, module._TERMINAL_MODE_SENTINEL, raising=False) + return module, Context, events + + +@pytest.mark.parametrize("route", ["before_owner", "before_worker"]) +def test_failed_start_releases_lease_before_environment_and_context(xr_module, bootstrap_context, route): + """An existing environment retains its context until the caller closes it.""" + module, context, events = bootstrap_context + bootstrap = module.OvxrSessionBootstrap(object()) + if route == "before_owner": + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + session.close() + else: + # Capture calls host.abort when validation fails before its worker exists. + host = xr_module._IsaacLabOvxrApplicationInterface.__new__(xr_module._IsaacLabOvxrApplicationInterface) + host._bootstrap = bootstrap + host.abort() + assert context.current is not None + events.append("environment-close") + context.clear_instance() + assert events == ["lease-release", "environment-close", "context-close"] + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + + +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +@pytest.mark.parametrize("cleanup_fails", [False, True]) +def test_bootstrap_construction_failure_restores_terminal_mode(bootstrap_context, error_type, cleanup_fails): + """Failed or interrupted construction restores process state and preserves its cause.""" + module, context, events = bootstrap_context + error = context.failure = error_type("renderer construction failed") + if cleanup_fails: + context.cleanup_failure = RuntimeError("context cleanup failed") + with pytest.raises(error_type) as caught: + module.OvxrSessionBootstrap(object()) + assert caught.value is error + assert events == ["context-close"] + assert context.current is None + assert not hasattr(builtins, module._TERMINAL_MODE_SENTINEL) + if cleanup_fails: + assert any("context cleanup failed" in note for note in error.__notes__) + + +def test_caller_renderer_settings_precede_renderer_creation(xr_module, monkeypatch): + settings = " --/rtx/rendermode=MinimalRendering " + events = [] + bootstrap = _FakeBootstrap() + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda value: events.append(value)) + ) + + def create_bootstrap(*_args): + assert len(events) == 1 + assert events[0] == settings.strip() + return bootstrap + + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", create_bootstrap) + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg(renderer_settings=settings)) + + +def test_prepare_rejects_existing_context_before_global_settings(xr_module, monkeypatch): + from isaaclab.sim import SimulationContext + + monkeypatch.setattr(SimulationContext, "instance", lambda: object()) + monkeypatch.setitem( + sys.modules, + "ovxr", + SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed existing renderer settings")), + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated second bootstrap")) + with pytest.raises(RuntimeError, match="must create the SimulationContext"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + +@pytest.mark.parametrize("terminal_mode", [False, None, 1]) +def test_prepare_rejects_extension_mode_before_global_settings(xr_module, monkeypatch, terminal_mode): + monkeypatch.setattr(builtins, "ISAAC_LAUNCHED_FROM_TERMINAL", terminal_mode, raising=False) + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(RuntimeError, match="extension mode"): + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + +def test_config_rejects_unknown_session_mode(): + with pytest.raises(ValueError, match="session_mode must be 'ar' or 'vr'"): + OvLibrariesXrCfg(session_mode="automatic") + + +@pytest.mark.parametrize("scale", [0.0, -1.0, float("nan"), float("inf")]) +def test_config_rejects_invalid_scene_scale(scale): + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + OvLibrariesXrCfg(meters_per_unit=scale) + + +@pytest.mark.parametrize("camera_paths", [("/Camera/mono",), ("/Camera/left", "")]) +def test_config_rejects_non_stereo_camera_paths(camera_paths): + with pytest.raises(ValueError, match="exactly two"): + OvLibrariesXrCfg(camera_paths=camera_paths) + + +def test_prepare_revalidates_changed_scale_before_global_settings(xr_module, monkeypatch): + cfg = xr_module.OvLibrariesXrCfg() + cfg.meters_per_unit = float("nan") + monkeypatch.setitem( + sys.modules, "ovxr", SimpleNamespace(apply_renderer_settings=lambda _: pytest.fail("changed renderer settings")) + ) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: pytest.fail("allocated bootstrap")) + with pytest.raises(ValueError, match="meters_per_unit must be finite and positive"): + xr_module.OvLibrariesXrSession.prepare(object(), cfg) + + +@pytest.mark.parametrize("count", [-1, 1.5, True]) +def test_start_rejects_invalid_bootstrap_count_before_owner_creation(xr_module, monkeypatch, count): + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + with pytest.raises(ValueError, match="max_blank_frames must be a non-negative integer"): + session.start_render_worker(SimpleNamespace(), max_blank_frames=count) + assert bootstrap.events == [] + + +@pytest.mark.parametrize("abort_fails", [False, True]) +def test_prepare_aborts_partial_bootstrap_and_preserves_first_error(xr_module, monkeypatch, abort_fails): + bootstrap = _FakeBootstrap() + first_error = RuntimeError("preparation failed") + original_cause = ValueError("invalid session input") + first_error.add_note("Original preparation diagnostic") + abort_error = RuntimeError("abort failed") + monkeypatch.setitem(sys.modules, "ovxr", _fake_ovxr()) + monkeypatch.setattr(xr_module, "OvxrSessionBootstrap", lambda *_: bootstrap) + + def fail(*_args, **_kwargs): + raise first_error from original_cause + + def abort(): + bootstrap.aborted = True + if abort_fails: + raise abort_error + + monkeypatch.setattr(bootstrap, "abort", abort) + monkeypatch.setattr(xr_module.OvLibrariesXrSession, "__init__", fail) + + with pytest.raises(RuntimeError) as caught: + xr_module.OvLibrariesXrSession.prepare(object(), xr_module.OvLibrariesXrCfg()) + + assert bootstrap.aborted + assert caught.value is first_error + assert caught.value.__cause__ is original_cause + assert caught.value.__notes__ == ["Original preparation diagnostic"] + ( + [f"Cleanup after failed XR preparation also failed: {abort_error!r}"] if abort_fails else [] + ) + + +def test_public_xr_exports_resolve_to_implemented_types(xr_module): + import isaaclab_teleop + + assert isaaclab_teleop.OvLibrariesXrCfg is xr_module.OvLibrariesXrCfg + assert isaaclab_teleop.OvLibrariesXrSession is xr_module.OvLibrariesXrSession + + +def test_poisoned_publication_still_retires_in_capture_order(xr_module, monkeypatch, real_lease_type): + """After a failed scene write, Capture's OvxrFramePump.close must still retire the pair and the env.""" + host, bootstrap, env = _make_host(xr_module, monkeypatch) + controller_type = sys.modules[real_lease_type.__module__].RenderLeaseController + stage = SimpleNamespace(floor=10) + stage.get_attribute_write_floor = lambda: contextlib.nullcontext( + SimpleNamespace(wait=lambda: None, fetch=lambda: stage.floor) + ) + native = object() + + def failed_transforms(): + raise ValueError("geometry callback failed") + + renderer = SimpleNamespace( + _closed=False, + _initialized_scene=False, + backend=None, + scene=None, + _update_transforms=failed_transforms, + _update_geometries=lambda: None, + _visual_material_writer_ref=None, + _publish_visual_materials=contextlib.nullcontext, + ) + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + controller = controller_type(renderer) + lease = controller.acquire() + bootstrap.publisher = lease.capture_scene_publisher() + # Stand in for clone preparation and Stage binding of the reserved pair. + renderer.backend = SimpleNamespace(renderer=native, attached=True, _render_lease_owner=renderer) + renderer.scene = SimpleNamespace(stage=stage, ordinal=10, _render_lease_owner=renderer) + renderer._initialized_scene = True + controller._generation = 1 + lease._bind(native, stage, 1) + bootstrap.lease = lease + + def close_bootstrap(): + lease.release() + bootstrap.closed = True + + bootstrap.close = close_bootstrap + host.attach_on_render_thread(host.offer_render_owner_transfer()) + with host.publication_guard(), pytest.raises(RuntimeError, match="recreate the exact pair"): + host.publish_complete_scene("step-1") + + # OvxrFramePump.close(): close publication, destroy the context in the owner lane, then retire. + with host.publication_guard(): + host.close_publication() + with host.publication_guard(): + with host.owner_lane(): + pass + host.close_on_render_thread() + host.close_environment_on_render_thread() + assert bootstrap.closed + assert controller.active is None + assert bootstrap.events[-1] == "environment-close" + + +class _FakeAnchorState: + """Minimal :class:`OvLibrariesXrSession.ExternalAnchorState` with scripted epochs.""" + + external_anchor_paths = ("/World/XrAnchor", "/World/XrRenderAnchor") + external_reference_path = "/World/Robot" + + def __init__(self): + self.epoch = (0, True) + self.commits = [] + self.accept = True + + @property + def external_epoch(self): + return self.epoch + + def commit_external_pose(self, reference, rendering_dt, epoch, commit): + if not self.accept or epoch != self.epoch[0]: + return False + import numpy as np + + self.commits.append((reference.copy(), rendering_dt, epoch)) + commit(np.stack([reference, reference])) + self.epoch = (self.epoch[0], False) + return True + + +def _make_anchor_host(xr_module, monkeypatch, anchor): + import numpy as np + + bootstrap = _FakeBootstrap() + bootstrap.get_world_transform = lambda _path: np.eye(4) + host = xr_module._IsaacLabOvxrApplicationInterface( + bootstrap, SimpleNamespace(), xr_module.OvLibrariesXrCfg(), external_anchor_state=anchor + ) + host._scene_publisher = bootstrap.publisher + host._ordinal_ledger = _FakeOrdinalLedger(None) + writes = [] + monkeypatch.setattr(host, "_write_anchor_matrices", lambda matrices, ordinal: writes.append(ordinal)) + return host, writes + + +def test_anchor_seed_is_sealed_with_scene_and_frames_use_the_xr_clock(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + times = iter((5.0, 5.25)) + monkeypatch.setattr(xr_module.time, "monotonic", lambda: next(times)) + + seed = host.publish_complete_scene("initial") + assert writes == [seed] == host._ordinal_ledger.published + assert anchor.commits[-1][1] == 0.0 # The seed has no elapsed interval. + host.prepare_frame_scene() + host.prepare_frame_scene() + assert [commit[1] for commit in anchor.commits] == [0.0, 0.0, 0.25] + assert writes == host._ordinal_ledger.published + + +def test_anchor_epoch_change_during_seed_fails_closed(xr_module, monkeypatch): + anchor = _FakeAnchorState() + anchor.accept = False + host, writes = _make_anchor_host(xr_module, monkeypatch, anchor) + with pytest.raises(RuntimeError, match="anchor reset changed"): + host.publish_complete_scene("initial") + assert writes == [] + assert host._ordinal_ledger.abandoned == [11] + with pytest.raises(RuntimeError, match="publication is not available"): + host.publish_complete_scene("initial") + + +def test_anchor_state_readoption_requires_the_same_paths(xr_module, monkeypatch): + anchor = _FakeAnchorState() + host, _writes = _make_anchor_host(xr_module, monkeypatch, anchor) + host.publish_complete_scene("initial") + replacement = _FakeAnchorState() + host.set_external_anchor_state(replacement) + assert host._anchor_snapshot is None and host._last_anchor_frame_time is None + moved = _FakeAnchorState() + moved.external_anchor_paths = ("/World/Other",) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(moved) + with pytest.raises(ValueError, match="preserve its anchor paths"): + host.set_external_anchor_state(None) + + +def test_start_rejects_anchor_state_without_the_protocol_before_owner_creation(xr_module, monkeypatch): + """A policy lacking the external-anchor API must fail on the caller, before the worker claims the lease.""" + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + with pytest.raises(TypeError, match="ExternalAnchorState"): + session.start_render_worker(SimpleNamespace(), external_anchor_state=SimpleNamespace(reset=lambda: None)) + assert bootstrap.events == [] + assert isinstance(_FakeAnchorState(), xr_module.OvLibrariesXrSession.ExternalAnchorState) + + +@pytest.mark.parametrize("reference_path", [None, "World/Robot"]) +def test_start_rejects_relative_anchor_reference_before_owner_creation(xr_module, monkeypatch, reference_path): + """Publication reads the reference prim on the worker; a missing path must fail before the lease is claimed.""" + external = importlib.import_module("isaaccapture.ovxr_session_manager.ovxr_session") + monkeypatch.setattr(external, "OvxrSession", lambda *_args, **_kwargs: pytest.fail("created an XR owner")) + bootstrap = _FakeBootstrap() + session = xr_module.OvLibrariesXrSession(bootstrap, xr_module.OvLibrariesXrCfg()) + anchor = _FakeAnchorState() + anchor.external_reference_path = reference_path + with pytest.raises(ValueError, match="absolute prim path"): + session.start_render_worker(SimpleNamespace(), external_anchor_state=anchor) + assert bootstrap.events == [] + + +def test_config_imports_without_native_ovxr(): + """Launchers build the configuration in a fresh process where Capture and the native OV packages are absent.""" + script = ( + "import sys\n" + "for name in ('isaaccapture', 'ovxr', 'ovrtx', 'ovstage'):\n" + " sys.modules[name] = None\n" + "from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg\n" + "assert OvLibrariesXrCfg(session_mode='vr').session_mode == 'vr'\n" + ) + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True, timeout=60) + assert result.returncode == 0, result.stderr diff --git a/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py new file mode 100644 index 000000000000..a70bd378a34b --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovrtx_transfer_native_harness.py @@ -0,0 +1,182 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Native scene publication and renderer ownership transfer.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import threading +from dataclasses import dataclass +from pathlib import Path + +import pytest + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1", + reason="native OVRTX transfer harness is explicitly opt-in", + ), +] + +_TASK = "Isaac-Cartpole" +"""Manager-based task with an ``ovphysx`` physics preset and no camera sensors.""" + + +@dataclass +class _CaseResult: + offer_thread: int + claim_thread: int | None = None + renderer_id: int | None = None + stage_id: int | None = None + ordinal: int | None = None + error: BaseException | None = None + + +def _make_env_and_bootstrap(task: str): + import gymnasium as gym + from isaaclab_ov.physics import OvPhysxCfg + from isaaclab_ov.renderers import OVRTXRendererCfg + from isaaclab_teleop.ovxr_session import OvxrSessionBootstrap + + import isaaclab_tasks # noqa: F401 -- registers the task + from isaaclab_tasks.utils.hydra import resolve_presets + from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry + + # parse_env_cfg resolves every preset to its default, so select OvPhysX while resolving. + cfg = resolve_presets(load_cfg_from_registry(task, "env_cfg_entry_point"), selected=("ovphysx",)) + assert isinstance(cfg.sim.physics, OvPhysxCfg) + cfg.sim.device = "cuda:0" + cfg.scene.num_envs = 1 + bootstrap = OvxrSessionBootstrap(cfg.sim, OVRTXRendererCfg()) + try: + return bootstrap, gym.make(task, cfg=cfg).unwrapped + except BaseException: + bootstrap.abort() + raise + + +def _run_case(task: str, *, transfer: bool) -> _CaseResult: + from isaaclab_teleop.ov_libraries_xr import _IsaacLabOvxrApplicationInterface + from isaaclab_teleop.ovxr_cfg import OvLibrariesXrCfg + + bootstrap, env = _make_env_and_bootstrap(task) + result = _CaseResult(offer_thread=threading.get_native_id()) + host = _IsaacLabOvxrApplicationInterface(bootstrap, env, OvLibrariesXrCfg(auto_launch_cloudxr=False)) + capability = host.offer_render_owner_transfer() + ready, published = threading.Event(), threading.Event() + + def attach(): + binding = host.attach_on_render_thread(capability) + result.renderer_id, result.stage_id = id(binding.renderer), id(binding.stage) + return binding + + def update_stage(binding): + with host.publication_guard(), host.owner_lane(): + binding.renderer.update_from_stage(result.ordinal) + + def close(): + host.close_publication() + try: + host.close_on_render_thread() + finally: + host.close_environment_on_render_thread() + + if not transfer: + try: + binding = attach() + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-control", 0)) + update_stage(binding) + finally: + close() + return result + + with pytest.raises(RuntimeError, match="transfer is pending"): + _ = bootstrap._lease.ovstage + + def worker(): + try: + result.claim_thread = threading.get_native_id() + binding = attach() + ready.set() + if not published.wait(60): + raise TimeoutError("application did not complete its scene publication") + if result.error is None: + update_stage(binding) + with pytest.raises(RuntimeError, match="capability is invalid"): + bootstrap.claim_render_owner_transfer(capability) + except BaseException as exc: + if result.error is None: + result.error = exc + finally: + ready.set() + try: + close() + except BaseException as exc: + if result.error is None: + result.error = exc + + thread = threading.Thread(target=worker, name="ovrtx-native-transfer-worker") + thread.start() + try: + if not ready.wait(90): + raise TimeoutError("renderer claimant did not finish binding") + if result.error is not None: + raise result.error + with host.publication_guard(): + result.ordinal = host.publish_complete_scene(("native-transfer", 0)) + except BaseException as exc: + result.error = exc + finally: + published.set() + thread.join(timeout=90) + assert not thread.is_alive(), "native transfer worker timed out" + if result.error is not None: + raise result.error + assert result.claim_thread is not None and result.claim_thread != result.offer_thread + with pytest.raises(RuntimeError, match="(owner thread|released)"): + _ = bootstrap._lease.ovstage + return result + + +def _run_native_subprocess(task: str, transfer: bool) -> None: + # A native non-daemon thread cannot be bounded by Thread.join alone. + # Keep the native lifetime in a child that the standard subprocess timeout owns. + completed = subprocess.run( + [sys.executable, str(Path(__file__).resolve()), task, "transfer" if transfer else "same-thread"], + env={**os.environ, "ISAAC_LAB_OVRTX_USE_OVSTAGE": "1"}, + capture_output=True, + text=True, + timeout=180, + check=False, + ) + print(completed.stdout) + assert completed.returncode == 0, completed.stderr + + +def test_same_thread_ovxr_session_native_control(): + """One-thread production scene publication remains viable.""" + _run_native_subprocess(_TASK, False) + + +def test_cross_thread_preuse_owner_transfer_native(): + """The application publishes while the render thread binds, renders and closes the pair.""" + _run_native_subprocess(_TASK, True) + + +if __name__ == "__main__": + if os.environ.get("ISAACLAB_RUN_NATIVE_OVRTX_TRANSFER") != "1": + raise RuntimeError("native transfer harness requires its explicit opt-in") + result = _run_case(sys.argv[1], transfer=sys.argv[2] == "transfer") + assert result.renderer_id is not None and result.stage_id is not None + assert result.ordinal is not None + print( + f"offer={result.offer_thread} claim={result.claim_thread} " + f"renderer={result.renderer_id} stage={result.stage_id} ordinal={result.ordinal}" + ) diff --git a/source/isaaclab_teleop/test/test_ovxr_session.py b/source/isaaclab_teleop/test/test_ovxr_session.py new file mode 100644 index 000000000000..30be9dad763f --- /dev/null +++ b/source/isaaclab_teleop/test/test_ovxr_session.py @@ -0,0 +1,128 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""OVXR bootstrap retirement across the teleoperation/renderer boundary.""" + +import builtins +from types import SimpleNamespace + +import pytest + +from isaaclab.sim import SimulationCfg + +pytest.importorskip("ovrtx") +from isaaclab_ov.renderers import OVRTXRenderer, OVRTXRendererCfg + + +@pytest.mark.parametrize("operation", ["close", "abort"]) +def test_bootstrap_failed_transfer_cancellation_keeps_owner_retryable(bootstrap, operation): + """A wrong-thread close cannot destroy a pending lease or prevent its owner retiring it.""" + import threading + + from isaaclab_teleop import ovxr_session + + bootstrap.offer_render_owner_transfer() + context = ovxr_session.SimulationContext.instance() + errors = [] + + def wrong_owner(): + try: + getattr(bootstrap, operation)() + except RuntimeError as error: + errors.append(str(error)) + + thread = threading.Thread(target=wrong_owner) + thread.start() + thread.join(5) + assert not thread.is_alive() + assert len(errors) == 1 and "owner thread must cancel" in errors[0] + assert not bootstrap.closed + assert ovxr_session.SimulationContext.instance() is context + # The real owner must still be able to retire the real renderer lease. + getattr(bootstrap, operation)() + assert bootstrap.closed + assert ovxr_session.SimulationContext.instance() is (None if operation == "abort" else context) + + +@pytest.fixture +def bootstrap(monkeypatch): + """Run the real bootstrap constructor and renderer lease; only the physics-backed context is replaced.""" + from isaaclab_teleop import ovxr_session + + from isaaclab.app import get_settings_manager + from isaaclab.renderers import RenderContext + + monkeypatch.setenv("ISAAC_LAB_OVRTX_USE_OVSTAGE", "1") + monkeypatch.delenv("ISAAC_LAB_SHARE_OVSTAGE", raising=False) + monkeypatch.delattr(builtins, ovxr_session._TERMINAL_MODE_SENTINEL, raising=False) + renderer = OVRTXRenderer( + OVRTXRendererCfg(), sim=SimpleNamespace(device="cpu", get_scene_data_provider=lambda: None) + ) + + class Context: + current = None + + def __init__(self, cfg): + self.cfg = cfg + self.render_context = RenderContext([(renderer.cfg, renderer)]) + type(self).current = self + + @classmethod + def instance(cls): + return cls.current + + @classmethod + def clear_instance(cls): + cls.current = None + + def get_or_create_backend(self, _cfg): + return renderer + + monkeypatch.setattr(ovxr_session, "SimulationContext", Context) + # SimulationContext resolves a bare "cuda" device from this setting. + settings = get_settings_manager() + previous_cuda_device = settings.get("/physics/cudaDevice") + settings.set("/physics/cudaDevice", 1) + try: + bootstrap = ovxr_session.OvxrSessionBootstrap(SimulationCfg(device="cuda:1", dt=0.01)) + yield bootstrap + bootstrap.abort() + assert renderer._render_lease.active is None + finally: + settings.set("/physics/cudaDevice", previous_cuda_device) + + +def _environment(bootstrap, *, num_envs=1, device="cuda:1", dt=0.01): + return SimpleNamespace( + sim=bootstrap.simulation_context, + scene=SimpleNamespace(stage=object()), + num_envs=num_envs, + cfg=SimpleNamespace(sim=SimulationCfg(device=device, dt=dt)), + ) + + +@pytest.mark.parametrize( + ("num_envs", "device", "dt", "error"), + [ + (1, "cuda", 0.01, None), + (2, "cuda:1", 0.01, "exactly one environment"), + (1, "cuda:0", 0.01, r"sim\.device='cuda:0'"), + (1, "cuda:1", 0.02, r"sim\.dt=0\.02"), + ], +) +def test_environment_validation_resolves_devices_and_requires_one_environment(bootstrap, num_envs, device, dt, error): + """The caller-side check resolves a bare CUDA device like SimulationContext and rejects late worker failures.""" + env = _environment(bootstrap, num_envs=num_envs, device=device, dt=dt) + if error is None: + bootstrap.validate_environment(env) + else: + with pytest.raises(RuntimeError, match=error): + bootstrap.validate_environment(env) + + +def test_binding_requires_the_claimed_owner_transfer(bootstrap): + """Binding before the claim names the missing claim, even when no camera deferred the Stage yet.""" + with pytest.raises(RuntimeError, match="requires a claimed owner transfer"): + bootstrap.bind_environment(_environment(bootstrap))