diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index cb31c998f0..b784d26351 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -162,6 +162,7 @@ jobs: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DBUILD_PLUGIN_OAK_CAMERA=ON \ -DBUILD_VIZ=ON \ + -DBUILD_OVXR_CAMERA_QUADS=ON \ -DENABLE_CLOUDXR_BUNDLE_CHECK=ON \ -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake \ "${coverage_args[@]}" @@ -341,6 +342,17 @@ jobs: isaaccapture-sbom check --wheel "${wheel}" done + - name: Verify camera support in release wheel artifacts + if: matrix.build_type == 'Release' + run: | + set -euo pipefail + shopt -s nullglob + wheels=(install/wheels/isaaccapture-*.whl) + [[ "${#wheels[@]}" -gt 0 ]] || { echo "No Isaac Capture wheels found"; exit 1; } + for wheel in "${wheels[@]}"; do + python3 scripts/check_camera_quad_wheel.py "${wheel}" --camera-quads + done + - name: Create tarball to preserve permissions if: matrix.build_type == 'Release' run: tar -cvf isaacteleop-install.tar -C install . diff --git a/CMakeLists.txt b/CMakeLists.txt index ded7797a6b..521ec03c30 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -74,6 +74,7 @@ option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; s option(BUILD_EXAMPLES "Build examples" ON) option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF) option(BUILD_TESTING "Build unit tests" ON) +option(BUILD_OVXR_CAMERA_QUADS "Build optional application-owned OVXR camera quad support" OFF) # Televiz (src/viz) needs Vulkan, the CUDA Toolkit, and glslangValidator. # Only Vulkan and CUDA gate the default -- without them the machine cannot build # viz at all, so a core-only source build still configures. glslangValidator is diff --git a/docs/source/getting_started/ovxr.rst b/docs/source/getting_started/ovxr.rst index 2b6f1de545..4ce1a7cb69 100644 --- a/docs/source/getting_started/ovxr.rst +++ b/docs/source/getting_started/ovxr.rst @@ -41,14 +41,68 @@ from the Isaac Capture wheel. These packages retain their own licenses: CloudXR native libraries are bundled in Isaac Capture wheels and remain subject to their separate NVIDIA license and CloudXR EULA acceptance. +Optional camera panels +---------------------- + +Camera picture-in-picture panels require the native camera helper. The Ubuntu +release workflow includes it in its wheels and checks the packaged artifact. +Installing those wheels requires no CMake configuration. + +For a source build, follow the source-build guide for your application's Python +version, then enable camera support and build the wheel: + +.. code-block:: bash + + cmake -S . -B build -DBUILD_OVXR_CAMERA_QUADS=ON + cmake --build build --target python_package python_stubs python_wheel --parallel + +Install the generated wheel with its ``ovxr`` extra using the instructions +above. The extra requires CUDA 13 Python bindings for GPU camera output; +``BUILD_OVXR_CAMERA_QUADS`` controls whether the wheel contains the native helper. +The helper is disabled by default. + +Use a CUDA 13-compatible NVIDIA driver (R580 or newer), meeting any stricter +requirements of the selected bindings and renderer. See the +`CUDA compatibility requirements `_. +Installing Python packages or a CUDA Toolkit does not update the host driver. + +Install the Vulkan development SDK before configuring camera support. Linux +builds also require the CUDA Toolkit. CMake discovers these installed dependencies. +Use ``CUDAToolkit_ROOT`` to select a CUDA Toolkit installation and ``VULKAN_SDK`` +to help locate a Vulkan SDK. The GPU driver and Vulkan runtime loader are required +on the host. + Use from another application ---------------------------- ``isaaccapture.ovxr_session_manager.ovxr_session`` provides ``OvxrSession`` and ``OvxrSessionConfig``. Implement ``OvxrApplicationInterface`` to supply the application's renderer, Stage and ordinal ledger in an ``OvxrSessionBinding``. -``camera_paths`` is required and must contain both stereo cameras. Set -``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``. +The binding also supplies optional ``frame_consumers`` after the application has +resolved its render-product paths. Capture does not depend on an Isaac Lab task +or scene configuration. ``camera_paths`` is required and must contain both stereo +cameras. Set ``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``. + +``CameraPanelConsumer`` implements the ``OvxrFrameConsumer`` lifecycle. Configure panels with a tuple of +``CameraPanelConfig`` values, each pairing a render-product path with its placement +and image bound. + +Camera-panel attachment failure disables panels and logs the reason without +blocking teleoperation input. An incomplete GPU copy disables the affected panel; +the consumer rechecks its copy stream before releasing the borrowed output. +If completion still cannot be proved, the session reports a failure and retains +the borrowed resources. Other camera mailboxes are still given a cleanup attempt. + +Consumers receive ``FrameOutputs`` on the XR owner thread. Use ``latest_color`` +to select an output, then ``copy_host_rgba`` or ``map_gpu`` to consume it without +unpacking native result structures. Selected handles are valid only during that +callback. CPU snapshots are shared within that frame; GPU resources retain their +completion and retirement rules. A consumer may contribute composition layers that remain valid +through submission. ``on_openxr_ready`` may be called repeatedly and must be +idempotent. ``finish_frame`` releases borrowed outputs before submission or +cancellation; ``close`` must tolerate partial startup and retain resources if +safe retirement fails. The camera consumer owns one native panel collection, +which is loaded directly and is not an OpenXR API layer. For a uv-managed application, declare Isaac Capture with its ``ovxr`` extra and select the source-built wheel in the application's project configuration: diff --git a/pyproject.toml b/pyproject.toml index c88d5779ec..b7f75e9bd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ ui = [ ] ovxr = [ "ovxr==0.1.0.5842", + "cuda-bindings>=13,<14", ] retargeters = [ "dex-retargeting>=0.4.6,<0.6.0", diff --git a/scripts/check_camera_quad_wheel.py b/scripts/check_camera_quad_wheel.py new file mode 100644 index 0000000000..00ac948ef4 --- /dev/null +++ b/scripts/check_camera_quad_wheel.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Check optional camera-helper delivery in an actual project wheel. + +Run against wheels built with BUILD_OVXR_CAMERA_QUADS ON and OFF, including +ON -> OFF -> ON in the same build directory to detect stale packaged data. +""" + +import argparse +import hashlib +import json +from pathlib import Path +from zipfile import ZipFile + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("wheel", type=Path) + parser.add_argument( + "--camera-quads", action=argparse.BooleanOptionalAction, required=True + ) + args = parser.parse_args() + prefix = "isaaccapture/ovxr_session_manager/native/" + expected = { + prefix + "libisaaccapture_camera_panels.so", + } + with ZipFile(args.wheel) as wheel: + members = { + name + for name in wheel.namelist() + if name.startswith(prefix) and not name.endswith("/") + } + wanted = expected if args.camera_quads else set() + if members != wanted: + raise RuntimeError( + f"Camera helper package differs: expected {sorted(wanted)}, got {sorted(members)}" + ) + if any("/openxr_layer/" in name for name in wheel.namelist()): + raise RuntimeError("Wheel contains obsolete OpenXR API-layer packaging") + print( + json.dumps( + { + "wheel": str(args.wheel), + "sha256": hashlib.sha256(args.wheel.read_bytes()).hexdigest(), + "camera_quads": args.camera_quads, + "result": "PASS", + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index b627490ba8..9cc4772373 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -38,6 +38,10 @@ add_subdirectory(deviceio_session) # Build MCAP library (depends on deviceio for DeviceIOSession and tracker interfaces) add_subdirectory(mcap) +if(BUILD_OVXR_CAMERA_QUADS) + add_subdirectory(ovxr_pip_layer) +endif() + # CloudXR runtime helper (stages the vendored CloudXR runtime SDK bundle). # Its pure-Python source lives in src/python/isaaccapture/cloudxr/, as does the # retargeting engine's and the session manager's. diff --git a/src/core/ovxr_pip_layer/CMakeLists.txt b/src/core/ovxr_pip_layer/CMakeLists.txt new file mode 100644 index 0000000000..2b9f1e328c --- /dev/null +++ b/src/core/ovxr_pip_layer/CMakeLists.txt @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +add_subdirectory(cpp) diff --git a/src/core/ovxr_pip_layer/cpp/CMakeLists.txt b/src/core/ovxr_pip_layer/cpp/CMakeLists.txt new file mode 100644 index 0000000000..26ebb48eaf --- /dev/null +++ b/src/core/ovxr_pip_layer/cpp/CMakeLists.txt @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +find_package(Vulkan REQUIRED) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + find_package(CUDAToolkit REQUIRED) +endif() + +add_library(ovxr_pip_layer SHARED pip_layer.cpp) +target_compile_features(ovxr_pip_layer PRIVATE cxx_std_20) +target_include_directories(ovxr_pip_layer PRIVATE + $) +target_link_libraries(ovxr_pip_layer PRIVATE Vulkan::Vulkan log_bridge::log_bridge_core) +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + # Keep static logger/fmt symbols private to the helper's existing C ABI. + set(_pip_exports "${CMAKE_CURRENT_BINARY_DIR}/ovxr_pip_exports.map") + file(GENERATE OUTPUT "${_pip_exports}" + CONTENT "{ global: isaacteleop_pip_*; local: *; };\n") + target_link_options(ovxr_pip_layer PRIVATE "-Wl,--version-script,${_pip_exports}") + set_property(TARGET ovxr_pip_layer APPEND PROPERTY LINK_DEPENDS "${_pip_exports}") +endif() +if(CUDAToolkit_FOUND AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_compile_definitions(ovxr_pip_layer PRIVATE ISAAC_TELEOP_PIP_CUDA) + target_link_libraries(ovxr_pip_layer PRIVATE CUDA::cudart_static) +endif() +set_target_properties(ovxr_pip_layer PROPERTIES + OUTPUT_NAME isaaccapture_camera_panels + LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python_package/$/isaaccapture/ovxr_session_manager/native") + +add_library(ovxr_pip_layer::ovxr_pip_layer ALIAS ovxr_pip_layer) diff --git a/src/core/ovxr_pip_layer/cpp/pip_layer.cpp b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp new file mode 100644 index 0000000000..9b7d4d5c90 --- /dev/null +++ b/src/core/ovxr_pip_layer/cpp/pip_layer.cpp @@ -0,0 +1,943 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Application-owned camera quads on a borrowed OVXR session/device. The frame +// owner uploads and returns layer descriptors before OVXR submits. This helper +// never creates a session or intercepts OpenXR frame calls. +#define XR_USE_GRAPHICS_API_VULKAN +#include + +// Keep Vulkan declarations before OpenXR platform structs, including after formatting. +#include +#include + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +# include +# include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ +constexpr size_t kMaxCameras = 3; + +// Integer C ABI mirrored by _UploadResult in openxr_quad.py. +enum UploadResult : int +{ + Success = 0, + InvalidArgument = -1, + NotAttached = -2, + CameraUnavailable = -3, + SwapchainFailed = -4, + AllocationFailed = -5, + UploadFailed = -6, + CompletionUnproved = -7, + CudaUnavailable = -8, + UploadModeMismatch = -9, +}; + +struct Dispatch +{ + PFN_xrGetInstanceProcAddr gipa = nullptr; + PFN_xrCreateReferenceSpace createSpace = nullptr; + PFN_xrDestroySpace destroySpace = nullptr; + PFN_xrEnumerateSwapchainFormats formats = nullptr; + PFN_xrCreateSwapchain createSwapchain = nullptr; + PFN_xrDestroySwapchain destroySwapchain = nullptr; + PFN_xrEnumerateSwapchainImages images = nullptr; + PFN_xrAcquireSwapchainImage acquire = nullptr; + PFN_xrWaitSwapchainImage wait = nullptr; + PFN_xrReleaseSwapchainImage release = nullptr; +}; + +struct CameraSession +{ + XrInstance instance = XR_NULL_HANDLE; + XrSession session = XR_NULL_HANDLE; + VkPhysicalDevice physical = VK_NULL_HANDLE; + VkDevice device = VK_NULL_HANDLE; + VkQueue queue = VK_NULL_HANDLE; + uint32_t queueFamily = 0; + XrSpace view = XR_NULL_HANDLE; + XrSpace local = XR_NULL_HANDLE; + XrSwapchain swapchain = XR_NULL_HANDLE; + std::vector images; + VkBuffer staging = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + void* pixels = nullptr; + VkCommandPool pool = VK_NULL_HANDLE; + VkCommandBuffer command = VK_NULL_HANDLE; + VkFence fence = VK_NULL_HANDLE; + uint32_t width = 0; + uint32_t height = 0; + uint32_t acquiredIndex = 0; + bool acquired = false; + bool waited = false; + bool fencePending = false; + bool retired = false; + bool ready = false; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + cudaExternalMemory_t cudaMemory = nullptr; + void* cudaPixels = nullptr; + cudaStream_t cudaStream = nullptr; + int cudaDevice = -1; + int vulkanCudaDevice = -1; + bool cudaPending = false; +#endif +}; + +struct Panel +{ + bool enabled = true; + bool headLocked = true; + XrPosef pose{ { 0, 0, 0, 1 }, { 0, -0.15f, -0.8f } }; + XrExtent2Df size{ 0.48f, 0.30f }; +}; + +// Each frame consumer owns one helper; calls stay on its XR owner thread. +struct PanelSession +{ + Dispatch dispatch{}; + std::array cameras{}; + std::array panels{}; + std::array quads{}; + bool attached = false; +}; +constexpr uint32_t kMaxImageDimension = 4096; + +template +bool resolve(PFN_xrGetInstanceProcAddr gipa, XrInstance instance, const char* name, Function& target) +{ + PFN_xrVoidFunction raw = nullptr; + if (XR_FAILED(gipa(instance, name, &raw)) || !raw) + { + return false; + } + target = reinterpret_cast(raw); + return true; +} + +bool vkOk(VkResult result) +{ + return result == VK_SUCCESS; +} + +bool releaseImageResources(CameraSession& camera) +{ + if (camera.device) + { +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaDevice >= 0) + { + // Failed completion quarantines both the borrowed input and our + // external allocation. Never free it merely because teardown ran. + if (cudaSetDevice(camera.cudaDevice) != cudaSuccess || + (camera.cudaPending && cudaStreamSynchronize(camera.cudaStream) != cudaSuccess)) + { + camera.retired = true; + return false; + } + camera.cudaPending = false; + } +#endif + if (camera.fence) + { + if (camera.fencePending) + { + // This helper owns only its copy, not every queue of the + // borrowed device. Retain storage until its own fence proves + // completion; device-wide idle is not our synchronization seam. + if (vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX) != VK_SUCCESS) + { + isaaccapture::Logger::get("isaaccapture.ovxr_pip_layer.CameraSession") + ->error("IsaacTeleop PiP: GPU completion unproved; retaining upload allocations"); + camera.retired = true; + camera.ready = false; + return false; + } + } + vkDestroyFence(camera.device, camera.fence, nullptr); + camera.fence = VK_NULL_HANDLE; + } + if (camera.pool) + { + vkDestroyCommandPool(camera.device, camera.pool, nullptr); + camera.pool = VK_NULL_HANDLE; + camera.command = VK_NULL_HANDLE; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaPixels) + { + if (cudaFree(camera.cudaPixels) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaPixels = nullptr; + } + if (camera.cudaMemory) + { + if (cudaDestroyExternalMemory(camera.cudaMemory) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaMemory = nullptr; + } + if (camera.cudaStream) + { + if (cudaStreamDestroy(camera.cudaStream) != cudaSuccess) + { + camera.retired = true; + return false; + } + camera.cudaStream = nullptr; + } + camera.cudaDevice = -1; +#endif + if (camera.pixels) + { + vkUnmapMemory(camera.device, camera.memory); + } + if (camera.staging) + { + vkDestroyBuffer(camera.device, camera.staging, nullptr); + } + if (camera.memory) + { + vkFreeMemory(camera.device, camera.memory, nullptr); + } + } + camera.fence = VK_NULL_HANDLE; + camera.pool = VK_NULL_HANDLE; + camera.command = VK_NULL_HANDLE; + camera.pixels = nullptr; + camera.staging = VK_NULL_HANDLE; + camera.memory = VK_NULL_HANDLE; + camera.fencePending = false; + camera.ready = false; + return true; +} + +bool releaseSessionResources(CameraSession& camera, const Dispatch& dispatch) +{ + if (!releaseImageResources(camera)) + { + return false; + } + if (camera.swapchain) + { + // OpenXR destruction requires completed application graphics work, not + // reacquiring/releasing a runtime image. Our owned copies retired above. + if (XR_FAILED(dispatch.destroySwapchain(camera.swapchain))) + { + return false; + } + camera.swapchain = XR_NULL_HANDLE; + } + camera.acquired = camera.waited = false; + for (auto* space : { &camera.view, &camera.local }) + { + if (*space && XR_FAILED(dispatch.destroySpace(*space))) + { + return false; + } + *space = XR_NULL_HANDLE; + } + camera.images.clear(); + return true; +} + +bool createSwapchain(CameraSession& camera, const Dispatch& dispatch, uint32_t width, uint32_t height) +{ + if (camera.acquired || camera.retired) + { + return false; + } + if (!releaseImageResources(camera)) + { + return false; + } + if (camera.swapchain) + { + if (XR_FAILED(dispatch.destroySwapchain(camera.swapchain))) + { + camera.retired = true; + camera.ready = false; + return false; // Retain the old handle for explicit owner cleanup. + } + camera.swapchain = XR_NULL_HANDLE; + camera.images.clear(); + } + // A new handle is not usable until enumeration succeeds. Invalidate the + // old extent so a failed resize cannot admit a later old-size upload. + camera.width = camera.height = 0; + uint32_t count = 0; + if (XR_FAILED(dispatch.formats(camera.session, 0, &count, nullptr)) || !count) + { + return false; + } + std::vector formats(count); + if (XR_FAILED(dispatch.formats(camera.session, count, &count, formats.data()))) + { + return false; + } + auto chosen = std::find(formats.begin(), formats.end(), VK_FORMAT_R8G8B8A8_UNORM); + if (chosen == formats.end()) + { + return false; // No color conversion or unverified swapchain format fallback. + } + XrSwapchainCreateInfo info{ XR_TYPE_SWAPCHAIN_CREATE_INFO }; + info.usageFlags = XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT | XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT; + info.format = *chosen; + info.sampleCount = 1; + info.width = width; + info.height = height; + info.faceCount = 1; + info.arraySize = 1; + info.mipCount = 1; + if (XR_FAILED(dispatch.createSwapchain(camera.session, &info, &camera.swapchain))) + { + return false; + } + count = 0; + if (XR_FAILED(dispatch.images(camera.swapchain, 0, &count, nullptr)) || !count) + { + return false; + } + camera.images.resize(count); + for (auto& image : camera.images) + { + image = {}; + image.type = XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR; + } + if (XR_FAILED(dispatch.images( + camera.swapchain, count, &count, reinterpret_cast(camera.images.data())))) + { + return false; + } + camera.width = width; + camera.height = height; + return true; +} + +bool createUploadResources(CameraSession& camera, bool gpu = false) +{ + const auto fail = [&]() + { + releaseImageResources(camera); // No GPU submission exists yet. + return false; + }; + const VkDeviceSize length = static_cast(camera.width) * camera.height * 4; + VkBufferCreateInfo bufferInfo{ VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO }; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkExternalMemoryBufferCreateInfo external{ VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO }; + external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + if (gpu) + { + bufferInfo.pNext = &external; + } +#endif + bufferInfo.size = length; + bufferInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + if (!vkOk(vkCreateBuffer(camera.device, &bufferInfo, nullptr, &camera.staging))) + { + return fail(); + } + VkMemoryRequirements requirements{}; + vkGetBufferMemoryRequirements(camera.device, camera.staging, &requirements); + VkPhysicalDeviceMemoryProperties properties{}; + vkGetPhysicalDeviceMemoryProperties(camera.physical, &properties); + uint32_t type = UINT32_MAX; + for (uint32_t i = 0; i < properties.memoryTypeCount; ++i) + { + const VkMemoryPropertyFlags flags = + gpu ? VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT : + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + if ((requirements.memoryTypeBits & (1u << i)) && (properties.memoryTypes[i].propertyFlags & flags) == flags) + { + type = i; + break; + } + } + if (type == UINT32_MAX) + { + return fail(); + } + VkMemoryAllocateInfo allocation{ VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkMemoryDedicatedAllocateInfo dedicated{ VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO }; + dedicated.buffer = camera.staging; + VkExportMemoryAllocateInfo exportInfo{ VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO }; + exportInfo.pNext = &dedicated; + exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + if (gpu) + { + allocation.pNext = &exportInfo; + } +#endif + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = type; + if (!vkOk(vkAllocateMemory(camera.device, &allocation, nullptr, &camera.memory)) || + !vkOk(vkBindBufferMemory(camera.device, camera.staging, camera.memory, 0)) || + (!gpu && !vkOk(vkMapMemory(camera.device, camera.memory, 0, length, 0, &camera.pixels)))) + { + return fail(); + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (gpu) + { + const auto getFd = reinterpret_cast(vkGetDeviceProcAddr(camera.device, "vkGetMemoryFdKHR")); + VkMemoryGetFdInfoKHR fdInfo{ VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR }; + fdInfo.memory = camera.memory; + fdInfo.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; + int fd = -1; + if (!getFd || getFd(camera.device, &fdInfo, &fd) != VK_SUCCESS) + { + return fail(); + } + cudaExternalMemoryHandleDesc memory{}; + memory.type = cudaExternalMemoryHandleTypeOpaqueFd; + memory.handle.fd = fd; + memory.size = requirements.size; + memory.flags = cudaExternalMemoryDedicated; + if (cudaImportExternalMemory(&camera.cudaMemory, &memory) != cudaSuccess) + { + close(fd); // Successful import alone transfers fd ownership to CUDA. + return fail(); + } + cudaExternalMemoryBufferDesc buffer{}; + buffer.size = length; + if (cudaExternalMemoryGetMappedBuffer(&camera.cudaPixels, camera.cudaMemory, &buffer) != cudaSuccess || + cudaStreamCreateWithFlags(&camera.cudaStream, cudaStreamNonBlocking) != cudaSuccess) + { + return fail(); + } + } +#endif + VkCommandPoolCreateInfo poolInfo{ VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO }; + poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; + poolInfo.queueFamilyIndex = camera.queueFamily; + if (!vkOk(vkCreateCommandPool(camera.device, &poolInfo, nullptr, &camera.pool))) + { + return fail(); + } + VkCommandBufferAllocateInfo commandInfo{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + commandInfo.commandPool = camera.pool; + commandInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + commandInfo.commandBufferCount = 1; + VkFenceCreateInfo fenceInfo{ VK_STRUCTURE_TYPE_FENCE_CREATE_INFO }; + if (!vkOk(vkAllocateCommandBuffers(camera.device, &commandInfo, &camera.command)) || + !vkOk(vkCreateFence(camera.device, &fenceInfo, nullptr, &camera.fence))) + { + return fail(); + } + return true; +} + +bool upload(CameraSession& camera, const Dispatch& dispatch, const void* rgba) +{ + camera.ready = false; + if (camera.retired) + { + return false; + } + if (camera.fencePending) + { + if (!vkOk(vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX))) + { + camera.retired = true; + return false; + } + camera.fencePending = false; + } + const VkDeviceSize length = static_cast(camera.width) * camera.height * 4; + if (rgba) + { + std::memcpy(camera.pixels, rgba, static_cast(length)); + } + if (!camera.acquired) + { + XrSwapchainImageAcquireInfo acquireInfo{ XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO }; + if (dispatch.acquire(camera.swapchain, &acquireInfo, &camera.acquiredIndex) != XR_SUCCESS) + { + return false; + } + camera.acquired = true; + } + const auto release = [&]() -> bool + { + if (camera.waited) + { + XrSwapchainImageReleaseInfo info{ XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO }; + if (dispatch.release(camera.swapchain, &info) != XR_SUCCESS) + { + camera.retired = true; + return false; + } + camera.waited = false; + camera.acquired = false; + } + return true; + }; + if (!camera.waited) + { + XrSwapchainImageWaitInfo waitInfo{ XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO }; + waitInfo.timeout = XR_INFINITE_DURATION; + const XrResult waited = dispatch.wait(camera.swapchain, &waitInfo); + if (waited == XR_TIMEOUT_EXPIRED) + { + return false; // Retry this same acquisition on the next publication. + } + if (waited != XR_SUCCESS) + { + camera.retired = true; + return false; + } + camera.waited = true; + } + if (camera.acquiredIndex >= camera.images.size()) + { + release(); + camera.retired = true; + return false; + } + if (!vkOk(vkResetFences(camera.device, 1, &camera.fence)) || !vkOk(vkResetCommandPool(camera.device, camera.pool, 0))) + { + release(); + return false; + } + VkCommandBufferBeginInfo begin{ VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + if (!vkOk(vkBeginCommandBuffer(camera.command, &begin))) + { + release(); + return false; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + VkBufferMemoryBarrier fromCuda{ VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER }; + if (camera.cudaPixels) + { + fromCuda.srcQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + fromCuda.dstQueueFamilyIndex = camera.queueFamily; + fromCuda.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + fromCuda.buffer = camera.staging; + fromCuda.size = VK_WHOLE_SIZE; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, + nullptr, 1, &fromCuda, 0, nullptr); + } +#endif + const VkImage image = camera.images[camera.acquiredIndex].image; + VkImageMemoryBarrier toTransfer{ VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER }; + toTransfer.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + toTransfer.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toTransfer.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toTransfer.image = image; + toTransfer.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + toTransfer.subresourceRange = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1 }; + toTransfer.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, 0, nullptr, 0, nullptr, 1, &toTransfer); + VkBufferImageCopy copy{}; + copy.imageSubresource = { VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1 }; + copy.imageExtent = { camera.width, camera.height, 1 }; + vkCmdCopyBufferToImage(camera.command, camera.staging, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, ©); +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaPixels) + { + VkBufferMemoryBarrier toCuda = fromCuda; + toCuda.srcQueueFamilyIndex = camera.queueFamily; + toCuda.dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL; + toCuda.srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCuda.dstAccessMask = 0; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, 0, + nullptr, 1, &toCuda, 0, nullptr); + } +#endif + VkImageMemoryBarrier toColor = toTransfer; + toColor.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toColor.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + toColor.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toColor.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + vkCmdPipelineBarrier(camera.command, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, 0, nullptr, 0, nullptr, 1, &toColor); + if (!vkOk(vkEndCommandBuffer(camera.command))) + { + release(); + return false; + } + VkSubmitInfo submit{ VK_STRUCTURE_TYPE_SUBMIT_INFO }; + submit.commandBufferCount = 1; + submit.pCommandBuffers = &camera.command; + if (!vkOk(vkQueueSubmit(camera.queue, 1, &submit, camera.fence))) + { + camera.retired = true; + release(); + return false; + } + camera.fencePending = true; + // Complete the bounded owned copy before the runtime reuses this image. + if (!vkOk(vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX))) + { + camera.retired = true; + return false; + } + camera.fencePending = false; + if (!release()) + { + return false; + } + camera.ready = true; + return true; +} + +} // namespace + +extern "C" PanelSession* isaacteleop_pip_create() +{ + return new (std::nothrow) PanelSession; +} + +extern "C" uint32_t isaacteleop_pip_max_cameras() +{ + return kMaxCameras; +} +extern "C" uint32_t isaacteleop_pip_max_image_dimension() +{ + return kMaxImageDimension; +} + +// Explicit application binding. All entry points are called on the OVXR owner +// thread, serialized with that context's queue and frame operations. +extern "C" int isaacteleop_pip_attach(PanelSession* owner, + XrInstance instance, + XrSession session, + PFN_xrGetInstanceProcAddr gipa, + VkPhysicalDevice physical, + VkDevice device, + VkQueue queue, + uint32_t queueFamily) +{ + if (!instance || !session || !gipa || !physical || !device || !queue || !owner || owner->attached) + { + return -1; + } + Dispatch dispatch{}; + dispatch.gipa = gipa; + if (!resolve(gipa, instance, "xrCreateReferenceSpace", dispatch.createSpace) || + !resolve(gipa, instance, "xrDestroySpace", dispatch.destroySpace) || + !resolve(gipa, instance, "xrEnumerateSwapchainFormats", dispatch.formats) || + !resolve(gipa, instance, "xrCreateSwapchain", dispatch.createSwapchain) || + !resolve(gipa, instance, "xrDestroySwapchain", dispatch.destroySwapchain) || + !resolve(gipa, instance, "xrEnumerateSwapchainImages", dispatch.images) || + !resolve(gipa, instance, "xrAcquireSwapchainImage", dispatch.acquire) || + !resolve(gipa, instance, "xrWaitSwapchainImage", dispatch.wait) || + !resolve(gipa, instance, "xrReleaseSwapchainImage", dispatch.release)) + { + return -2; + } + // Register before allocation so even failed partial setup has an explicit + // detach owner. Error -3 tells the binding to retain this cleanup obligation. + owner->dispatch = dispatch; + owner->attached = true; + auto& cameras = owner->cameras; + for (auto& camera : cameras) + { + camera.instance = instance; + camera.session = session; + camera.physical = physical; + camera.device = device; + camera.queue = queue; + camera.queueFamily = queueFamily; + XrReferenceSpaceCreateInfo space{ XR_TYPE_REFERENCE_SPACE_CREATE_INFO }; + space.referenceSpaceType = XR_REFERENCE_SPACE_TYPE_VIEW; + space.poseInReferenceSpace.orientation.w = 1.0f; + if (XR_FAILED(dispatch.createSpace(session, &space, &camera.view))) + { + return -3; + } + } + return 0; +} + +extern "C" int isaacteleop_pip_destroy(PanelSession* owner) +{ + if (!owner) + return 0; + if (owner->attached) + { + for (auto& camera : owner->cameras) + { + if (!releaseSessionResources(camera, owner->dispatch)) + return -1; // Retain the owner until all borrowed reads retire. + } + } + delete owner; + return 0; +} + +extern "C" const XrCompositionLayerBaseHeader* isaacteleop_pip_layer(PanelSession* owner, uint32_t cameraIndex) +{ + if (!owner || !owner->attached || cameraIndex >= kMaxCameras) + { + return nullptr; + } + const auto& camera = owner->cameras[cameraIndex]; + const auto& panel = owner->panels[cameraIndex]; + if (!camera.ready || camera.retired || !panel.enabled || !camera.swapchain || + !(panel.headLocked ? camera.view : camera.local)) + { + return nullptr; + } + auto& quad = owner->quads[cameraIndex]; + quad = { XR_TYPE_COMPOSITION_LAYER_QUAD }; + quad.layerFlags = 0; // Opaque camera panels, independent of projection alpha. + quad.space = panel.headLocked ? camera.view : camera.local; + quad.eyeVisibility = XR_EYE_VISIBILITY_BOTH; + quad.subImage.swapchain = camera.swapchain; + quad.subImage.imageRect.extent = { static_cast(camera.width), static_cast(camera.height) }; + quad.pose = panel.pose; + quad.size = panel.size; + return reinterpret_cast(&quad); +} + +extern "C" int isaacteleop_pip_configure( + PanelSession* owner, uint32_t cameraIndex, int enabled, int headLocked, const float* pose7, const float* size2) +{ + if (!owner || cameraIndex >= kMaxCameras || !pose7 || !size2 || !std::isfinite(size2[0]) || + !std::isfinite(size2[1]) || size2[0] <= 0 || size2[1] <= 0) + { + return -1; + } + float orientationLength = 0.0f; + for (int i = 0; i < 7; ++i) + { + if (!std::isfinite(pose7[i])) + { + return -1; + } + if (i >= 3) + { + orientationLength += pose7[i] * pose7[i]; + } + } + if (orientationLength < 0.99f || orientationLength > 1.01f) + { + return -1; + } + auto& panel = owner->panels[cameraIndex]; + panel.enabled = enabled != 0; + panel.headLocked = headLocked != 0; + panel.pose = { { pose7[3], pose7[4], pose7[5], pose7[6] }, { pose7[0], pose7[1], pose7[2] } }; + panel.size = { size2[0], size2[1] }; + return 0; +} + +extern "C" int isaacteleop_pip_publish_rgba( + PanelSession* owner, uint32_t cameraIndex, const void* rgba, uint32_t width, uint32_t height, size_t length) +{ + if (!owner || cameraIndex >= kMaxCameras || !rgba || !width || !height || width > kMaxImageDimension || + height > kMaxImageDimension || static_cast(width) * height * 4 != length) + { + return UploadResult::InvalidArgument; + } + if (!owner->attached) + { + return UploadResult::NotAttached; + } + CameraSession& camera = owner->cameras[cameraIndex]; + const auto& panel = owner->panels[cameraIndex]; + if (!camera.device || !camera.queue || camera.retired || !(panel.headLocked ? camera.view : camera.local)) + { + return UploadResult::CameraUnavailable; + } + const Dispatch& dispatch = owner->dispatch; + if ((camera.width != width || camera.height != height || !camera.swapchain) && + !createSwapchain(camera, dispatch, width, height)) + { + return UploadResult::SwapchainFailed; + } +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (camera.cudaDevice >= 0) + { + return UploadResult::UploadModeMismatch; // Do not replace live CUDA aliases with CPU staging resources. + } +#endif + if ((!camera.staging || !camera.memory || !camera.pixels || !camera.pool || !camera.command || !camera.fence) && + !createUploadResources(camera)) + { + return UploadResult::AllocationFailed; + } + return upload(camera, dispatch, rgba) ? UploadResult::Success : UploadResult::UploadFailed; +} + + +// Application API, not an OVXR export. The caller owns a completed OVRTX map +// until this synchronous upload returns. CompletionUnproved retains that map +// until completion is proved; the affected panel retires. +extern "C" int isaacteleop_pip_supports_cuda() +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + return 1; +#else + return 0; +#endif +} + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +namespace +{ +cudaError_t copyCameraArray(CameraSession& camera, const void* array, uint32_t width, uint32_t height, int sourceDevice) +{ + if (sourceDevice == camera.cudaDevice) + { + return cudaMemcpy2DFromArrayAsync(camera.cudaPixels, static_cast(width) * 4, + static_cast(array), 0, 0, static_cast(width) * 4, + height, cudaMemcpyDeviceToDevice, camera.cudaStream); + } + // RenderProducts can be assigned to a different GPU from the XR binding. + // Preserve their assignment; copy to the binding's device, not host pixels. + cudaMemcpy3DPeerParms copy{}; + copy.srcArray = const_cast(static_cast(array)); + copy.srcDevice = sourceDevice; + copy.dstPtr = { camera.cudaPixels, static_cast(width) * 4, static_cast(width) * 4, height }; + copy.dstDevice = camera.cudaDevice; + // With a CUDA array, extent.width is in array elements (RGBA pixels). + copy.extent = { width, height, 1 }; + return cudaMemcpy3DPeerAsync(©, camera.cudaStream); +} + +int copyCameraArrayAndWait(CameraSession& camera, const void* array, uint32_t width, uint32_t height, int sourceDevice) +{ + camera.cudaPending = true; + const auto copied = copyCameraArray(camera, array, width, height, sourceDevice); + // CUDA calls may surface an earlier asynchronous error. Even an enqueue + // failure must prove that this stream has stopped reading the borrowed map. + if (cudaStreamSynchronize(camera.cudaStream) != cudaSuccess) + { + camera.retired = true; + return UploadResult::CompletionUnproved; + } + camera.cudaPending = false; + return copied == cudaSuccess ? UploadResult::Success : UploadResult::UploadFailed; +} +} // namespace +#endif + +// Completion-only recovery never retries a copy or destroys native resources. +extern "C" int isaacteleop_pip_wait_cuda(PanelSession* owner, uint32_t cameraIndex) +{ + if (!owner || cameraIndex >= kMaxCameras) + return UploadResult::InvalidArgument; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + auto& camera = owner->cameras[cameraIndex]; + if (camera.cudaPending) + { + if (cudaSetDevice(camera.cudaDevice) != cudaSuccess || cudaStreamSynchronize(camera.cudaStream) != cudaSuccess) + return UploadResult::CompletionUnproved; + camera.cudaPending = false; + } +#endif + return UploadResult::Success; +} + +extern "C" int isaacteleop_pip_publish_cuda_array( + PanelSession* owner, uint32_t cameraIndex, const void* array, uint32_t width, uint32_t height, int device) +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + if (!owner || cameraIndex >= kMaxCameras || !array || !width || !height || width > kMaxImageDimension || + height > kMaxImageDimension || device < 0) + { + return UploadResult::InvalidArgument; + } + if (!owner->attached) + { + return UploadResult::NotAttached; + } + CameraSession& camera = owner->cameras[cameraIndex]; + if (!camera.device || !camera.queue || camera.retired) + { + return UploadResult::CameraUnavailable; + } + // Resolve the borrowed Vulkan device once. A camera output's CUDA ordinal + // is a source identity, not proof it is the destination Vulkan device. + if (camera.vulkanCudaDevice < 0) + { + VkPhysicalDeviceIDProperties id{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES }; + VkPhysicalDeviceProperties2 properties{ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2 }; + properties.pNext = &id; + vkGetPhysicalDeviceProperties2(camera.physical, &properties); + int count = 0; + if (cudaGetDeviceCount(&count) != cudaSuccess) + { + return UploadResult::CudaUnavailable; + } + for (int candidate = 0; candidate < count; ++candidate) + { + cudaDeviceProp properties{}; + if (cudaGetDeviceProperties(&properties, candidate) == cudaSuccess && + std::memcmp(id.deviceUUID, properties.uuid.bytes, VK_UUID_SIZE) == 0) + { + camera.vulkanCudaDevice = candidate; + break; + } + } + if (camera.vulkanCudaDevice < 0) + { + return UploadResult::CudaUnavailable; + } + } + if (cudaSetDevice(camera.vulkanCudaDevice) != cudaSuccess) + { + return UploadResult::CudaUnavailable; + } + const Dispatch& dispatch = owner->dispatch; + if ((camera.width != width || camera.height != height || !camera.swapchain) && + !createSwapchain(camera, dispatch, width, height)) + { + return UploadResult::SwapchainFailed; + } + if (!camera.cudaPixels || !camera.pool || !camera.command || !camera.fence) + { + if (!releaseImageResources(camera)) + { + return UploadResult::CompletionUnproved; + } + camera.cudaDevice = camera.vulkanCudaDevice; + if (!createUploadResources(camera, true)) + { + return UploadResult::AllocationFailed; + } + } + if (camera.fencePending) + { + if (vkWaitForFences(camera.device, 1, &camera.fence, VK_TRUE, UINT64_MAX) != VK_SUCCESS) + { + camera.retired = true; + return UploadResult::UploadFailed; + } + camera.fencePending = false; + } + camera.ready = false; + const int copied = copyCameraArrayAndWait(camera, array, width, height, device); + if (copied != UploadResult::Success) + { + return copied; + } + // No external semaphore wait is introduced: preserve the current host-wait + // scheduling policy. This is a GPU-only copy, not end-to-end zero-copy. + return upload(camera, dispatch, nullptr) ? UploadResult::Success : UploadResult::UploadFailed; +#else + return UploadResult::CudaUnavailable; +#endif +} diff --git a/src/core/python/CMakeLists.txt b/src/core/python/CMakeLists.txt index b2edfa611f..c55fd4a7d0 100644 --- a/src/core/python/CMakeLists.txt +++ b/src/core/python/CMakeLists.txt @@ -41,6 +41,13 @@ foreach(_license_file IN LISTS _license_files_out) endforeach() # Generate pyproject.toml from template with ISAAC_TELEOP_PYPROJECT_VERSION +if(BUILD_OVXR_CAMERA_QUADS) + set(OVXR_CAMERA_PACKAGE_DATA_BLOCK + "\"isaaccapture.ovxr_session_manager\" = [\"native/*.so\"]\n") +else() + set(OVXR_CAMERA_PACKAGE_DATA_BLOCK "") +endif() + configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/pyproject.toml.in" "${CMAKE_CURRENT_BINARY_DIR}/pyproject.toml" @@ -87,6 +94,10 @@ add_custom_target(python_package ALL COMMENT "Preparing Python package structure" ) +if(TARGET ovxr_pip_layer) + add_dependencies(python_package ovxr_pip_layer) +endif() + # viz_py only exists when BUILD_VIZ=ON. Use add_dependencies (not a # DEPENDS genex) so the build-order constraint is unconditional under # all generators — Windows ninja runs the wheel build in parallel @@ -260,7 +271,17 @@ if(SKBUILD) USE_SOURCE_PERMISSIONS PATTERN "*.py" EXCLUDE PATTERN "__pycache__" EXCLUDE + # Reused staging can retain a formerly enabled camera helper. The option, + # not old files in the build tree, owns its inclusion in a scikit wheel. + PATTERN "native" EXCLUDE ) + if(BUILD_OVXR_CAMERA_QUADS) + install(TARGETS ovxr_pip_layer + LIBRARY DESTINATION "isaaccapture/ovxr_session_manager/native" + COMPONENT isaacteleop_wheel + ) + + endif() install( DIRECTORY "${ROBOTIC_GROUNDING_STAGED_DIR}" DESTINATION "." diff --git a/src/core/python/pyproject.toml.in b/src/core/python/pyproject.toml.in index 05eec7faf9..33c1855baf 100644 --- a/src/core/python/pyproject.toml.in +++ b/src/core/python/pyproject.toml.in @@ -102,6 +102,7 @@ robotic_grounding = [ "assets/xmls/sharpawave/NOTICE", "assets/xmls/sharpawave/README.md", ] +@OVXR_CAMERA_PACKAGE_DATA_BLOCK@ [tool.uv] # Only use managed Python installations (not system Python) diff --git a/src/core/python/requirements-ovxr.txt b/src/core/python/requirements-ovxr.txt index 735767ae6b..076edbaff8 100644 --- a/src/core/python/requirements-ovxr.txt +++ b/src/core/python/requirements-ovxr.txt @@ -3,3 +3,5 @@ # Application overrides: docs/source/getting_started/ovxr.rst. ovxr==0.1.0.5842 +# GPU camera output uses CUDA 13 bindings, matching the Isaac Lab runtime. +cuda-bindings>=13,<14 diff --git a/src/core/python/setup.py b/src/core/python/setup.py index a32e276ab9..c99a3cb283 100644 --- a/src/core/python/setup.py +++ b/src/core/python/setup.py @@ -8,7 +8,10 @@ we need to override setuptools to recognize this as a platform wheel, not pure Python. """ +from pathlib import Path + from setuptools import setup +from setuptools.command.build_py import build_py from setuptools.dist import Distribution @@ -19,4 +22,21 @@ def has_ext_modules(self): return True -setup(distclass=BinaryDistribution) +class BuildPy(build_py): + """Retire optional camera data left by a previous build configuration.""" + + def run(self): + super().run() + root = Path(self.build_lib) / "isaaccapture/ovxr_session_manager" + # Retire outputs from the former API-layer layout on incremental builds. + for name in ( + "libXrApiLayer_isaacteleop_pip.so", + "XrApiLayer_isaacteleop_pip.json", + ): + (root / "openxr_layer" / name).unlink(missing_ok=True) + if not self.package_data.get("isaaccapture.ovxr_session_manager"): + # Setuptools reuses build_lib; omitted package data is not deleted. + (root / "native/libisaaccapture_camera_panels.so").unlink(missing_ok=True) + + +setup(distclass=BinaryDistribution, cmdclass={"build_py": BuildPy}) diff --git a/src/python/isaaccapture/ovxr_session_manager/__init__.py b/src/python/isaaccapture/ovxr_session_manager/__init__.py index 8528115853..d85ab4d2ad 100644 --- a/src/python/isaaccapture/ovxr_session_manager/__init__.py +++ b/src/python/isaaccapture/ovxr_session_manager/__init__.py @@ -5,8 +5,11 @@ from .ovxr_session import ( OvxrSessionBinding, + OvxrScenePublication, + OvxrResourceOwner, OvxrSessionConfig, OvxrApplicationInterface, + OvxrFrameConsumer, OvxrSession, StageOrdinalLedger, ) @@ -15,10 +18,14 @@ "OvxrSessionBinding", "OvxrSessionConfig", "OvxrApplicationInterface", + "OvxrFrameConsumer", "OvxrSession", "StageOrdinalLedger", ] -from .interfaces import OvxrScenePublication, OvxrResourceOwner +from .camera_panels import CameraPanelConfig, CameraPanelConsumer +from .camera_mailbox import FrameOutputs + +__all__ += ["CameraPanelConfig", "CameraPanelConsumer", "FrameOutputs"] __all__ += ["OvxrScenePublication", "OvxrResourceOwner"] diff --git a/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py b/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py new file mode 100644 index 0000000000..3c003cf710 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/camera_mailbox.py @@ -0,0 +1,233 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded completed camera outputs for later application presentation. + +CPU snapshots own their bytes. GPU snapshots retain an OVRTX map until +next production or teardown; they may be consumed only on the render thread, +with synchronous copy completion. Cadence gaps retain the last completed image. +No borrowed result/output handle survives the callback. +""" + +from __future__ import annotations + +import ctypes +import threading +from dataclasses import dataclass +from typing import Any, Callable, TypeVar + +from .gpu_output import MappedGpuImage, MappedOutput, OutputMapCleanupError, _api + +_Image = TypeVar("_Image") + + +@dataclass(frozen=True) +class CameraImage: + width: int + height: int + rgba: bytes | None + sensor_time: float + revision: int + gpu: MappedGpuImage | None = None + + +def _text(value: Any) -> str: + if not value.ptr: + return "" + return ctypes.string_at(value.ptr, value.length).decode("utf-8") + + +def _latest_color_handle(outputs: Any, product_path: str) -> tuple[int, float] | None: + """Select the latest *complete* LdrColor frame, not the first array entry.""" + latest: tuple[int, float] | None = None + produced = False + for product_index in range(outputs.output_count): + product = outputs.outputs[product_index] + if _text(product.render_product_path) != product_path: + continue + for frame_index in range(product.output_frame_count): + produced = True + frame = product.output_frames[frame_index] + for var_index in range(frame.render_var_count): + var = frame.output_render_vars[var_index] + if _text(var.source_name) != "LdrColor" or not var.output_handle: + continue + candidate = (int(var.output_handle), float(frame.frame_end_time)) + if latest is None or candidate[1] >= latest[1]: + latest = candidate + if produced and latest is None: + raise RuntimeError(f"{product_path} produced a frame without LdrColor") + return latest + + +def _copy_host_rgba( + renderer: int, handle: int, max_bytes: int +) -> tuple[int, int, bytes]: + """Map one completed OVRTX output to CPU and copy before unmap.""" + b, lib = _api() + description = b.ovrtx_map_output_description_t( + device_type=b.Device.CPU, sync_stream=None + ) + mapped = b.ovrtx_render_var_output_t() + result = lib.ovrtx_map_render_var_output( + renderer, + handle, + ctypes.byref(description), + b.OVRTX_TIMEOUT_INFINITE, + ctypes.byref(mapped), + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX camera output map failed") + lease = MappedOutput(renderer=renderer, mapped=mapped, owner=threading.get_ident()) + failure = None + try: + if mapped.status != b.EventStatus.COMPLETED or mapped.num_tensors != 1: + raise RuntimeError( + "OVRTX camera output is incomplete or has unexpected tensors" + ) + tensor = mapped.tensors[0].dl.contents + shape = tuple(int(tensor.shape[i]) for i in range(tensor.ndim)) + dtype = tensor.dtype + if dtype.code.value != 1 or dtype.bits != 8 or not tensor.data: + raise RuntimeError("OVRTX camera output is not CPU uint8") + if len(shape) == 3 and shape[2] == 4 and dtype.lanes == 1: + height, width, _ = shape + expected_strides = (width * 4, 4, 1) + elif len(shape) == 2 and dtype.lanes == 4: + height, width = shape + expected_strides = (width, 1) + else: + raise RuntimeError(f"unexpected OVRTX camera tensor shape {shape}") + length = width * height * 4 + if width <= 0 or height <= 0 or length > max_bytes: + raise RuntimeError("OVRTX camera output exceeds bounded mailbox size") + if ( + tensor.strides + and tuple(tensor.strides[i] for i in range(tensor.ndim)) != expected_strides + ): + raise RuntimeError("OVRTX camera output is not contiguous RGBA8") + return width, height, ctypes.string_at(tensor.data + tensor.byte_offset, length) + except BaseException as error: + failure = error + raise + finally: + try: + lease.close() + except Exception as cleanup: + # Keep the native handle even when validation or the host copy failed. + raise OutputMapCleanupError(lease, failure, cleanup) from failure + + +class FrameOutputs: + """Select and copy completed color outputs during the owner-thread callback. + + Output handles are opaque and valid only within this frame. CPU snapshots + are shared; GPU mappings belong to the consumer and must be retired by it. + """ + + def __init__(self, renderer: int, outputs: Any) -> None: + self._renderer = renderer + self._outputs = outputs + self._host_images = {} + + def latest_color(self, product_path: str) -> tuple[int, float] | None: + """Return the latest color handle and sensor time, or None for a cadence gap.""" + return _latest_color_handle(self._outputs.contents, product_path) + + def map_gpu( + self, handle: int, max_bytes: int, mapper: Callable[[int, int, int], _Image] + ) -> _Image: + """Map an output using the consumer's borrowed or owned GPU copy policy.""" + return mapper(self._renderer, handle, max_bytes) + + def copy_host_rgba(self, handle: int, max_bytes: int) -> tuple[int, int, bytes]: + """Copy once per output and enforce each consumer's byte bound on reuse.""" + if handle not in self._host_images: + self._host_images[handle] = _copy_host_rgba( + self._renderer, handle, max_bytes + ) + width, height, rgba = self._host_images[handle] + if len(rgba) > max_bytes: + raise RuntimeError("OVRTX camera output exceeds bounded mailbox size") + return width, height, rgba + + +class CompletedCameraMailbox: + """Latest camera image and explicit error/omission state for one product.""" + + def __init__( + self, + product_path: str, + max_image_bytes: int, + *, + gpu_mapper=None, + ) -> None: + if not product_path.startswith("/") or max_image_bytes <= 0: + raise ValueError( + "camera mailbox requires an absolute product path and positive byte bound" + ) + self.product_path = product_path + self.max_image_bytes = max_image_bytes + self._gpu_mapper = gpu_mapper + self._mapped = None + self._gpu_error = None + self._lock = threading.Lock() + self._image: CameraImage | None = None + self._revision = 0 + self._error: str | None = None + + def snapshot(self) -> CameraImage | None: + with self._lock: + return self._image + + def error(self) -> str | None: + with self._lock: + return self._error + + def on_outputs(self, frame: FrameOutputs) -> None: + """Consume on the render thread; contain errors and retain only owned data/maps.""" + try: + if self._gpu_error is not None: + raise RuntimeError(self._gpu_error) + selected = frame.latest_color(self.product_path) + if selected is None: + return + handle, sensor_time = selected + gpu = None + if self._gpu_mapper is not None: + self.close() + gpu = frame.map_gpu(handle, self.max_image_bytes, self._gpu_mapper) + self._mapped = gpu + width, height, rgba = gpu.width, gpu.height, None + else: + width, height, rgba = frame.copy_host_rgba(handle, self.max_image_bytes) + if len(rgba) != width * height * 4 or len(rgba) > self.max_image_bytes: + raise RuntimeError("camera copier returned invalid RGBA8 size") + with self._lock: + self._revision += 1 + self._image = CameraImage( + width, + height, + bytes(rgba) if rgba is not None else None, + sensor_time, + self._revision, + gpu, + ) + self._error = None + except Exception as exc: + if isinstance(exc, OutputMapCleanupError): + self._mapped = exc.image + self._gpu_error = str(exc) + with self._lock: + self._image = None + self._error = str(exc) + + def close(self) -> None: + """Frame owner only, after synchronous quad consumption, before renderer destruction.""" + if self._mapped is not None: + try: + self._mapped.close() + except Exception as error: + self._gpu_error = str(error) + raise + self._mapped = None diff --git a/src/python/isaaccapture/ovxr_session_manager/camera_panels.py b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py new file mode 100644 index 0000000000..832736c634 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/camera_panels.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Camera picture-in-picture as an application-neutral OVXR frame consumer.""" + +from __future__ import annotations + +import logging +import math +from dataclasses import dataclass +from typing import Literal + +from .camera_mailbox import CompletedCameraMailbox, FrameOutputs +from .gpu_output import map_cuda_array_rgba +from .openxr_quad import OpenXRCameraPanels +from .ovxr_session import OvxrFrameConsumer + +_LOGGER = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class CameraPanelConfig: + """One host-produced camera and its head-locked presentation policy.""" + + render_product_path: str + max_image_bytes: int = 64 * 1024 * 1024 + panel_width_m: float = 0.48 + position: tuple[float, float, float] = (0.0, -0.15, -0.8) + orientation_xyzw: tuple[float, float, float, float] = (0.0, 0.0, 0.0, 1.0) + space: Literal["head_locked"] = "head_locked" + label: str | None = None + + def __post_init__(self) -> None: + if not self.render_product_path.startswith("/"): + raise ValueError("camera render product path must be absolute") + if self.max_image_bytes <= 0: + raise ValueError("camera panel max_image_bytes must be positive") + if not math.isfinite(self.panel_width_m) or self.panel_width_m <= 0.0: + raise ValueError("camera panel panel_width_m must be finite and positive") + if len(self.position) != 3 or not all( + math.isfinite(value) for value in self.position + ): + raise ValueError("camera panel position must contain three finite values") + if len(self.orientation_xyzw) != 4 or not all( + math.isfinite(value) for value in self.orientation_xyzw + ): + raise ValueError("camera panel orientation must contain four finite values") + norm = math.sqrt(sum(value * value for value in self.orientation_xyzw)) + if norm <= 0.0: + raise ValueError("camera panel orientation must have non-zero norm") + object.__setattr__( + self, + "orientation_xyzw", + tuple(value / norm for value in self.orientation_xyzw), + ) + if self.space != "head_locked": + raise ValueError("camera panel space must be 'head_locked'") + + +class CameraPanelConsumer(OvxrFrameConsumer): + """Own camera uploads and composition layers until the XR owner retires them.""" + + def __init__(self, panels: tuple[CameraPanelConfig, ...]) -> None: + self.panels = tuple(panels) + paths = self.render_product_paths + if len(paths) != len(set(paths)): + raise ValueError("camera panels require unique paths") + self._native = None + self._mailboxes = [] + self._layers = () + self._attach_attempted = False + self._enabled = False + + @property + def render_product_paths(self) -> tuple[str, ...]: + return tuple(panel.render_product_path for panel in self.panels) + + def on_session_started(self, binding, config, request_shutdown) -> None: + self._native = OpenXRCameraPanels(self.panels) + for panel in self.panels: + self._mailboxes.append( + CompletedCameraMailbox( + panel.render_product_path, + panel.max_image_bytes, + gpu_mapper=map_cuda_array_rgba + if self._native.supports_gpu + else None, + ) + ) + + def on_openxr_ready(self, openxr, vulkan) -> bool: + if not self._attach_attempted: + self._attach_attempted = True + try: + if vulkan is None: + raise RuntimeError("Vulkan handles are unavailable") + self._native.attach(openxr, vulkan) + except Exception: + # Partial native attachment stays owned until normal close. + _LOGGER.exception( + "Camera panels disabled; teleoperation remains available" + ) + else: + self._enabled = True + return True + + def on_outputs(self, frame: FrameOutputs) -> None: + if not self._enabled: + return + layers = [] + self._layers = () + for index, mailbox in enumerate(self._mailboxes): + try: + mailbox.on_outputs(frame) + self._native.publish( + index, mailbox.snapshot(), invalid=mailbox.error() is not None + ) + if layer := self._native.layer(index): + layers.append(layer) + except Exception: + _LOGGER.exception("OpenXR camera quad publication failed") + try: + self._native.hide(index) + except Exception: + _LOGGER.exception("OpenXR camera quad hiding failed") + self._layers = tuple(layers) + + def composition_layers(self) -> tuple[int, ...]: + return self._layers + + def finish_frame(self) -> None: + first_error = None + operations = [self._native.finish_frame] if self._native is not None else [] + operations.extend(mailbox.close for mailbox in self._mailboxes) + for finish in operations: + try: + finish() + except Exception as error: + # Retain unresolved maps without skipping independent mailboxes. + if first_error is None: + first_error = error + else: + first_error.add_note(str(error)) + if first_error is not None: + raise first_error + + def close(self) -> None: + self.finish_frame() + if self._native is not None: + self._native.close() + self._native = None + self._mailboxes.clear() + self._layers = () diff --git a/src/python/isaaccapture/ovxr_session_manager/frame_pump.py b/src/python/isaaccapture/ovxr_session_manager/frame_pump.py index f13eba25bc..f5b0dc8c63 100644 --- a/src/python/isaaccapture/ovxr_session_manager/frame_pump.py +++ b/src/python/isaaccapture/ovxr_session_manager/frame_pump.py @@ -9,6 +9,8 @@ from collections.abc import Callable, Iterable from .interfaces import OvxrApplicationInterface, OvxrSessionBinding, OvxrSessionConfig from .runtime_owner import CloudXRRuntimeOwner +from .camera_mailbox import FrameOutputs +from .gpu_output import completed_outputs, native_renderer_handle _LOGGER = logging.getLogger(__name__) @@ -46,9 +48,11 @@ def __init__( self._context_lifecycle_lock = threading.Lock() self._last_ordinal: int | None = None self._first_ordinal_validated = False + self._frame_consumers = () def attach(self, capability: object) -> None: self._binding = self._host.attach_on_render_thread(capability) + self._frame_consumers = tuple(self._binding.frame_consumers) @staticmethod def validate_provider() -> None: @@ -77,6 +81,13 @@ def create_context(self) -> None: if self._shutdown_requested(): return ovxr = _import_ovxr() + paths = tuple( + dict.fromkeys( + path + for consumer in self._frame_consumers + for path in consumer.render_product_paths + ) + ) extensions = list( dict.fromkeys( ( @@ -91,9 +102,14 @@ def create_context(self) -> None: ovstage=self._binding.stage, application_name=self._config.application_name, display_mode=ovxr.DisplayMode.STEREO, + # The server mode must reach OVXR; the browser AR preference cannot + # enable transparent rendering in an opaque server projection. session_mode={"ar": ovxr.SessionMode.AR, "vr": ovxr.SessionMode.VR}[ self._config.session_mode ], + # Empty means OVXR creates its default eye products. Supplying + # those names explicitly instead means existing products and + # changes the production/authoring contract. render_products=list(self._config.render_products), camera_paths=list(self._config.camera_paths), create_render_products=self._config.create_render_products, @@ -110,6 +126,12 @@ def create_context(self) -> None: ) with self._host.owner_lane(): self._context = ovxr.Context(desc) + if self._frame_consumers: + self._context.set_frame_render_products(list(paths)) + for consumer in self._frame_consumers: + consumer.on_session_started( + self._binding, self._config, self.request_shutdown + ) def request_exit(self) -> None: """Wake an in-flight OpenXR frame wait without stopping CloudXR.""" @@ -147,6 +169,10 @@ def external_handles(self): return None from isaaccapture.oxr import OpenXRSessionHandles + vulkan = self._context.get_vulkan_handles() if self._frame_consumers else None + for consumer in self._frame_consumers: + if not consumer.on_openxr_ready(handles, vulkan): + return None return OpenXRSessionHandles( handles.xr_instance, handles.xr_session, @@ -194,6 +220,8 @@ def render(self, ordinal: int) -> bool: raise self._binding.ordinal_ledger.resolve_external( frame_ordinal, + # Publication is independent of the eventual bind result: + # a later failure must not reuse an already sealed ordinal. expected_published=sealed, ) if result not in ( @@ -206,7 +234,19 @@ def render(self, ordinal: int) -> bool: ) if result != ovxr.Result.SUCCESS: return True - result, _frame = self._context.submit_frame(active_plan) + layers = [] + if self._frame_consumers and active_plan.step_result: + try: + renderer = native_renderer_handle(self._binding.renderer) + outputs = completed_outputs(renderer, active_plan.step_result) + layers = self._on_completed_outputs(renderer, outputs) + except Exception: + _LOGGER.exception("Optional bound-frame output observation failed") + finally: + self._finish_frame_outputs() + result, _frame = self._context.submit_frame( + active_plan, additional_layers=layers + ) if result not in ( ovxr.Result.SUCCESS, ovxr.Result.TIMEOUT, @@ -215,6 +255,17 @@ def render(self, ordinal: int) -> bool: raise RuntimeError( f"OVXR submit_frame failed: {ovxr.result_name(result)}: {self._context.last_error}" ) + if ( + result == ovxr.Result.SUCCESS + and _frame.submitted + and self._frame_consumers + ): + # Submit consumed the plan; consumer callbacks must not + # cancel it even if a callback fails. + active_plan = None + self._pending_plan = None + for consumer in self._frame_consumers: + consumer.on_frame_submitted() active_plan = None self._pending_plan = None except BaseException as error: @@ -222,7 +273,10 @@ def render(self, ordinal: int) -> bool: raise finally: if active_plan is not None: + # An unproved copy/unmap keeps the plan and context alive. Never + # consume borrowed output storage merely to recover a frame. try: + self._finish_frame_outputs() self._context.cancel_frame(active_plan) self._pending_plan = None except BaseException as cleanup_error: @@ -243,15 +297,28 @@ def close(self) -> None: self._host.close_publication() except BaseException as exc: first_error = exc - if self._pending_plan is not None: - try: - self._context.cancel_frame(self._pending_plan) - except BaseException as exc: - # Native destroy remains responsible for the pending frame. - if first_error is None: - first_error = exc - else: - self._pending_plan = None + try: + for consumer in reversed(self._frame_consumers): + # A failed completion/unmap must keep the renderer alive: never + # destroy storage that an optional consumer might still access. + consumer.close() + if self._pending_plan is not None: + self._finish_frame_outputs() + try: + self._context.cancel_frame(self._pending_plan) + except BaseException as exc: + # All external mappings are retired. Native destroy remains + # responsible for draining the context's pending frame. + if first_error is None: + first_error = exc + else: + self._pending_plan = None + except BaseException as retirement_error: + if first_error is not None: + first_error.add_note(f"Consumer retirement failed: {retirement_error}") + # Keep the primary error and the failed retirement cause visible. + raise first_error from retirement_error + raise context_retired = self._context is None if self._context is not None: assert self._binding is not None @@ -321,6 +388,21 @@ def close(self) -> None: if first_error is not None: raise first_error + def _on_completed_outputs(self, renderer, outputs) -> list[int]: + frame = FrameOutputs(renderer, outputs) + layers = [] + for consumer in self._frame_consumers: + try: + consumer.on_outputs(frame) + layers.extend(consumer.composition_layers()) + except Exception: + _LOGGER.exception("Optional frame consumer failed") + return layers + + def _finish_frame_outputs(self) -> None: + for consumer in self._frame_consumers: + consumer.finish_frame() + def retry_runtime_stop(self) -> None: """Retry service cleanup only after native borrowers have retired.""" if self._context is None: diff --git a/src/python/isaaccapture/ovxr_session_manager/gpu_output.py b/src/python/isaaccapture/ovxr_session_manager/gpu_output.py new file mode 100644 index 0000000000..3e9478fc29 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/gpu_output.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Public OVRTX mappings and bounded application-owned CUDA snapshots.""" + +from __future__ import annotations + +import ctypes +import threading +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Any + + +@lru_cache(maxsize=1) +def _api(): + from ovrtx._src import bindings as b + + root = b.ovrtx_loaded_bin_dir() + if root is None: + raise RuntimeError("OVRTX loaded provider is unavailable") + lib = ctypes.CDLL(str(Path(root) / b.OVRTX_LOADER_LIB_NAME)) + lib.ovrtx_map_render_var_output.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint64, + ctypes.POINTER(b.ovrtx_map_output_description_t), + b.ovrtx_timeout_t, + ctypes.POINTER(b.ovrtx_render_var_output_t), + ] + lib.ovrtx_map_render_var_output.restype = b.ovrtx_result_t + lib.ovrtx_unmap_render_var_output.argtypes = [ + ctypes.c_void_p, + ctypes.c_uint64, + b.ovrtx_cuda_sync_t, + ] + lib.ovrtx_unmap_render_var_output.restype = b.ovrtx_result_t + lib.ovrtx_fetch_results.argtypes = [ + ctypes.c_void_p, + b.ovrtx_step_result_handle_t, + b.ovrtx_timeout_t, + ctypes.POINTER(b.ovrtx_render_product_set_outputs_t), + ] + lib.ovrtx_fetch_results.restype = b.ovrtx_result_t + return b, lib + + +def native_renderer_handle(renderer) -> int: + """Normalize the borrowed renderer object before calling the OVRTX C API.""" + # Accept Renderer objects and raw native pointers at this boundary. + handle = getattr(renderer, "handle", getattr(renderer, "_handle", renderer)) + value = ctypes.cast(handle, ctypes.c_void_p).value + if not value: + raise ValueError("borrowed OVRTX renderer has no native handle") + return value + + +def completed_outputs(renderer: int, step_result: int): + """Borrow completed output metadata until OVXR consumes the bound plan.""" + b, lib = _api() + outputs = b.ovrtx_render_product_set_outputs_t() + result = lib.ovrtx_fetch_results( + renderer, step_result, b.OVRTX_TIMEOUT_INFINITE, ctypes.byref(outputs) + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX completed output fetch failed") + return ctypes.pointer(outputs) + + +def _check_cuda(result): + if int(result[0]): + raise RuntimeError(f"GPU output synchronization failed: CUDA {result[0]}") + return result[1] if len(result) > 1 else None + + +def _wait_ready(cuda, sync) -> None: + # Both fields are independently valid in the public OVRTX contract. + if sync.wait_event: + _check_cuda(cuda.cudaEventSynchronize(sync.wait_event)) + if sync.stream: + stream = cuda.cudaStreamLegacy if sync.stream == 1 else sync.stream + _check_cuda(cuda.cudaStreamSynchronize(stream)) + + +class OutputMapCleanupError(RuntimeError): + """The caller must retain this map and stop admission until owner cleanup.""" + + def __init__(self, image, cause, cleanup): + super().__init__(f"Output map failed: {cause}; cleanup failed: {cleanup}") + self.image = image + + +@dataclass(kw_only=True) +class MappedOutput: + """One native output map retained until its owner proves unmap succeeded.""" + + renderer: int + mapped: Any + owner: int + closed: bool = False + busy: bool = False + + def close(self) -> None: + """Owner only, after every consumer has completed its last source read.""" + if threading.get_ident() != self.owner: + raise RuntimeError("Output must be unmapped on its renderer owner") + if self.busy: + raise RuntimeError("Output consumer completion is unproved") + if not self.closed: + b, lib = _api() + result = lib.ovrtx_unmap_render_var_output( + self.renderer, self.mapped.map_handle, b.ovrtx_cuda_sync_t() + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX output unmap failed") + self.closed = True + + +@dataclass +class MappedGpuImage(MappedOutput): + width: int + height: int + pointer: int + pitch: int + device: int + is_array: bool = False + + +def map_cuda_array_rgba(renderer: int, handle: int, max_bytes: int) -> MappedGpuImage: + """Retain the native image array, avoiding OVRTX array-to-linear conversion.""" + from cuda.bindings import runtime as cuda + + b, lib = _api() + mapped = b.ovrtx_render_var_output_t() + desc = b.ovrtx_map_output_description_t( + device_type=b.Device.CUDA_ARRAY, sync_stream=None + ) + result = lib.ovrtx_map_render_var_output( + renderer, + handle, + ctypes.byref(desc), + b.OVRTX_TIMEOUT_INFINITE, + ctypes.byref(mapped), + ) + if result.status != b.OVRTX_API_SUCCESS: + raise RuntimeError("OVRTX GPU output map failed") + image = MappedGpuImage( + 0, 0, 0, 0, 0, renderer=renderer, mapped=mapped, owner=threading.get_ident() + ) + try: + if mapped.status != b.EventStatus.COMPLETED or mapped.num_tensors != 1: + raise RuntimeError( + "OVRTX GPU output is incomplete or has unexpected tensors" + ) + dl = mapped.tensors[0].dl.contents + shape = tuple(int(dl.shape[i]) for i in range(dl.ndim)) + if dl.device.device_type.value != 2 or not dl.data: + raise ValueError("mirror GPU output must be CUDA memory") + image.device = int(dl.device.device_id) + _check_cuda(cuda.cudaSetDevice(image.device)) + if ( + dl.dtype.code.value != 3 + or dl.byte_offset + or len(shape) != 3 + or shape[2] != 4 + ): + raise ValueError("mirror GPU output must be an opaque RGBA CUDA array") + info = cuda.cudaArrayGetInfo(int(dl.data)) + _check_cuda(info) + channel, extent = info[1:3] + height, width, _ = shape + if (channel.x, channel.y, channel.z, channel.w, int(channel.f)) != ( + 8, + 8, + 8, + 8, + 1, + ) or (extent.width, extent.height) != (width, height): + raise ValueError( + "camera CUDA array is not unsigned RGBA8 at the declared size" + ) + pitch = width * 4 + image.is_array = True + if width <= 0 or height <= 0 or pitch < width * 4 or pitch * height > max_bytes: + raise ValueError("mirror GPU output exceeds bounded image size") + image.width, image.height, image.pitch = width, height, pitch + image.pointer = int(dl.data) + int(dl.byte_offset) + # Wait for producer completion before reading the mapped image. + _wait_ready(cuda, mapped.cuda_sync) + return image + except BaseException as cause: + try: + image.close() + except Exception as cleanup: + raise OutputMapCleanupError(image, cause, cleanup) from cause + raise diff --git a/src/python/isaaccapture/ovxr_session_manager/interfaces.py b/src/python/isaaccapture/ovxr_session_manager/interfaces.py index 48783edbbe..c23de8e73e 100644 --- a/src/python/isaaccapture/ovxr_session_manager/interfaces.py +++ b/src/python/isaaccapture/ovxr_session_manager/interfaces.py @@ -5,11 +5,12 @@ from __future__ import annotations import math -from abc import abstractmethod +from abc import ABC, abstractmethod from collections.abc import Callable, Hashable from contextlib import AbstractContextManager from dataclasses import dataclass from typing import Any, Literal, Protocol +from .camera_mailbox import FrameOutputs class OvxrScenePublication(Protocol): @@ -93,6 +94,7 @@ class OvxrSessionBinding: renderer: object stage: object ordinal_ledger: StageOrdinalLedger + frame_consumers: tuple[OvxrFrameConsumer, ...] = () prepare_frame_scene: Callable[[], None] | None = None def __post_init__(self) -> None: @@ -104,7 +106,11 @@ def __post_init__(self) -> None: @dataclass class OvxrSessionConfig: - """Configure an application-owned renderer/Stage XR session.""" + """Application-neutral OVXR and CloudXR configuration. + + The supported OVXR package supplies bound-frame outputs. Optional PiP and + output consumers copy those outputs on the sole frame owner before retirement. + """ camera_paths: tuple[str, ...] application_name: str = "isaacteleop-ovxr-session" @@ -141,3 +147,43 @@ def __post_init__(self) -> None: ) if len(self.camera_paths) != 2 or any(not path for path in self.camera_paths): raise ValueError("OVXR stereo requires exactly two non-empty camera paths") + + +class OvxrFrameConsumer(ABC): + """Optional presentation on the OVXR owner thread. + + Outputs are borrowed until finish_frame, before submit/cancel. Composition + layers remain owned by the consumer through submit. on_openxr_ready may be + called repeatedly and must be idempotent. close must tolerate + partial startup and retain resources when retirement cannot be proved. + """ + + @property + def render_product_paths(self) -> tuple[str, ...]: + return () + + def on_session_started(self, binding, config, request_shutdown) -> None: + return None + + def on_openxr_ready(self, openxr, vulkan) -> bool: + """Bind borrowed XR resources; return False while required handles are absent.""" + return True + + @abstractmethod + def on_outputs(self, frame: FrameOutputs) -> None: + """Consume this completed frame without retaining its output descriptors.""" + raise NotImplementedError + + def composition_layers(self) -> tuple[int, ...]: + """Borrow native layer pointers valid through the following submission.""" + return () + + def on_frame_submitted(self) -> None: + return None + + def finish_frame(self) -> None: + return None + + @abstractmethod + def close(self) -> None: + raise NotImplementedError diff --git a/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py new file mode 100644 index 0000000000..f85e602547 --- /dev/null +++ b/src/python/isaaccapture/ovxr_session_manager/openxr_quad.py @@ -0,0 +1,225 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Application-owned camera panels on borrowed OpenXR/Vulkan resources.""" + +from __future__ import annotations + +import ctypes +import logging +from enum import IntEnum +from pathlib import Path + +from .camera_mailbox import CameraImage + +_LOGGER = logging.getLogger(__name__) + + +class _UploadResult(IntEnum): + """Integer C ABI shared with UploadResult in pip_layer.cpp.""" + + SUCCESS = 0 + INVALID_ARGUMENT = -1 + NOT_ATTACHED = -2 + CAMERA_UNAVAILABLE = -3 + SWAPCHAIN_FAILED = -4 + ALLOCATION_FAILED = -5 + UPLOAD_FAILED = -6 + COMPLETION_UNPROVED = -7 + CUDA_UNAVAILABLE = -8 + UPLOAD_MODE_MISMATCH = -9 + + +class OpenXRCameraPanels: + """One native owner for panel spaces, swapchains and upload completion.""" + + def __init__(self, panels: tuple, *, library_path: str | None = None) -> None: + library = ( + Path(library_path) + if library_path + else Path(__file__).parent / "native/libisaaccapture_camera_panels.so" + ) + self._native = ctypes.CDLL(str(library.resolve(strict=True))) + pointer = ctypes.c_void_p + uint = ctypes.c_uint32 + signatures = { + "create": ([], pointer), + "max_cameras": ([], uint), + "max_image_dimension": ([], uint), + "supports_cuda": ([], ctypes.c_int), + "attach": ([pointer] * 7 + [uint], ctypes.c_int), + "destroy": ([pointer], ctypes.c_int), + "wait_cuda": ([pointer, uint], ctypes.c_int), + "layer": ([pointer, uint], pointer), + "configure": ( + [ + pointer, + uint, + ctypes.c_int, + ctypes.c_int, + ctypes.POINTER(ctypes.c_float), + ctypes.POINTER(ctypes.c_float), + ], + ctypes.c_int, + ), + "publish_rgba": ( + [pointer, uint, pointer, uint, uint, ctypes.c_size_t], + ctypes.c_int, + ), + "publish_cuda_array": ( + [pointer, uint, pointer, uint, uint, ctypes.c_int], + ctypes.c_int, + ), + } + for name, (args, result) in signatures.items(): + function = getattr(self._native, f"isaacteleop_pip_{name}") + function.argtypes, function.restype = args, result + self._panels = tuple(panels) + limit = self._native.isaacteleop_pip_max_cameras() + if len(panels) > limit: + raise ValueError(f"native camera helper supports at most {limit} cameras") + self._max_dimension = self._native.isaacteleop_pip_max_image_dimension() + self.supports_gpu = bool(self._native.isaacteleop_pip_supports_cuda()) + self._visible = [False] * len(panels) + self._published_revision = [0] * len(panels) + self.last_errors = [None] * len(panels) + self._attached = False + self._pending_uploads = {} + self._retired_panels = set() + self._handle = self._native.isaacteleop_pip_create() + if not self._handle: + raise MemoryError("could not create camera panel owner") + + def attach(self, openxr, vulkan) -> None: + if self._attached: + return + if self.supports_gpu and not (vulkan.external_memory_handle_types & 1): + raise RuntimeError( + "camera GPU upload requires enabled opaque-FD external memory" + ) + result = self._native.isaacteleop_pip_attach( + self._handle, + openxr.xr_instance, + openxr.xr_session, + openxr.xr_get_instance_proc_addr, + vulkan.vk_physical_device, + vulkan.vk_device, + vulkan.vk_queue, + vulkan.queue_family_index, + ) + if result != 0: + # The owner also retains partial attachment until close succeeds. + raise RuntimeError(f"camera session binding failed: {result}") + self._attached = True + + def layer(self, index: int): + return self._native.isaacteleop_pip_layer(self._handle, index) + + def _set_panel(self, index: int, enabled: bool, width: int, height: int) -> None: + panel = self._panels[index] + pose = (ctypes.c_float * 7)(*panel.position, *panel.orientation_xyzw) + size = (ctypes.c_float * 2)( + panel.panel_width_m, panel.panel_width_m * height / width + ) + result = self._native.isaacteleop_pip_configure( + self._handle, index, int(enabled), 1, pose, size + ) + if result != 0: + raise RuntimeError(f"OpenXR camera quad configuration failed: {result}") + self._visible[index] = enabled + + def publish( + self, index: int, image: CameraImage | None, *, invalid: bool = False + ) -> bool: + if index in self._retired_panels: + return False + if invalid: + self.hide(index) + self.last_errors[index] = "completed camera output was invalid" + return False + if image is None or image.revision <= self._published_revision[index]: + return False + try: + if ( + not 0 < image.width <= self._max_dimension + or not 0 < image.height <= self._max_dimension + ): + raise ValueError("camera image exceeds native helper dimensions") + self._set_panel(index, True, image.width, image.height) + if image.gpu is not None: + gpu = image.gpu + if gpu.closed or gpu.busy or not gpu.is_array or not self.supports_gpu: + raise RuntimeError("camera GPU array is unavailable") + gpu.busy = True + result = self._native.isaacteleop_pip_publish_cuda_array( + self._handle, + index, + gpu.pointer, + image.width, + image.height, + gpu.device, + ) + # Keep the borrowed map until the native copy has completed. + if result != _UploadResult.COMPLETION_UNPROVED: + gpu.busy = False + else: + self._pending_uploads[index] = gpu + self._retired_panels.add(index) + _LOGGER.warning( + "Camera panel %s disabled after incomplete GPU copy", index + ) + else: + pixels = ctypes.create_string_buffer(image.rgba) + result = self._native.isaacteleop_pip_publish_rgba( + self._handle, + index, + pixels, + image.width, + image.height, + len(image.rgba), + ) + except Exception: + self.hide(index) + raise + if result != _UploadResult.SUCCESS: + self.hide(index) + self.last_errors[index] = ( + f"OpenXR camera quad upload failed: {_UploadResult(result).name} ({result})" + ) + return False + self._published_revision[index] = image.revision + self.last_errors[index] = None + return True + + def hide(self, index: int) -> None: + if self._visible[index]: + self._set_panel(index, False, 1, 1) + + def finish_frame(self) -> None: + first_error = None + for index, gpu in tuple(self._pending_uploads.items()): + result = self._native.isaacteleop_pip_wait_cuda(self._handle, index) + if result != _UploadResult.SUCCESS: + error = RuntimeError( + f"camera {index} GPU copy completion is unproved: " + f"{_UploadResult(result).name} ({result})" + ) + if first_error is None: + first_error = error + else: + first_error.add_note(str(error)) + continue + gpu.busy = False + del self._pending_uploads[index] + if first_error is not None: + raise first_error + + def close(self) -> None: + if self._handle: + result = self._native.isaacteleop_pip_destroy(self._handle) + if result != 0: + raise RuntimeError( + "camera GPU completion/cleanup is unproved; retain OVXR context" + ) + self._handle = None + self._attached = False diff --git a/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py b/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py index efd567ae44..eb574a4295 100644 --- a/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py +++ b/src/python/isaaccapture/ovxr_session_manager/ovxr_session.py @@ -17,6 +17,7 @@ StageOrdinalLedger as StageOrdinalLedger, OvxrSessionBinding as OvxrSessionBinding, OvxrSessionConfig as OvxrSessionConfig, + OvxrFrameConsumer as OvxrFrameConsumer, OvxrScenePublication as OvxrScenePublication, OvxrResourceOwner as OvxrResourceOwner, ) @@ -340,7 +341,7 @@ def request_shutdown(self) -> None: self._request_worker_shutdown() def wait_for_shutdown(self, timeout_s: float) -> bool: - """Wait interruptibly on the same event used by signals and mirror close.""" + """Wait interruptibly for a shutdown request.""" return self._shutdown_requested.wait(timeout_s) @property diff --git a/tests/cpp/core/CMakeLists.txt b/tests/cpp/core/CMakeLists.txt index 56c3b9100b..30980a5985 100644 --- a/tests/cpp/core/CMakeLists.txt +++ b/tests/cpp/core/CMakeLists.txt @@ -9,3 +9,7 @@ add_subdirectory(live_trackers) add_subdirectory(tracker_metadata) add_subdirectory(log_bridge) add_subdirectory(plugin_manager) + +if(BUILD_OVXR_CAMERA_QUADS) + add_subdirectory(ovxr_pip_layer) +endif() diff --git a/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt b/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt new file mode 100644 index 0000000000..64a2f5adfb --- /dev/null +++ b/tests/cpp/core/ovxr_pip_layer/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Keep white-box lifecycle checks without exposing a plugin's private source +# tree as an interface include path. COPYONLY tracks both authoritative inputs. +set(_pip_whitebox_dir "${CMAKE_CURRENT_BINARY_DIR}/whitebox") +file(MAKE_DIRECTORY "${_pip_whitebox_dir}") +configure_file( + "${PROJECT_SOURCE_DIR}/src/core/ovxr_pip_layer/cpp/pip_layer.cpp" + "${_pip_whitebox_dir}/pip_layer.cpp" COPYONLY) +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/pip_layer_contract_test.cpp" + "${_pip_whitebox_dir}/pip_layer_contract_test.cpp" COPYONLY) +add_executable(ovxr_pip_layer_contract_test "${_pip_whitebox_dir}/pip_layer_contract_test.cpp") +target_compile_features(ovxr_pip_layer_contract_test PRIVATE cxx_std_20) +target_include_directories(ovxr_pip_layer_contract_test PRIVATE + $) +target_link_libraries(ovxr_pip_layer_contract_test PRIVATE Vulkan::Vulkan log_bridge::log_bridge_core) +if(CUDAToolkit_FOUND AND CMAKE_SYSTEM_NAME STREQUAL "Linux") + target_compile_definitions(ovxr_pip_layer_contract_test PRIVATE ISAAC_TELEOP_PIP_CUDA) + target_link_libraries(ovxr_pip_layer_contract_test PRIVATE CUDA::cudart) +endif() +add_test(NAME ovxr_pip_layer_contract_test COMMAND ovxr_pip_layer_contract_test) diff --git a/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp new file mode 100644 index 0000000000..c819ad88ee --- /dev/null +++ b/tests/cpp/core/ovxr_pip_layer/pip_layer_contract_test.cpp @@ -0,0 +1,443 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Single translation unit so the application layer's private state is exercised +// against mocked OpenXR dispatch without a GPU or a second XR session. +// The package is built in Release; its contract checks must still execute. +#undef NDEBUG +#include "pip_layer.cpp" + +#include +#include +#include + +namespace +{ +int acquireCalls = 0; +int waitCalls = 0; +int releaseCalls = 0; +bool firstWaitTimeout = false; +bool releaseFails = false; +bool destroySwapchainFails = false; +bool destroySpaceFails = false; +int createSpaceCalls = 0; +int vkMapCalls = 0; +int vkBufferDestroys = 0; +int vkFenceWaits = 0; +bool vkFenceWaitFails = false; +bool vkIdleFails = false; +std::vector destroyOrder; +std::vector forwardedSwapchains; + +XrResult XRAPI_CALL acquire(XrSwapchain, const XrSwapchainImageAcquireInfo*, uint32_t* index) +{ + ++acquireCalls; + *index = 7; // Forces an exit after the swapchain state transition. + return XR_SUCCESS; +} +XrResult XRAPI_CALL wait(XrSwapchain, const XrSwapchainImageWaitInfo*) +{ + ++waitCalls; + return firstWaitTimeout && waitCalls == 1 ? XR_TIMEOUT_EXPIRED : XR_SUCCESS; +} +XrResult XRAPI_CALL release(XrSwapchain, const XrSwapchainImageReleaseInfo*) +{ + ++releaseCalls; + return releaseFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL destroySwapchain(XrSwapchain) +{ + destroyOrder.push_back(1); + return destroySwapchainFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL unavailableFormats(XrSession, uint32_t, uint32_t*, int64_t*) +{ + return XR_ERROR_RUNTIME_FAILURE; +} +int resizeCreates = 0; +bool imageEnumerationFails = true; +XrResult XRAPI_CALL availableFormats(XrSession, uint32_t capacity, uint32_t* count, int64_t* formats) +{ + *count = 1; + if (capacity) + formats[0] = VK_FORMAT_R8G8B8A8_UNORM; + return XR_SUCCESS; +} +XrResult XRAPI_CALL createResizeSwapchain(XrSession, const XrSwapchainCreateInfo*, XrSwapchain* swapchain) +{ + *swapchain = reinterpret_cast(uintptr_t{ 100 } + ++resizeCreates); + return XR_SUCCESS; +} +XrResult XRAPI_CALL resizeImages(XrSwapchain, uint32_t capacity, uint32_t* count, XrSwapchainImageBaseHeader* images) +{ + *count = 1; + if (capacity) + { + if (imageEnumerationFails) + return XR_ERROR_RUNTIME_FAILURE; + reinterpret_cast(images)[0].image = VkImage(99); + } + return XR_SUCCESS; +} +XrResult XRAPI_CALL destroySpace(XrSpace) +{ + destroyOrder.push_back(2); + return destroySpaceFails ? XR_ERROR_RUNTIME_FAILURE : XR_SUCCESS; +} +XrResult XRAPI_CALL createSpace(XrSession, const XrReferenceSpaceCreateInfo*, XrSpace* space) +{ + if (++createSpaceCalls == 2) + return XR_ERROR_RUNTIME_FAILURE; + *space = reinterpret_cast(uintptr_t{ 30 }); + return XR_SUCCESS; +} +XrResult XRAPI_CALL testGipa(XrInstance, const char* name, PFN_xrVoidFunction* function) +{ + const std::pair functions[] = { + { "xrCreateReferenceSpace", reinterpret_cast(createSpace) }, + { "xrDestroySpace", reinterpret_cast(destroySpace) }, + { "xrEnumerateSwapchainFormats", reinterpret_cast(availableFormats) }, + { "xrCreateSwapchain", reinterpret_cast(createResizeSwapchain) }, + { "xrDestroySwapchain", reinterpret_cast(destroySwapchain) }, + { "xrEnumerateSwapchainImages", reinterpret_cast(resizeImages) }, + { "xrAcquireSwapchainImage", reinterpret_cast(acquire) }, + { "xrWaitSwapchainImage", reinterpret_cast(wait) }, + { "xrReleaseSwapchainImage", reinterpret_cast(release) }, + }; + for (const auto& [entry, address] : functions) + { + if (entry == name) + { + *function = address; + return XR_SUCCESS; + } + } + return XR_ERROR_FUNCTION_UNSUPPORTED; +} +} // namespace + +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateBuffer(VkDevice, + const VkBufferCreateInfo*, + const VkAllocationCallbacks*, + VkBuffer* buffer) +{ + *buffer = VkBuffer(11); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkGetBufferMemoryRequirements(VkDevice, VkBuffer, VkMemoryRequirements* req) +{ + req->size = 64; + req->memoryTypeBits = 1; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkGetPhysicalDeviceMemoryProperties(VkPhysicalDevice, + VkPhysicalDeviceMemoryProperties* props) +{ + props->memoryTypeCount = 1; + props->memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkAllocateMemory(VkDevice, + const VkMemoryAllocateInfo*, + const VkAllocationCallbacks*, + VkDeviceMemory* memory) +{ + *memory = VkDeviceMemory(12); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkBindBufferMemory(VkDevice, VkBuffer, VkDeviceMemory, VkDeviceSize) +{ + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL +vkMapMemory(VkDevice, VkDeviceMemory, VkDeviceSize, VkDeviceSize, VkMemoryMapFlags, void** data) +{ + static uint32_t bytes = 0; + ++vkMapCalls; + if (vkMapCalls == 1) + { + return VK_ERROR_MEMORY_MAP_FAILED; + } + *data = &bytes; + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateCommandPool(VkDevice, + const VkCommandPoolCreateInfo*, + const VkAllocationCallbacks*, + VkCommandPool* pool) +{ + *pool = VkCommandPool(13); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkAllocateCommandBuffers(VkDevice, + const VkCommandBufferAllocateInfo*, + VkCommandBuffer* command) +{ + *command = VkCommandBuffer(14); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkCreateFence(VkDevice, + const VkFenceCreateInfo*, + const VkAllocationCallbacks*, + VkFence* fence) +{ + *fence = VkFence(15); + return VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkWaitForFences(VkDevice, uint32_t, const VkFence*, VkBool32, uint64_t) +{ + ++vkFenceWaits; + return vkFenceWaitFails ? VK_TIMEOUT : VK_SUCCESS; +} +extern "C" VKAPI_ATTR VkResult VKAPI_CALL vkDeviceWaitIdle(VkDevice) +{ + return vkIdleFails ? VK_TIMEOUT : VK_SUCCESS; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyCommandPool(VkDevice, VkCommandPool, const VkAllocationCallbacks*) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkUnmapMemory(VkDevice, VkDeviceMemory) +{ +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkDestroyBuffer(VkDevice, VkBuffer, const VkAllocationCallbacks*) +{ + ++vkBufferDestroys; +} +extern "C" VKAPI_ATTR void VKAPI_CALL vkFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) +{ +} + +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) +namespace +{ +int sameDeviceCopies = 0; +int peerCopies = 0; +cudaError_t copyResult = cudaSuccess; +cudaError_t syncResult = cudaSuccess; +int cudaSyncCalls = 0; +int currentCudaDevice = -1; +cudaStream_t lastCudaStream = nullptr; +cudaMemcpy3DPeerParms peerCopy{}; +} +extern "C" cudaError_t CUDARTAPI cudaMemcpy2DFromArrayAsync( + void*, size_t pitch, cudaArray_const_t, size_t, size_t, size_t width, size_t height, cudaMemcpyKind kind, cudaStream_t) +{ + ++sameDeviceCopies; + assert(pitch == 640 * 4 && width == pitch && height == 480 && kind == cudaMemcpyDeviceToDevice); + return copyResult; +} +extern "C" cudaError_t CUDARTAPI cudaMemcpy3DPeerAsync(const cudaMemcpy3DPeerParms* parameters, cudaStream_t) +{ + ++peerCopies; + peerCopy = *parameters; + return copyResult; +} +extern "C" cudaError_t CUDARTAPI cudaSetDevice(int device) +{ + currentCudaDevice = device; + return cudaSuccess; +} +extern "C" cudaError_t CUDARTAPI cudaStreamSynchronize(cudaStream_t stream) +{ + lastCudaStream = stream; + ++cudaSyncCalls; + return syncResult; +} +#endif + +int main() +{ +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + CameraSession copyCamera{}; + copyCamera.cudaDevice = 0; + copyCamera.cudaPixels = reinterpret_cast(uintptr_t{ 42 }); + const void* sourceArray = reinterpret_cast(uintptr_t{ 43 }); + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 0) == cudaSuccess); + assert(sameDeviceCopies == 1 && peerCopies == 0); + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 1) == cudaSuccess); + assert(sameDeviceCopies == 1 && peerCopies == 1); + assert(peerCopy.srcDevice == 1 && peerCopy.dstDevice == 0); + assert(peerCopy.srcArray == sourceArray && peerCopy.dstPtr.ptr == copyCamera.cudaPixels); + assert(peerCopy.extent.width == 640 && peerCopy.extent.height == 480 && peerCopy.extent.depth == 1); + assert(peerCopy.dstPtr.pitch == 640 * 4 && peerCopy.dstPtr.xsize == 640 * 4); + copyResult = cudaErrorInvalidDevice; + assert(copyCameraArray(copyCamera, sourceArray, 640, 480, 1) == cudaErrorInvalidDevice); + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == UploadResult::UploadFailed); + assert(cudaSyncCalls == 1 && !copyCamera.cudaPending && !copyCamera.retired); + syncResult = cudaErrorUnknown; + assert(copyCameraArrayAndWait(copyCamera, sourceArray, 640, 480, 1) == UploadResult::CompletionUnproved); + assert(cudaSyncCalls == 2 && copyCamera.cudaPending && copyCamera.retired); + PanelSession pendingOwner{}; + pendingOwner.cameras[1] = copyCamera; + pendingOwner.cameras[1].cudaStream = reinterpret_cast(uintptr_t{ 91 }); + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == UploadResult::CompletionUnproved); + assert(pendingOwner.cameras[1].cudaPending); + assert(currentCudaDevice == copyCamera.cudaDevice); + assert(lastCudaStream == pendingOwner.cameras[1].cudaStream); + copyResult = syncResult = cudaSuccess; + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == 0); + assert(!pendingOwner.cameras[1].cudaPending && pendingOwner.cameras[1].retired); + const int completedSyncCalls = cudaSyncCalls; + assert(isaacteleop_pip_wait_cuda(&pendingOwner, 1) == 0); + assert(cudaSyncCalls == completedSyncCalls); + assert(isaacteleop_pip_wait_cuda(nullptr, 0) == UploadResult::InvalidArgument); + assert(isaacteleop_pip_wait_cuda(&pendingOwner, kMaxCameras) == UploadResult::InvalidArgument); +#endif + const auto instance = reinterpret_cast(uintptr_t{ 1 }); + const auto session = reinterpret_cast(uintptr_t{ 2 }); + Dispatch dispatch{}; + dispatch.acquire = acquire; + dispatch.wait = wait; + dispatch.release = release; + dispatch.destroySwapchain = destroySwapchain; + dispatch.destroySpace = destroySpace; + dispatch.formats = unavailableFormats; + auto* owner = isaacteleop_pip_create(); + assert(owner); + owner->dispatch = dispatch; + owner->attached = true; + CameraSession camera{}; + camera.instance = instance; + camera.session = session; + camera.view = reinterpret_cast(uintptr_t{ 3 }); + camera.local = reinterpret_cast(uintptr_t{ 4 }); + camera.swapchain = reinterpret_cast(uintptr_t{ 5 }); + camera.width = camera.height = 1; + camera.ready = true; + // Resize cannot forget an old swapchain whose destruction failed. No new + // GPU allocation is needed to exercise this native ownership boundary. + CameraSession resizing = camera; + destroySwapchainFails = true; + const auto previousSwapchain = resizing.swapchain; + assert(!createSwapchain(resizing, dispatch, 2, 2)); + assert(resizing.swapchain == previousSwapchain && resizing.retired && !resizing.ready); + destroySwapchainFails = false; + // A partially created resized swapchain must not advertise the old extent. + // Return to the old size after failed enumeration: it still must recreate. + resizing = camera; + Dispatch resizeDispatch = dispatch; + resizeDispatch.formats = availableFormats; + resizeDispatch.createSwapchain = createResizeSwapchain; + resizeDispatch.images = resizeImages; + assert(!createSwapchain(resizing, resizeDispatch, 2, 2)); + assert(resizing.width == 0 && resizing.height == 0 && !resizing.ready); + imageEnumerationFails = false; + assert(createSwapchain(resizing, resizeDispatch, 1, 1)); + assert(resizeCreates == 2 && resizing.width == 1 && resizing.images[0].image == VkImage(99)); + assert(releaseSessionResources(resizing, resizeDispatch)); + destroyOrder.clear(); + owner->cameras[0] = camera; + + assert(isaacteleop_pip_attach(owner, instance, session, nullptr, {}, {}, {}, 0) == -1); + const auto* layer = isaacteleop_pip_layer(owner, 0); + assert(layer && layer->type == XR_TYPE_COMPOSITION_LAYER_QUAD && layer->layerFlags == 0); + assert(isaacteleop_pip_layer(owner, 1) == nullptr); + assert(isaacteleop_pip_layer(owner, 3) == nullptr); + const auto* quad = reinterpret_cast(layer); + assert(quad->subImage.swapchain == camera.swapchain && quad->eyeVisibility == XR_EYE_VISIBILITY_BOTH); + + // All three descriptors stay alive through the caller's submit, in caller order. + for (size_t i = 1; i < kMaxCameras; ++i) + { + owner->cameras[i] = camera; + owner->cameras[i].swapchain = reinterpret_cast(uintptr_t{ 5 } + i); + owner->cameras[i].view = reinterpret_cast(uintptr_t{ 30 } + i); + owner->cameras[i].local = XR_NULL_HANDLE; + } + const auto* second = isaacteleop_pip_layer(owner, 1); + const auto* third = isaacteleop_pip_layer(owner, 2); + assert(second && third && second != layer && second != third); + assert(quad->subImage.swapchain == camera.swapchain); + owner->panels[1].enabled = false; + assert(isaacteleop_pip_layer(owner, 1) == nullptr); + assert(isaacteleop_pip_layer(owner, 0) == layer && isaacteleop_pip_layer(owner, 2) == third); + + auto& owned = owner->cameras[0]; + uint32_t rgba = 0; + owned.pixels = &rgba; +#if defined(ISAAC_TELEOP_PIP_CUDA) && defined(__linux__) + owned.device = VkDevice(24); + owned.cudaDevice = 1; + owned.queue = VkQueue(25); + owned.width = owned.height = 1; + assert(isaacteleop_pip_publish_rgba(owner, 0, &rgba, 1, 1, 4) == UploadResult::UploadModeMismatch); + assert(owned.pixels == &rgba); // Reject without overwriting any resource. + owned.cudaDevice = -1; + owned.device = VK_NULL_HANDLE; +#endif + firstWaitTimeout = true; + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 1 && waitCalls == 1 && releaseCalls == 0); + assert(owned.acquired && !owned.waited && !owned.ready); + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 1 && waitCalls == 2 && releaseCalls == 1); + assert(!owned.acquired && !owned.ready); + + // Model a fresh session for the independent release-failure cell. + owned.retired = false; + releaseFails = true; + assert(!upload(owned, dispatch, &rgba)); + assert(acquireCalls == 2 && releaseCalls == 2); + assert(owned.retired && !owned.ready); + + CameraSession partial{}; + partial.device = VkDevice(21); + partial.physical = VkPhysicalDevice(22); + partial.width = partial.height = 1; + assert(!createUploadResources(partial)); + assert(!partial.staging && !partial.memory && !partial.pixels); + assert(vkBufferDestroys == 1); + assert(createUploadResources(partial)); + assert(partial.staging && partial.memory && partial.pixels && partial.pool && partial.fence); + partial.fencePending = true; + vkFenceWaitFails = true; + vkIdleFails = true; + assert(!releaseImageResources(partial)); + assert(partial.staging && partial.memory && partial.fence); + assert(vkBufferDestroys == 1); + vkIdleFails = false; + assert(!releaseImageResources(partial)); // Another queue's idle is not our completion contract. + assert(partial.staging && partial.memory && partial.fence); + vkFenceWaitFails = false; + assert(releaseImageResources(partial)); + assert(!partial.staging && !partial.memory && !partial.fence); + assert(vkBufferDestroys == 2 && vkFenceWaits == 3); + + // This is application-owned storage in this test, not Vulkan storage. + owned.pixels = nullptr; + const int waitsBeforeClose = waitCalls; + const int releasesBeforeClose = releaseCalls; + destroySwapchainFails = true; + assert(isaacteleop_pip_destroy(owner) == -1); + assert(owner->attached); // Failed destruction still retains the owner. + destroySwapchainFails = false; + assert(isaacteleop_pip_destroy(owner) == 0); + // No application work remains. Destroying an acquired swapchain does not + // require waiting for a runtime image that may never become available. + assert(waitCalls == waitsBeforeClose && releaseCalls == releasesBeforeClose); + owner = nullptr; + assert((destroyOrder == std::vector{ 1, 1, 2, 2, 1, 2, 1, 2 })); + assert(isaacteleop_pip_layer(owner, 0) == nullptr); + assert(isaacteleop_pip_destroy(owner) == 0); + // Independent owners do not share panel policy. + owner = isaacteleop_pip_create(); + auto* other = isaacteleop_pip_create(); + owner->panels[0].enabled = false; + assert(other->panels[0].enabled); + assert(isaacteleop_pip_max_cameras() == owner->cameras.size()); + assert(isaacteleop_pip_max_image_dimension() > 0); + assert(isaacteleop_pip_destroy(other) == 0); + // Partial reference-space creation cannot lose the only cleanup owner. + destroySpaceFails = true; + assert(isaacteleop_pip_attach(owner, instance, session, testGipa, VkPhysicalDevice(1), VkDevice(2), VkQueue(3), 0) == + -3); + assert(owner->attached); + assert(owner->cameras[0].view); + assert(isaacteleop_pip_destroy(owner) == -1); + assert(owner->cameras[0].view); + destroySpaceFails = false; + assert(isaacteleop_pip_destroy(owner) == 0); + owner = nullptr; + std::puts("pip layer contract: PASS"); +} diff --git a/tests/python/core/ovxr_session_manager/test_camera_mailbox.py b/tests/python/core/ovxr_session_manager/test_camera_mailbox.py new file mode 100644 index 0000000000..7fa006ab92 --- /dev/null +++ b/tests/python/core/ovxr_session_manager/test_camera_mailbox.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pure callback lifetime and output-selection tests for the borrowed camera step.""" + +import ctypes +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from isaaccapture.ovxr_session_manager import camera_mailbox +from isaaccapture.ovxr_session_manager.camera_mailbox import FrameOutputs + +from isaaccapture.ovxr_session_manager.camera_mailbox import CompletedCameraMailbox + + +def test_cpu_unmap_failure_retains_handle_and_blocks_more_maps(monkeypatch): + """Exercise the real CPU copier; failed unmap must not lose its native lease.""" + b = pytest.importorskip("ovrtx._src.bindings") + from isaaccapture.ovxr_session_manager import gpu_output + + gpu_output._api.cache_clear() + + def map_output(_renderer, _handle, _description, _timeout, output): + mapped = ctypes.cast( + output, ctypes.POINTER(b.ovrtx_render_var_output_t) + ).contents + mapped.map_handle = 71 + mapped.num_tensors = 0 # Validation fails after a successful native map. + mapped.status = b.EventStatus.COMPLETED + return SimpleNamespace(status=b.OVRTX_API_SUCCESS) + + native = SimpleNamespace( + ovrtx_map_render_var_output=Mock(side_effect=map_output), + ovrtx_unmap_render_var_output=Mock( + side_effect=[ + SimpleNamespace(status=1), + SimpleNamespace(status=b.OVRTX_API_SUCCESS), + ] + ), + ovrtx_fetch_results=Mock(), + ) + monkeypatch.setattr(b, "ovrtx_loaded_bin_dir", lambda: "/declared/provider") + monkeypatch.setattr(ctypes, "CDLL", lambda _path: native) + try: + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is None and mailbox.error() is not None + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(2.0, 2)))) + assert native.ovrtx_map_render_var_output.call_count == 1 + mailbox.close() + assert native.ovrtx_unmap_render_var_output.call_count == 2 + assert [ + call.args[1] for call in native.ovrtx_unmap_render_var_output.call_args_list + ] == [71, 71] + mailbox.close() + assert native.ovrtx_unmap_render_var_output.call_count == 2 + finally: + gpu_output._api.cache_clear() + + +def _string(text): + encoded = text.encode() + return SimpleNamespace(ptr=ctypes.c_char_p(encoded), length=len(encoded)) + + +def _frame(time, handle, *, color=True): + vars_ = [ + SimpleNamespace( + source_name=_string("LdrColor" if color else "XRDepth"), + output_handle=handle, + ) + ] + return SimpleNamespace( + frame_end_time=time, render_var_count=len(vars_), output_render_vars=vars_ + ) + + +def _outputs(*frames): + product = SimpleNamespace( + render_product_path=_string("/Camera"), + output_frame_count=len(frames), + output_frames=list(frames), + ) + return SimpleNamespace(contents=SimpleNamespace(output_count=1, outputs=[product])) + + +def test_newest_completed_image_and_zero_output_retention(monkeypatch): + seen = [] + + def copy(_renderer, handle, _limit): + seen.append(handle) + return 1, 1, bytes((handle, 2, 3, 255)) + + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", copy) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(3.0, 3), _frame(4.0, 4)))) + image = mailbox.snapshot() + assert image is not None + assert (image.sensor_time, image.revision, image.rgba) == ( + 4.0, + 1, + b"\x04\x02\x03\xff", + ) + mailbox.on_outputs(FrameOutputs(1, _outputs())) + assert mailbox.snapshot() is image + assert seen == [4] + + +def test_produced_but_invalid_image_is_omitted_and_error_is_contained(monkeypatch): + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", lambda *_: (1, 1, b"rgba")) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is not None + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(2.0, 2, color=False)))) + assert mailbox.snapshot() is None + assert "without LdrColor" in mailbox.error() + + +def test_mapping_failure_is_contained_without_reusing_stale_pixels(monkeypatch): + def fail(*_args): + raise RuntimeError("map failed") + + monkeypatch.setattr(camera_mailbox, "_copy_host_rgba", fail) + mailbox = CompletedCameraMailbox("/Camera", 4) + mailbox.on_outputs(FrameOutputs(1, _outputs(_frame(1.0, 1)))) + assert mailbox.snapshot() is None + assert mailbox.error() == "map failed" + + +def test_gpu_camera_retains_cadence_gaps_and_retires_on_owner(): + import threading + + class FakeMap: + width = height = 1 + + def __init__(self, handle): + self.pointer = handle + self.closed = False + self.owner = threading.get_ident() + + def close(self): + assert threading.get_ident() == self.owner + assert not self.closed + self.closed = True + + def outputs(**handles): + return _outputs(*[_frame(float(h), h) for h in handles.values()]) + + made = [] + + def mapper(_renderer, handle, _limit): + image = FakeMap(handle) + made.append(image) + return image + + mailbox = CompletedCameraMailbox("/Render/Left", 64, gpu_mapper=mapper) + mailbox.product_path = "/Camera" + mailbox.on_outputs(FrameOutputs(1, outputs(Left=1))) + first = mailbox.snapshot() + assert first.gpu is made[0] and first.rgba is None + mailbox.on_outputs(FrameOutputs(1, outputs())) + assert mailbox.snapshot() is first and not made[0].closed + mailbox.on_outputs(FrameOutputs(1, outputs(Left=2))) + assert made[0].closed and mailbox.snapshot().gpu is made[1] + mailbox.close() + assert made[1].closed diff --git a/tests/python/core/ovxr_session_manager/test_openxr_quad.py b/tests/python/core/ovxr_session_manager/test_openxr_quad.py new file mode 100644 index 0000000000..a149bf3238 --- /dev/null +++ b/tests/python/core/ovxr_session_manager/test_openxr_quad.py @@ -0,0 +1,300 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import os +from types import SimpleNamespace + +import pytest + +from isaaccapture.ovxr_session_manager.camera_mailbox import CameraImage +from isaaccapture.ovxr_session_manager.camera_panels import CameraPanelConfig +import isaaccapture.ovxr_session_manager.openxr_quad as quad_module +from isaaccapture.ovxr_session_manager.openxr_quad import _UploadResult + + +class _Function: + def __init__(self): + self.calls = [] + self.result = 0 + + def __call__(self, *args): + self.calls.append(args) + return self.result + + +def _quad(tmp_path, monkeypatch, *, panels=None, supports_gpu=False): + library = tmp_path / "layer.so" + library.touch() + native = SimpleNamespace( + isaacteleop_pip_create=_Function(), + isaacteleop_pip_max_cameras=_Function(), + isaacteleop_pip_max_image_dimension=_Function(), + isaacteleop_pip_attach=_Function(), + isaacteleop_pip_destroy=_Function(), + isaacteleop_pip_wait_cuda=_Function(), + isaacteleop_pip_layer=_Function(), + isaacteleop_pip_configure=_Function(), + isaacteleop_pip_publish_rgba=_Function(), + isaacteleop_pip_supports_cuda=_Function(), + isaacteleop_pip_publish_cuda_array=_Function(), + ) + monkeypatch.setattr(quad_module.ctypes, "CDLL", lambda _path: native) + native.isaacteleop_pip_supports_cuda.result = int(supports_gpu) + native.isaacteleop_pip_create.result = 123 + native.isaacteleop_pip_max_cameras.result = 3 + native.isaacteleop_pip_max_image_dimension.result = 4096 + return quad_module.OpenXRCameraPanels( + panels or (CameraPanelConfig("/Camera/Head"),), library_path=str(library) + ), native + + +def test_default_config_is_accepted_by_native_quad(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + assert quad.publish(0, CameraImage(1, 1, bytes(4), 1.0, 1)) + assert native.isaacteleop_pip_configure.calls[-1][3] == 1 + + +def test_head_locked_image_aspect_and_invalid_frame_hides_quad( + tmp_path, + monkeypatch, +): + quad, native = _quad( + tmp_path, + monkeypatch, + panels=(CameraPanelConfig("/Camera/Head", panel_width_m=0.5),), + ) + with pytest.raises(ValueError, match="head_locked"): + CameraPanelConfig("/Camera/Head", space="viewer_start") + image = CameraImage(640, 480, bytes(640 * 480 * 4), 1.0, 1) + assert quad.publish(0, image) + _handle, index, enabled, head_locked, _pose, size = ( + native.isaacteleop_pip_configure.calls[-1] + ) + assert (index, enabled, head_locked) == (0, 1, 1) + assert size[0] == pytest.approx(0.5) + assert size[1] == pytest.approx(0.375) + assert not quad.publish(0, None, invalid=True) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + assert quad.last_errors[0] is not None + + +@pytest.mark.parametrize("result", [-1, -2, -3]) +def test_failed_binding_retains_only_its_partial_cleanup_owner( + tmp_path, monkeypatch, result +): + quad, native = _quad(tmp_path, monkeypatch) + native.isaacteleop_pip_attach.result = result + with pytest.raises(RuntimeError, match="binding failed"): + quad.attach( + SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3), + SimpleNamespace( + vk_physical_device=4, vk_device=5, vk_queue=6, queue_family_index=0 + ), + ) + quad.close() + assert len(native.isaacteleop_pip_destroy.calls) == 1 + + +def test_upload_failure_hides_last_valid_panel(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + native.isaacteleop_pip_publish_rgba.result = _UploadResult.UPLOAD_FAILED + assert not quad.publish(0, CameraImage(1, 1, b"\0" * 4, 1.0, 1)) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + + +def test_pixel_buffer_failure_hides_last_valid_panel(tmp_path, monkeypatch): + quad, native = _quad(tmp_path, monkeypatch) + assert quad.publish(0, CameraImage(1, 1, b"\0" * 4, 1.0, 1)) + monkeypatch.setattr( + quad_module.ctypes, + "create_string_buffer", + lambda _pixels: (_ for _ in ()).throw(MemoryError()), + ) + with pytest.raises(MemoryError): + quad.publish(0, CameraImage(1, 1, b"\0" * 4, 2.0, 2)) + assert native.isaacteleop_pip_configure.calls[-1][2] == 0 + + +def test_explicit_binding_preserves_loader_and_retains_failed_cleanup( + tmp_path, monkeypatch +): + quad, native = _quad(tmp_path, monkeypatch) + monkeypatch.setenv("XR_ENABLE_API_LAYERS", "XR_APILAYER_OTHER") + monkeypatch.delenv("XR_API_LAYER_PATH", raising=False) + xr = SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3) + vk = SimpleNamespace( + vk_physical_device=4, vk_device=5, vk_queue=6, queue_family_index=7 + ) + quad.attach(xr, vk) + assert native.isaacteleop_pip_attach.calls == [(123, 1, 2, 3, 4, 5, 6, 7)] + assert os.environ["XR_ENABLE_API_LAYERS"] == "XR_APILAYER_OTHER" + assert "XR_API_LAYER_PATH" not in os.environ + native.isaacteleop_pip_destroy.result = -1 + with pytest.raises(RuntimeError, match="retain OVXR"): + quad.close() + assert quad._attached + native.isaacteleop_pip_destroy.result = 0 + quad.close() + assert not quad._attached + + +def test_three_panels_keep_independent_revisions(tmp_path, monkeypatch): + owner, native = _quad( + tmp_path, + monkeypatch, + panels=tuple(CameraPanelConfig(f"/Camera/{i}") for i in range(3)), + ) + for index in range(3): + assert owner.publish(index, CameraImage(1, 1, b"abcd", 1.0, 1)) + assert [call[1] for call in native.isaacteleop_pip_publish_rgba.calls] == [0, 1, 2] + owner.publish(1, None, invalid=True) + assert native.isaacteleop_pip_configure.calls[-1][1:3] == (1, 0) + assert not owner.publish(0, CameraImage(1, 1, b"abcd", 1.0, 1)) + assert owner.publish(2, CameraImage(1, 1, b"efgh", 2.0, 2)) + + +def test_old_native_layer_is_rejected_before_upload(tmp_path, monkeypatch): + _quad(tmp_path, monkeypatch) + monkeypatch.setattr(quad_module.ctypes, "CDLL", lambda _path: SimpleNamespace()) + with pytest.raises(AttributeError, match="isaacteleop_pip_create"): + quad_module.OpenXRCameraPanels( + (CameraPanelConfig("/Camera/Head"),), + library_path=str(tmp_path / "layer.so"), + ) + + +@pytest.mark.parametrize( + "result", [0, _UploadResult.UPLOAD_FAILED, _UploadResult.COMPLETION_UNPROVED] +) +def test_gpu_quad_completion_controls_source_retirement(tmp_path, monkeypatch, result): + quad, native = _quad(tmp_path, monkeypatch) + quad.supports_gpu = True + gpu = SimpleNamespace( + closed=False, busy=False, is_array=True, pointer=123, device=1 + ) + image = CameraImage(720, 450, None, 1.0, 1, gpu) + native.isaacteleop_pip_publish_cuda_array.result = result + assert quad.publish(0, image) == (result == 0) + assert gpu.busy == (result == _UploadResult.COMPLETION_UNPROVED) + assert len(native.isaacteleop_pip_publish_cuda_array.calls) == 1 + assert native.isaacteleop_pip_publish_rgba.calls == [] + + +@pytest.mark.parametrize("wait_result", [0, _UploadResult.COMPLETION_UNPROVED]) +def test_failed_gpu_copy_finishes_before_source_retirement( + tmp_path, monkeypatch, wait_result +): + quad, native = _quad(tmp_path, monkeypatch) + quad.supports_gpu = True + gpu = SimpleNamespace( + closed=False, busy=False, is_array=True, pointer=123, device=1 + ) + image = CameraImage(720, 450, None, 1.0, 1, gpu) + native.isaacteleop_pip_publish_cuda_array.result = _UploadResult.COMPLETION_UNPROVED + assert not quad.publish(0, image) + native.isaacteleop_pip_wait_cuda.result = wait_result + if wait_result: + with pytest.raises(RuntimeError, match="copy completion is unproved"): + quad.finish_frame() + assert gpu.busy + native.isaacteleop_pip_wait_cuda.result = 0 + quad.finish_frame() + assert not gpu.busy + # Retirement disables this panel; a later frame must not retry its copy. + assert not quad.publish(0, CameraImage(720, 450, None, 2.0, 2, gpu)) + assert len(native.isaacteleop_pip_publish_cuda_array.calls) == 1 + quad.close() + + +@pytest.mark.parametrize("completion_recovers", [True, False]) +def test_incomplete_panel_copy_retires_only_safe_mailboxes( + tmp_path, monkeypatch, completion_recovers +): + """Drive real panel/mailbox owners, mocking only native output and copy APIs.""" + import ctypes + import threading + + from isaaccapture.ovxr_session_manager import camera_panels, gpu_output + from isaaccapture.ovxr_session_manager.camera_mailbox import FrameOutputs + + panels = tuple(CameraPanelConfig(f"/Camera/{i}") for i in range(2)) + quad, native = _quad(tmp_path, monkeypatch, panels=panels, supports_gpu=True) + monkeypatch.setattr(camera_panels, "OpenXRCameraPanels", lambda _: quad) + unmapped = [] + bindings = SimpleNamespace(OVRTX_API_SUCCESS=0, ovrtx_cuda_sync_t=lambda: None) + api = SimpleNamespace( + ovrtx_unmap_render_var_output=lambda _renderer, handle, _sync: ( + unmapped.append(handle) or SimpleNamespace(status=0) + ) + ) + monkeypatch.setattr(gpu_output, "_api", lambda: (bindings, api)) + + def map_image(renderer, handle, _limit): + return gpu_output.MappedGpuImage( + width=1, + height=1, + pointer=handle, + pitch=4, + device=0, + is_array=True, + renderer=renderer, + mapped=SimpleNamespace(map_handle=handle), + owner=threading.get_ident(), + ) + + monkeypatch.setattr(camera_panels, "map_cuda_array_rgba", map_image) + consumer = camera_panels.CameraPanelConsumer(panels) + consumer.on_session_started(None, None, None) + consumer.on_openxr_ready( + SimpleNamespace(xr_instance=1, xr_session=2, xr_get_instance_proc_addr=3), + SimpleNamespace( + vk_physical_device=4, + vk_device=5, + vk_queue=6, + queue_family_index=0, + external_memory_handle_types=1, + ), + ) + + def text(value): + return SimpleNamespace(ptr=ctypes.c_char_p(value), length=len(value)) + + outputs = SimpleNamespace( + contents=SimpleNamespace( + output_count=2, + outputs=[ + SimpleNamespace( + render_product_path=text(panel.render_product_path.encode()), + output_frame_count=1, + output_frames=[ + SimpleNamespace( + frame_end_time=1.0, + render_var_count=1, + output_render_vars=[ + SimpleNamespace( + source_name=text(b"LdrColor"), output_handle=i + 1 + ) + ], + ) + ], + ) + for i, panel in enumerate(panels) + ], + ) + ) + native.isaacteleop_pip_publish_cuda_array = lambda _owner, index, *_: ( + _UploadResult.COMPLETION_UNPROVED if index == 0 else 0 + ) + native.isaacteleop_pip_wait_cuda.result = ( + 0 if completion_recovers else _UploadResult.COMPLETION_UNPROVED + ) + consumer.on_outputs(FrameOutputs(1, outputs)) + if not completion_recovers: + with pytest.raises(RuntimeError, match="completion is unproved"): + consumer.close() + assert unmapped == [2] + assert not native.isaacteleop_pip_destroy.calls + native.isaacteleop_pip_wait_cuda.result = 0 + consumer.close() + assert sorted(unmapped) == [1, 2] + assert len(native.isaacteleop_pip_destroy.calls) == 1 diff --git a/tests/python/core/ovxr_session_manager/test_ovxr_session.py b/tests/python/core/ovxr_session_manager/test_ovxr_session.py index 9c79e130e3..b74fc6a5e1 100644 --- a/tests/python/core/ovxr_session_manager/test_ovxr_session.py +++ b/tests/python/core/ovxr_session_manager/test_ovxr_session.py @@ -4,12 +4,21 @@ """Focused ownership and lifecycle tests for external OVXR presentation.""" from __future__ import annotations + import threading import time from contextlib import contextmanager +from dataclasses import replace from types import ModuleType, SimpleNamespace from typing import ClassVar + import pytest + +from isaaccapture.ovxr_session_manager import camera_panels +from isaaccapture.ovxr_session_manager.camera_panels import ( + CameraPanelConfig, + CameraPanelConsumer, +) import isaaccapture.ovxr_session_manager.ovxr_session as ovxr_session from isaaccapture.ovxr_session_manager import frame_pump, runtime_owner @@ -183,6 +192,7 @@ def __init__(self): self.events = [] self.publication_closed = False self.owner_lane_entries = [] + self.frame_consumers = () def offer_render_owner_transfer(self): self.offer_thread = threading.get_native_id() @@ -193,7 +203,10 @@ def attach_on_render_thread(self, _capability): self.events.append("attach") _FakeContext.ordinal_ledger = self.ordinal_ledger return ovxr_session.OvxrSessionBinding( - self.renderer, self.stage, self.ordinal_ledger + self.renderer, + self.stage, + self.ordinal_ledger, + frame_consumers=self.frame_consumers, ) @contextmanager @@ -239,6 +252,11 @@ def abort(self): self.events.append("abort") +def _session_with_consumers(host, config, *, frame_consumers): + host.frame_consumers = frame_consumers + return ovxr_session.OvxrSession(host, config) + + @pytest.fixture(autouse=True) def _runtime_stubs(monkeypatch): _FakeContext.operations = [] @@ -255,6 +273,18 @@ def _runtime_stubs(monkeypatch): _FakeContext.last_desc = None _FakeContext.selected_products = None monkeypatch.setattr(frame_pump, "_import_ovxr", _fake_ovxr) + monkeypatch.setattr( + camera_panels, + "OpenXRCameraPanels", + lambda panels: SimpleNamespace( + supports_gpu=False, + finish_frame=lambda: None, + attach=lambda _xr, _vk: None, + layer=lambda _index: None, + publish=lambda _index, _image, **kw: True, + close=lambda: None, + ), + ) oxr = ModuleType("isaaccapture.oxr") oxr.OpenXRSessionHandles = lambda *values: values monkeypatch.setitem(__import__("sys").modules, "isaaccapture.oxr", oxr) @@ -634,6 +664,60 @@ def test_bind_timeout_reconciles_a_sealed_ordinal_then_reserves_a_fresh_one(): assert (first, True, True) in host.ordinal_ledger.external_resolutions +def test_frame_scene_callback_seals_before_the_next_bind_ordinal(): + class Host(_FakeHost): + def attach_on_render_thread(self, capability): + binding = super().attach_on_render_thread(capability) + + def prepare(): + ordinal = self.ordinal_ledger.reserve() + self.ordinal_ledger.publish(ordinal) + self.events.append(("frame-scene", ordinal)) + + return replace(binding, prepare_frame_scene=prepare) + + host = Host() + owner = ovxr_session.OvxrSession(host, _config()) + owner.start() + owner.request_render("physics") + _wait_for_operation("submit") + owner.close() + prepared = [ + e[1] for e in host.events if isinstance(e, tuple) and e[0] == "frame-scene" + ] + assert prepared + assert _FakeContext.bind_ordinals == [ordinal + 1 for ordinal in prepared] + + +def test_shutdown_wait_wakes_on_the_existing_lifecycle_event(): + owner = ovxr_session.OvxrSession(_FakeHost(), _config()) + woke = threading.Event() + thread = threading.Thread(target=lambda: (owner.wait_for_shutdown(30), woke.set())) + thread.start() + try: + owner.request_shutdown() + assert woke.wait(1) + assert owner.wait_for_shutdown(30) + finally: + thread.join(timeout=1) + assert not thread.is_alive() + + +def test_unsupported_provider_rejects_before_host_transfer(monkeypatch): + runtime = _fake_ovxr() + runtime.Context = type( + "OldContext", (_FakeContext,), {"set_frame_render_products": None} + ) + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: runtime) + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + with pytest.raises( + RuntimeError, match="required Context methods: set_frame_render_products" + ): + owner.start() + assert host.offer_thread is None + + def test_missing_core_phase_method_still_rejects_before_host_transfer(monkeypatch): runtime = _fake_ovxr() runtime.Context = type("BrokenContext", (_FakeContext,), {"cancel_frame": None}) @@ -816,6 +900,45 @@ def wait(self, timeout=None): assert _FakeContext.operations.count(("destroy", host.attach_thread)) == 1 +def test_close_times_out_without_blocking_on_startup_teardown_lock(monkeypatch): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + entered = threading.Event() + resume = threading.Event() + destroy = _FakeContext.destroy + failure = RuntimeError("startup failed") + + def blocked_destroy(context, *, stage_ordinal): + entered.set() + assert resume.wait(timeout=2.0) + destroy(context, stage_ordinal=stage_ordinal) + + def fail_start(): + raise failure + + monkeypatch.setattr(_FakeContext, "destroy", blocked_destroy) + monkeypatch.setattr(owner._pump, "bootstrap", fail_start) + monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) + try: + with pytest.raises(RuntimeError) as caught: + owner.start() + assert caught.value is failure + assert entered.wait(timeout=1.0) + closing, errors = _call_in_thread(owner.close) + closing.join(timeout=0.5) + assert not closing.is_alive() + assert len(errors) == 1 + assert isinstance(errors[0], TimeoutError) + assert owner._pump._context is not None + assert not owner.retirement_complete + finally: + resume.set() + owner._worker.join(timeout=1.0) + with pytest.raises(RuntimeError) as caught: + owner.close() + assert caught.value is failure + + @pytest.mark.parametrize("abort_error_type", [None, RuntimeError, KeyboardInterrupt]) def test_worker_thread_start_failure_rolls_back_host_and_preserves_first_error( monkeypatch, abort_error_type, caplog @@ -874,6 +997,82 @@ def test_binding_rejects_an_incomplete_borrowed_tuple(): ovxr_session.OvxrSessionBinding(object(), object(), None) +def test_camera_panel_registers_one_borrowed_product_without_a_second_frame_source(): + host = _FakeHost() + path = "/Camera/Head" + config = _config() + panel = CameraPanelConfig( + path, + max_image_bytes=720 * 450 * 4, + panel_width_m=0.5, + position=(0.1, -0.2, -0.8), + space="head_locked", + ) + host.frame_consumers = (CameraPanelConsumer((panel,)),) + owner = ovxr_session.OvxrSession(host, config) + + owner.start() + desc = _FakeContext.last_desc + owner.close() + + assert _FakeContext.selected_products == (path,) + assert not hasattr(desc, "enable_raster_layer") + assert not hasattr(desc, "auxiliary_render_product") + assert [name for name, _thread in _FakeContext.operations].count("submit") == 0 + + +def test_auxiliary_panel_space_remains_application_policy(): + host = _FakeHost() + path = "/Camera/Head" + config = _config() + panel = CameraPanelConfig( + path, + max_image_bytes=720 * 450 * 4, + space="head_locked", + ) + host.frame_consumers = (CameraPanelConsumer((panel,)),) + owner = ovxr_session.OvxrSession(host, config) + + owner.start() + owner.close() + + assert panel.space == "head_locked" + assert _FakeContext.selected_products == (path,) + assert not hasattr(_FakeContext.last_desc, "auxiliary_panel_space") + + +@pytest.mark.parametrize("space", ["stage", "viewer_start", "world"]) +def test_camera_panel_rejects_unsupported_panel_space(space): + with pytest.raises(ValueError, match="space"): + CameraPanelConfig("/Camera/Head", max_image_bytes=720 * 450 * 4, space=space) + + +def test_camera_panel_has_bounded_public_defaults(): + config = CameraPanelConfig( + "/Camera/Head", + ) + + assert config.max_image_bytes == 64 * 1024 * 1024 + + +@pytest.mark.parametrize( + ("kwargs", "message"), + [ + ({"max_image_bytes": 0}, "max_image_bytes"), + ({"panel_width_m": 0.0}, "panel_width_m"), + ({"orientation_xyzw": (0.0, 0.0, 0.0, 0.0)}, "orientation"), + ], +) +def test_camera_panel_rejects_invalid_policy(kwargs, message): + with pytest.raises(ValueError, match=message): + CameraPanelConfig("/Camera/Head", **kwargs) + + +def test_camera_panel_requires_an_absolute_host_product(): + with pytest.raises(ValueError, match="absolute"): + CameraPanelConfig("relative") + + def test_cloudxr_and_ovxr_are_created_and_destroyed_on_the_owner_thread(monkeypatch): host = _FakeHost() launcher_events = [] @@ -907,6 +1106,228 @@ def launcher_factory(_path, *, accept_eula=False): ] +def test_injected_consumers_follow_frame_and_retirement_order(monkeypatch): + events = [] + threads = [] + outputs = SimpleNamespace(contents=SimpleNamespace(output_count=0, outputs=[])) + caller_thread = threading.get_native_id() + host = _FakeHost() + + class Context(_FakeContext): + def begin_frame(self): + result, plan = super().begin_frame() + plan.step_result = 1 + return result, plan + + class Consumer(ovxr_session.OvxrFrameConsumer): + def __init__(self, name): + self.name = name + + def record(self, event): + events.append((event, self.name)) + threads.append(threading.get_native_id()) + + def on_session_started(self, _binding, _config, request_shutdown): + assert any(name == "create" for name, _ in _FakeContext.operations) + self.request_shutdown = request_shutdown + self.record("start") + + def on_outputs(self, frame): + assert frame.latest_color("/missing") is None + self.record("outputs") + + def finish_frame(self): + self.record("finish") + + def on_frame_submitted(self): + assert any(name == "submit" for name, _ in _FakeContext.operations) + self.record("submitted") + self.request_shutdown() + + def close(self): + assert not any(name == "destroy" for name, _ in _FakeContext.operations) + self.record("close") + + api = _fake_ovxr() + api.Context = Context + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: api) + monkeypatch.setattr(frame_pump, "native_renderer_handle", lambda _renderer: 7) + monkeypatch.setattr(frame_pump, "completed_outputs", lambda *_args: outputs) + owner = _session_with_consumers( + host, _config(), frame_consumers=(Consumer("first"), Consumer("second")) + ) + with owner: + owner.start() + owner.request_render("step-1") + _wait_for_operation("submit") + assert events == [ + ("start", "first"), + ("start", "second"), + ("outputs", "first"), + ("outputs", "second"), + ("finish", "first"), + ("finish", "second"), + ("submitted", "first"), + ("submitted", "second"), + ("close", "second"), + ("close", "first"), + ] + assert caller_thread != owner.render_worker_native_thread_id + assert set(threads) == {owner.render_worker_native_thread_id} + assert not any(name == "cancel" for name, _ in _FakeContext.operations) + + +def test_consumer_start_failure_retires_borrowed_owners(): + host = _FakeHost() + failure = RuntimeError("consumer start failed") + + class Consumer(ovxr_session.OvxrFrameConsumer): + def on_session_started(self, *_args): + raise failure + + def on_outputs(self, frame): + pass + + def close(self): + host.events.append("consumer-close") + + owner = _session_with_consumers(host, _config(), frame_consumers=(Consumer(),)) + with pytest.raises(RuntimeError) as caught: + owner.start() + assert caught.value is failure + with pytest.raises(RuntimeError): + owner.close() + assert "consumer-close" in host.events + assert "destroy" in [name for name, _ in _FakeContext.operations] + + +def test_quad_detach_failure_reaches_close_and_retains_borrowed_owners(monkeypatch): + host = _FakeHost() + panel = CameraPanelConsumer((CameraPanelConfig("/Render/Head"),)) + owner = _session_with_consumers(host, _config(), frame_consumers=(panel,)) + owner.start() + context = owner._pump._context + quad = panel._native + launcher = _ControlledLauncher() + owner._pump._runtime._launcher = launcher + failure = RuntimeError("native quad detach failed") + + def fail_detach(): + raise failure + + quad.close = fail_detach + with pytest.raises(RuntimeError, match="native quad detach failed") as caught: + owner.close() + + assert caught.value is failure + assert not owner._worker.is_alive() + assert owner._shutdown_requested.is_set() + assert owner._phase is ovxr_session._Phase.CLOSED + assert not owner.retirement_complete + with pytest.raises(RuntimeError) as repeated: + owner.close() + assert repeated.value is failure + assert owner._pump._context is context + assert panel._native is quad + assert owner._pump._runtime._launcher is launcher + assert launcher.events == [] + assert "destroy" not in [name for name, _thread in _FakeContext.operations] + assert not any( + event[0] in {"host-close", "environment-close"} + for event in host.events + if isinstance(event, tuple) + ) + + +def test_shutdown_timeout_preserves_runtime_until_frame_worker_retires(monkeypatch): + host = _FakeHost() + frame_wait_entered = threading.Event() + release_frame = threading.Event() + launcher = _ControlledLauncher() + + class BlockingContext(_FakeContext): + def begin_frame(self): + frame_wait_entered.set() + assert release_frame.wait(timeout=2.0) + return super().begin_frame() + + fake_ovxr = _fake_ovxr() + fake_ovxr.Context = BlockingContext + monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: fake_ovxr) + + def launcher_factory(*args, **kwargs): + return launcher + + monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + owner.start() + assert owner.request_render("step-1") == 10 + assert frame_wait_entered.wait(timeout=1.0) + context = owner._pump._context + failure = RuntimeError("request exit failed") + _FakeContext.request_exit_error = failure + try: + with pytest.raises( + TimeoutError, match="OVXR session worker did not stop" + ) as error: + owner.close() + assert error.value.__cause__ is failure + assert owner._pump._context is context + assert owner._worker.is_alive() + assert not owner.retirement_complete + assert launcher.events == [] + assert "destroy" not in [name for name, _thread in _FakeContext.operations] + finally: + release_frame.set() + owner._worker.join(timeout=1.0) + assert not owner._worker.is_alive() + with pytest.raises(RuntimeError) as error: + owner.close() + assert error.value is failure + assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] + + +def test_cloudxr_stop_can_finish_without_runtime_interruption(monkeypatch, caplog): + host = _FakeHost() + launcher = _ControlledLauncher(block_stop=True) + + def launcher_factory(_path, *, accept_eula=False): + return launcher + + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + owner.start() + + close_thread, close_errors = _call_in_thread(owner.close) + try: + assert launcher.stop_started.wait(timeout=1.0) + assert owner._pump._runtime._launcher is launcher + time.sleep(0.03) + assert close_thread.is_alive() + assert "interrupt" not in [name for name, _thread in launcher.events] + assert "interrupting the owned OpenXR runtime" not in caplog.text + finally: + launcher.allow_stop.set() + close_thread.join(timeout=1.0) + + assert not close_thread.is_alive() + assert close_errors == [] + assert owner._pump._runtime._launcher is None + assert launcher.events == [ + ("stop-start", owner.render_worker_native_thread_id), + ("stop-end", owner.render_worker_native_thread_id), + ] + + def test_destroy_failure_retains_borrowed_providers_and_forbids_retry(monkeypatch): host = _FakeHost() launcher = _ControlledLauncher() @@ -1062,21 +1483,57 @@ def launcher_factory(_path, *, accept_eula=False): ] -@pytest.mark.parametrize("failure_type", [RuntimeError, _WorkerAbort]) -def test_render_failure_waits_for_borrowers_and_preserves_first_error( - monkeypatch, failure_type -): +def test_partial_startup_failure_retires_context_before_cloudxr_stop(monkeypatch): host = _FakeHost() - owner = ovxr_session.OvxrSession(host, _config()) - owner.start() - failure = failure_type("bind failed") - _FakeContext.bind_error = failure - failed = threading.Event() - original = owner._record_worker_failure - - def record(error): - original(error) - failed.set() + launcher = _ControlledLauncher(block_stop=True) + + def launcher_factory(_path, *, accept_eula=False): + return launcher + + _FakeContext.create_error = RuntimeError("create failed") + config = _config() + config.cloudxr_env_file = "/cloudxr.env" + config.auto_launch_cloudxr = True + owner = ovxr_session.OvxrSession( + host, config, runtime_launcher_factory=launcher_factory + ) + + start_thread, start_errors = _call_in_thread(owner.start) + assert launcher.stop_started.wait(timeout=1.0) + close_thread, close_errors = _call_in_thread(owner.close) + try: + time.sleep(0.03) + assert "interrupt" not in [name for name, _thread in launcher.events] + finally: + launcher.allow_stop.set() + start_thread.join(timeout=1.0) + close_thread.join(timeout=1.0) + + assert not start_thread.is_alive() + assert not close_thread.is_alive() + assert [str(error) for error in start_errors] == ["create failed"] + assert [str(error) for error in close_errors] == ["create failed"] + assert launcher.events == [ + ("stop-start", owner.render_worker_native_thread_id), + ("stop-end", owner.render_worker_native_thread_id), + ] + + +@pytest.mark.parametrize("failure_type", [RuntimeError, _WorkerAbort]) +def test_render_failure_waits_for_borrowers_and_preserves_first_error( + monkeypatch, failure_type +): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner.start() + failure = failure_type("bind failed") + _FakeContext.bind_error = failure + failed = threading.Event() + original = owner._record_worker_failure + + def record(error): + original(error) + failed.set() monkeypatch.setattr(owner, "_record_worker_failure", record) @@ -1157,6 +1614,8 @@ class PacedContext(_FakeContext): def begin_frame(self): self.frames += 1 + # The runtime owns the next-frame wait, including while physics is + # paused. Block in that boundary to check shutdown as well as pace. if self.frames == 4: fourth_frame.set() assert release_frame.wait(1.0) @@ -1259,6 +1718,8 @@ def get_openxr_handles(self): if publish_scene: owner.request_render("paused-scene") _wait_for_operation("submit") + # Scene rendering may proceed before DeviceIO handles exist, exactly + # as for an initial failed peer followed by a later headset connection. peer_ready.set() with owner._worker_condition: ready = owner._worker_condition.wait_for( @@ -1283,72 +1744,281 @@ def test_config_rejects_unknown_session_mode(): _config(session_mode="automatic") -@pytest.mark.parametrize( - "scale", [float("nan"), float("inf"), float("-inf"), 0.0, -1.0] -) -def test_world_scale_rejects_nonfinite_or_nonpositive_values(scale): - with pytest.raises(ValueError): - _config(meters_per_unit=scale) +def test_three_products_keep_selection_order_and_one_native_owner(monkeypatch): + events = [] + paths = ("/Render/Head", "/Render/Left", "/Render/Right") + panels = tuple( + CameraPanelConfig(path, label=label) + for path, label in zip(paths, ("Head", "Left", "Right")) + ) + def create(configs): + assert configs == panels + return SimpleNamespace( + supports_gpu=False, + finish_frame=lambda: None, + attach=lambda *_a: events.append("attach"), + close=lambda: events.append("close"), + ) -@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) -def test_cancel_failure_does_not_skip_context_and_service_cleanup( - monkeypatch, error_type -): + monkeypatch.setattr(camera_panels, "OpenXRCameraPanels", create) + consumer = CameraPanelConsumer(panels) + owner = _session_with_consumers(_FakeHost(), _config(), frame_consumers=(consumer,)) + owner.start() + assert _FakeContext.selected_products == paths + assert tuple(box.product_path for box in consumer._mailboxes) == paths + owner.close() + assert events[-1] == "close" + + +def test_camera_products_reject_duplicate_identity(): + with pytest.raises(ValueError, match="unique"): + CameraPanelConsumer((CameraPanelConfig("/A"), CameraPanelConfig("/A"))) + + +@pytest.mark.parametrize("release_fails", [False, True]) +def test_bound_output_reads_finish_before_submit_or_cancel(monkeypatch, release_fails): host = _FakeHost() - launcher = _ControlledLauncher() + import ctypes - def launcher_factory(*args, **kwargs): - return launcher + host.renderer = SimpleNamespace(_handle=ctypes.c_void_p(123)) + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + plan = SimpleNamespace(step_result=77) + events = [] + + def bind(_plan, _time, *, stage_ordinal): + host.ordinal_ledger.seal_external(stage_ordinal) + events.append("bind") + return 0, True + + def release(): + events.append("unmap") + if release_fails: + raise RuntimeError("source read completion unproved") + + def submit(_plan, *, additional_layers): + assert events == ["bind", "observe", "copy", "unmap"] + assert additional_layers == [123] + events.append("submit") + return 0, SimpleNamespace(submitted=True) - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + submit_frame=submit, + cancel_frame=lambda _plan: events.append("cancel"), ) - failure = error_type("cancel failed") - def cancel(_self, _plan): + class Consumer(ovxr_session.OvxrFrameConsumer): + def on_outputs(self, frame): + events.extend(["observe", "copy"]) + + def composition_layers(self): + return (123,) + + def finish_frame(self): + release() + + def close(self): + release() + + owner._pump._frame_consumers = (Consumer(),) + + def fetch(renderer, result): + assert renderer == 123 and result == 77 + return object() + + monkeypatch.setattr(frame_pump, "completed_outputs", fetch) + if release_fails: + with pytest.raises(RuntimeError, match="completion unproved"): + owner._pump.render(9) + assert "submit" not in events and "cancel" not in events + assert owner._pump._pending_plan is plan + else: + owner._pump.render(9) + assert events[-1] == "submit" and owner._pump._pending_plan is None + + +def test_no_render_plan_does_not_read_outputs_or_append_layers(monkeypatch): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + plan = SimpleNamespace(step_result=0) + submitted = [] + + def bind(_plan, _time, *, stage_ordinal): + return 0, False + + def unexpected(*args): + pytest.fail("no-render frame attempted an output read or quad operation") + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + submit_frame=lambda _plan, **kw: ( + submitted.append(kw) or 0, + SimpleNamespace(submitted=False), + ), + cancel_frame=unexpected, + ) + + class Consumer(ovxr_session.OvxrFrameConsumer): + on_outputs = unexpected + composition_layers = unexpected + + def close(self): + pass + + owner._pump._frame_consumers = (Consumer(),) + monkeypatch.setattr(frame_pump, "completed_outputs", unexpected) + owner._pump.render(9) + assert submitted == [{"additional_layers": []}] + + +@pytest.mark.parametrize("cleanup_boundary", ["outputs", "cancel"]) +def test_frame_cleanup_preserves_the_original_bind_error(cleanup_boundary): + host = _FakeHost() + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + + failure = RuntimeError("original bind failure") + plan = SimpleNamespace(step_result=0) + + def bind(*_args, **_kwargs): raise failure - monkeypatch.setattr(_FakeContext, "cancel_frame", cancel) - owner.start() - owner._pump._pending_plan = SimpleNamespace(step_result=0) - with pytest.raises(error_type) as caught: - owner.close() - assert caught.value is failure - assert "destroy" in [name for name, _thread in _FakeContext.operations] - assert owner._pump._context is None - assert owner._pump._pending_plan is None - assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] - assert [event[0] for event in host.events if isinstance(event, tuple)][-2:] == [ - "host-close", - "environment-close", + def cleanup(): + raise RuntimeError("cleanup failed") + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, plan), + bind_frame=bind, + cancel_frame=lambda _plan: cleanup() if cleanup_boundary == "cancel" else None, + ) + if cleanup_boundary == "outputs": + owner._pump._finish_frame_outputs = cleanup + with pytest.raises(RuntimeError, match="original bind failure") as error: + owner._pump.render(9) + assert error.value is failure + assert "cleanup failed" in " ".join(error.value.__notes__) + assert owner._pump._pending_plan is plan + + +def test_consumer_retirement_failure_preserves_earlier_publication_error(): + host = _FakeHost() + failure = RuntimeError("publication close failed") + + def close_publication(): + raise failure + + def close_map(): + raise RuntimeError("map retirement failed") + + host.close_publication = close_publication + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._frame_consumers = (SimpleNamespace(close=close_map),) + with pytest.raises(RuntimeError, match="publication close failed") as error: + owner._pump.close() + assert error.value is failure + assert "map retirement failed" in " ".join(error.value.__notes__) + assert not host.events # Borrowed objects must not retire past a live map. + + +@pytest.mark.parametrize("hide_fails", [False, True]) +@pytest.mark.parametrize("failure_site", ["fetch", "publish"]) +def test_failed_output_hides_cached_quad_until_successful_refresh( + monkeypatch, hide_fails, failure_site +): + import ctypes + + host = _FakeHost() + host.renderer = SimpleNamespace(_handle=ctypes.c_void_p(123)) + owner = ovxr_session.OvxrSession(host, _config()) + owner._pump._binding = host.attach_on_render_thread(object()) + events = [] + + def bind(_plan, _time, *, stage_ordinal): + host.ordinal_ledger.seal_external(stage_ordinal) + return 0, True + + failing = [True] + + def fetch(*_args): + if failing[0] and failure_site == "fetch": + raise RuntimeError("fetch failed") + return object() + + owner._pump._context = SimpleNamespace( + begin_frame=lambda: (0, SimpleNamespace(step_result=77)), + bind_frame=bind, + submit_frame=lambda *_a, **kw: ( + events.append(kw["additional_layers"]) or 0, + SimpleNamespace(submitted=True), + ), + ) + visible = [True] + + def hide(): + if hide_fails: + raise RuntimeError("optional hide failed") + visible[0] = False + + def publish(*_args, **_kwargs): + if failing[0] and failure_site == "publish": + raise RuntimeError("publish failed") + visible[0] = True + + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + consumer._enabled = True + consumer._native = SimpleNamespace( + finish_frame=lambda: None, + hide=lambda _index: hide(), + publish=publish, + layer=lambda _index: 123 if visible[0] else 0, + ) + consumer._mailboxes = [ + SimpleNamespace( + on_outputs=lambda *_a, **_kw: None, + close=lambda: None, + snapshot=object, + error=lambda: None, + ) ] - assert not owner.retirement_complete # Cleanup does not erase the original error. + owner._pump._frame_consumers = (consumer,) + monkeypatch.setattr(frame_pump, "completed_outputs", fetch) + owner._pump.render(9) + assert events == [[]] + failing[0] = False + owner._pump.render(9) + assert events == [[], [123]] -def test_shutdown_timeout_preserves_runtime_until_frame_worker_retires(monkeypatch): +@pytest.mark.parametrize( + "scale", [float("nan"), float("inf"), float("-inf"), 0.0, -1.0] +) +def test_world_scale_rejects_nonfinite_or_nonpositive_values(scale): + with pytest.raises(ValueError): + _config(meters_per_unit=scale) + + +def test_slow_context_retirement_keeps_owned_runtime_alive(monkeypatch): host = _FakeHost() - frame_wait_entered = threading.Event() - release_frame = threading.Event() launcher = _ControlledLauncher() + entered = threading.Event() + resume = threading.Event() + destroy = _FakeContext.destroy - class BlockingContext(_FakeContext): - def begin_frame(self): - frame_wait_entered.set() - assert release_frame.wait(timeout=2.0) - return super().begin_frame() + def slow_destroy(context, *, stage_ordinal): + entered.set() + assert resume.wait(timeout=3.0) + destroy(context, stage_ordinal=stage_ordinal) - fake_ovxr = _fake_ovxr() - fake_ovxr.Context = BlockingContext - monkeypatch.setattr(frame_pump, "_import_ovxr", lambda: fake_ovxr) + monkeypatch.setattr(_FakeContext, "destroy", slow_destroy) def launcher_factory(*args, **kwargs): return launcher - monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) config = _config() config.cloudxr_env_file = "/cloudxr.env" config.auto_launch_cloudxr = True @@ -1356,29 +2026,19 @@ def launcher_factory(*args, **kwargs): host, config, runtime_launcher_factory=launcher_factory ) owner.start() - assert owner.request_render("step-1") == 10 - assert frame_wait_entered.wait(timeout=1.0) - context = owner._pump._context - failure = RuntimeError("request exit failed") - _FakeContext.request_exit_error = failure + closing, errors = _call_in_thread(owner.close) try: - with pytest.raises( - TimeoutError, match="OVXR session worker did not stop" - ) as error: - owner.close() - assert error.value.__cause__ is failure - assert owner._pump._context is context - assert owner._worker.is_alive() - assert not owner.retirement_complete + assert entered.wait(timeout=1.0) + # Slow native retirement must keep its runtime alive until completion. + time.sleep(1.1) + assert closing.is_alive() assert launcher.events == [] - assert "destroy" not in [name for name, _thread in _FakeContext.operations] finally: - release_frame.set() - owner._worker.join(timeout=1.0) - assert not owner._worker.is_alive() - with pytest.raises(RuntimeError) as error: - owner.close() - assert error.value is failure + resume.set() + closing.join(timeout=2.0) + assert not closing.is_alive() + assert errors == [] + assert owner.retirement_complete assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] @@ -1421,40 +2081,42 @@ def __exit__(self, *_args): assert context is not None -def test_partial_startup_failure_retires_context_before_cloudxr_stop(monkeypatch): +@pytest.mark.parametrize("error_type", [RuntimeError, KeyboardInterrupt]) +def test_cancel_failure_does_not_skip_context_and_service_cleanup( + monkeypatch, error_type +): host = _FakeHost() - launcher = _ControlledLauncher(block_stop=True) + launcher = _ControlledLauncher() - def launcher_factory(_path, *, accept_eula=False): + def launcher_factory(*args, **kwargs): return launcher - _FakeContext.create_error = RuntimeError("create failed") config = _config() config.cloudxr_env_file = "/cloudxr.env" config.auto_launch_cloudxr = True owner = ovxr_session.OvxrSession( host, config, runtime_launcher_factory=launcher_factory ) + failure = error_type("cancel failed") - start_thread, start_errors = _call_in_thread(owner.start) - assert launcher.stop_started.wait(timeout=1.0) - close_thread, close_errors = _call_in_thread(owner.close) - try: - time.sleep(0.03) - assert "interrupt" not in [name for name, _thread in launcher.events] - finally: - launcher.allow_stop.set() - start_thread.join(timeout=1.0) - close_thread.join(timeout=1.0) + def cancel(_self, _plan): + raise failure - assert not start_thread.is_alive() - assert not close_thread.is_alive() - assert [str(error) for error in start_errors] == ["create failed"] - assert [str(error) for error in close_errors] == ["create failed"] - assert launcher.events == [ - ("stop-start", owner.render_worker_native_thread_id), - ("stop-end", owner.render_worker_native_thread_id), + monkeypatch.setattr(_FakeContext, "cancel_frame", cancel) + owner.start() + owner._pump._pending_plan = SimpleNamespace(step_result=0) + with pytest.raises(error_type) as caught: + owner.close() + assert caught.value is failure + assert "destroy" in [name for name, _thread in _FakeContext.operations] + assert owner._pump._context is None + assert owner._pump._pending_plan is None + assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] + assert [event[0] for event in host.events if isinstance(event, tuple)][-2:] == [ + "host-close", + "environment-close", ] + assert not owner.retirement_complete # Cleanup does not erase the original error. def test_owner_lane_exit_failure_after_destroy_still_releases_retired_owners( @@ -1492,121 +2154,6 @@ def failing_lane(): ] -def test_cloudxr_stop_can_finish_without_runtime_interruption(monkeypatch, caplog): - host = _FakeHost() - launcher = _ControlledLauncher(block_stop=True) - - def launcher_factory(_path, *, accept_eula=False): - return launcher - - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory - ) - owner.start() - - close_thread, close_errors = _call_in_thread(owner.close) - try: - assert launcher.stop_started.wait(timeout=1.0) - assert owner._pump._runtime._launcher is launcher - time.sleep(0.03) - assert close_thread.is_alive() - assert "interrupt" not in [name for name, _thread in launcher.events] - assert "interrupting the owned OpenXR runtime" not in caplog.text - finally: - launcher.allow_stop.set() - close_thread.join(timeout=1.0) - - assert not close_thread.is_alive() - assert close_errors == [] - assert owner._pump._runtime._launcher is None - assert launcher.events == [ - ("stop-start", owner.render_worker_native_thread_id), - ("stop-end", owner.render_worker_native_thread_id), - ] - - -def test_close_times_out_without_blocking_on_startup_teardown_lock(monkeypatch): - host = _FakeHost() - owner = ovxr_session.OvxrSession(host, _config()) - entered = threading.Event() - resume = threading.Event() - destroy = _FakeContext.destroy - failure = RuntimeError("startup failed") - - def blocked_destroy(context, *, stage_ordinal): - entered.set() - assert resume.wait(timeout=2.0) - destroy(context, stage_ordinal=stage_ordinal) - - def fail_start(): - raise failure - - monkeypatch.setattr(_FakeContext, "destroy", blocked_destroy) - monkeypatch.setattr(owner._pump, "bootstrap", fail_start) - monkeypatch.setattr(ovxr_session, "_WORKER_SHUTDOWN_TIMEOUT_SECONDS", 0.01) - try: - with pytest.raises(RuntimeError) as caught: - owner.start() - assert caught.value is failure - assert entered.wait(timeout=1.0) - closing, errors = _call_in_thread(owner.close) - closing.join(timeout=0.5) - assert not closing.is_alive() - assert len(errors) == 1 - assert isinstance(errors[0], TimeoutError) - assert owner._pump._context is not None - assert not owner.retirement_complete - finally: - resume.set() - owner._worker.join(timeout=1.0) - with pytest.raises(RuntimeError) as caught: - owner.close() - assert caught.value is failure - - -def test_slow_context_retirement_keeps_owned_runtime_alive(monkeypatch): - host = _FakeHost() - launcher = _ControlledLauncher() - entered = threading.Event() - resume = threading.Event() - destroy = _FakeContext.destroy - - def slow_destroy(context, *, stage_ordinal): - entered.set() - assert resume.wait(timeout=3.0) - destroy(context, stage_ordinal=stage_ordinal) - - monkeypatch.setattr(_FakeContext, "destroy", slow_destroy) - - def launcher_factory(*args, **kwargs): - return launcher - - config = _config() - config.cloudxr_env_file = "/cloudxr.env" - config.auto_launch_cloudxr = True - owner = ovxr_session.OvxrSession( - host, config, runtime_launcher_factory=launcher_factory - ) - owner.start() - closing, errors = _call_in_thread(owner.close) - try: - assert entered.wait(timeout=1.0) - # Slow native retirement must keep its runtime alive until completion. - time.sleep(1.1) - assert closing.is_alive() - assert launcher.events == [] - finally: - resume.set() - closing.join(timeout=2.0) - assert not closing.is_alive() - assert errors == [] - assert owner.retirement_complete - assert [name for name, _thread in launcher.events] == ["stop-start", "stop-end"] - - @pytest.mark.parametrize("axis, expected", [("Z", 4), ("Y", 5)]) def test_host_up_axis_reaches_openxr_context(axis, expected): owner = ovxr_session.OvxrSession(_FakeHost(), _config(up_axis=axis)) @@ -1665,3 +2212,102 @@ def fail_frame(_pump): finally: with pytest.raises(RuntimeError): owner.close() + + +@pytest.mark.parametrize("failure", ["vulkan_missing", "external_memory", "attach"]) +def test_unavailable_camera_panels_do_not_block_input_handles( + monkeypatch, tmp_path, caplog, failure +): + import ctypes + from unittest.mock import Mock + from isaaccapture.ovxr_session_manager.openxr_quad import OpenXRCameraPanels + + library = tmp_path / "panels.so" + library.touch() + native = SimpleNamespace( + **{ + "isaacteleop_pip_" + name: Mock(return_value=value) + for name, value in { + "create": 123, + "max_cameras": 3, + "max_image_dimension": 4096, + "supports_cuda": 1, + "attach": -3 if failure == "attach" else 0, + "destroy": 0, + "wait_cuda": 0, + "layer": 0, + "configure": 0, + "publish_rgba": 0, + "publish_cuda_array": 0, + }.items() + } + ) + monkeypatch.setattr(ctypes, "CDLL", lambda _path: native) + monkeypatch.setattr( + camera_panels, + "OpenXRCameraPanels", + lambda panels: OpenXRCameraPanels(panels, library_path=str(library)), + ) + vulkan = ( + None + if failure == "vulkan_missing" + else SimpleNamespace( + external_memory_handle_types=0 if failure == "external_memory" else 1, + vk_physical_device=1, + vk_device=2, + vk_queue=3, + queue_family_index=0, + ) + ) + attempts = [] + + def get_vulkan(self): + attempts.append(None) + if len(attempts) > 3: + raise RuntimeError("optional panels stalled core readiness") + return vulkan + + monkeypatch.setattr(_FakeContext, "get_vulkan_handles", get_vulkan) + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + owner = _session_with_consumers(_FakeHost(), _config(), frame_consumers=(consumer,)) + try: + assert owner.start() + assert owner.external_oxr_handles() == (11, 22, 33, 44) + assert consumer.composition_layers() == () + assert "Camera panels disabled" in caplog.text + finally: + owner.close() + assert native.isaacteleop_pip_attach.call_count == int(failure == "attach") + native.isaacteleop_pip_destroy.assert_called_once() + + +@pytest.mark.parametrize("native_fails", [False, True]) +def test_camera_completion_failure_still_retires_independent_mailboxes(native_fails): + events = [] + pending = [True] + + def finish_native(): + events.append("wait") + if native_fails and pending[0]: + raise RuntimeError("native completion failed") + + def blocked(): + events.append("blocked") + if pending[0]: + raise RuntimeError("map still busy") + + consumer = CameraPanelConsumer((CameraPanelConfig("/Camera/Head"),)) + consumer._native = SimpleNamespace( + finish_frame=finish_native, close=lambda: events.append("native close") + ) + consumer._mailboxes = [ + SimpleNamespace(close=blocked), + SimpleNamespace(close=lambda: events.append("independent")), + ] + with pytest.raises(RuntimeError): + consumer.close() + assert events == ["wait", "blocked", "independent"] + assert consumer._native is not None + pending[0] = False + consumer.close() + assert events[-4:] == ["wait", "blocked", "independent", "native close"]