Add camera PiP and shared OVXR frame consumers

.github/workflows/build-ubuntu.yml

@@ -162,6 +162,7 @@ jobs:
           -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
           -DBUILD_PLUGIN_OAK_CAMERA=ON \
           -DBUILD_VIZ=ON \
+          -DBUILD_OVXR_CAMERA_QUADS=ON \
           -DENABLE_CLOUDXR_BUNDLE_CHECK=ON \
           -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake \
           "${coverage_args[@]}"
@@ -341,6 +342,17 @@ jobs:
             isaaccapture-sbom check --wheel "${wheel}"
         done
 
+    - name: Verify camera support in release wheel artifacts
+      if: matrix.build_type == 'Release'
+      run: |
+        set -euo pipefail
+        shopt -s nullglob
+        wheels=(install/wheels/isaaccapture-*.whl)
+        [[ "${#wheels[@]}" -gt 0 ]] || { echo "No Isaac Capture wheels found"; exit 1; }
+        for wheel in "${wheels[@]}"; do
+          python3 scripts/check_camera_quad_wheel.py "${wheel}" --camera-quads
+        done
+
     - name: Create tarball to preserve permissions
       if: matrix.build_type == 'Release'
       run: tar -cvf isaacteleop-install.tar -C install .

CMakeLists.txt

@@ -74,6 +74,7 @@ option(BUILD_PLUGIN_SHARPA_AVATAR "Build Sharpa AvatarExo Glove plugin (Linux; s
 option(BUILD_EXAMPLES "Build examples" ON)
 option(BUILD_EXAMPLE_TELEOP_ROS2 "Build only the teleop_ros2 ROS 2 reference integration (e.g. for Docker)" OFF)
 option(BUILD_TESTING "Build unit tests" ON)
+option(BUILD_OVXR_CAMERA_QUADS "Build optional application-owned OVXR camera quad support" OFF)
 # Televiz (src/viz) needs Vulkan, the CUDA Toolkit, and glslangValidator.
 # Only Vulkan and CUDA gate the default -- without them the machine cannot build
 # viz at all, so a core-only source build still configures. glslangValidator is

docs/source/getting_started/ovxr.rst

@@ -41,14 +41,68 @@ from the Isaac Capture wheel. These packages retain their own licenses:
 CloudXR native libraries are bundled in Isaac Capture wheels and remain subject
 to their separate NVIDIA license and CloudXR EULA acceptance.
 
+Optional camera panels
+----------------------
+
+Camera picture-in-picture panels require the native camera helper. The Ubuntu
+release workflow includes it in its wheels and checks the packaged artifact.
+Installing those wheels requires no CMake configuration.
+
+For a source build, follow the source-build guide for your application's Python
+version, then enable camera support and build the wheel:
+
+.. code-block:: bash
+
+   cmake -S . -B build -DBUILD_OVXR_CAMERA_QUADS=ON
+   cmake --build build --target python_package python_stubs python_wheel --parallel
+
+Install the generated wheel with its ``ovxr`` extra using the instructions
+above. The extra requires CUDA 13 Python bindings for GPU camera output;
+``BUILD_OVXR_CAMERA_QUADS`` controls whether the wheel contains the native helper.
+The helper is disabled by default.
+
+Use a CUDA 13-compatible NVIDIA driver (R580 or newer), meeting any stricter
+requirements of the selected bindings and renderer. See the
+`CUDA compatibility requirements <https://docs.nvidia.com/deploy/cuda-compatibility/minor-version-compatibility.html>`_.
+Installing Python packages or a CUDA Toolkit does not update the host driver.
+
+Install the Vulkan development SDK before configuring camera support. Linux
+builds also require the CUDA Toolkit. CMake discovers these installed dependencies.
+Use ``CUDAToolkit_ROOT`` to select a CUDA Toolkit installation and ``VULKAN_SDK``
+to help locate a Vulkan SDK. The GPU driver and Vulkan runtime loader are required
+on the host.
+
 Use from another application
 ----------------------------
 
 ``isaaccapture.ovxr_session_manager.ovxr_session`` provides ``OvxrSession``
 and ``OvxrSessionConfig``. Implement ``OvxrApplicationInterface`` to supply the
 application's renderer, Stage and ordinal ledger in an ``OvxrSessionBinding``.
-``camera_paths`` is required and must contain both stereo cameras. Set
-``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``.
+The binding also supplies optional ``frame_consumers`` after the application has
+resolved its render-product paths. Capture does not depend on an Isaac Lab task
+or scene configuration. ``camera_paths`` is required and must contain both stereo
+cameras. Set ``up_axis="Y"`` for a Y-up Stage; the default is ``"Z"``.
+
+``CameraPanelConsumer`` implements the ``OvxrFrameConsumer`` lifecycle. Configure panels with a tuple of
+``CameraPanelConfig`` values, each pairing a render-product path with its placement
+and image bound.
+
+Camera-panel attachment failure disables panels and logs the reason without
+blocking teleoperation input. An incomplete GPU copy disables the affected panel;
+the consumer rechecks its copy stream before releasing the borrowed output.
+If completion still cannot be proved, the session reports a failure and retains
+the borrowed resources. Other camera mailboxes are still given a cleanup attempt.
+
+Consumers receive ``FrameOutputs`` on the XR owner thread. Use ``latest_color``
+to select an output, then ``copy_host_rgba`` or ``map_gpu`` to consume it without
+unpacking native result structures. Selected handles are valid only during that
+callback. CPU snapshots are shared within that frame; GPU resources retain their
+completion and retirement rules. A consumer may contribute composition layers that remain valid
+through submission. ``on_openxr_ready`` may be called repeatedly and must be
+idempotent. ``finish_frame`` releases borrowed outputs before submission or
+cancellation; ``close`` must tolerate partial startup and retain resources if
+safe retirement fails. The camera consumer owns one native panel collection,
+which is loaded directly and is not an OpenXR API layer.
 
 For a uv-managed application, declare Isaac Capture with its ``ovxr`` extra and
 select the source-built wheel in the application's project configuration:

pyproject.toml

@@ -52,6 +52,7 @@ ui = [
 ]
 ovxr = [
     "ovxr==0.1.0.5842",
+    "cuda-bindings>=13,<14",
 ]
 retargeters = [
     "dex-retargeting>=0.4.6,<0.6.0",

scripts/check_camera_quad_wheel.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Check optional camera-helper delivery in an actual project wheel.
+
+Run against wheels built with BUILD_OVXR_CAMERA_QUADS ON and OFF, including
+ON -> OFF -> ON in the same build directory to detect stale packaged data.
+"""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+from zipfile import ZipFile
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("wheel", type=Path)
+    parser.add_argument(
+        "--camera-quads", action=argparse.BooleanOptionalAction, required=True
+    )
+    args = parser.parse_args()
+    prefix = "isaaccapture/ovxr_session_manager/native/"
+    expected = {
+        prefix + "libisaaccapture_camera_panels.so",
+    }
+    with ZipFile(args.wheel) as wheel:
+        members = {
+            name
+            for name in wheel.namelist()
+            if name.startswith(prefix) and not name.endswith("/")
+        }
+        wanted = expected if args.camera_quads else set()
+        if members != wanted:
+            raise RuntimeError(
+                f"Camera helper package differs: expected {sorted(wanted)}, got {sorted(members)}"
+            )
+        if any("/openxr_layer/" in name for name in wheel.namelist()):
+            raise RuntimeError("Wheel contains obsolete OpenXR API-layer packaging")
+    print(
+        json.dumps(
+            {
+                "wheel": str(args.wheel),
+                "sha256": hashlib.sha256(args.wheel.read_bytes()).hexdigest(),
+                "camera_quads": args.camera_quads,
+                "result": "PASS",
+            },
+            indent=2,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()

src/core/CMakeLists.txt

@@ -38,6 +38,10 @@ add_subdirectory(deviceio_session)
 # Build MCAP library (depends on deviceio for DeviceIOSession and tracker interfaces)
 add_subdirectory(mcap)
 
+if(BUILD_OVXR_CAMERA_QUADS)
+    add_subdirectory(ovxr_pip_layer)
+endif()
+
 # CloudXR runtime helper (stages the vendored CloudXR runtime SDK bundle).
 # Its pure-Python source lives in src/python/isaaccapture/cloudxr/, as does the
 # retargeting engine's and the session manager's.

src/core/ovxr_pip_layer/CMakeLists.txt

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+add_subdirectory(cpp)

src/core/ovxr_pip_layer/cpp/CMakeLists.txt

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+find_package(Vulkan REQUIRED)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    find_package(CUDAToolkit REQUIRED)
+endif()
+
+add_library(ovxr_pip_layer SHARED pip_layer.cpp)
+target_compile_features(ovxr_pip_layer PRIVATE cxx_std_20)
+target_include_directories(ovxr_pip_layer PRIVATE
+    $<TARGET_PROPERTY:openxr_loader,INTERFACE_INCLUDE_DIRECTORIES>)
+target_link_libraries(ovxr_pip_layer PRIVATE Vulkan::Vulkan log_bridge::log_bridge_core)
+if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    # Keep static logger/fmt symbols private to the helper's existing C ABI.
+    set(_pip_exports "${CMAKE_CURRENT_BINARY_DIR}/ovxr_pip_exports.map")
+    file(GENERATE OUTPUT "${_pip_exports}"
+        CONTENT "{ global: isaacteleop_pip_*; local: *; };\n")
+    target_link_options(ovxr_pip_layer PRIVATE "-Wl,--version-script,${_pip_exports}")
+    set_property(TARGET ovxr_pip_layer APPEND PROPERTY LINK_DEPENDS "${_pip_exports}")
+endif()
+if(CUDAToolkit_FOUND AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    target_compile_definitions(ovxr_pip_layer PRIVATE ISAAC_TELEOP_PIP_CUDA)
+    target_link_libraries(ovxr_pip_layer PRIVATE CUDA::cudart_static)
+endif()
+set_target_properties(ovxr_pip_layer PROPERTIES
+    OUTPUT_NAME isaaccapture_camera_panels
+    LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python_package/$<CONFIG>/isaaccapture/ovxr_session_manager/native")
+
+add_library(ovxr_pip_layer::ovxr_pip_layer ALIAS ovxr_pip_layer)