From 1b180474b5e15ab32eb6f116e8dfe7e3e84bd60f Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 19:53:16 -0700 Subject: [PATCH 01/37] feat(stereo-camera): XR_DXR_stereo_camera header, type-value block 1004999300-310, catalog note (ADR-043) Type values sit at 1004999300-310 (not 290-300 as first drafted): on the rebase onto post-lift main, 290 had been taken by XR_TYPE_WEAVE_SUBMIT_OVERLAY_UNCHANGED_DXR (weave spec v14). Safe to move: spec v1, nothing shipped chains these. Co-Authored-By: Claude Fable 5.1 --- docs/specs/extensions/index.json | 5 + src/external/openxr_includes/openxr/README.md | 4 +- .../openxr/XR_DXR_stereo_camera.h | 371 ++++++++++++++++++ src/xrt/include/xrt/xrt_openxr_includes.h | 1 + 4 files changed, 380 insertions(+), 1 deletion(-) create mode 100644 src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h diff --git a/docs/specs/extensions/index.json b/docs/specs/extensions/index.json index c10cb4d566..dc343efa01 100644 --- a/docs/specs/extensions/index.json +++ b/docs/specs/extensions/index.json @@ -131,6 +131,11 @@ "group": "capture", "title": "Atlas Capture", "summary": "Vendor-neutral PNG snapshot of the multi-view atlas at a caller-selected compositor stage, for any app class" + }, + "XR_DXR_stereo_camera": { + "group": "capture", + "title": "Stereo Camera Source", + "summary": "The display's own stereo camera (usually the eye tracker's) as a runtime-owned, privacy-gated source: the vendor plug-in supplies frames without taking the device from the tracker, and the service fans newest-frame side-by-side pairs out to clients over shared memory" } } } diff --git a/src/external/openxr_includes/openxr/README.md b/src/external/openxr_includes/openxr/README.md index 89837363cf..9fc409a17e 100644 --- a/src/external/openxr_includes/openxr/README.md +++ b/src/external/openxr_includes/openxr/README.md @@ -49,7 +49,9 @@ Rules: | 1004999270–279 | `XR_DXR_lift` | 270 = `XR_TYPE_LIFT_PROPERTIES_DXR`, 271 = `XR_TYPE_LIFT_STREAM_CREATE_INFO_DXR`, 272 = `XR_TYPE_LIFT_FRAME_SUBMIT_INFO_DXR`, 273 = `XR_TYPE_LIFT_OPTIONS_DXR`, 274 = `XR_TYPE_LIFT_RESULT_DXR`, 275 = `XR_TYPE_WEAVE_SUBMIT_LIFT_RECTS_DXR`, 276 = `XR_TYPE_WEAVE_RECT_LIFT_DXR`, 277 = `XR_TYPE_LIFT_BLOB_DXR` (spec v1, ADR-042). 278 = `XR_OBJECT_TYPE_LIFT_STREAM_DXR` — an `XrObjectType`, not an `XrStructureType`. 279 = `XR_LIFT_NOT_READY_DXR` — a **success**-class `XrResult` (positive) | | 1004999280–289 | `XR_DXR_lift` (continued) | 280 = `XR_TYPE_LIFT_STREAM_STATS_DXR` (spec v1). A second decade because 270–279 is full | | 1004999290–299 | `XR_DXR_weave` (v14+ additions) | 290 = `XR_TYPE_WEAVE_SUBMIT_OVERLAY_UNCHANGED_DXR` (the v4 overlay atlas is unchanged since the previous submit, so a DP may reuse its lens prefilter of the 2D layer, spec v14, ADR-027 Amendment); 291 = `XR_TYPE_WEAVE_SUBMIT_OVERLAY_FILTER_DXR` (the caller's lens-filter strength for the 2D layer, 0..1, omitted = the DP's default, spec v15). The third weave decade (240–249 is full) | -| 1004999300+ | **next free** | | +| 1004999300–309 | `XR_DXR_stereo_camera` | 300 = `XR_TYPE_STEREO_CAMERA_PROPERTIES_DXR`, 301 = `XR_TYPE_STEREO_CAMERA_CALIBRATION_DXR`, 302 = `XR_TYPE_STEREO_CAMERA_STREAM_CREATE_INFO_DXR`, 303 = `XR_TYPE_STEREO_CAMERA_FRAME_DXR`, 304 = `XR_TYPE_STEREO_CAMERA_STREAM_INFO_DXR`, 305 = `XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR`, 306 = `XR_TYPE_STEREO_CAMERA_STREAM_STATS_DXR`, 307 = `XR_TYPE_EVENT_DATA_STEREO_CAMERA_STATE_CHANGED_DXR`, 308 = `XR_TYPE_EVENT_DATA_STEREO_CAMERAS_CHANGED_DXR` (spec v1, ADR-043). 309 = `XR_OBJECT_TYPE_STEREO_CAMERA_STREAM_DXR` — an `XrObjectType`, not an `XrStructureType`. **Relocated from 290–299** on the rebase onto post-lift `main`: 290 had meanwhile been taken by `XR_TYPE_WEAVE_SUBMIT_OVERLAY_UNCHANGED_DXR` (weave spec v14). Safe to move: spec v1, draft PR, no shipped app chains it | +| 1004999310–319 | `XR_DXR_stereo_camera` (continued) | 310 = `XR_STEREO_CAMERA_FRAME_NOT_READY_DXR` — a **success**-class `XrResult` (positive). A second decade because 300–309 is full | +| 1004999320+ | **next free** | | `XR_DXR_android_surface_binding` was implemented in #1037 and now has its own header and decade — see the row above for why its create-info type value sits diff --git a/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h new file mode 100644 index 0000000000..bb635fd762 --- /dev/null +++ b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h @@ -0,0 +1,371 @@ +// Copyright 2026, DisplayXR +// SPDX-License-Identifier: Apache-2.0 +// +// PROVISIONAL — DXR is DisplayXR's Khronos-registered OpenXR author ID, but +// the XR_DXR_* extensions in this header are NOT yet registered in the +// Khronos OpenXR registry: extension numbers and XrStructureType values sit +// in a provisional experimental block (1004999xxx) pending official +// assignment. Extension names are expected to be stable; numeric values are +// not. +// See GOVERNANCE.md. +// +/*! + * @file + * @brief Header for XR_DXR_stereo_camera extension + * @author David Fattal + * @ingroup external_openxr + * + * A display's STEREO CAMERA (usually the one its eye tracker looks through) as + * a runtime-provided source (ADR-043). The vendor display plug-in produces + * frames from its own stack without taking the device from the tracker; the + * runtime service owns the camera, opens it once and fans every frame out to + * each authorised stream; clients consume the newest frame over shared memory: + * + * xrEnumerateStereoCamerasDXR(instance, systemId, 0, &n, NULL); // + properties + * xrCreateStereoCameraStreamDXR(instance, &createInfo, &stream); + * xrStartStereoCameraStreamDXR(stream); // authorisation point + * xrGetStereoCameraStreamInfoDXR(stream, &info); // chain XrStereoCameraStreamTransportDXR + * // map info's section once; per frame (wake on the per-stream wake handle): + * if (xrAcquireStereoCameraFrameDXR(stream, &frame) == XR_SUCCESS) { read slot frame.slot } + * + * The frame is always ONE side-by-side image, 2*eyeWidth x eyeHeight, left eye + * in the left half, never mirrored, never woven (showing it in 3D is the + * weave path's job, ADR-007). + * + * Instance-level (a capture component has no session) and SERVICE-ONLY: an + * in-process instance enumerates zero cameras. The service is the single owner + * of each camera; an in-process runtime opening it too would recreate the + * exclusive-device fight this extension exists to end. + * + * Scope: stereo pairs. Names and struct shapes are kept open for other camera + * kinds (XrStereoCameraPropertiesDXR::viewCount, next chains), but v1 sources + * are two-view side-by-side only. + * + * Privacy: frames bypass the OS camera stack, so the runtime gates them itself + * (consent, foreground rule, in-use indicator, kill switch — spec §7). + * xrStartStereoCameraStreamDXR and xrGetStereoCameraCalibrationDXR return + * XR_ERROR_PERMISSION_INSUFFICIENT when refused. + * + * Spec: docs/specs/extensions/XR_DXR_stereo_camera.md. + * + * Version history: 1 = initial (enumerate + state, calibration raw/rectified, + * streams with start/stop, latest-wins acquire over a pinned 3-slot shared- + * memory ring with per-stream wake handles, stream stats). + */ +#ifndef XR_DXR_STEREO_CAMERA_H +#define XR_DXR_STEREO_CAMERA_H 1 + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define XR_DXR_stereo_camera 1 +#define XR_DXR_stereo_camera_SPEC_VERSION 1 +#define XR_DXR_STEREO_CAMERA_EXTENSION_NAME "XR_DXR_stereo_camera" + +// Reserved 1004999300..319 (relocated from 290..309 on the post-lift rebase: 290 was taken by XR_DXR_weave v14). Allocation registry: README.md in this directory. +#define XR_TYPE_STEREO_CAMERA_PROPERTIES_DXR ((XrStructureType)1004999300) +#define XR_TYPE_STEREO_CAMERA_CALIBRATION_DXR ((XrStructureType)1004999301) +#define XR_TYPE_STEREO_CAMERA_STREAM_CREATE_INFO_DXR ((XrStructureType)1004999302) +#define XR_TYPE_STEREO_CAMERA_FRAME_DXR ((XrStructureType)1004999303) +#define XR_TYPE_STEREO_CAMERA_STREAM_INFO_DXR ((XrStructureType)1004999304) +#define XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR ((XrStructureType)1004999305) +#define XR_TYPE_STEREO_CAMERA_STREAM_STATS_DXR ((XrStructureType)1004999306) +#define XR_TYPE_EVENT_DATA_STEREO_CAMERA_STATE_CHANGED_DXR ((XrStructureType)1004999307) +#define XR_TYPE_EVENT_DATA_STEREO_CAMERAS_CHANGED_DXR ((XrStructureType)1004999308) +//! XrObjectType of an XrStereoCameraStreamDXR (debug-utils naming). +#define XR_OBJECT_TYPE_STEREO_CAMERA_STREAM_DXR ((XrObjectType)1004999309) +//! SUCCESS-class result: no frame newer than the last one this stream acquired. +//! Not an error — keep showing the previous frame. +#define XR_STEREO_CAMERA_FRAME_NOT_READY_DXR ((XrResult)1004999310) + +//! Size of XrStereoCameraPropertiesDXR::persistentId, NUL included. +#define XR_STEREO_CAMERA_PERSISTENT_ID_MAX_SIZE_DXR 64 +//! Size of XrStereoCameraPropertiesDXR::displayName, NUL included. +#define XR_STEREO_CAMERA_DISPLAY_NAME_MAX_SIZE_DXR 128 +//! Size of XrStereoCameraPropertiesDXR::platformDeviceHint, NUL included. +#define XR_STEREO_CAMERA_PLATFORM_HINT_MAX_SIZE_DXR 256 +//! Slots in a stream's shared-memory ring (latest + pinned + one to write). +#define XR_STEREO_CAMERA_RING_SLOTS_DXR 3 + +XR_DEFINE_HANDLE(XrStereoCameraStreamDXR) + +typedef XrFlags64 XrStereoCameraFlagsDXR; +//! The camera is the eye tracker's; frames exist only while tracking runs. +static const XrStereoCameraFlagsDXR XR_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING_BIT_DXR = 0x00000001; +//! The camera faces the user (a 3D display's front camera). +static const XrStereoCameraFlagsDXR XR_STEREO_CAMERA_USER_FACING_BIT_DXR = 0x00000002; +//! Calibration is available for the ACTIVE device (RECTIFIED supported). +static const XrStereoCameraFlagsDXR XR_STEREO_CAMERA_CALIBRATED_BIT_DXR = 0x00000004; +//! The source already delivers a rectified pair. +static const XrStereoCameraFlagsDXR XR_STEREO_CAMERA_NATIVELY_RECTIFIED_BIT_DXR = 0x00000008; +//! Luma only (an IR / grey tracking sensor). +static const XrStereoCameraFlagsDXR XR_STEREO_CAMERA_MONOCHROME_BIT_DXR = 0x00000010; + +typedef enum XrStereoCameraStateDXR { + //! Frames flowing, or will flow on start. + XR_STEREO_CAMERA_STATE_AVAILABLE_DXR = 1, + //! Source up, no frame yet (tracker warming up). + XR_STEREO_CAMERA_STATE_WAITING_DXR = 2, + //! Temporarily no frames: tracker stopped, privacy lock, session locked. + XR_STEREO_CAMERA_STATE_SUSPENDED_DXR = 3, + //! Failed / removed. + XR_STEREO_CAMERA_STATE_UNAVAILABLE_DXR = 4, + XR_STEREO_CAMERA_STATE_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraStateDXR; + +typedef enum XrStereoCameraOutputDXR { + //! Parallel pair, zero disparity at infinity, no convergence shear. Needs + //! CALIBRATED or NATIVELY_RECTIFIED. + XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR = 1, + //! The source's pixels as delivered (native clients only; never web pages). + XR_STEREO_CAMERA_OUTPUT_RAW_DXR = 2, + XR_STEREO_CAMERA_OUTPUT_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraOutputDXR; + +typedef enum XrStereoCameraFormatDXR { + XR_STEREO_CAMERA_FORMAT_GRAY8_DXR = 1, + //! A MONOCHROME source in NV12 is luma + neutral (128) chroma. + XR_STEREO_CAMERA_FORMAT_NV12_DXR = 2, + XR_STEREO_CAMERA_FORMAT_BGRA8_DXR = 3, + XR_STEREO_CAMERA_FORMAT_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraFormatDXR; + +typedef enum XrStereoCameraTransportDXR { + //! Required; every platform. + XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR = 1, + //! Optional (Windows); not implemented by spec v1 runtimes. + XR_STEREO_CAMERA_TRANSPORT_D3D11_TEXTURE_DXR = 2, + //! Optional (Android); not implemented by spec v1 runtimes. + XR_STEREO_CAMERA_TRANSPORT_AHARDWAREBUFFER_DXR = 3, + XR_STEREO_CAMERA_TRANSPORT_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraTransportDXR; + +typedef enum XrStereoCameraDistortionModelDXR { + XR_STEREO_CAMERA_DISTORTION_NONE_DXR = 0, + //! OpenCV k1 k2 p1 p2 k3. + XR_STEREO_CAMERA_DISTORTION_RADTAN5_DXR = 1, + //! + k4 k5 k6 (rational). + XR_STEREO_CAMERA_DISTORTION_RADTAN8_DXR = 2, + //! Fisheye (Kannala-Brandt k1..k4). + XR_STEREO_CAMERA_DISTORTION_KB4_DXR = 3, + XR_STEREO_CAMERA_DISTORTION_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraDistortionModelDXR; + +//! One bit per XrStereoCameraFormatDXR value: bit (1 << value). +typedef XrFlags64 XrStereoCameraFormatFlagsDXR; +static const XrStereoCameraFormatFlagsDXR XR_STEREO_CAMERA_FORMAT_GRAY8_BIT_DXR = 0x00000002; +static const XrStereoCameraFormatFlagsDXR XR_STEREO_CAMERA_FORMAT_NV12_BIT_DXR = 0x00000004; +static const XrStereoCameraFormatFlagsDXR XR_STEREO_CAMERA_FORMAT_BGRA8_BIT_DXR = 0x00000008; + +//! One bit per XrStereoCameraTransportDXR value: bit (1 << value). +typedef XrFlags64 XrStereoCameraTransportFlagsDXR; +static const XrStereoCameraTransportFlagsDXR XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_BIT_DXR = 0x00000002; +static const XrStereoCameraTransportFlagsDXR XR_STEREO_CAMERA_TRANSPORT_D3D11_TEXTURE_BIT_DXR = 0x00000004; +static const XrStereoCameraTransportFlagsDXR XR_STEREO_CAMERA_TRANSPORT_AHARDWAREBUFFER_BIT_DXR = 0x00000008; + +/*! + * One camera, as enumerated. @c cameraId is valid for the service's lifetime + * and never reused for another camera within it. + */ +typedef struct XrStereoCameraPropertiesDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint64_t cameraId; + //! Stable per (device, consumer executable); never the raw serial (spec §7.5). + char persistentId[XR_STEREO_CAMERA_PERSISTENT_ID_MAX_SIZE_DXR]; + //! Plug-in supplied, human-readable ("Built-in 3D camera"). + char displayName[XR_STEREO_CAMERA_DISPLAY_NAME_MAX_SIZE_DXR]; + //! The OS's id of the physical device this source reads, "" if none — so a + //! capture stack can hide the raw (busy) duplicate. + char platformDeviceHint[XR_STEREO_CAMERA_PLATFORM_HINT_MAX_SIZE_DXR]; + XrStereoCameraFlagsDXR flags; + XrStereoCameraStateDXR state; + //! Views in the source. Always 2 in spec v1 (reserved for other camera kinds). + uint32_t viewCount; + //! Native per-eye size (e.g. 640x480). + XrExtent2Di eyeExtent; + //! What the source can deliver, Hz. + float maxFrameRate; + //! |T| of the pair, millimetres; 0 if uncalibrated. + float baselineMm; + //! Per eye, rectified, degrees; 0 if uncalibrated. + float horizontalFovDeg; + XrStereoCameraFormatFlagsDXR supportedFormats; + XrStereoCameraTransportFlagsDXR supportedTransports; +} XrStereoCameraPropertiesDXR; + +typedef struct XrStereoCameraIntrinsicsDXR { + //! The image these numbers describe (per eye). + XrExtent2Di imageExtent; + float fx, fy, cx, cy; + XrStereoCameraDistortionModelDXR model; + float coefficients[8]; +} XrStereoCameraIntrinsicsDXR; + +typedef struct XrStereoCameraCalibrationDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + //! IN: which output the numbers should describe. + XrStereoCameraOutputDXR output; + //! RAW: native + distortion. RECTIFIED: post-rectification, model NONE, equal fy/cy. + XrStereoCameraIntrinsicsDXR eye[2]; + //! RAW: the pair's extrinsics. RECTIFIED: pure +x translation of baselineMm/1000. + XrPosef rightFromLeft; + float baselineMm; +} XrStereoCameraCalibrationDXR; + +typedef struct XrStereoCameraStreamCreateInfoDXR { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint64_t cameraId; + XrStereoCameraOutputDXR output; + XrStereoCameraFormatDXR format; + XrStereoCameraTransportDXR transport; + //! <= 0 = the source's rate; the service decimates, never interpolates. + float maxFrameRate; +} XrStereoCameraStreamCreateInfoDXR; + +/*! + * The shared-memory ring of a started stream (chained on + * XrStereoCameraStreamInfoDXR). Handles are NEW handles duplicated into the + * caller (Windows HANDLE / POSIX fd): the caller closes them. The section is + * read-only for the caller. + */ +typedef struct XrStereoCameraStreamTransportDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + uint64_t sectionHandle; + uint64_t sectionSize; + uint32_t slotCount; + uint64_t slotStride; + //! Per-stream wake handle, signalled once per published frame. Windows: an + //! auto-reset event (SYNCHRONIZE access). POSIX: the read end of a + //! non-blocking pipe — one byte per frame; drain it before acquiring. + uint64_t wakeHandle; +} XrStereoCameraStreamTransportDXR; + +typedef struct XrStereoCameraStreamInfoDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + //! Full SBS extent (2*eyeWidth x eyeHeight). + XrExtent2Di extent; + XrStereoCameraFormatDXR format; + XrStereoCameraOutputDXR output; + XrStereoCameraTransportDXR transport; + //! The effective delivery cap, Hz. + float maxFrameRate; +} XrStereoCameraStreamInfoDXR; + +typedef struct XrStereoCameraFrameDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + //! Source frame counter, monotonic from 1; gaps = frames this stream missed. + uint64_t frameIndex; + //! Ring slot holding it; PINNED (byte-stable) until this stream's next acquire. + uint32_t slot; + XrExtent2Di extent; + XrStereoCameraFormatDXR format; + //! Plane pitches in bytes (NV12: Y, UV; single-plane formats: [0]). + uint32_t rowPitch[2]; + //! Byte offsets of the planes inside the slot. + uint64_t planeOffset[2]; + XrTime captureTime; + //! TRUE = sensor/exposure time; FALSE = arrival time at the plug-in. + XrBool32 captureTimeIsExposure; + //! What this frame is (RAW when rectification is unavailable). + XrStereoCameraOutputDXR output; + //! Bumps when calibration/rectification changed; re-query calibration. + uint32_t calibrationGeneration; +} XrStereoCameraFrameDXR; + +typedef struct XrStereoCameraStreamStatsDXR { + XrStructureType type; + void* XR_MAY_ALIAS next; + //! Rate the source delivered to the service, Hz (exponential average). + float sourceFrameRate; + //! Rate published to THIS stream after decimation, Hz. + float deliveredFrameRate; + uint64_t framesPublished; + //! Frames published but superseded before this stream acquired them. + uint64_t framesSkipped; + uint64_t framesAcquired; + //! Mean plug-in-timestamp -> publish latency, ns. + uint64_t meanLatencyNs; +} XrStereoCameraStreamStatsDXR; + +typedef struct XrEventDataStereoCameraStateChangedDXR { + XrStructureType type; + const void* XR_MAY_ALIAS next; + uint64_t cameraId; + XrStereoCameraStateDXR state; +} XrEventDataStereoCameraStateChangedDXR; + +typedef struct XrEventDataStereoCamerasChangedDXR { + XrStructureType type; + const void* XR_MAY_ALIAS next; +} XrEventDataStereoCamerasChangedDXR; + +typedef XrResult (XRAPI_PTR *PFN_xrEnumerateStereoCamerasDXR)( + XrInstance instance, XrSystemId systemId, uint32_t capacityInput, uint32_t* countOutput, + XrStereoCameraPropertiesDXR* cameras); +typedef XrResult (XRAPI_PTR *PFN_xrGetStereoCameraCalibrationDXR)( + XrInstance instance, uint64_t cameraId, XrStereoCameraCalibrationDXR* calibration); +typedef XrResult (XRAPI_PTR *PFN_xrCreateStereoCameraStreamDXR)( + XrInstance instance, const XrStereoCameraStreamCreateInfoDXR* createInfo, XrStereoCameraStreamDXR* stream); +typedef XrResult (XRAPI_PTR *PFN_xrDestroyStereoCameraStreamDXR)(XrStereoCameraStreamDXR stream); +typedef XrResult (XRAPI_PTR *PFN_xrStartStereoCameraStreamDXR)(XrStereoCameraStreamDXR stream); +typedef XrResult (XRAPI_PTR *PFN_xrStopStereoCameraStreamDXR)(XrStereoCameraStreamDXR stream); +typedef XrResult (XRAPI_PTR *PFN_xrGetStereoCameraStreamInfoDXR)( + XrStereoCameraStreamDXR stream, XrStereoCameraStreamInfoDXR* info); +typedef XrResult (XRAPI_PTR *PFN_xrAcquireStereoCameraFrameDXR)( + XrStereoCameraStreamDXR stream, XrStereoCameraFrameDXR* frame); +typedef XrResult (XRAPI_PTR *PFN_xrGetStereoCameraStreamStatsDXR)( + XrStereoCameraStreamDXR stream, XrStereoCameraStreamStatsDXR* stats); + +#ifndef XR_NO_PROTOTYPES +#ifdef XR_EXTENSION_PROTOTYPES + +//! Two-call idiom. An in-process instance, a plug-in with no camera, and the +//! kill switch all report count 0 — indistinguishable on purpose. +XRAPI_ATTR XrResult XRAPI_CALL xrEnumerateStereoCamerasDXR( + XrInstance instance, XrSystemId systemId, uint32_t capacityInput, uint32_t* countOutput, + XrStereoCameraPropertiesDXR* cameras); + +//! Calibration of the ACTIVE device. Consent-gated (spec §7.1, §7.5). +XRAPI_ATTR XrResult XRAPI_CALL xrGetStereoCameraCalibrationDXR( + XrInstance instance, uint64_t cameraId, XrStereoCameraCalibrationDXR* calibration); + +XRAPI_ATTR XrResult XRAPI_CALL xrCreateStereoCameraStreamDXR( + XrInstance instance, const XrStereoCameraStreamCreateInfoDXR* createInfo, XrStereoCameraStreamDXR* stream); + +XRAPI_ATTR XrResult XRAPI_CALL xrDestroyStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +//! The authorisation point, and where the service opens the source. +XRAPI_ATTR XrResult XRAPI_CALL xrStartStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +XRAPI_ATTR XrResult XRAPI_CALL xrStopStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +//! Valid after start. Chain XrStereoCameraStreamTransportDXR to receive the +//! ring section + wake handle (new handles each call; caller closes). +XRAPI_ATTR XrResult XRAPI_CALL xrGetStereoCameraStreamInfoDXR( + XrStereoCameraStreamDXR stream, XrStereoCameraStreamInfoDXR* info); + +//! Newest frame newer than the last one acquired, or XR_STEREO_CAMERA_FRAME_NOT_READY_DXR. +XRAPI_ATTR XrResult XRAPI_CALL xrAcquireStereoCameraFrameDXR( + XrStereoCameraStreamDXR stream, XrStereoCameraFrameDXR* frame); + +XRAPI_ATTR XrResult XRAPI_CALL xrGetStereoCameraStreamStatsDXR( + XrStereoCameraStreamDXR stream, XrStereoCameraStreamStatsDXR* stats); + +#endif /* XR_EXTENSION_PROTOTYPES */ +#endif /* !XR_NO_PROTOTYPES */ + +#ifdef __cplusplus +} +#endif + +#endif // XR_DXR_STEREO_CAMERA_H diff --git a/src/xrt/include/xrt/xrt_openxr_includes.h b/src/xrt/include/xrt/xrt_openxr_includes.h index 4016e59f60..74dd4afa25 100644 --- a/src/xrt/include/xrt/xrt_openxr_includes.h +++ b/src/xrt/include/xrt/xrt_openxr_includes.h @@ -87,3 +87,4 @@ typedef __eglMustCastToProperFunctionPointerType (*PFNEGLGETPROCADDRESSPROC)(con #include "openxr/XR_DXR_display_zones.h" #include "openxr/XR_DXR_weave.h" #include "openxr/XR_DXR_lift.h" +#include "openxr/XR_DXR_stereo_camera.h" From 8a70bee7cb88a7e0c77226a32b876b603425dc0d Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:34:00 -0700 Subject: [PATCH 02/37] =?UTF-8?q?feat(stereo-camera):=20u=5Fstereo=5Fcamer?= =?UTF-8?q?a=20=E2=80=94=20latest-wins=20pinned=20ring,=20decimator,=20NV1?= =?UTF-8?q?2/BGRA/GRAY8=20conversion,=20persistent=20id,=20disparity=20pro?= =?UTF-8?q?be?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure, OS-free building blocks shared by the service camera manager, the sim_display fake and displayxr-cli camera (ADR-043). Co-Authored-By: Claude Opus 5.5 --- src/xrt/auxiliary/util/CMakeLists.txt | 2 + src/xrt/auxiliary/util/u_stereo_camera.c | 394 +++++++++++++++++++++++ src/xrt/auxiliary/util/u_stereo_camera.h | 189 +++++++++++ 3 files changed, 585 insertions(+) create mode 100644 src/xrt/auxiliary/util/u_stereo_camera.c create mode 100644 src/xrt/auxiliary/util/u_stereo_camera.h diff --git a/src/xrt/auxiliary/util/CMakeLists.txt b/src/xrt/auxiliary/util/CMakeLists.txt index 6fa0f2db18..4c6484c67d 100644 --- a/src/xrt/auxiliary/util/CMakeLists.txt +++ b/src/xrt/auxiliary/util/CMakeLists.txt @@ -33,6 +33,8 @@ add_library( u_builders.h u_capture_dims.c u_capture_dims.h + u_stereo_camera.c + u_stereo_camera.h u_color_encoding.h u_debug.c u_debug.h diff --git a/src/xrt/auxiliary/util/u_stereo_camera.c b/src/xrt/auxiliary/util/u_stereo_camera.c new file mode 100644 index 0000000000..c811125694 --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_camera.c @@ -0,0 +1,394 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) building blocks — see u_stereo_camera.h. + * @ingroup aux_util + */ + +#include "util/u_stereo_camera.h" + +#include +#include + + +/* + * + * Ring. + * + */ + +void +u_stereo_camera_ring_init(struct u_stereo_camera_ring *r) +{ + memset(r, 0, sizeof(*r)); + r->latest = -1; + r->pinned = -1; + r->writing = -1; + r->latest_acquired = true; +} + +int32_t +u_stereo_camera_ring_begin_write(struct u_stereo_camera_ring *r) +{ + for (int32_t s = 0; s < U_STEREO_CAMERA_RING_SLOTS; s++) { + if (s != r->latest && s != r->pinned) { + r->writing = s; + return s; + } + } + // Unreachable with 3 slots: at most two are latest / pinned. + r->writing = -1; + return -1; +} + +void +u_stereo_camera_ring_publish(struct u_stereo_camera_ring *r, int32_t slot, uint64_t seq) +{ + if (slot < 0 || slot >= U_STEREO_CAMERA_RING_SLOTS || slot == r->pinned) { + r->writing = -1; + return; + } + if (r->latest >= 0 && !r->latest_acquired) { + r->skipped++; + } + r->latest = slot; + r->latest_seq = seq; + r->latest_acquired = false; + r->writing = -1; + r->published++; +} + +void +u_stereo_camera_ring_abort_write(struct u_stereo_camera_ring *r) +{ + r->writing = -1; +} + +bool +u_stereo_camera_ring_acquire(struct u_stereo_camera_ring *r, int32_t *out_slot, uint64_t *out_seq) +{ + if (r->latest < 0 || r->latest_acquired || r->latest_seq <= r->last_acquired_seq) { + return false; + } + r->pinned = r->latest; + r->last_acquired_seq = r->latest_seq; + r->latest_acquired = true; + r->acquired++; + *out_slot = r->pinned; + *out_seq = r->latest_seq; + return true; +} + +void +u_stereo_camera_ring_clear(struct u_stereo_camera_ring *r) +{ + r->pinned = -1; + r->latest = -1; + r->latest_acquired = true; +} + + +/* + * + * Decimator. + * + */ + +void +u_stereo_camera_decimator_init(struct u_stereo_camera_decimator *d, float max_rate, float source_rate) +{ + memset(d, 0, sizeof(*d)); + if (max_rate <= 0.0f || (source_rate > 0.0f && max_rate >= source_rate * 0.999f)) { + d->period_ns = 0; + return; + } + d->period_ns = (int64_t)(1e9 / (double)max_rate); +} + +bool +u_stereo_camera_decimator_accept(struct u_stereo_camera_decimator *d, int64_t t_ns) +{ + if (d->period_ns <= 0) { + return true; + } + if (!d->started) { + d->started = true; + d->next_due_ns = t_ns + d->period_ns; + return true; + } + // A tenth of a period of slack absorbs source jitter without letting two + // frames through for one due time. + if (t_ns < d->next_due_ns - d->period_ns / 10) { + return false; + } + d->next_due_ns += d->period_ns; + if (d->next_due_ns <= t_ns) { + // Fell behind (source paused / suspended): re-anchor, never burst. + d->next_due_ns = t_ns + d->period_ns; + } + return true; +} + + +/* + * + * Layout + conversion. + * + */ + +static uint32_t +align64(uint32_t v) +{ + return (v + 63u) & ~63u; +} + +bool +u_stereo_camera_layout(uint32_t format, uint32_t width, uint32_t height, struct u_stereo_camera_planes *out) +{ + memset(out, 0, sizeof(*out)); + if (width == 0 || height == 0) { + return false; + } + switch (format) { + case 1: // GRAY8 + out->plane_count = 1; + out->pitch[0] = align64(width); + out->size = (uint64_t)out->pitch[0] * height; + return true; + case 2: // NV12 + if ((width & 1u) != 0 || (height & 1u) != 0) { + return false; + } + out->plane_count = 2; + out->pitch[0] = align64(width); + out->pitch[1] = align64(width); + out->offset[1] = (uint64_t)out->pitch[0] * height; + out->size = out->offset[1] + (uint64_t)out->pitch[1] * (height / 2); + return true; + case 3: // BGRA8 + out->plane_count = 1; + out->pitch[0] = align64(width * 4u); + out->size = (uint64_t)out->pitch[0] * height; + return true; + default: return false; + } +} + +static inline uint8_t +clamp_u8(int v) +{ + return (uint8_t)(v < 0 ? 0 : (v > 255 ? 255 : v)); +} + +static inline uint8_t +luma_601(int r, int g, int b) +{ + return clamp_u8(16 + ((66 * r + 129 * g + 25 * b + 128) >> 8)); +} + +bool +u_stereo_camera_convert(uint32_t src_format, + const uint8_t *const src_planes[2], + const uint32_t src_pitches[2], + uint32_t dst_format, + uint8_t *dst, + const struct u_stereo_camera_planes *dl, + uint32_t width, + uint32_t height) +{ + if (src_planes == NULL || src_planes[0] == NULL || dst == NULL || dl == NULL || width == 0 || height == 0) { + return false; + } + uint8_t *d0 = dst + dl->offset[0]; + uint8_t *d1 = dst + dl->offset[1]; + const uint8_t *s0 = src_planes[0]; + const uint8_t *s1 = src_planes[1]; + + // Same format: row copy (pitches may differ). + if (src_format == dst_format) { + uint32_t row = src_format == 3 ? width * 4u : width; + for (uint32_t y = 0; y < height; y++) { + memcpy(d0 + (size_t)y * dl->pitch[0], s0 + (size_t)y * src_pitches[0], row); + } + if (src_format == 2) { + if (s1 == NULL) { + return false; + } + for (uint32_t y = 0; y < height / 2; y++) { + memcpy(d1 + (size_t)y * dl->pitch[1], s1 + (size_t)y * src_pitches[1], width); + } + } + return src_format >= 1 && src_format <= 3; + } + + // GRAY8 / NV12 luma are copied verbatim between each other (a grey sensor's + // value IS its luma); only the RGB legs apply BT.601. + if ((src_format == 1 || src_format == 2) && dst_format == 1) { + for (uint32_t y = 0; y < height; y++) { + memcpy(d0 + (size_t)y * dl->pitch[0], s0 + (size_t)y * src_pitches[0], width); + } + return true; + } + if (src_format == 1 && dst_format == 2) { + for (uint32_t y = 0; y < height; y++) { + memcpy(d0 + (size_t)y * dl->pitch[0], s0 + (size_t)y * src_pitches[0], width); + } + for (uint32_t y = 0; y < height / 2; y++) { + memset(d1 + (size_t)y * dl->pitch[1], 128, width); + } + return true; + } + if (src_format == 1 && dst_format == 3) { + for (uint32_t y = 0; y < height; y++) { + const uint8_t *s = s0 + (size_t)y * src_pitches[0]; + uint8_t *o = d0 + (size_t)y * dl->pitch[0]; + for (uint32_t x = 0; x < width; x++) { + o[4 * x + 0] = o[4 * x + 1] = o[4 * x + 2] = s[x]; + o[4 * x + 3] = 255; + } + } + return true; + } + if (src_format == 2 && dst_format == 3) { + if (s1 == NULL) { + return false; + } + for (uint32_t y = 0; y < height; y++) { + const uint8_t *ys = s0 + (size_t)y * src_pitches[0]; + const uint8_t *uv = s1 + (size_t)(y / 2) * src_pitches[1]; + uint8_t *o = d0 + (size_t)y * dl->pitch[0]; + for (uint32_t x = 0; x < width; x++) { + int c = (int)ys[x] - 16; + int dd = (int)uv[x & ~1u] - 128; + int e = (int)uv[(x & ~1u) + 1] - 128; + o[4 * x + 2] = clamp_u8((298 * c + 409 * e + 128) >> 8); + o[4 * x + 1] = clamp_u8((298 * c - 100 * dd - 208 * e + 128) >> 8); + o[4 * x + 0] = clamp_u8((298 * c + 516 * dd + 128) >> 8); + o[4 * x + 3] = 255; + } + } + return true; + } + if (src_format == 3 && dst_format == 1) { + for (uint32_t y = 0; y < height; y++) { + const uint8_t *s = s0 + (size_t)y * src_pitches[0]; + uint8_t *o = d0 + (size_t)y * dl->pitch[0]; + for (uint32_t x = 0; x < width; x++) { + // Full-range luma: a GRAY8 consumer wants intensity, not video levels. + o[x] = (uint8_t)((29 * s[4 * x + 0] + 150 * s[4 * x + 1] + 77 * s[4 * x + 2] + 128) >> 8); + } + } + return true; + } + if (src_format == 3 && dst_format == 2) { + if ((width & 1u) || (height & 1u)) { + return false; + } + for (uint32_t y = 0; y < height; y++) { + const uint8_t *s = s0 + (size_t)y * src_pitches[0]; + uint8_t *o = d0 + (size_t)y * dl->pitch[0]; + for (uint32_t x = 0; x < width; x++) { + o[x] = luma_601(s[4 * x + 2], s[4 * x + 1], s[4 * x + 0]); + } + } + for (uint32_t y = 0; y < height / 2; y++) { + const uint8_t *a = s0 + (size_t)(2 * y) * src_pitches[0]; + const uint8_t *b = a + src_pitches[0]; + uint8_t *o = d1 + (size_t)y * dl->pitch[1]; + for (uint32_t x = 0; x < width; x += 2) { + int bb = a[4 * x] + a[4 * x + 4] + b[4 * x] + b[4 * x + 4]; + int gg = a[4 * x + 1] + a[4 * x + 5] + b[4 * x + 1] + b[4 * x + 5]; + int rr = a[4 * x + 2] + a[4 * x + 6] + b[4 * x + 2] + b[4 * x + 6]; + bb = (bb + 2) / 4; + gg = (gg + 2) / 4; + rr = (rr + 2) / 4; + o[x] = clamp_u8(128 + ((-38 * rr - 74 * gg + 112 * bb + 128) >> 8)); + o[x + 1] = clamp_u8(128 + ((112 * rr - 94 * gg - 18 * bb + 128) >> 8)); + } + } + return true; + } + return false; +} + + +/* + * + * Identity + probe. + * + */ + +static uint64_t +fnv1a64(uint64_t h, const char *s) +{ + for (; s != NULL && *s != '\0'; s++) { + h ^= (uint8_t)*s; + h *= 0x100000001b3ull; + } + return h; +} + +void +u_stereo_camera_persistent_id(const char *device_identity, const char *consumer, char out[64]) +{ + uint64_t a = fnv1a64(0xcbf29ce484222325ull, "dxr-stereo-camera|"); + a = fnv1a64(a, device_identity); + a = fnv1a64(a, "|"); + a = fnv1a64(a, consumer); + // Second lane with a different basis so the id is 128 bits wide. + uint64_t b = fnv1a64(0x84222325cbf29ce4ull, consumer); + b = fnv1a64(b, "|"); + b = fnv1a64(b, device_identity); + snprintf(out, 64, "dxrcam-%016llx%016llx", (unsigned long long)a, (unsigned long long)b); +} + +bool +u_stereo_camera_estimate_disparity(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + float *out_disparity) +{ + if (gray == NULL || out_disparity == NULL || w == 0 || h == 0 || x0 + w > eye_width || y0 + h > height) { + return false; + } + uint32_t dmax = max_disparity < x0 ? max_disparity : x0; // right x = left x - d >= 0 + uint64_t best = UINT64_MAX; + uint32_t best_d = 0; + uint64_t sad[1024]; + if (dmax >= 1024) { + dmax = 1023; + } + for (uint32_t d = 0; d <= dmax; d++) { + uint64_t s = 0; + for (uint32_t y = y0; y < y0 + h; y++) { + const uint8_t *l = gray + (size_t)y * pitch + x0; + const uint8_t *r = gray + (size_t)y * pitch + eye_width + x0 - d; + for (uint32_t x = 0; x < w; x++) { + int diff = (int)l[x] - (int)r[x]; + s += (uint64_t)(diff < 0 ? -diff : diff); + } + } + sad[d] = s; + if (s < best) { + best = s; + best_d = d; + } + } + float sub = 0.0f; + if (best_d > 0 && best_d < dmax) { + double a = (double)sad[best_d - 1], b = (double)sad[best_d], c = (double)sad[best_d + 1]; + double den = a - 2.0 * b + c; + if (den > 0.0) { + sub = (float)(0.5 * (a - c) / den); + } + } + *out_disparity = (float)best_d + sub; + return true; +} diff --git a/src/xrt/auxiliary/util/u_stereo_camera.h b/src/xrt/auxiliary/util/u_stereo_camera.h new file mode 100644 index 0000000000..95ee0c3692 --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_camera.h @@ -0,0 +1,189 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) platform-neutral building blocks: + * the per-stream latest-wins pinned ring, the frame-rate decimator, + * plane layout + format conversion, the per-consumer persistent id, + * and a block-matching disparity probe. + * + * Pure logic — no threads, no OS handles — so the service's camera manager + * (ipc_server_stereo_camera.c), the sim_display fake and `displayxr-cli + * camera` share one tested implementation (tests_stereo_camera). + * + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/* + * + * Latest-wins ring with one consumer pin. + * + * One ring per stream. The camera thread writes into a slot that is neither + * the latest published slot nor the one the consumer has pinned, then + * publishes it. An acquire hands out the latest slot IF it is newer than the + * last one this stream acquired and pins it: the slot stays byte-stable until + * the next acquire. Three slots are therefore always enough. Frames the + * consumer did not acquire in time are skipped, never queued. + * + */ + +#define U_STEREO_CAMERA_RING_SLOTS 3 + +struct u_stereo_camera_ring +{ + int32_t latest; //!< slot of the newest published frame, -1 = none + int32_t pinned; //!< slot the consumer holds, -1 = none + int32_t writing; //!< slot handed out by begin_write, -1 = none + uint64_t latest_seq; //!< sequence of @ref latest (0 = none) + uint64_t last_acquired_seq; //!< sequence of the last acquire (0 = none) + bool latest_acquired; //!< the consumer has taken @ref latest + + uint64_t published; //!< frames published to this ring + uint64_t skipped; //!< published frames superseded before any acquire + uint64_t acquired; //!< successful acquires +}; + +void +u_stereo_camera_ring_init(struct u_stereo_camera_ring *r); + +//! A slot that is neither latest nor pinned. Always succeeds (3 slots). +int32_t +u_stereo_camera_ring_begin_write(struct u_stereo_camera_ring *r); + +//! Publish the slot from begin_write as the newest frame @p seq (> 0). +void +u_stereo_camera_ring_publish(struct u_stereo_camera_ring *r, int32_t slot, uint64_t seq); + +//! Abandon a begin_write (conversion failed): the slot stays free. +void +u_stereo_camera_ring_abort_write(struct u_stereo_camera_ring *r); + +/*! + * Pin and return the newest frame if newer than the last acquire. The + * previous pin is released first (it is only valid until the next acquire). + * @return false = NOT READY (the previous pin is kept). + */ +bool +u_stereo_camera_ring_acquire(struct u_stereo_camera_ring *r, int32_t *out_slot, uint64_t *out_seq); + +/*! + * Drop the pin and forget the latest frame — used on suspension so the last + * image does not linger for a consumer that may no longer see frames. + */ +void +u_stereo_camera_ring_clear(struct u_stereo_camera_ring *r); + + +/* + * + * Decimation: never interpolate, never burst. + * + */ + +struct u_stereo_camera_decimator +{ + int64_t period_ns; //!< 0 = pass everything + int64_t next_due_ns; + bool started; +}; + +//! @p max_rate <= 0 or >= @p source_rate = pass everything. +void +u_stereo_camera_decimator_init(struct u_stereo_camera_decimator *d, float max_rate, float source_rate); + +//! Should the frame stamped @p t_ns be delivered? +bool +u_stereo_camera_decimator_accept(struct u_stereo_camera_decimator *d, int64_t t_ns); + + +/* + * + * Plane layout + conversion (GRAY8 = 1, NV12 = 2, BGRA8 = 3; the + * xrt_stereo_camera_format values). + * + */ + +struct u_stereo_camera_planes +{ + uint32_t plane_count; + uint32_t pitch[2]; + uint64_t offset[2]; + uint64_t size; //!< bytes the whole image needs +}; + +/*! + * Tight layout of a @p width x @p height image, pitches rounded up to 64 bytes + * (GPU-upload and SIMD friendly). NV12 needs even width and height. + * @return false on an unknown format or odd NV12 extent. + */ +bool +u_stereo_camera_layout(uint32_t format, uint32_t width, uint32_t height, struct u_stereo_camera_planes *out); + +/*! + * Convert one SBS image between formats. BT.601 limited range for the + * luma<->RGB legs; a GRAY8 source becomes NV12 with neutral (128) chroma. + * @return false on an unsupported pair or bad arguments. + */ +bool +u_stereo_camera_convert(uint32_t src_format, + const uint8_t *const src_planes[2], + const uint32_t src_pitches[2], + uint32_t dst_format, + uint8_t *dst, + const struct u_stereo_camera_planes *dst_layout, + uint32_t width, + uint32_t height); + + +/* + * + * Identity + probe helpers. + * + */ + +/*! + * A per-(device, consumer) id: "dxrcam-" + 32 hex chars. Never the raw device + * identity (fingerprinting, spec §7.5). + * + * R1: an unkeyed 128-bit FNV-1a pair over (device_identity, consumer). R3 + * keys it with a persisted per-user secret so ids cannot be recomputed from a + * known serial. + */ +void +u_stereo_camera_persistent_id(const char *device_identity, const char *consumer, char out[64]); + +/*! + * Horizontal disparity (left x - right x, pixels, sub-pixel) of the region + * [x0, x0+w) x [y0, y0+h) of the LEFT eye of a GRAY8 SBS image, by SAD block + * matching over 0..@p max_disparity. Used by `displayxr-cli camera probe` and + * the tests to check a fake source's known disparity. + * @return false if the region does not fit. + */ +bool +u_stereo_camera_estimate_disparity(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + float *out_disparity); + +#ifdef __cplusplus +} +#endif From 494a768b37632c39a0e2a84fad871a1368764924 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:34:00 -0700 Subject: [PATCH 03/37] feat(stereo-camera): plug-in iface camera slots (XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA) + sim_display FAKE stereo camera Six optional slots appended to xrt_plugin_iface per ADR-020 (struct_size-gated, no ABI bump): enumerate, get_calibration (OpenCV R,T of the ACTIVE device), open (= tracker keep-alive), wait_frame, release_frame, close. They sit after a one-pointer placeholder for ADR-042's create_dp_d3d11_lift, which claims the offset right after the vk fingerprint on feat/lift-ext but is not on main yet: the camera slots are therefore at their final offsets whichever branch merges first, and tests_stereo_camera asserts the adjacency (against the real lift slot once it exists). sim_display fills the slots; SIM_DISPLAY_FAKE_STEREO_CAMERA=1 advertises one 640x480-per-eye 30 Hz GRAY8 camera rendering a random-dot pair with a 12 px background and a 40 px bar (68 deg HFOV, 50 mm, 2.0/0.6 m) plus a frame counter; _SIZE / _FPS / _FORMAT / _SUSPEND_PERIOD_MS knobs. xrt_instance gains the stereo_camera client aspect (IPC instance only) and get_active_plugin (native instance) for the service's camera manager. Co-Authored-By: Claude Opus 5.5 --- src/xrt/drivers/CMakeLists.txt | 4 + .../drivers/sim_display/sim_display_plugin.c | 13 + .../sim_display/sim_display_stereo_camera.c | 289 +++++++++++++++++ .../sim_display/sim_display_stereo_camera.h | 59 ++++ .../sim_display_stereo_camera_pattern.c | 136 ++++++++ .../sim_display_stereo_camera_pattern.h | 64 ++++ src/xrt/include/xrt/xrt_instance.h | 20 ++ src/xrt/include/xrt/xrt_plugin.h | 207 +++++++++++++ src/xrt/include/xrt/xrt_stereo_camera.h | 210 +++++++++++++ src/xrt/targets/common/target_instance.c | 17 + tests/CMakeLists.txt | 6 + tests/tests_stereo_camera.cpp | 291 ++++++++++++++++++ 12 files changed, 1316 insertions(+) create mode 100644 src/xrt/drivers/sim_display/sim_display_stereo_camera.c create mode 100644 src/xrt/drivers/sim_display/sim_display_stereo_camera.h create mode 100644 src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c create mode 100644 src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h create mode 100644 src/xrt/include/xrt/xrt_stereo_camera.h create mode 100644 tests/tests_stereo_camera.cpp diff --git a/src/xrt/drivers/CMakeLists.txt b/src/xrt/drivers/CMakeLists.txt index 1f412e8776..d57543e2f9 100644 --- a/src/xrt/drivers/CMakeLists.txt +++ b/src/xrt/drivers/CMakeLists.txt @@ -75,6 +75,10 @@ set(SIM_DISPLAY_SOURCES sim_display/sim_display_device.c sim_display/sim_display_interface.h sim_display/sim_display_zone_common.h + sim_display/sim_display_stereo_camera.c + sim_display/sim_display_stereo_camera.h + sim_display/sim_display_stereo_camera_pattern.c + sim_display/sim_display_stereo_camera_pattern.h ) # Vulkan display processor (anaglyph/blend shaders) diff --git a/src/xrt/drivers/sim_display/sim_display_plugin.c b/src/xrt/drivers/sim_display/sim_display_plugin.c index b2c8e80c5b..906ff323dc 100644 --- a/src/xrt/drivers/sim_display/sim_display_plugin.c +++ b/src/xrt/drivers/sim_display/sim_display_plugin.c @@ -34,6 +34,7 @@ #include "xrt/xrt_results.h" #include "sim_display_interface.h" +#include "sim_display_stereo_camera.h" #if defined(XRT_HAVE_VULKAN) || !defined(_WIN32) #include "vk/vk_helpers.h" // #1243: sizeof(struct vk_bundle) fingerprint #endif @@ -283,6 +284,18 @@ static struct xrt_plugin_iface g_sim_display_iface = { #endif .get_platform_state = sim_display_plugin_get_platform_state, + + /* + * ADR-043 stereo camera source. The slots are always filled; the FAKE + * camera itself only exists under SIM_DISPLAY_FAKE_STEREO_CAMERA=1 (the + * enumerate slot reports zero cameras otherwise). + */ + .stereo_camera_enumerate = sim_display_stereo_camera_enumerate, + .stereo_camera_get_calibration = sim_display_stereo_camera_get_calibration, + .stereo_camera_open = sim_display_stereo_camera_open, + .stereo_camera_wait_frame = sim_display_stereo_camera_wait_frame, + .stereo_camera_release_frame = sim_display_stereo_camera_release_frame, + .stereo_camera_close = sim_display_stereo_camera_close, }; diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c new file mode 100644 index 0000000000..74933342f3 --- /dev/null +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -0,0 +1,289 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief sim_display FAKE stereo camera provider (ADR-043). See the header. + * + * Paces itself on os_monotonic_get_ns (no thread of its own): wait_frame + * sleeps until the next frame's due time, renders the scene with the frame + * number burned in, and hands the runtime a pointer into its own buffer — + * exactly the shape a real plug-in has (a tracker publishes, the plug-in + * copies/decodes, the runtime's camera thread consumes). + * + * @ingroup drv_sim_display + */ + +#include "sim_display_stereo_camera.h" +#include "sim_display_stereo_camera_pattern.h" + +#include "os/os_time.h" +#include "util/u_logging.h" +#include "util/u_stereo_camera.h" + +#include +#include +#include +#include + +#define FAKE_HFOV_DEG 68.0 +#define FAKE_BASELINE_MM 50.0 +#define FAKE_BG_DEPTH_M 2.0 +#define FAKE_BAR_DEPTH_M 0.6 + +struct fake_config +{ + bool enabled; + uint32_t eye_w, eye_h; + float fps; + uint32_t format; + int64_t suspend_period_ns; +}; + +//! Completes the plug-in-owned opaque type. +struct xrt_plugin_stereo_camera +{ + struct fake_config cfg; + struct sim_stereo_camera_scene scene; + uint8_t *gray; + uint32_t gray_pitch; + uint8_t *converted; + struct u_stereo_camera_planes conv_layout; + int64_t t0_ns; + int64_t period_ns; + uint64_t seq; +}; + +static const struct fake_config * +fake_config(void) +{ + static struct fake_config cfg; + static bool read = false; + if (read) { + return &cfg; + } + read = true; + const char *e = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA"); + cfg.enabled = e != NULL && e[0] != '\0' && e[0] != '0'; + cfg.eye_w = 640; + cfg.eye_h = 480; + cfg.fps = 30.0f; + cfg.format = XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8; + const char *sz = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_SIZE"); + unsigned w = 0, h = 0; + if (sz != NULL && sscanf(sz, "%ux%u", &w, &h) == 2 && w >= 64 && h >= 64 && w <= 4096 && h <= 4096) { + cfg.eye_w = w & ~1u; // NV12 needs even extents + cfg.eye_h = h & ~1u; + } + const char *fps = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_FPS"); + if (fps != NULL && atof(fps) >= 1.0 && atof(fps) <= 240.0) { + cfg.fps = (float)atof(fps); + } + const char *fmt = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_FORMAT"); + if (fmt != NULL) { + if (strcmp(fmt, "nv12") == 0) { + cfg.format = XRT_PLUGIN_STEREO_CAMERA_FORMAT_NV12; + } else if (strcmp(fmt, "bgra") == 0 || strcmp(fmt, "bgra8") == 0) { + cfg.format = XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8; + } + } + const char *sp = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_SUSPEND_PERIOD_MS"); + if (sp != NULL && atol(sp) > 0) { + cfg.suspend_period_ns = (int64_t)atol(sp) * 1000000; + } + if (cfg.enabled) { + U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, format %u%s", cfg.eye_w, + cfg.eye_h, cfg.fps, cfg.format, cfg.suspend_period_ns > 0 ? ", suspend square-wave" : ""); + } + return &cfg; +} + +static double +fake_fx(uint32_t eye_w) +{ + return (eye_w / 2.0) / tan(FAKE_HFOV_DEG * 0.5 * 3.14159265358979323846 / 180.0); +} + +uint32_t +sim_display_stereo_camera_enumerate(struct xrt_plugin_instance *inst, + uint32_t capacity, + struct xrt_plugin_stereo_camera_info *out) +{ + (void)inst; + const struct fake_config *cfg = fake_config(); + if (!cfg->enabled) { + return 0; + } + if (capacity >= 1 && out != NULL) { + uint32_t sz = out->struct_size; + struct xrt_plugin_stereo_camera_info info; + memset(&info, 0, sizeof(info)); + info.struct_size = (uint32_t)sizeof(info); + snprintf(info.display_name, sizeof(info.display_name), "Sim 3D camera (fake)"); + snprintf(info.device_identity, sizeof(info.device_identity), "sim-display-fake-stereo-camera-0001"); + info.flags = XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING | XRT_PLUGIN_STEREO_CAMERA_USER_FACING | + XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED; + if (cfg->format == XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { + info.flags |= XRT_PLUGIN_STEREO_CAMERA_MONOCHROME; + } + info.eye_width = cfg->eye_w; + info.eye_height = cfg->eye_h; + info.max_frame_rate = cfg->fps; + info.native_format = cfg->format; + // Never write past the runtime's struct_size (ADR-020). + memcpy(out, &info, sz < sizeof(info) ? sz : sizeof(info)); + out->struct_size = sz; + } + return 1; +} + +xrt_result_t +sim_display_stereo_camera_get_calibration(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera_calibration *out) +{ + (void)inst; + const struct fake_config *cfg = fake_config(); + if (!cfg->enabled || index != 0 || out == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + uint32_t sz = out->struct_size; + struct xrt_plugin_stereo_camera_calibration c; + memset(&c, 0, sizeof(c)); + c.struct_size = (uint32_t)sizeof(c); + c.image_width = cfg->eye_w; + c.image_height = cfg->eye_h; + double fx = fake_fx(cfg->eye_w); + for (int e = 0; e < 2; e++) { + c.k[e][0] = fx; + c.k[e][1] = fx; + c.k[e][2] = (cfg->eye_w - 1) * 0.5; + c.k[e][3] = (cfg->eye_h - 1) * 0.5; + } + c.distortion_model = XRT_PLUGIN_STEREO_CAMERA_DISTORTION_NONE; + c.rotation_right_from_left[0][0] = 1.0; + c.rotation_right_from_left[1][1] = 1.0; + c.rotation_right_from_left[2][2] = 1.0; + // OpenCV convention (x_R = R x_L + T): a right camera at +x of the left + // one has T = (-B, 0, 0). + c.translation_right_from_left_mm[0] = -FAKE_BASELINE_MM; + memcpy(out, &c, sz < sizeof(c) ? sz : sizeof(c)); + out->struct_size = sz; + return XRT_SUCCESS; +} + +xrt_result_t +sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera **out_cam) +{ + (void)inst; + const struct fake_config *cfg = fake_config(); + if (!cfg->enabled || index != 0 || out_cam == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + struct xrt_plugin_stereo_camera *cam = calloc(1, sizeof(*cam)); + if (cam == NULL) { + return XRT_ERROR_ALLOCATION; + } + cam->cfg = *cfg; + sim_stereo_camera_scene_init(&cam->scene, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), FAKE_BASELINE_MM, + FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M); + cam->gray_pitch = 2 * cfg->eye_w; + cam->gray = calloc((size_t)cam->gray_pitch * cfg->eye_h, 1); + if (cfg->format != XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { + u_stereo_camera_layout(cfg->format, 2 * cfg->eye_w, cfg->eye_h, &cam->conv_layout); + cam->converted = calloc((size_t)cam->conv_layout.size, 1); + } + if (cam->gray == NULL || (cfg->format != XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 && cam->converted == NULL)) { + free(cam->gray); + free(cam->converted); + free(cam); + return XRT_ERROR_ALLOCATION; + } + cam->period_ns = (int64_t)(1e9 / cfg->fps); + cam->t0_ns = os_monotonic_get_ns(); + U_LOG_W("sim_display: FAKE stereo camera opened (bg disparity %u px, bar disparity %u px)", + cam->scene.bg_disparity, cam->scene.bar_disparity); + *out_cam = cam; + return XRT_SUCCESS; +} + +static void +sleep_ns(int64_t ns) +{ + if (ns > 0) { + os_nanosleep(ns); + } +} + +uint32_t +sim_display_stereo_camera_wait_frame(struct xrt_plugin_stereo_camera *cam, + int64_t timeout_ns, + struct xrt_plugin_stereo_camera_frame *out) +{ + if (cam == NULL || out == NULL) { + return XRT_PLUGIN_STEREO_CAMERA_WAIT_ERROR; + } + int64_t now = os_monotonic_get_ns(); + + // SUSPEND square wave: odd half-periods are "tracker stopped". + if (cam->cfg.suspend_period_ns > 0 && ((now - cam->t0_ns) / cam->cfg.suspend_period_ns) % 2 == 1) { + sleep_ns(timeout_ns < 20000000 ? timeout_ns : 20000000); + return XRT_PLUGIN_STEREO_CAMERA_WAIT_SUSPENDED; + } + + int64_t due = cam->t0_ns + (int64_t)cam->seq * cam->period_ns; + if (now - due > cam->period_ns) { + // Back from a pause: re-anchor instead of bursting the missed frames. + cam->t0_ns = now - (int64_t)cam->seq * cam->period_ns; + due = now; + } + if (due - now > timeout_ns) { + sleep_ns(timeout_ns); + return XRT_PLUGIN_STEREO_CAMERA_WAIT_TIMEOUT; + } + sleep_ns(due - now); + + cam->seq++; + sim_stereo_camera_render_gray(&cam->scene, cam->seq, cam->gray, cam->gray_pitch); + + memset(out, 0, sizeof(*out)); + out->sequence = cam->seq; + out->time_ns = due; // the synthetic "exposure" + out->time_is_exposure = true; + out->width = 2 * cam->cfg.eye_w; + out->height = cam->cfg.eye_h; + out->format = cam->cfg.format; + if (cam->cfg.format == XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { + out->planes[0] = cam->gray; + out->pitches[0] = cam->gray_pitch; + } else { + const uint8_t *src[2] = {cam->gray, NULL}; + const uint32_t pitches[2] = {cam->gray_pitch, 0}; + u_stereo_camera_convert(XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8, src, pitches, cam->cfg.format, + cam->converted, &cam->conv_layout, out->width, out->height); + for (uint32_t p = 0; p < cam->conv_layout.plane_count; p++) { + out->planes[p] = cam->converted + cam->conv_layout.offset[p]; + out->pitches[p] = cam->conv_layout.pitch[p]; + } + } + return XRT_PLUGIN_STEREO_CAMERA_WAIT_OK; +} + +void +sim_display_stereo_camera_release_frame(struct xrt_plugin_stereo_camera *cam) +{ + (void)cam; // the buffer is ours and is only rewritten by the next wait_frame +} + +void +sim_display_stereo_camera_close(struct xrt_plugin_stereo_camera *cam) +{ + if (cam == NULL) { + return; + } + U_LOG_W("sim_display: FAKE stereo camera closed after %llu frames", (unsigned long long)cam->seq); + free(cam->gray); + free(cam->converted); + free(cam); +} diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.h b/src/xrt/drivers/sim_display/sim_display_stereo_camera.h new file mode 100644 index 0000000000..94f1f8809e --- /dev/null +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.h @@ -0,0 +1,59 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief sim_display FAKE stereo camera provider (ADR-043) — the plug-in + * iface camera slots, filled under SIM_DISPLAY_FAKE_STEREO_CAMERA=1. + * + * Knobs (read by the process that loads the plug-in — the SERVICE): + * SIM_DISPLAY_FAKE_STEREO_CAMERA=1 advertise one camera + * SIM_DISPLAY_FAKE_STEREO_CAMERA_SIZE=WxH per-eye size (default 640x480) + * SIM_DISPLAY_FAKE_STEREO_CAMERA_FPS=N source rate (default 30) + * SIM_DISPLAY_FAKE_STEREO_CAMERA_FORMAT=gray8|nv12|bgra native format (default gray8) + * SIM_DISPLAY_FAKE_STEREO_CAMERA_SUSPEND_PERIOD_MS=N square-wave SUSPENDED/AVAILABLE + * + * The camera: SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | + * NATIVELY_RECTIFIED (+ MONOCHROME for gray8), 68 deg horizontal FOV per eye, + * 50 mm baseline, background at 2.0 m and a bar at 0.6 m — at 640 px per eye + * that is fx = 474.4 px and disparities of 12 px and 40 px. + * + * @ingroup drv_sim_display + */ + +#pragma once + +#include "xrt/xrt_plugin.h" + +#ifdef __cplusplus +extern "C" { +#endif + +uint32_t +sim_display_stereo_camera_enumerate(struct xrt_plugin_instance *inst, + uint32_t capacity, + struct xrt_plugin_stereo_camera_info *out); + +xrt_result_t +sim_display_stereo_camera_get_calibration(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera_calibration *out); + +xrt_result_t +sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera **out_cam); + +uint32_t +sim_display_stereo_camera_wait_frame(struct xrt_plugin_stereo_camera *cam, + int64_t timeout_ns, + struct xrt_plugin_stereo_camera_frame *out); + +void +sim_display_stereo_camera_release_frame(struct xrt_plugin_stereo_camera *cam); + +void +sim_display_stereo_camera_close(struct xrt_plugin_stereo_camera *cam); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c new file mode 100644 index 0000000000..5fc21237e3 --- /dev/null +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c @@ -0,0 +1,136 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief sim_display FAKE stereo camera: the synthetic SBS scene. + * @ingroup drv_sim_display + */ + +#include "sim_display_stereo_camera_pattern.h" + +#include +#include +#include + +//! 5x7 digits, one byte per row, bit 4 = leftmost column. +static const uint8_t k_digits[10][7] = { + {0x0E, 0x11, 0x13, 0x15, 0x19, 0x11, 0x0E}, // 0 + {0x04, 0x0C, 0x04, 0x04, 0x04, 0x04, 0x0E}, // 1 + {0x0E, 0x11, 0x01, 0x02, 0x04, 0x08, 0x1F}, // 2 + {0x1F, 0x02, 0x04, 0x02, 0x01, 0x11, 0x0E}, // 3 + {0x02, 0x06, 0x0A, 0x12, 0x1F, 0x02, 0x02}, // 4 + {0x1F, 0x10, 0x1E, 0x01, 0x01, 0x11, 0x0E}, // 5 + {0x06, 0x08, 0x10, 0x1E, 0x11, 0x11, 0x0E}, // 6 + {0x1F, 0x01, 0x02, 0x04, 0x08, 0x08, 0x08}, // 7 + {0x0E, 0x11, 0x11, 0x0E, 0x11, 0x11, 0x0E}, // 8 + {0x0E, 0x11, 0x11, 0x0F, 0x01, 0x02, 0x0C}, // 9 +}; + +#define COUNTER_DIGITS 8 +#define COUNTER_SCALE 3 +#define COUNTER_X 8 +#define COUNTER_Y 8 + +static inline uint32_t +hash3(uint32_t x, uint32_t y, uint32_t seed) +{ + uint32_t h = x * 0x8da6b343u ^ y * 0xd8163841u ^ seed * 0xcb1ab31fu; + h ^= h >> 13; + h *= 0x5bd1e995u; + h ^= h >> 15; + return h; +} + +//! Value noise on 2-pixel cells: detail a block matcher locks onto, without +//! single-pixel aliasing. +static inline uint8_t +texel(uint32_t u, uint32_t v, uint32_t seed, uint8_t lo, uint8_t span) +{ + return (uint8_t)(lo + hash3(u >> 1, v >> 1, seed) % span); +} + +void +sim_stereo_camera_scene_init(struct sim_stereo_camera_scene *s, + uint32_t eye_width, + uint32_t eye_height, + double fx_px, + double baseline_mm, + double bg_depth_m, + double bar_depth_m) +{ + memset(s, 0, sizeof(*s)); + s->eye_width = eye_width; + s->eye_height = eye_height; + s->bg_disparity = (uint32_t)lround(fx_px * (baseline_mm / 1000.0) / bg_depth_m); + s->bar_disparity = (uint32_t)lround(fx_px * (baseline_mm / 1000.0) / bar_depth_m); + s->bar_x0 = eye_width * 3 / 8; + s->bar_x1 = eye_width * 5 / 8; + s->bar_y0 = eye_height / 4; + s->bar_y1 = eye_height * 3 / 4; +} + +static void +render_eye(const struct sim_stereo_camera_scene *s, uint8_t *dst, uint32_t pitch, uint32_t x_off, int right) +{ + for (uint32_t y = 0; y < s->eye_height; y++) { + uint8_t *row = dst + (size_t)y * pitch + x_off; + bool in_bar_rows = y >= s->bar_y0 && y < s->bar_y1; + for (uint32_t x = 0; x < s->eye_width; x++) { + // A feature at scene coordinate u shows at x = u in the left eye and at + // x = u - d in the right eye, so the right eye samples u = x + d. + uint32_t ub = right ? x + s->bg_disparity : x; + uint32_t ubar = right ? x + s->bar_disparity : x; + if (in_bar_rows && ubar >= s->bar_x0 && ubar < s->bar_x1) { + row[x] = texel(ubar, y, 0xBA5u, 128, 128); + } else { + row[x] = texel(ub, y, 0x0B6u, 0, 112); + } + } + } +} + +static void +render_counter(uint8_t *dst, uint32_t pitch, uint32_t x_off, uint32_t eye_w, uint32_t eye_h, uint64_t n) +{ + const uint32_t cw = 6 * COUNTER_SCALE, ch = 9 * COUNTER_SCALE; + const uint32_t box_w = COUNTER_DIGITS * cw + COUNTER_SCALE, box_h = ch; + if (COUNTER_X + box_w > eye_w || COUNTER_Y + box_h > eye_h) { + return; + } + for (uint32_t y = 0; y < box_h; y++) { + memset(dst + (size_t)(COUNTER_Y + y) * pitch + x_off + COUNTER_X, 0, box_w); + } + char digits[COUNTER_DIGITS]; + for (int i = COUNTER_DIGITS - 1; i >= 0; i--) { + digits[i] = (char)(n % 10); + n /= 10; + } + for (uint32_t i = 0; i < COUNTER_DIGITS; i++) { + const uint8_t *g = k_digits[(int)digits[i]]; + uint32_t gx = COUNTER_X + COUNTER_SCALE + i * cw; + for (uint32_t r = 0; r < 7; r++) { + for (uint32_t c = 0; c < 5; c++) { + if (((g[r] >> (4 - c)) & 1u) == 0) { + continue; + } + for (uint32_t sy = 0; sy < COUNTER_SCALE; sy++) { + uint8_t *p = dst + (size_t)(COUNTER_Y + COUNTER_SCALE + r * COUNTER_SCALE + sy) * pitch + + x_off + gx + c * COUNTER_SCALE; + memset(p, 255, COUNTER_SCALE); + } + } + } + } +} + +void +sim_stereo_camera_render_gray(const struct sim_stereo_camera_scene *s, + uint64_t frame_number, + uint8_t *dst, + uint32_t pitch) +{ + render_eye(s, dst, pitch, 0, 0); + render_eye(s, dst, pitch, s->eye_width, 1); + render_counter(dst, pitch, 0, s->eye_width, s->eye_height, frame_number); + render_counter(dst, pitch, s->eye_width, s->eye_width, s->eye_height, frame_number); +} diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h new file mode 100644 index 0000000000..81b4953522 --- /dev/null +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h @@ -0,0 +1,64 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief sim_display FAKE stereo camera (ADR-043): the synthetic SBS scene. + * + * Pure CPU rendering, no OS dependencies, so the unit tests pin it host-side + * (tests_stereo_camera) next to the disparity probe that measures it. + * + * The scene is a random-dot stereogram with two planes at KNOWN integer + * disparities (left x - right x, pixels): a textured background and a nearer + * textured bar in the middle of the frame. A frame counter is burned into the + * top-left of BOTH halves at zero disparity. The pair is parallel and + * undistorted (the R1 fake reports NATIVELY_RECTIFIED); a distorted, + * misaligned variant for the rectifier's tests is R2's. + * + * @ingroup drv_sim_display + */ + +#pragma once + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct sim_stereo_camera_scene +{ + uint32_t eye_width; + uint32_t eye_height; + uint32_t bg_disparity; //!< background plane, pixels + uint32_t bar_disparity; //!< near bar, pixels + //! The bar's rectangle in LEFT-eye coordinates. + uint32_t bar_x0, bar_y0, bar_x1, bar_y1; +}; + +/*! + * Scene for a pinhole pair of focal @p fx_px and baseline @p baseline_mm, with + * the background at @p bg_depth_m and the bar at @p bar_depth_m: disparities + * are round(fx * B / Z). + */ +void +sim_stereo_camera_scene_init(struct sim_stereo_camera_scene *s, + uint32_t eye_width, + uint32_t eye_height, + double fx_px, + double baseline_mm, + double bg_depth_m, + double bar_depth_m); + +/*! + * Render frame @p frame_number into a GRAY8 SBS image (2*eye_width x + * eye_height, @p pitch bytes per row, left eye left). + */ +void +sim_stereo_camera_render_gray(const struct sim_stereo_camera_scene *s, + uint64_t frame_number, + uint8_t *dst, + uint32_t pitch); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/include/xrt/xrt_instance.h b/src/xrt/include/xrt/xrt_instance.h index 61767f5bd8..9a93cd75ae 100644 --- a/src/xrt/include/xrt/xrt_instance.h +++ b/src/xrt/include/xrt/xrt_instance.h @@ -23,6 +23,9 @@ extern "C" { struct xrt_prober; struct xrt_device; struct xrt_instance_android; +struct xrt_stereo_camera_client; +struct xrt_plugin_iface; +struct xrt_plugin_instance; struct xrt_space_overseer; struct xrt_system; struct xrt_system_devices; @@ -245,6 +248,23 @@ struct xrt_instance * @see xrt_instance_android */ struct xrt_instance_android *android_instance; + + /*! + * XR_DXR_stereo_camera (ADR-043) client aspect. Set by the IPC client + * instance only; NULL for an in-process instance (which enumerates zero + * cameras — the service is each camera's single owner). + */ + struct xrt_stereo_camera_client *stereo_camera; + + /*! + * Optional: the plug-in that claimed the system for THIS instance, for + * service-side owners of plug-in-provided sources (the IPC server's + * stereo camera manager, ADR-043). Set by the native instance; NULL on + * the IPC client instance. Returns false when no plug-in is active. + */ + bool (*get_active_plugin)(struct xrt_instance *xinst, + const struct xrt_plugin_iface **out_iface, + struct xrt_plugin_instance **out_inst); }; /*! diff --git a/src/xrt/include/xrt/xrt_plugin.h b/src/xrt/include/xrt/xrt_plugin.h index 8e11724f37..ceb4268518 100644 --- a/src/xrt/include/xrt/xrt_plugin.h +++ b/src/xrt/include/xrt/xrt_plugin.h @@ -43,6 +43,7 @@ #include "xrt/xrt_display_processor_metal.h" #include +#include #include #ifdef __cplusplus @@ -628,6 +629,128 @@ struct xrt_plugin_host_iface */ #define XRT_PLUGIN_HOST_HAS_ANDROID_PACKAGE_VISIBILITY 1 +/* + * + * Stereo camera source (ADR-043, XR_DXR_stereo_camera). + * + * A display's stereo camera — usually the one its eye tracker looks through — + * produced by the plug-in from its own stack WITHOUT taking the device from + * the tracker. The runtime service owns threads, fan-out, rectification, + * format conversion, transport, consent and the in-use indicator; the plug-in + * owns reading + decoding frames and the calibration of ITS ACTIVE device. + * Graphics-API-neutral on purpose: a camera is a sensor, not a weaver, and it + * must not share a lifetime with a display processor that is recreated on + * presenter changes. Slots on @ref xrt_plugin_iface (appended, ADR-020). + * + */ + +//! @ref xrt_plugin_stereo_camera_info::flags — values equal the +//! XrStereoCameraFlagsDXR bits (static-asserted in the state tracker). +#define XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING (1u << 0) +#define XRT_PLUGIN_STEREO_CAMERA_USER_FACING (1u << 1) +#define XRT_PLUGIN_STEREO_CAMERA_CALIBRATED (1u << 2) +#define XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED (1u << 3) +#define XRT_PLUGIN_STEREO_CAMERA_MONOCHROME (1u << 4) + +//! Pixel formats a plug-in delivers (values equal XrStereoCameraFormatDXR). +enum xrt_plugin_stereo_camera_format +{ + XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 = 1, + XRT_PLUGIN_STEREO_CAMERA_FORMAT_NV12 = 2, + XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8 = 3, +}; + +//! Lens models (values equal XrStereoCameraDistortionModelDXR). +enum xrt_plugin_stereo_camera_distortion +{ + XRT_PLUGIN_STEREO_CAMERA_DISTORTION_NONE = 0, + XRT_PLUGIN_STEREO_CAMERA_DISTORTION_RADTAN5 = 1, + XRT_PLUGIN_STEREO_CAMERA_DISTORTION_RADTAN8 = 2, + XRT_PLUGIN_STEREO_CAMERA_DISTORTION_KB4 = 3, +}; + +//! Result of @ref xrt_plugin_iface::stereo_camera_wait_frame. +enum xrt_plugin_stereo_camera_wait +{ + //! @p out holds a new frame; call stereo_camera_release_frame when done. + XRT_PLUGIN_STEREO_CAMERA_WAIT_OK = 0, + //! No frame within the timeout (source up; e.g. tracker warming up). + XRT_PLUGIN_STEREO_CAMERA_WAIT_TIMEOUT = 1, + //! The source is temporarily not delivering (tracker stopped). + XRT_PLUGIN_STEREO_CAMERA_WAIT_SUSPENDED = 2, + //! The source failed or went away; the runtime closes it. + XRT_PLUGIN_STEREO_CAMERA_WAIT_ERROR = 3, +}; + +/*! + * One camera, as a plug-in describes it. The runtime sets @ref struct_size to + * its own size before the call; the plug-in must not write past it. + */ +struct xrt_plugin_stereo_camera_info +{ + uint32_t struct_size; + //! Human-readable, UTF-8 ("Built-in 3D camera"). + char display_name[128]; + //! Stable device key (e.g. the panel serial). The SERVICE hashes it per + //! consumer; it is never exported to clients. + char device_identity[128]; + //! The OS's id of the physical device the frames come from, or "". + char platform_device_hint[256]; + //! XRT_PLUGIN_STEREO_CAMERA_* bits. + uint32_t flags; + //! Native per-eye size. + uint32_t eye_width; + uint32_t eye_height; + //! What the source can deliver, Hz. + float max_frame_rate; + //! @ref xrt_plugin_stereo_camera_format the plug-in decodes to. + uint32_t native_format; +}; + +/*! + * RAW calibration of the ACTIVE device (the one this plug-in weaves for and + * whose tracker it reads) — never "the first calibration on the machine". + */ +struct xrt_plugin_stereo_camera_calibration +{ + uint32_t struct_size; + //! Per eye, pixels. + uint32_t image_width; + uint32_t image_height; + //! Per eye: fx fy cx cy, pixels. + double k[2][4]; + //! @ref xrt_plugin_stereo_camera_distortion. + uint32_t distortion_model; + double distortion[2][8]; + double rotation_right_from_left[3][3]; + double translation_right_from_left_mm[3]; +}; + +/*! + * One frame handed to the runtime. Planes stay valid until + * stereo_camera_release_frame. Always ONE side-by-side image, left eye left. + */ +struct xrt_plugin_stereo_camera_frame +{ + //! Monotonic per open, from 1. + uint64_t sequence; + //! os_monotonic_get_ns() domain. + int64_t time_ns; + //! true = sensor/exposure time; false = arrival time in the plug-in. + bool time_is_exposure; + //! Full SBS extent (2 * eye_width x eye_height). + uint32_t width; + uint32_t height; + //! @ref xrt_plugin_stereo_camera_format. + uint32_t format; + const uint8_t *planes[2]; + uint32_t pitches[2]; +}; + +//! Opaque plug-in-owned open camera. +struct xrt_plugin_stereo_camera; + + /*! * The plug-in's vtable. Filled in by the plug-in inside its * `xrtPluginNegotiate` implementation and handed back to the runtime via @@ -925,6 +1048,67 @@ struct xrt_plugin_iface * @ref struct_size; no XRT_PLUGIN_API_VERSION_CURRENT bump). */ bool (*get_platform_state)(struct xrt_plugin_platform_status *out_status); + + /*! + * @name Stereo camera source (ADR-043, XR_DXR_stereo_camera) + * + * Optional, all-or-nothing: the runtime uses the camera slots only when + * @ref struct_size covers @ref stereo_camera_close and every one of them + * is non-NULL. Appended per ADR-020 (append-only within a major; gated by + * @ref struct_size; no XRT_PLUGIN_API_VERSION_CURRENT bump) AFTER + * @ref get_platform_state (ADR-045), the last member before this block. + * Announced by @ref XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA. + * + * Threading: enumerate / get_calibration are cheap and callable from any + * thread. wait_frame / release_frame are called from ONE runtime-owned + * camera thread per open camera. Nothing throws across this boundary. + * @{ + */ + + /*! + * Fill up to @p capacity entries of @p out (each with struct_size preset + * by the runtime) and return the number of cameras. @p capacity 0 / NULL + * @p out = count query. Stable indices for the plug-in instance's life. + */ + uint32_t (*stereo_camera_enumerate)(struct xrt_plugin_instance *inst, + uint32_t capacity, + struct xrt_plugin_stereo_camera_info *out); + + /*! + * RAW calibration of camera @p index, for the ACTIVE device. Return + * XRT_ERROR_FEATURE_NOT_SUPPORTED when it cannot be resolved — the camera + * must then not report XRT_PLUGIN_STEREO_CAMERA_CALIBRATED. + */ + xrt_result_t (*stereo_camera_get_calibration)(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera_calibration *out); + + /*! + * Open camera @p index. Doubles as a tracker keep-alive: while any camera + * is open, a SHARED_WITH_EYE_TRACKING source keeps its tracker running. + * Must never open / reconfigure / re-time the device away from the tracker. + */ + xrt_result_t (*stereo_camera_open)(struct xrt_plugin_instance *inst, + uint32_t index, + struct xrt_plugin_stereo_camera **out_cam); + + /*! + * Block up to @p timeout_ns for the next frame. Returns an @ref + * xrt_plugin_stereo_camera_wait. On OK the frame's planes stay valid until + * @ref stereo_camera_release_frame. Must not wait on a wake object shared + * with other readers of the vendor channel (poll it instead; ADR-043). + */ + uint32_t (*stereo_camera_wait_frame)(struct xrt_plugin_stereo_camera *cam, + int64_t timeout_ns, + struct xrt_plugin_stereo_camera_frame *out); + + //! Release the frame of the last OK wait_frame. + void (*stereo_camera_release_frame)(struct xrt_plugin_stereo_camera *cam); + + //! Close; drops the keep-alive when it was the last open camera. + void (*stereo_camera_close)(struct xrt_plugin_stereo_camera *cam); + + /*! @} */ }; /*! @@ -943,6 +1127,29 @@ struct xrt_plugin_iface */ #define XRT_PLUGIN_HAS_PLATFORM_STATE 1 +/*! + * Defined when @ref xrt_plugin_iface carries the stereo camera slots + * (ADR-043), so a plug-in built against an older runtime header can + * #ifdef-guard filling them. + */ +#define XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA 1 + +/*! + * True when @p iface fills the whole stereo camera slot set (and its + * struct_size covers it). + */ +static inline bool +xrt_plugin_iface_has_stereo_camera(const struct xrt_plugin_iface *iface) +{ + if (iface == NULL || iface->struct_size < offsetof(struct xrt_plugin_iface, stereo_camera_close) + + sizeof(iface->stereo_camera_close)) { + return false; + } + return iface->stereo_camera_enumerate != NULL && iface->stereo_camera_get_calibration != NULL && + iface->stereo_camera_open != NULL && iface->stereo_camera_wait_frame != NULL && + iface->stereo_camera_release_frame != NULL && iface->stereo_camera_close != NULL; +} + /* * diff --git a/src/xrt/include/xrt/xrt_stereo_camera.h b/src/xrt/include/xrt/xrt_stereo_camera.h new file mode 100644 index 0000000000..40b05c3537 --- /dev/null +++ b/src/xrt/include/xrt/xrt_stereo_camera.h @@ -0,0 +1,210 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) runtime-side value types and the + * client aspect of @ref xrt_instance. + * + * The value structs are plain fixed-size POD: they ride the IPC wire as-is + * (proto.json) and are translated 1:1 to the XR structs by the state tracker. + * Enum values equal the XR ones (static-asserted in oxr_stereo_camera.c). + * + * @ingroup xrt_iface + */ + +#pragma once + +#include "xrt/xrt_compiler.h" +#include "xrt/xrt_handles.h" +#include "xrt/xrt_results.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! Most cameras the service exposes (per plug-in). +#define XRT_STEREO_CAMERA_MAX_CAMERAS 4 +//! Streams per camera, service-wide (spec §8: XR_ERROR_LIMIT_REACHED past it). +#define XRT_STEREO_CAMERA_MAX_STREAMS_PER_CAMERA 8 +//! Slots in a stream's ring: latest + pinned + one to write. +#define XRT_STEREO_CAMERA_RING_SLOTS 3 + +enum xrt_stereo_camera_state +{ + XRT_STEREO_CAMERA_STATE_AVAILABLE = 1, + XRT_STEREO_CAMERA_STATE_WAITING = 2, + XRT_STEREO_CAMERA_STATE_SUSPENDED = 3, + XRT_STEREO_CAMERA_STATE_UNAVAILABLE = 4, +}; + +enum xrt_stereo_camera_output +{ + XRT_STEREO_CAMERA_OUTPUT_RECTIFIED = 1, + XRT_STEREO_CAMERA_OUTPUT_RAW = 2, +}; + +//! Same values as xrt_plugin_stereo_camera_format and XrStereoCameraFormatDXR. +enum xrt_stereo_camera_format +{ + XRT_STEREO_CAMERA_FORMAT_GRAY8 = 1, + XRT_STEREO_CAMERA_FORMAT_NV12 = 2, + XRT_STEREO_CAMERA_FORMAT_BGRA8 = 3, +}; + +enum xrt_stereo_camera_transport +{ + XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY = 1, + XRT_STEREO_CAMERA_TRANSPORT_D3D11_TEXTURE = 2, + XRT_STEREO_CAMERA_TRANSPORT_AHARDWAREBUFFER = 3, +}; + +//! Format / transport sets are one bit per enum value: (1 << value). +#define XRT_STEREO_CAMERA_BIT(v) (1ull << (uint32_t)(v)) + +/*! + * One enumerated camera (XrStereoCameraPropertiesDXR). + */ +struct xrt_stereo_camera_properties +{ + uint64_t camera_id; + char persistent_id[64]; + char display_name[128]; + char platform_device_hint[256]; + uint64_t flags; //!< XRT_PLUGIN_STEREO_CAMERA_* bits (= XrStereoCameraFlagsDXR) + uint32_t state; //!< enum xrt_stereo_camera_state + uint32_t view_count; + uint32_t eye_width; + uint32_t eye_height; + float max_frame_rate; + float baseline_mm; + float horizontal_fov_deg; + uint32_t reserved; + uint64_t supported_formats; //!< XRT_STEREO_CAMERA_BIT(format) + uint64_t supported_transports; //!< XRT_STEREO_CAMERA_BIT(transport) +}; + +struct xrt_stereo_camera_intrinsics +{ + uint32_t width; + uint32_t height; + float fx, fy, cx, cy; + uint32_t model; //!< xrt_plugin_stereo_camera_distortion + float coefficients[8]; +}; + +/*! + * Calibration as a client sees it (XrStereoCameraCalibrationDXR). + */ +struct xrt_stereo_camera_calibration +{ + uint32_t output; //!< enum xrt_stereo_camera_output these numbers describe + uint32_t reserved; + struct xrt_stereo_camera_intrinsics eye[2]; + float orientation[4]; //!< right-from-left rotation, quaternion x y z w + float position[3]; //!< right-from-left translation, metres + float baseline_mm; +}; + +/*! + * What a client asks for at stream create. + */ +struct xrt_stereo_camera_stream_request +{ + uint64_t camera_id; + uint32_t output; //!< enum xrt_stereo_camera_output + uint32_t format; //!< enum xrt_stereo_camera_format + uint32_t transport; //!< enum xrt_stereo_camera_transport + float max_frame_rate; +}; + +/*! + * A started stream's layout (XrStereoCameraStreamInfoDXR + transport). + */ +struct xrt_stereo_camera_stream_layout +{ + uint32_t width; //!< full SBS + uint32_t height; + uint32_t format; + uint32_t output; + uint32_t transport; + float max_frame_rate; //!< effective delivery cap + uint64_t section_size; + uint64_t slot_stride; + uint32_t slot_count; + uint32_t reserved; +}; + +/*! + * One acquired frame (XrStereoCameraFrameDXR). @ref capture_time_ns is on the + * os_monotonic clock; the state tracker converts it to XrTime. + */ +struct xrt_stereo_camera_frame_info +{ + uint64_t frame_index; + uint32_t slot; + uint32_t width; + uint32_t height; + uint32_t format; + uint32_t row_pitch[2]; + uint64_t plane_offset[2]; + int64_t capture_time_ns; + uint32_t time_is_exposure; + uint32_t output; + uint32_t calibration_generation; + uint32_t reserved; +}; + +struct xrt_stereo_camera_stream_stats +{ + float source_frame_rate; + float delivered_frame_rate; + uint64_t frames_published; + uint64_t frames_skipped; + uint64_t frames_acquired; + uint64_t mean_latency_ns; +}; + +/*! + * Client aspect of an @ref xrt_instance: set by the IPC client instance only + * (the camera's single owner is the service); NULL for an in-process instance, + * which therefore enumerates zero cameras. Streams are service-side ids owned + * by the instance's connection. + */ +struct xrt_stereo_camera_client +{ + xrt_result_t (*count)(struct xrt_stereo_camera_client *c, uint32_t *out_count); + xrt_result_t (*get_properties)(struct xrt_stereo_camera_client *c, + uint32_t index, + struct xrt_stereo_camera_properties *out_props); + xrt_result_t (*get_calibration)(struct xrt_stereo_camera_client *c, + uint64_t camera_id, + uint32_t output, + struct xrt_stereo_camera_calibration *out_calib); + xrt_result_t (*stream_create)(struct xrt_stereo_camera_client *c, + const struct xrt_stereo_camera_stream_request *req, + uint64_t *out_stream_id); + xrt_result_t (*stream_destroy)(struct xrt_stereo_camera_client *c, uint64_t stream_id); + xrt_result_t (*stream_start)(struct xrt_stereo_camera_client *c, uint64_t stream_id); + xrt_result_t (*stream_stop)(struct xrt_stereo_camera_client *c, uint64_t stream_id); + //! @p out_section / @p out_wake are NEW handles owned by the caller. + xrt_result_t (*stream_get_transport)(struct xrt_stereo_camera_client *c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_layout *out_layout, + xrt_shmem_handle_t *out_section, + xrt_graphics_sync_handle_t *out_wake); + //! @p out_ready false = nothing newer than the last acquire. + xrt_result_t (*acquire)(struct xrt_stereo_camera_client *c, + uint64_t stream_id, + bool *out_ready, + struct xrt_stereo_camera_frame_info *out_frame); + xrt_result_t (*stats)(struct xrt_stereo_camera_client *c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_stats *out_stats); +}; + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/targets/common/target_instance.c b/src/xrt/targets/common/target_instance.c index c8779b399f..e0dfbddd91 100644 --- a/src/xrt/targets/common/target_instance.c +++ b/src/xrt/targets/common/target_instance.c @@ -935,6 +935,22 @@ t_instance_create_system(struct xrt_instance *xinst, * */ +/*! + * xrt_instance::get_active_plugin — the plug-in that claimed the system. The + * service's stereo camera manager (ADR-043) reads its camera slots through it. + */ +static bool +t_instance_get_active_plugin(struct xrt_instance *xinst, + const struct xrt_plugin_iface **out_iface, + struct xrt_plugin_instance **out_inst) +{ + (void)xinst; + const struct xrt_plugin_iface *iface = target_plugin_get_active(); + *out_iface = iface; + *out_inst = iface != NULL ? target_plugin_get_active_instance() : NULL; + return iface != NULL; +} + #ifdef XRT_FEATURE_HYBRID_MODE // In hybrid mode, export as native_instance_create to avoid symbol conflict // with ipc_instance_create from the IPC client library @@ -960,6 +976,7 @@ xrt_instance_create(struct xrt_instance_info *ii, struct xrt_instance **out_xins tinst->base.create_system = t_instance_create_system; tinst->base.get_prober = t_instance_get_prober; tinst->base.destroy = t_instance_destroy; + tinst->base.get_active_plugin = t_instance_get_active_plugin; tinst->xp = xp; tinst->base.startup_timestamp = os_monotonic_get_ns(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0b5829ce45..073006fb21 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -67,6 +67,7 @@ set(tests tests_comp_layer_view_camera tests_comp_color_policy tests_m_matrix_projection + tests_stereo_camera ) if(XRT_BUILD_DRIVER_QWERTY) list(APPEND tests tests_qwerty_camera_profile tests_qwerty_velocity) @@ -703,3 +704,8 @@ add_test(NAME tests_ipc_proto COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURC # lift mailbox tests (the module that emits it is Windows/D3D11-only). target_sources(tests_lift_mailbox PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_fake_ply.c) target_include_directories(tests_lift_mailbox PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display) + +# ADR-043: the sim_display FAKE stereo camera's scene is pinned host-side next to +# the ring / conversion / disparity-probe tests (it compiles into the plug-in). +target_sources(tests_stereo_camera PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c) +target_include_directories(tests_stereo_camera PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display) diff --git a/tests/tests_stereo_camera.cpp b/tests/tests_stereo_camera.cpp new file mode 100644 index 0000000000..b530ec45cf --- /dev/null +++ b/tests/tests_stereo_camera.cpp @@ -0,0 +1,291 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R1: the platform-neutral pieces the + * service's camera manager is built from, pinned host-side. + * + * - the per-stream latest-wins ring: a pinned slot is never written, the + * writer always finds a free slot, NOT READY until something newer lands, + * skipped frames are counted, suspension clears the pin; + * - the decimator: exact ratios (30 -> 15, 30 -> 20), no burst after a pause; + * - plane layout + format conversion round trips; + * - the persistent id: stable, per consumer, never the raw identity; + * - the sim_display FAKE scene: the disparity probe measures exactly the + * disparities the scene was built with, and the frame counter differs + * between frames; + * - the plug-in iface: the camera slots sit right after the ADR-042 lift + * slot (or its placeholder), so the two branches cannot share an offset. + */ + +#include "catch_amalgamated.hpp" + +#include "util/u_stereo_camera.h" +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_stereo_camera.h" + +#include "sim_display_stereo_camera_pattern.h" + +#include +#include +#include +#include +#include + +TEST_CASE("stereo camera ring: latest-wins with one pin", "[stereo_camera]") +{ + u_stereo_camera_ring r; + u_stereo_camera_ring_init(&r); + + int32_t slot = -1; + uint64_t seq = 0; + CHECK_FALSE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); // nothing yet + + int32_t w1 = u_stereo_camera_ring_begin_write(&r); + REQUIRE(w1 >= 0); + u_stereo_camera_ring_publish(&r, w1, 1); + REQUIRE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); + CHECK(slot == w1); + CHECK(seq == 1); + CHECK_FALSE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); // not newer + + // Many publishes while the consumer holds its pin: the writer must never + // be handed the pinned slot, and every unacquired frame counts as skipped. + for (uint64_t s = 2; s < 50; s++) { + int32_t w = u_stereo_camera_ring_begin_write(&r); + REQUIRE(w >= 0); + REQUIRE(w != r.pinned); + REQUIRE(w != r.latest); + u_stereo_camera_ring_publish(&r, w, s); + } + CHECK(r.skipped == 47); // 2..48 superseded, 49 is the latest + REQUIRE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); + CHECK(seq == 49); + CHECK(r.acquired == 2); + + // Suspension: pin dropped, latest forgotten, next publish is not a skip. + u_stereo_camera_ring_clear(&r); + CHECK(r.pinned == -1); + CHECK_FALSE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); + int32_t w = u_stereo_camera_ring_begin_write(&r); + u_stereo_camera_ring_publish(&r, w, 50); + CHECK(r.skipped == 47); + REQUIRE(u_stereo_camera_ring_acquire(&r, &slot, &seq)); + CHECK(seq == 50); +} + +TEST_CASE("stereo camera ring: a pinned slot stays byte-stable", "[stereo_camera]") +{ + // Simulate the payload: each slot holds the sequence last written into it. + u_stereo_camera_ring r; + u_stereo_camera_ring_init(&r); + uint64_t payload[U_STEREO_CAMERA_RING_SLOTS] = {0, 0, 0}; + int32_t slot = -1; + uint64_t seq = 0, held = 0; + int32_t held_slot = -1; + for (uint64_t s = 1; s <= 1000; s++) { + int32_t w = u_stereo_camera_ring_begin_write(&r); + payload[w] = s; + u_stereo_camera_ring_publish(&r, w, s); + if (held_slot >= 0) { + REQUIRE(payload[held_slot] == held); // never overwritten under the pin + } + if (s % 7 == 0 && u_stereo_camera_ring_acquire(&r, &slot, &seq)) { + REQUIRE(payload[slot] == seq); + held = seq; + held_slot = slot; + } + } +} + +TEST_CASE("stereo camera decimator", "[stereo_camera]") +{ + const int64_t p30 = 33333333; + auto count = [&](float max_rate, int frames, int64_t pause_at = -1) { + u_stereo_camera_decimator d; + u_stereo_camera_decimator_init(&d, max_rate, 30.0f); + int n = 0; + int64_t t = 1000000000; + for (int i = 0; i < frames; i++) { + if (i == pause_at) { + t += 2000000000; // a 2 s gap (suspended source) + } + n += u_stereo_camera_decimator_accept(&d, t) ? 1 : 0; + t += p30; + } + return n; + }; + CHECK(count(0.0f, 300) == 300); + CHECK(count(30.0f, 300) == 300); + CHECK(count(60.0f, 300) == 300); // never interpolates up + CHECK(count(15.0f, 300) == 150); + CHECK(count(20.0f, 300) == 200); + CHECK(count(10.0f, 300) == 100); + // After a pause the gate re-anchors: no burst of "missed" frames. + int with_pause = count(15.0f, 300, 150); + CHECK(with_pause >= 150); + CHECK(with_pause <= 151); +} + +TEST_CASE("stereo camera layout", "[stereo_camera]") +{ + u_stereo_camera_planes l; + REQUIRE(u_stereo_camera_layout(1, 1280, 480, &l)); + CHECK(l.plane_count == 1); + CHECK(l.pitch[0] == 1280); + CHECK(l.size == 1280ull * 480); + REQUIRE(u_stereo_camera_layout(2, 1280, 480, &l)); + CHECK(l.plane_count == 2); + CHECK(l.offset[1] == 1280ull * 480); + CHECK(l.size == 1280ull * 480 * 3 / 2); + REQUIRE(u_stereo_camera_layout(3, 1000, 10, &l)); + CHECK(l.pitch[0] == 4032); // 4000 rounded up to 64 + CHECK_FALSE(u_stereo_camera_layout(2, 1281, 480, &l)); // odd NV12 + CHECK_FALSE(u_stereo_camera_layout(9, 64, 64, &l)); +} + +TEST_CASE("stereo camera format conversion", "[stereo_camera]") +{ + const uint32_t W = 64, H = 8; + std::vector gray(W * H); + for (uint32_t i = 0; i < W * H; i++) { + gray[i] = (uint8_t)(i * 7); + } + const uint8_t *src[2] = {gray.data(), nullptr}; + const uint32_t sp[2] = {W, 0}; + + SECTION("gray -> nv12 keeps luma, neutral chroma") + { + u_stereo_camera_planes l; + REQUIRE(u_stereo_camera_layout(2, W, H, &l)); + std::vector out(l.size); + REQUIRE(u_stereo_camera_convert(1, src, sp, 2, out.data(), &l, W, H)); + CHECK(std::memcmp(out.data(), gray.data(), W) == 0); + for (uint64_t i = l.offset[1]; i < l.size; i++) { + REQUIRE(out[i] == 128); + } + // and back to gray + u_stereo_camera_planes g; + REQUIRE(u_stereo_camera_layout(1, W, H, &g)); + std::vector back(g.size); + const uint8_t *nsrc[2] = {out.data(), out.data() + l.offset[1]}; + REQUIRE(u_stereo_camera_convert(2, nsrc, l.pitch, 1, back.data(), &g, W, H)); + CHECK(std::memcmp(back.data(), gray.data(), W * H) == 0); + } + SECTION("gray -> bgra -> gray is lossless") + { + u_stereo_camera_planes b; + REQUIRE(u_stereo_camera_layout(3, W, H, &b)); + std::vector bgra(b.size); + REQUIRE(u_stereo_camera_convert(1, src, sp, 3, bgra.data(), &b, W, H)); + CHECK(bgra[3] == 255); + u_stereo_camera_planes g; + REQUIRE(u_stereo_camera_layout(1, W, H, &g)); + std::vector back(g.size); + const uint8_t *bsrc[2] = {bgra.data(), nullptr}; + REQUIRE(u_stereo_camera_convert(3, bsrc, b.pitch, 1, back.data(), &g, W, H)); + CHECK(std::memcmp(back.data(), gray.data(), W * H) == 0); + } + SECTION("bgra -> nv12 -> bgra stays within video-range rounding") + { + u_stereo_camera_planes b; + REQUIRE(u_stereo_camera_layout(3, W, H, &b)); + std::vector bgra(b.size, 0); + for (uint32_t y = 0; y < H; y++) { + for (uint32_t x = 0; x < W; x++) { + uint8_t *p = &bgra[y * b.pitch[0] + 4 * x]; + p[0] = 60; // uniform colour: chroma subsampling is exact + p[1] = 120; + p[2] = 200; + p[3] = 255; + } + } + u_stereo_camera_planes n; + REQUIRE(u_stereo_camera_layout(2, W, H, &n)); + std::vector nv(n.size); + const uint8_t *bs[2] = {bgra.data(), nullptr}; + REQUIRE(u_stereo_camera_convert(3, bs, b.pitch, 2, nv.data(), &n, W, H)); + std::vector rt(b.size); + const uint8_t *ns[2] = {nv.data(), nv.data() + n.offset[1]}; + REQUIRE(u_stereo_camera_convert(2, ns, n.pitch, 3, rt.data(), &b, W, H)); + for (int c = 0; c < 3; c++) { + int d = (int)rt[c] - (int)bgra[c]; + CHECK(d >= -3); + CHECK(d <= 3); + } + } + CHECK_FALSE(u_stereo_camera_convert(1, src, sp, 7, gray.data(), nullptr, W, H)); +} + +TEST_CASE("stereo camera persistent id", "[stereo_camera]") +{ + char a[64], b[64], c[64]; + u_stereo_camera_persistent_id("SERIAL-123", "C:/apps/chrome.exe", a); + u_stereo_camera_persistent_id("SERIAL-123", "C:/apps/chrome.exe", b); + u_stereo_camera_persistent_id("SERIAL-123", "C:/apps/other.exe", c); + CHECK(std::strcmp(a, b) == 0); + CHECK(std::strcmp(a, c) != 0); + CHECK(std::strncmp(a, "dxrcam-", 7) == 0); + CHECK(std::strlen(a) == 7 + 32); + CHECK(std::strstr(a, "SERIAL") == nullptr); +} + +TEST_CASE("sim_display fake scene has the disparities it was built with", "[stereo_camera]") +{ + sim_stereo_camera_scene s; + // 640 px per eye, 68 deg HFOV, 50 mm, 2.0 m / 0.6 m — the fake's defaults. + const double fx = 320.0 / std::tan(34.0 * 3.14159265358979323846 / 180.0); + sim_stereo_camera_scene_init(&s, 640, 480, fx, 50.0, 2.0, 0.6); + CHECK(s.bg_disparity == 12); + CHECK(s.bar_disparity == 40); + + const uint32_t pitch = 1280; + std::vector f1(pitch * 480), f2(pitch * 480); + sim_stereo_camera_render_gray(&s, 1, f1.data(), pitch); + sim_stereo_camera_render_gray(&s, 2, f2.data(), pitch); + CHECK(std::memcmp(f1.data(), f2.data(), f1.size()) != 0); // the counter moved + + float d = 0.0f; + // Inside the bar (left-eye coordinates). + REQUIRE(u_stereo_camera_estimate_disparity(f1.data(), pitch, 640, 480, 272, 200, 64, 64, 64, &d)); + CHECK(d == Catch::Approx(40.0f).margin(0.5f)); + // Background, right of the bar, below the counter. + REQUIRE(u_stereo_camera_estimate_disparity(f1.data(), pitch, 640, 480, 480, 300, 64, 64, 64, &d)); + CHECK(d == Catch::Approx(12.0f).margin(0.5f)); + // Out-of-range region is refused. + CHECK_FALSE(u_stereo_camera_estimate_disparity(f1.data(), pitch, 640, 480, 600, 0, 64, 64, 64, &d)); +} + +TEST_CASE("plug-in iface: camera slots are appended after lift and platform-state (ADR-020)", "[stereo_camera]") +{ + // ADR-020 append-only: the camera block comes AFTER every slot that was on + // main before it — lift's factory (ADR-042) and then the platform-state + // probe (ADR-045), which is the last pre-camera member. Pin both so a + // reorder (or a slot squeezed in between) fails here, not on a vendor box. + static_assert(sizeof(((xrt_plugin_iface *)0)->get_platform_state) == sizeof(void *), "one slot"); + static_assert(sizeof(((xrt_plugin_iface *)0)->create_dp_d3d11_lift) == sizeof(void *), "one slot"); + const size_t lift_end = + offsetof(xrt_plugin_iface, create_dp_d3d11_lift) + sizeof(((xrt_plugin_iface *)0)->create_dp_d3d11_lift); + const size_t platform_state_end = + offsetof(xrt_plugin_iface, get_platform_state) + sizeof(((xrt_plugin_iface *)0)->get_platform_state); + // lift sits right after the #1243 vk fingerprint, platform-state right + // after lift, and the camera block right after platform-state. + CHECK(offsetof(xrt_plugin_iface, create_dp_d3d11_lift) == + offsetof(xrt_plugin_iface, vk_bundle_fn_table_offset) + sizeof(uint32_t)); + CHECK(offsetof(xrt_plugin_iface, get_platform_state) == lift_end); + CHECK(offsetof(xrt_plugin_iface, stereo_camera_enumerate) == platform_state_end); + CHECK(offsetof(xrt_plugin_iface, stereo_camera_close) + sizeof(void *) == sizeof(xrt_plugin_iface)); + + xrt_plugin_iface iface{}; + iface.struct_size = (uint32_t)offsetof(xrt_plugin_iface, stereo_camera_enumerate); + CHECK_FALSE(xrt_plugin_iface_has_stereo_camera(&iface)); // an older plug-in + CHECK_FALSE(xrt_plugin_iface_has_stereo_camera(nullptr)); +} + +TEST_CASE("stereo camera value types agree with the plug-in encodings", "[stereo_camera]") +{ + STATIC_REQUIRE((int)XRT_STEREO_CAMERA_FORMAT_GRAY8 == (int)XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8); + STATIC_REQUIRE((int)XRT_STEREO_CAMERA_FORMAT_NV12 == (int)XRT_PLUGIN_STEREO_CAMERA_FORMAT_NV12); + STATIC_REQUIRE((int)XRT_STEREO_CAMERA_FORMAT_BGRA8 == (int)XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8); + STATIC_REQUIRE(XRT_STEREO_CAMERA_RING_SLOTS == U_STEREO_CAMERA_RING_SLOTS); +} From 90dde1a1b06821432aba3fb3a1bf3e1d2f62ff5e Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:40:12 -0700 Subject: [PATCH 04/37] feat(stereo-camera): service camera manager + IPC protocol (stereo_camera_* calls) ipc_server_stereo_camera: the service opens each plug-in camera once, on the first stream start, and closes it 2 s after the last stop. One runtime-owned thread per open camera pulls wait_frame and fans each frame out to every started stream: per stream a 3-slot latest-wins pinned ring in shared memory (u_stereo_camera_ring), format conversion, decimation, and a per-stream wake handle (auto-reset event on Windows, non-blocking pipe on POSIX). States WAITING / AVAILABLE / SUSPENDED / UNAVAILABLE follow the plug-in; a suspension clears every pin. Streams are owned by the connection and die with it. Privacy hooks, deny by default: start and calibration reads are refused (XRT_ERROR_NOT_AUTHORIZED) unless DXR_STEREO_CAMERA_DEV_ALLOW=1 is set in the SERVICE environment (R3 replaces it with OS + DisplayXR consent); DXR_STEREO_CAMERA=0 is the kill switch (zero cameras); RAW is refused to PRESENT_OWNER clients (the browser); the foreground rule and in-use indicator are marked hook points. Windows hands out a FILE_MAP_READ section duplicate and a SYNCHRONIZE-only event duplicate. The D3D11 / AHardwareBuffer transports are designed in the file comment (same ring, texture slots + fence / sync_file). Client side: ipc_client_stereo_camera (connection-level calls) and the xrt_stereo_camera_client aspect the IPC instance now carries. Co-Authored-By: Claude Opus 5.5 --- src/xrt/ipc/CMakeLists.txt | 4 + src/xrt/ipc/client/ipc_client_instance.c | 7 + src/xrt/ipc/client/ipc_client_stereo_camera.c | 266 ++++ src/xrt/ipc/client/ipc_client_stereo_camera.h | 86 ++ src/xrt/ipc/server/ipc_server.h | 13 + .../ipc/server/ipc_server_per_client_thread.c | 4 + src/xrt/ipc/server/ipc_server_process.c | 7 + src/xrt/ipc/server/ipc_server_stereo_camera.c | 1297 +++++++++++++++++ src/xrt/ipc/server/ipc_server_stereo_camera.h | 50 + src/xrt/ipc/shared/ipc_protocol.h | 1 + src/xrt/ipc/shared/proto.json | 88 ++ 11 files changed, 1823 insertions(+) create mode 100644 src/xrt/ipc/client/ipc_client_stereo_camera.c create mode 100644 src/xrt/ipc/client/ipc_client_stereo_camera.h create mode 100644 src/xrt/ipc/server/ipc_server_stereo_camera.c create mode 100644 src/xrt/ipc/server/ipc_server_stereo_camera.h diff --git a/src/xrt/ipc/CMakeLists.txt b/src/xrt/ipc/CMakeLists.txt index 3ae3b601f9..845865f079 100644 --- a/src/xrt/ipc/CMakeLists.txt +++ b/src/xrt/ipc/CMakeLists.txt @@ -70,6 +70,8 @@ add_library( client/ipc_client_connection.c client/ipc_client_lift.c client/ipc_client_lift.h + client/ipc_client_stereo_camera.c + client/ipc_client_stereo_camera.h client/ipc_client_device.c client/ipc_client_hmd.c client/ipc_client_instance.c @@ -121,6 +123,8 @@ add_library( server/ipc_server_peer_creds.c server/ipc_server_peer_creds.h server/ipc_server_process.c + server/ipc_server_stereo_camera.c + server/ipc_server_stereo_camera.h ) target_include_directories( ipc_server diff --git a/src/xrt/ipc/client/ipc_client_instance.c b/src/xrt/ipc/client/ipc_client_instance.c index 223174c19c..9cce353915 100644 --- a/src/xrt/ipc/client/ipc_client_instance.c +++ b/src/xrt/ipc/client/ipc_client_instance.c @@ -30,6 +30,7 @@ #include "client/ipc_client.h" #include "client/ipc_client_interface.h" #include "client/ipc_client_connection.h" +#include "client/ipc_client_stereo_camera.h" #include "ipc_client_generated.h" @@ -229,6 +230,9 @@ ipc_client_instance_destroy(struct xrt_instance *xinst) { struct ipc_client_instance *ii = ipc_client_instance(xinst); + free(ii->base.stereo_camera); + ii->base.stereo_camera = NULL; + // service considers us to be connected until fd is closed ipc_client_connection_fini(&ii->ipc_c); @@ -334,6 +338,9 @@ ipc_instance_create(const struct xrt_instance_info *i_info, struct xrt_instance ii->base.startup_timestamp = ii->ipc_c.ism->startup_timestamp; + // XR_DXR_stereo_camera (ADR-043): cameras are reached over this connection. + ii->base.stereo_camera = ipc_client_stereo_camera_client_create(&ii->ipc_c); + *out_xinst = &ii->base; return XRT_SUCCESS; diff --git a/src/xrt/ipc/client/ipc_client_stereo_camera.c b/src/xrt/ipc/client/ipc_client_stereo_camera.c new file mode 100644 index 0000000000..a87c9d6db8 --- /dev/null +++ b/src/xrt/ipc/client/ipc_client_stereo_camera.c @@ -0,0 +1,266 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) client-side IPC calls (see + * ipc_client_stereo_camera.h). + * @ingroup ipc_client + */ + +#include "xrt/xrt_config_os.h" + +#include "util/u_misc.h" + +#include "shared/ipc_protocol.h" +#include "shared/ipc_shmem.h" +#include "client/ipc_client.h" +#include "client/ipc_client_connection.h" +#include "client/ipc_client_stereo_camera.h" +#include "ipc_client_generated.h" + +#include +#include + +xrt_result_t +ipc_client_stereo_camera_count(struct ipc_connection *ipc_c, uint32_t *out_count) +{ + *out_count = 0; + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_count(ipc_c, out_count); +} + +xrt_result_t +ipc_client_stereo_camera_get_properties(struct ipc_connection *ipc_c, + uint32_t index, + struct xrt_stereo_camera_properties *out_props) +{ + U_ZERO(out_props); + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + xrt_result_t xret = ipc_call_stereo_camera_get_properties(ipc_c, index, out_props); + out_props->persistent_id[sizeof(out_props->persistent_id) - 1] = '\0'; + out_props->display_name[sizeof(out_props->display_name) - 1] = '\0'; + out_props->platform_device_hint[sizeof(out_props->platform_device_hint) - 1] = '\0'; + return xret; +} + +xrt_result_t +ipc_client_stereo_camera_get_calibration(struct ipc_connection *ipc_c, + uint64_t camera_id, + uint32_t output, + struct xrt_stereo_camera_calibration *out_calib) +{ + U_ZERO(out_calib); + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_get_calibration(ipc_c, camera_id, output, out_calib); +} + +xrt_result_t +ipc_client_stereo_camera_stream_create(struct ipc_connection *ipc_c, + const struct xrt_stereo_camera_stream_request *req, + uint64_t *out_stream_id) +{ + *out_stream_id = 0; + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_stream_create(ipc_c, req, out_stream_id); +} + +xrt_result_t +ipc_client_stereo_camera_stream_destroy(struct ipc_connection *ipc_c, uint64_t stream_id) +{ + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_stream_destroy(ipc_c, stream_id); +} + +xrt_result_t +ipc_client_stereo_camera_stream_start(struct ipc_connection *ipc_c, uint64_t stream_id) +{ + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_stream_start(ipc_c, stream_id); +} + +xrt_result_t +ipc_client_stereo_camera_stream_stop(struct ipc_connection *ipc_c, uint64_t stream_id) +{ + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_stream_stop(ipc_c, stream_id); +} + +xrt_result_t +ipc_client_stereo_camera_stream_get_transport(struct ipc_connection *ipc_c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_layout *out_layout, + xrt_shmem_handle_t *out_section, + xrt_graphics_sync_handle_t *out_wake) +{ + U_ZERO(out_layout); + *out_section = XRT_SHMEM_HANDLE_INVALID; + *out_wake = XRT_GRAPHICS_SYNC_HANDLE_INVALID; + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + xrt_shmem_handle_t section = XRT_SHMEM_HANDLE_INVALID; + xrt_result_t xret = ipc_call_stereo_camera_stream_get_section(ipc_c, stream_id, out_layout, §ion, 1); + if (xret != XRT_SUCCESS) { + return xret; + } + xrt_graphics_sync_handle_t wake = XRT_GRAPHICS_SYNC_HANDLE_INVALID; + xret = ipc_call_stereo_camera_stream_get_wake(ipc_c, stream_id, &wake, 1); + if (xret != XRT_SUCCESS) { + // The section crossed; do not leak it. + void *none = NULL; + ipc_shmem_destroy(§ion, &none, 0); + return xret; + } + *out_section = section; + *out_wake = wake; + return XRT_SUCCESS; +} + +xrt_result_t +ipc_client_stereo_camera_acquire(struct ipc_connection *ipc_c, + uint64_t stream_id, + bool *out_ready, + struct xrt_stereo_camera_frame_info *out_frame) +{ + *out_ready = false; + U_ZERO(out_frame); + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_acquire(ipc_c, stream_id, out_ready, out_frame); +} + +xrt_result_t +ipc_client_stereo_camera_stats(struct ipc_connection *ipc_c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_stats *out_stats) +{ + U_ZERO(out_stats); + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_stream_stats(ipc_c, stream_id, out_stats); +} + + +/* + * + * xrt_stereo_camera_client aspect. + * + */ + +struct ipc_stereo_camera_client +{ + struct xrt_stereo_camera_client base; + struct ipc_connection *ipc_c; +}; + +static inline struct ipc_connection * +conn(struct xrt_stereo_camera_client *c) +{ + return ((struct ipc_stereo_camera_client *)c)->ipc_c; +} + +static xrt_result_t +c_count(struct xrt_stereo_camera_client *c, uint32_t *out_count) +{ + return ipc_client_stereo_camera_count(conn(c), out_count); +} + +static xrt_result_t +c_get_properties(struct xrt_stereo_camera_client *c, uint32_t index, struct xrt_stereo_camera_properties *out) +{ + return ipc_client_stereo_camera_get_properties(conn(c), index, out); +} + +static xrt_result_t +c_get_calibration(struct xrt_stereo_camera_client *c, + uint64_t camera_id, + uint32_t output, + struct xrt_stereo_camera_calibration *out) +{ + return ipc_client_stereo_camera_get_calibration(conn(c), camera_id, output, out); +} + +static xrt_result_t +c_stream_create(struct xrt_stereo_camera_client *c, + const struct xrt_stereo_camera_stream_request *req, + uint64_t *out_stream_id) +{ + return ipc_client_stereo_camera_stream_create(conn(c), req, out_stream_id); +} + +static xrt_result_t +c_stream_destroy(struct xrt_stereo_camera_client *c, uint64_t stream_id) +{ + return ipc_client_stereo_camera_stream_destroy(conn(c), stream_id); +} + +static xrt_result_t +c_stream_start(struct xrt_stereo_camera_client *c, uint64_t stream_id) +{ + return ipc_client_stereo_camera_stream_start(conn(c), stream_id); +} + +static xrt_result_t +c_stream_stop(struct xrt_stereo_camera_client *c, uint64_t stream_id) +{ + return ipc_client_stereo_camera_stream_stop(conn(c), stream_id); +} + +static xrt_result_t +c_stream_get_transport(struct xrt_stereo_camera_client *c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_layout *out_layout, + xrt_shmem_handle_t *out_section, + xrt_graphics_sync_handle_t *out_wake) +{ + return ipc_client_stereo_camera_stream_get_transport(conn(c), stream_id, out_layout, out_section, out_wake); +} + +static xrt_result_t +c_acquire(struct xrt_stereo_camera_client *c, + uint64_t stream_id, + bool *out_ready, + struct xrt_stereo_camera_frame_info *out_frame) +{ + return ipc_client_stereo_camera_acquire(conn(c), stream_id, out_ready, out_frame); +} + +static xrt_result_t +c_stats(struct xrt_stereo_camera_client *c, uint64_t stream_id, struct xrt_stereo_camera_stream_stats *out) +{ + return ipc_client_stereo_camera_stats(conn(c), stream_id, out); +} + +struct xrt_stereo_camera_client * +ipc_client_stereo_camera_client_create(struct ipc_connection *ipc_c) +{ + struct ipc_stereo_camera_client *c = U_TYPED_CALLOC(struct ipc_stereo_camera_client); + c->ipc_c = ipc_c; + c->base.count = c_count; + c->base.get_properties = c_get_properties; + c->base.get_calibration = c_get_calibration; + c->base.stream_create = c_stream_create; + c->base.stream_destroy = c_stream_destroy; + c->base.stream_start = c_stream_start; + c->base.stream_stop = c_stream_stop; + c->base.stream_get_transport = c_stream_get_transport; + c->base.acquire = c_acquire; + c->base.stats = c_stats; + return &c->base; +} diff --git a/src/xrt/ipc/client/ipc_client_stereo_camera.h b/src/xrt/ipc/client/ipc_client_stereo_camera.h new file mode 100644 index 0000000000..ab8bf10789 --- /dev/null +++ b/src/xrt/ipc/client/ipc_client_stereo_camera.h @@ -0,0 +1,86 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) client side: connection-level calls, + * shared by the IPC instance's @ref xrt_stereo_camera_client aspect + * and `displayxr-cli camera`. + * + * Streams belong to the IPC connection (not a session), so every call takes a + * bare @ref ipc_connection. + * + * @ingroup ipc_client + */ + +#pragma once + +#include "xrt/xrt_handles.h" +#include "xrt/xrt_results.h" +#include "xrt/xrt_stereo_camera.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct ipc_connection; + +xrt_result_t +ipc_client_stereo_camera_count(struct ipc_connection *ipc_c, uint32_t *out_count); + +xrt_result_t +ipc_client_stereo_camera_get_properties(struct ipc_connection *ipc_c, + uint32_t index, + struct xrt_stereo_camera_properties *out_props); + +xrt_result_t +ipc_client_stereo_camera_get_calibration(struct ipc_connection *ipc_c, + uint64_t camera_id, + uint32_t output, + struct xrt_stereo_camera_calibration *out_calib); + +xrt_result_t +ipc_client_stereo_camera_stream_create(struct ipc_connection *ipc_c, + const struct xrt_stereo_camera_stream_request *req, + uint64_t *out_stream_id); + +xrt_result_t +ipc_client_stereo_camera_stream_destroy(struct ipc_connection *ipc_c, uint64_t stream_id); + +xrt_result_t +ipc_client_stereo_camera_stream_start(struct ipc_connection *ipc_c, uint64_t stream_id); + +xrt_result_t +ipc_client_stereo_camera_stream_stop(struct ipc_connection *ipc_c, uint64_t stream_id); + +//! The started stream's ring section + wake handle (NEW handles, caller-owned). +xrt_result_t +ipc_client_stereo_camera_stream_get_transport(struct ipc_connection *ipc_c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_layout *out_layout, + xrt_shmem_handle_t *out_section, + xrt_graphics_sync_handle_t *out_wake); + +xrt_result_t +ipc_client_stereo_camera_acquire(struct ipc_connection *ipc_c, + uint64_t stream_id, + bool *out_ready, + struct xrt_stereo_camera_frame_info *out_frame); + +xrt_result_t +ipc_client_stereo_camera_stats(struct ipc_connection *ipc_c, + uint64_t stream_id, + struct xrt_stereo_camera_stream_stats *out_stats); + +/*! + * The @ref xrt_stereo_camera_client aspect over @p ipc_c (which must outlive + * it). Caller frees with free(). + */ +struct xrt_stereo_camera_client * +ipc_client_stereo_camera_client_create(struct ipc_connection *ipc_c); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/ipc/server/ipc_server.h b/src/xrt/ipc/server/ipc_server.h index fa29e255b7..43d3aedaf0 100644 --- a/src/xrt/ipc/server/ipc_server.h +++ b/src/xrt/ipc/server/ipc_server.h @@ -60,6 +60,7 @@ extern "C" { #define IPC_MAX_CLIENT_SPACES 128 struct xrt_instance; +struct ipc_server_stereo_camera; struct xrt_compositor; struct xrt_compositor_native; @@ -218,6 +219,14 @@ struct ipc_client_state * every stream it owns destroyed — at client teardown. */ uint64_t lift_owner; + + /*! + * XR_DXR_stereo_camera (ADR-043): this connection's stream-owner token, + * taken from the camera manager on first stream create (0 = never used a + * camera). A token, not the ics pointer: the thread slot is reused by later + * connections. Every stream it owns is destroyed at client teardown. + */ + uint64_t stereo_camera_owner; }; enum ipc_thread_state @@ -509,6 +518,10 @@ struct ipc_server struct os_mutex lock; } global_state; + + //! XR_DXR_stereo_camera (ADR-043): the camera manager (never NULL once + //! the server is initialised; it may expose zero cameras). + struct ipc_server_stereo_camera *stereo_camera; }; diff --git a/src/xrt/ipc/server/ipc_server_per_client_thread.c b/src/xrt/ipc/server/ipc_server_per_client_thread.c index 9ad3f07849..d0e9459ad0 100644 --- a/src/xrt/ipc/server/ipc_server_per_client_thread.c +++ b/src/xrt/ipc/server/ipc_server_per_client_thread.c @@ -22,6 +22,7 @@ DEBUG_GET_ONCE_NUM_OPTION(test_exit_on_disconnect, "DXR_TEST_EXIT_ON_DISCONNECT" #include "shared/ipc_shmem.h" #include "shared/ipc_utils.h" #include "server/ipc_server.h" +#include "server/ipc_server_stereo_camera.h" #include "ipc_server_generated.h" #include "xrt/xrt_config_have.h" @@ -127,6 +128,9 @@ common_shutdown(volatile struct ipc_client_state *ics) // XR_DXR_lift (ADR-042): streams are owned by the connection, not a session. ipc_server_client_lift_release(ics); + // XR_DXR_stereo_camera (ADR-043): streams are owned by the connection. + ipc_server_client_stereo_camera_release(ics); + // Make sure undestroyed spaces are unreferenced for (uint32_t i = 0; i < IPC_MAX_CLIENT_SPACES; i++) { // Cast away volatile. diff --git a/src/xrt/ipc/server/ipc_server_process.c b/src/xrt/ipc/server/ipc_server_process.c index 726e6fb34d..4e44b32e30 100644 --- a/src/xrt/ipc/server/ipc_server_process.c +++ b/src/xrt/ipc/server/ipc_server_process.c @@ -34,6 +34,7 @@ #include "shared/ipc_protocol.h" #include "shared/ipc_shmem.h" #include "server/ipc_server.h" +#include "server/ipc_server_stereo_camera.h" #include "server/ipc_server_interface.h" #include "server/ipc_server_peer_creds.h" @@ -298,6 +299,9 @@ teardown_all(struct ipc_server *s) { u_var_remove_root(s); + // Before the instance: the camera threads call into the plug-in. + ipc_server_stereo_camera_destroy(&s->stereo_camera); + xrt_syscomp_destroy(&s->xsysc); teardown_idevs(s); @@ -619,6 +623,9 @@ init_all(struct ipc_server *s, enum u_logging_level log_level) xret = xrt_instance_create_system(s->xinst, &s->xsys, &s->xsysd, &s->xso, &s->xsysc); IPC_CHK_WITH_GOTO(s, xret, "xrt_instance_create_system", error); + // XR_DXR_stereo_camera (ADR-043): the service owns every plug-in camera. + s->stereo_camera = ipc_server_stereo_camera_create(s->xinst); + // Always succeeds. init_idevs(s); init_tracking_origins(s); diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c new file mode 100644 index 0000000000..b7f3282d3a --- /dev/null +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -0,0 +1,1297 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043): the service's camera manager and its + * IPC handlers. See ipc_server_stereo_camera.h. + * + * Locking: ONE manager mutex guards cameras, streams and rings. Plug-in calls + * that may block (open, wait_frame, close) are made with it RELEASED; the + * per-frame fan-out (conversion into each stream's slot) runs under it — a + * 1280x480 NV12 copy is well under a millisecond, and it keeps an acquire from + * ever observing a half-written slot. Only the camera thread closes a source. + * + * Privacy (spec §7) — R1 ships the HOOK POINTS, deny-by-default: + * - authorise_locked(): every start and every calibration read. Allowed in R1 + * only with DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment; + * otherwise XRT_ERROR_NOT_AUTHORIZED (XR_ERROR_PERMISSION_INSUFFICIENT). + * R3 replaces the env check with OS consent + the DisplayXR consent store + + * delegating-client registration. + * - client_visible_locked(): the foreground rule, evaluated per published + * frame. R1 always true; R3 wires session visibility / OS lock. + * - output_allowed(): RAW is refused to PRESENT_OWNER clients (the browser, + * which exposes cameras to web pages) — maintainer decision for R1. + * - DXR_STEREO_CAMERA=0: kill switch, zero cameras enumerated. + * - the in-use indicator (tray badge naming consumers) is R3; R1 logs one + * WARN per stream start/stop naming the peer executable. + * + * Transports — R1 implements SHARED_MEMORY only. The GPU transports are + * designed to reuse this ring's state machine unchanged, only the slot storage + * differs: + * - D3D11_TEXTURE (Windows): TODO(L1/B1). Three D3D11 textures on the + * service's lift/weave device created SHARED_NTHANDLE|SHARED_KEYEDMUTEX-free + * with one ID3D11Fence; the publish step becomes an UpdateSubresource into + * the free slot + Signal(fence, seq); acquire returns (slot, fence value); + * the NT handles + fence are handed out on the first acquire and on + * reallocation, the XR_DXR_weave output-export pattern (legacy DXGI handles + * low-bit tagged for Low-IL callers). The browser's capture device imports + * them and waits on the fence instead of reading the section. + * - Vulkan / AHARDWAREBUFFER (Android, and desktop Linux via dma-buf): TODO(A1). + * Three AHardwareBuffers (or VkImages exported as dma-buf / opaque fd) per + * stream, sent once over the IPC socket like weave's dma-buf outputs + * (#1699), with a sync_file per publish instead of a fence value. NV12 per + * eye maps to AHARDWAREBUFFER_FORMAT_Y8Cb8Cr8_420; the tablet's per-lens + * 1280x720 NV12 pair makes this transport the default there. + * + * @ingroup ipc_server + */ + +#include "server/ipc_server.h" +#include "server/ipc_server_peer_creds.h" +#include "server/ipc_server_stereo_camera.h" +#include "ipc_server_generated.h" + +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_stereo_camera.h" + +#include "os/os_threading.h" +#include "os/os_time.h" +#include "shared/ipc_shmem.h" +#include "util/u_debug.h" +#include "util/u_logging.h" +#include "util/u_misc.h" +#include "util/u_stereo_camera.h" + +#include +#include +#include + +#ifdef XRT_OS_WINDOWS +#include +#else +#include +#include +#include +#endif + +DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_enabled, "DXR_STEREO_CAMERA", true) +DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_dev_allow, "DXR_STEREO_CAMERA_DEV_ALLOW", false) + +#define MAX_STREAMS (XRT_STEREO_CAMERA_MAX_CAMERAS * XRT_STEREO_CAMERA_MAX_STREAMS_PER_CAMERA) +#define LINGER_NS (2000ll * 1000 * 1000) +#define WAIT_TIMEOUT_NS (100ll * 1000 * 1000) +#define SUSPEND_AFTER_NS (1000ll * 1000 * 1000) +#define REOPEN_BACKOFF_NS (1000ll * 1000 * 1000) +#define STATS_LOG_NS (5000ll * 1000 * 1000) + + +/* + * + * Wake handle: an auto-reset event (Windows) / a non-blocking pipe (POSIX). + * + */ + +struct scam_wake +{ +#ifdef XRT_OS_WINDOWS + HANDLE event; +#else + int fds[2]; //!< [0] = read end (sent to the client), [1] = write end +#endif +}; + +static bool +wake_create(struct scam_wake *w) +{ +#ifdef XRT_OS_WINDOWS + w->event = CreateEventW(NULL, FALSE, FALSE, NULL); + return w->event != NULL; +#else + w->fds[0] = w->fds[1] = -1; + if (pipe(w->fds) != 0) { + return false; + } + for (int i = 0; i < 2; i++) { + int fl = fcntl(w->fds[i], F_GETFL, 0); + fcntl(w->fds[i], F_SETFL, fl | O_NONBLOCK); + fcntl(w->fds[i], F_SETFD, FD_CLOEXEC); + } + return true; +#endif +} + +static void +wake_signal(struct scam_wake *w) +{ +#ifdef XRT_OS_WINDOWS + if (w->event != NULL) { + SetEvent(w->event); + } +#else + if (w->fds[1] >= 0) { + const char b = 1; + // EAGAIN = the consumer is not draining: it already has a pending wake. + ssize_t r = write(w->fds[1], &b, 1); + (void)r; + } +#endif +} + +static void +wake_destroy(struct scam_wake *w) +{ +#ifdef XRT_OS_WINDOWS + if (w->event != NULL) { + CloseHandle(w->event); + w->event = NULL; + } +#else + for (int i = 0; i < 2; i++) { + if (w->fds[i] >= 0) { + close(w->fds[i]); + w->fds[i] = -1; + } + } +#endif +} + + +/* + * + * Structs. + * + */ + +struct scam_stream +{ + bool used; + uint64_t id; + uint64_t owner; //!< ics->stereo_camera_owner of the creating connection + volatile struct ipc_client_state *ics; + uint32_t camera; //!< index into mgr->cams + struct xrt_stereo_camera_stream_request req; + uint32_t output; //!< what frames actually are (RAW when rectification is unavailable) + char consumer[260]; + + bool started; + bool allocated; + uint32_t width, height; + struct u_stereo_camera_planes layout; + uint64_t slot_stride; + uint64_t section_size; + xrt_shmem_handle_t section; + uint8_t *map; + struct scam_wake wake; + struct u_stereo_camera_ring ring; + struct u_stereo_camera_decimator dec; + struct xrt_stereo_camera_frame_info slot_info[U_STEREO_CAMERA_RING_SLOTS]; + +#ifdef XRT_OS_WINDOWS + //! Restricted duplicates handed to the last get_section / get_wake reply. + //! The generated dispatch duplicates them into the peer AFTER the handler + //! returns, so they are closed on the next call / stream destroy. + HANDLE sent_section_dup; + HANDLE sent_wake_dup; +#endif + + // stats + float delivered_rate; + int64_t last_publish_ns; + uint64_t latency_sum_ns; + uint64_t latency_count; +}; + +struct scam_camera +{ + uint32_t index; //!< plug-in index + uint64_t camera_id; + struct xrt_plugin_stereo_camera_info info; + bool have_calib; + struct xrt_plugin_stereo_camera_calibration calib; + uint32_t state; + uint32_t calibration_generation; + + struct xrt_plugin_stereo_camera *src; //!< owned by the camera thread + uint32_t started_count; + int64_t linger_deadline_ns; + int64_t last_frame_ns; + int64_t retry_after_ns; + float source_rate; + int64_t last_stats_log_ns; + + bool thread_started; + struct os_thread thread; + struct os_cond cond; + struct ipc_server_stereo_camera *mgr; +}; + +struct ipc_server_stereo_camera +{ + struct os_mutex lock; + bool shutting_down; + const struct xrt_plugin_iface *iface; + struct xrt_plugin_instance *inst; + uint32_t camera_count; + struct scam_camera cams[XRT_STEREO_CAMERA_MAX_CAMERAS]; + struct scam_stream streams[MAX_STREAMS]; + uint64_t next_stream_id; + uint64_t next_owner; +}; + + +/* + * + * Helpers. + * + */ + +static const char * +state_str(uint32_t s) +{ + switch (s) { + case XRT_STEREO_CAMERA_STATE_AVAILABLE: return "AVAILABLE"; + case XRT_STEREO_CAMERA_STATE_WAITING: return "WAITING"; + case XRT_STEREO_CAMERA_STATE_SUSPENDED: return "SUSPENDED"; + case XRT_STEREO_CAMERA_STATE_UNAVAILABLE: return "UNAVAILABLE"; + default: return "?"; + } +} + +static struct ipc_server_stereo_camera * +mgr_of(volatile struct ipc_client_state *ics) +{ + return (ics != NULL && ics->server != NULL) ? ics->server->stereo_camera : NULL; +} + +static uint64_t +owner_of(struct ipc_server_stereo_camera *m, volatile struct ipc_client_state *ics) +{ + if (ics->stereo_camera_owner == 0) { + ics->stereo_camera_owner = ++m->next_owner; + } + return ics->stereo_camera_owner; +} + +static struct scam_camera * +camera_by_id_locked(struct ipc_server_stereo_camera *m, uint64_t id) +{ + for (uint32_t i = 0; i < m->camera_count; i++) { + if (m->cams[i].camera_id == id) { + return &m->cams[i]; + } + } + return NULL; +} + +static struct scam_stream * +stream_by_id_locked(struct ipc_server_stereo_camera *m, volatile struct ipc_client_state *ics, uint64_t id) +{ + if (ics->stereo_camera_owner == 0) { + return NULL; + } + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (s->used && s->id == id && s->owner == ics->stereo_camera_owner) { + return s; + } + } + return NULL; +} + +//! Who may use cameras at all: apps, the browser, and the diagnostic CLI. +static xrt_result_t +require_camera_client(volatile struct ipc_client_state *ics, const char *what) +{ + uint32_t cls = ics->client_state.client_class; + if (cls != XRT_CLIENT_CLASS_APP && cls != XRT_CLIENT_CLASS_PRESENT_OWNER && cls != XRT_CLIENT_CLASS_DIAG) { + U_LOG_W("%s: denied — pid %ld is class %s (stereo camera: APP / PRESENT_OWNER / DIAG).", what, + ics->peer_pid, ipc_server_client_class_str(cls)); + return XRT_ERROR_NOT_AUTHORIZED; + } + return XRT_SUCCESS; +} + +/*! + * Spec §7.1 authorisation HOOK. R1: deny by default; the dev override is the + * only way in. R3 replaces the body: OS camera consent of the OS-derived peer + * (Windows CapabilityAccessManager ConsentStore\webcam, Android CAMERA of the + * peer uid), then the DisplayXR per-executable consent (HKCU allow list / tray + * prompt; a registered consent-delegating browser passes per executable and + * prompts per origin itself). + */ +static xrt_result_t +authorise_locked(volatile struct ipc_client_state *ics, const struct scam_camera *cam, const char *what) +{ + (void)cam; + if (debug_get_bool_option_stereo_camera_dev_allow()) { + return XRT_SUCCESS; + } + U_LOG_W("%s: refused for pid %ld — stereo camera consent is not implemented yet (R3); set " + "DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment for development.", + what, ics->peer_pid); + return XRT_ERROR_NOT_AUTHORIZED; +} + +/*! + * Spec §7.2 foreground-rule HOOK, evaluated per published frame. R1: always + * visible. R3: an app's session must be VISIBLE/FOCUSED; a delegating client + * follows its own rule; every stream is suspended while the OS session is + * locked / switched away (Android: android_package_is_visible of the peer). + */ +static bool +client_visible_locked(volatile struct ipc_client_state *ics) +{ + (void)ics; + return true; +} + +//! Maintainer decision (R1): raw frames never reach web pages — the browser +//! (PRESENT_OWNER) gets RECTIFIED only; native clients may ask for RAW. +static bool +output_allowed(volatile struct ipc_client_state *ics, uint32_t output) +{ + return output != XRT_STEREO_CAMERA_OUTPUT_RAW || + ics->client_state.client_class != XRT_CLIENT_CLASS_PRESENT_OWNER; +} + +static bool +camera_can_rectify(const struct scam_camera *cam) +{ + // R1 has no rectifier (u_stereo_rectify is R2): only a natively rectified + // source yields RECTIFIED frames; a CALIBRATED one is delivered RAW-flagged. + return (cam->info.flags & XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED) != 0; +} + +static void +set_state_locked(struct scam_camera *cam, uint32_t state) +{ + if (cam->state == state) { + return; + } + U_LOG_W("stereo camera %llu \"%s\": %s -> %s", (unsigned long long)cam->camera_id, cam->info.display_name, + state_str(cam->state), state_str(state)); + cam->state = state; + if (state == XRT_STEREO_CAMERA_STATE_SUSPENDED || state == XRT_STEREO_CAMERA_STATE_UNAVAILABLE) { + // Spec §7.2: the last image must not linger for a suspended consumer. + struct ipc_server_stereo_camera *m = cam->mgr; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (s->used && s->camera == cam->index) { + u_stereo_camera_ring_clear(&s->ring); + } + } + } + // TODO(R1 follow-up): queue XrEventDataStereoCameraStateChangedDXR to the + // owning connections (needs a server->client instance-event channel). +} + +static void +mat3_to_quat(const double r[3][3], float out[4]) +{ + double tr = r[0][0] + r[1][1] + r[2][2]; + double x, y, z, w; + if (tr > 0.0) { + double s = sqrt(tr + 1.0) * 2.0; + w = 0.25 * s; + x = (r[2][1] - r[1][2]) / s; + y = (r[0][2] - r[2][0]) / s; + z = (r[1][0] - r[0][1]) / s; + } else if (r[0][0] > r[1][1] && r[0][0] > r[2][2]) { + double s = sqrt(1.0 + r[0][0] - r[1][1] - r[2][2]) * 2.0; + w = (r[2][1] - r[1][2]) / s; + x = 0.25 * s; + y = (r[0][1] + r[1][0]) / s; + z = (r[0][2] + r[2][0]) / s; + } else if (r[1][1] > r[2][2]) { + double s = sqrt(1.0 + r[1][1] - r[0][0] - r[2][2]) * 2.0; + w = (r[0][2] - r[2][0]) / s; + x = (r[0][1] + r[1][0]) / s; + y = 0.25 * s; + z = (r[1][2] + r[2][1]) / s; + } else { + double s = sqrt(1.0 + r[2][2] - r[0][0] - r[1][1]) * 2.0; + w = (r[1][0] - r[0][1]) / s; + x = (r[0][2] + r[2][0]) / s; + y = (r[1][2] + r[2][1]) / s; + z = 0.25 * s; + } + out[0] = (float)x; + out[1] = (float)y; + out[2] = (float)z; + out[3] = (float)w; +} + +static double +baseline_mm(const struct xrt_plugin_stereo_camera_calibration *c) +{ + const double *t = c->translation_right_from_left_mm; + return sqrt(t[0] * t[0] + t[1] * t[1] + t[2] * t[2]); +} + + +/* + * + * Stream storage. + * + */ + +static void +stream_free_storage(struct scam_stream *s) +{ + if (s->map != NULL || s->section != XRT_SHMEM_HANDLE_INVALID) { + void *map = s->map; + ipc_shmem_destroy(&s->section, &map, (size_t)s->section_size); + s->map = NULL; + } + wake_destroy(&s->wake); +#ifdef XRT_OS_WINDOWS + if (s->sent_section_dup != NULL) { + CloseHandle(s->sent_section_dup); + s->sent_section_dup = NULL; + } + if (s->sent_wake_dup != NULL) { + CloseHandle(s->sent_wake_dup); + s->sent_wake_dup = NULL; + } +#endif + s->allocated = false; +} + +static xrt_result_t +stream_allocate_locked(struct scam_stream *s, const struct scam_camera *cam) +{ + if (s->allocated) { + return XRT_SUCCESS; + } + s->width = 2 * cam->info.eye_width; + s->height = cam->info.eye_height; + if (!u_stereo_camera_layout(s->req.format, s->width, s->height, &s->layout)) { + return XRT_ERROR_ALLOCATION; + } + s->slot_stride = (s->layout.size + 4095u) & ~(uint64_t)4095u; + s->section_size = s->slot_stride * U_STEREO_CAMERA_RING_SLOTS; + void *map = NULL; + s->section = XRT_SHMEM_HANDLE_INVALID; + xrt_result_t xret = ipc_shmem_create((size_t)s->section_size, &s->section, &map); + if (xret != XRT_SUCCESS) { + return xret; + } + s->map = map; + memset(s->map, 0, (size_t)s->section_size); + // TODO(R3): hand POSIX consumers a read-only fd (Linux: reopen + // /proc/self/fd/N O_RDONLY; macOS: shm_open the name O_RDONLY before unlink; + // Android: an ASharedMemory with PROT_READ set via ASharedMemory_setProt). + if (!wake_create(&s->wake)) { + stream_free_storage(s); + return XRT_ERROR_ALLOCATION; + } + u_stereo_camera_ring_init(&s->ring); + s->allocated = true; + return XRT_SUCCESS; +} + + +/* + * + * Camera thread. + * + */ + +static void +publish_locked(struct ipc_server_stereo_camera *m, + struct scam_camera *cam, + const struct xrt_plugin_stereo_camera_frame *f, + int64_t now) +{ + if (cam->last_frame_ns > 0) { + double dt = (double)(now - cam->last_frame_ns) * 1e-9; + if (dt > 0.0) { + float inst = (float)(1.0 / dt); + cam->source_rate = cam->source_rate <= 0.0f ? inst : cam->source_rate * 0.9f + inst * 0.1f; + } + } + cam->last_frame_ns = now; + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_AVAILABLE); + + if (f->width != 2 * cam->info.eye_width || f->height != cam->info.eye_height) { + U_LOG_W("stereo camera %llu: frame %ux%u does not match the advertised %ux%u SBS — dropped", + (unsigned long long)cam->camera_id, f->width, f->height, 2 * cam->info.eye_width, + cam->info.eye_height); + return; + } + + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (!s->used || !s->started || !s->allocated || s->camera != cam->index) { + continue; + } + if (!client_visible_locked(s->ics)) { + u_stereo_camera_ring_clear(&s->ring); + continue; + } + if (!u_stereo_camera_decimator_accept(&s->dec, f->time_ns)) { + continue; + } + int32_t slot = u_stereo_camera_ring_begin_write(&s->ring); + if (slot < 0) { + continue; + } + uint8_t *dst = s->map + (uint64_t)slot * s->slot_stride; + if (!u_stereo_camera_convert(f->format, f->planes, f->pitches, s->req.format, dst, &s->layout, f->width, + f->height)) { + u_stereo_camera_ring_abort_write(&s->ring); + continue; + } + struct xrt_stereo_camera_frame_info *fi = &s->slot_info[slot]; + U_ZERO(fi); + fi->frame_index = f->sequence; + fi->slot = (uint32_t)slot; + fi->width = f->width; + fi->height = f->height; + fi->format = s->req.format; + fi->row_pitch[0] = s->layout.pitch[0]; + fi->row_pitch[1] = s->layout.pitch[1]; + fi->plane_offset[0] = s->layout.offset[0]; + fi->plane_offset[1] = s->layout.offset[1]; + fi->capture_time_ns = f->time_ns; + fi->time_is_exposure = f->time_is_exposure ? 1u : 0u; + fi->output = s->output; + fi->calibration_generation = cam->calibration_generation; + u_stereo_camera_ring_publish(&s->ring, slot, f->sequence); + + int64_t pub = os_monotonic_get_ns(); + if (pub > f->time_ns) { + s->latency_sum_ns += (uint64_t)(pub - f->time_ns); + s->latency_count++; + } + if (s->last_publish_ns > 0 && pub > s->last_publish_ns) { + float inst = (float)(1e9 / (double)(pub - s->last_publish_ns)); + s->delivered_rate = s->delivered_rate <= 0.0f ? inst : s->delivered_rate * 0.9f + inst * 0.1f; + } + s->last_publish_ns = pub; + wake_signal(&s->wake); + } + + if (now - cam->last_stats_log_ns >= STATS_LOG_NS) { + cam->last_stats_log_ns = now; + U_LOG_I("stereo camera %llu: source %.1f Hz, %u started stream(s)", (unsigned long long)cam->camera_id, + cam->source_rate, cam->started_count); + } +} + +static void * +camera_thread(void *ptr) +{ + struct scam_camera *cam = (struct scam_camera *)ptr; + struct ipc_server_stereo_camera *m = cam->mgr; + const struct xrt_plugin_iface *iface = m->iface; + + os_mutex_lock(&m->lock); + while (!m->shutting_down) { + int64_t now = os_monotonic_get_ns(); + + // Idle: close the source once the linger expires, then sleep until a + // stream starts (or the manager shuts down). + if (cam->started_count == 0 && (cam->src == NULL || now >= cam->linger_deadline_ns)) { + if (cam->src != NULL) { + struct xrt_plugin_stereo_camera *src = cam->src; + cam->src = NULL; + os_mutex_unlock(&m->lock); + iface->stereo_camera_close(src); + os_mutex_lock(&m->lock); + U_LOG_W("stereo camera %llu: source closed (no started stream for %lld ms)", + (unsigned long long)cam->camera_id, (long long)(LINGER_NS / 1000000)); + cam->last_frame_ns = 0; + cam->source_rate = 0.0f; + if (cam->state != XRT_STEREO_CAMERA_STATE_UNAVAILABLE) { + cam->state = XRT_STEREO_CAMERA_STATE_AVAILABLE; // idle: "will flow on start" + } + continue; + } + os_cond_wait(&cam->cond, &m->lock); + continue; + } + + if (cam->src == NULL) { + if (now < cam->retry_after_ns) { + os_cond_wait_timeout_ns(&cam->cond, &m->lock, (uint64_t)(cam->retry_after_ns - now)); + continue; + } + struct xrt_plugin_stereo_camera *src = NULL; + os_mutex_unlock(&m->lock); + xrt_result_t xret = iface->stereo_camera_open(m->inst, cam->index, &src); + os_mutex_lock(&m->lock); + if (xret != XRT_SUCCESS || src == NULL) { + U_LOG_W("stereo camera %llu: plug-in open failed (%d); retrying in 1 s", + (unsigned long long)cam->camera_id, (int)xret); + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_UNAVAILABLE); + cam->retry_after_ns = os_monotonic_get_ns() + REOPEN_BACKOFF_NS; + continue; + } + if (m->shutting_down) { + os_mutex_unlock(&m->lock); + iface->stereo_camera_close(src); + os_mutex_lock(&m->lock); + break; + } + cam->src = src; + cam->last_frame_ns = 0; + U_LOG_W("stereo camera %llu: source opened", (unsigned long long)cam->camera_id); + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_WAITING); + } + + struct xrt_plugin_stereo_camera *src = cam->src; + struct xrt_plugin_stereo_camera_frame f; + memset(&f, 0, sizeof(f)); + os_mutex_unlock(&m->lock); + uint32_t w = iface->stereo_camera_wait_frame(src, WAIT_TIMEOUT_NS, &f); + os_mutex_lock(&m->lock); + now = os_monotonic_get_ns(); + + switch (w) { + case XRT_PLUGIN_STEREO_CAMERA_WAIT_OK: + publish_locked(m, cam, &f, now); + iface->stereo_camera_release_frame(src); + break; + case XRT_PLUGIN_STEREO_CAMERA_WAIT_TIMEOUT: + if (cam->last_frame_ns == 0) { + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_WAITING); + } else if (now - cam->last_frame_ns > SUSPEND_AFTER_NS) { + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_SUSPENDED); + } + break; + case XRT_PLUGIN_STEREO_CAMERA_WAIT_SUSPENDED: + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_SUSPENDED); + cam->last_frame_ns = 0; + break; + default: + U_LOG_W("stereo camera %llu: plug-in reported a source error; closing, retry in 1 s", + (unsigned long long)cam->camera_id); + set_state_locked(cam, XRT_STEREO_CAMERA_STATE_UNAVAILABLE); + cam->src = NULL; + os_mutex_unlock(&m->lock); + iface->stereo_camera_close(src); + os_mutex_lock(&m->lock); + cam->retry_after_ns = os_monotonic_get_ns() + REOPEN_BACKOFF_NS; + break; + } + } + if (cam->src != NULL) { + struct xrt_plugin_stereo_camera *src = cam->src; + cam->src = NULL; + os_mutex_unlock(&m->lock); + iface->stereo_camera_close(src); + os_mutex_lock(&m->lock); + } + os_mutex_unlock(&m->lock); + return NULL; +} + + +/* + * + * Manager lifecycle. + * + */ + +struct ipc_server_stereo_camera * +ipc_server_stereo_camera_create(struct xrt_instance *xinst) +{ + struct ipc_server_stereo_camera *m = U_TYPED_CALLOC(struct ipc_server_stereo_camera); + os_mutex_init(&m->lock); + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + m->streams[i].section = XRT_SHMEM_HANDLE_INVALID; + } + + if (!debug_get_bool_option_stereo_camera_enabled()) { + U_LOG_W("stereo camera: disabled by DXR_STEREO_CAMERA=0 (kill switch) — zero cameras"); + return m; + } + const struct xrt_plugin_iface *iface = NULL; + struct xrt_plugin_instance *inst = NULL; + if (xinst == NULL || xinst->get_active_plugin == NULL || !xinst->get_active_plugin(xinst, &iface, &inst) || + !xrt_plugin_iface_has_stereo_camera(iface)) { + U_LOG_I("stereo camera: the active plug-in provides no camera source"); + return m; + } + m->iface = iface; + m->inst = inst; + + struct xrt_plugin_stereo_camera_info infos[XRT_STEREO_CAMERA_MAX_CAMERAS]; + memset(infos, 0, sizeof(infos)); + for (uint32_t i = 0; i < XRT_STEREO_CAMERA_MAX_CAMERAS; i++) { + infos[i].struct_size = (uint32_t)sizeof(infos[i]); + } + uint32_t n = iface->stereo_camera_enumerate(inst, XRT_STEREO_CAMERA_MAX_CAMERAS, infos); + if (n > XRT_STEREO_CAMERA_MAX_CAMERAS) { + n = XRT_STEREO_CAMERA_MAX_CAMERAS; + } + for (uint32_t i = 0; i < n; i++) { + struct scam_camera *cam = &m->cams[m->camera_count]; + struct xrt_plugin_stereo_camera_info *info = &infos[i]; + info->display_name[sizeof(info->display_name) - 1] = '\0'; + info->device_identity[sizeof(info->device_identity) - 1] = '\0'; + info->platform_device_hint[sizeof(info->platform_device_hint) - 1] = '\0'; + if (info->eye_width == 0 || info->eye_height == 0 || (info->eye_width & 1u) || (info->eye_height & 1u) || + info->native_format < XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 || + info->native_format > XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8) { + U_LOG_W("stereo camera: plug-in camera %u has a malformed description (%ux%u, format %u) — " + "skipped", + i, info->eye_width, info->eye_height, info->native_format); + continue; + } + cam->index = i; + cam->camera_id = (uint64_t)m->camera_count + 1; // never reused in this service lifetime + cam->info = *info; + cam->mgr = m; + cam->state = XRT_STEREO_CAMERA_STATE_AVAILABLE; + os_cond_init(&cam->cond); + if (info->flags & XRT_PLUGIN_STEREO_CAMERA_CALIBRATED) { + cam->calib.struct_size = (uint32_t)sizeof(cam->calib); + if (iface->stereo_camera_get_calibration(inst, i, &cam->calib) == XRT_SUCCESS) { + cam->have_calib = true; + } else { + U_LOG_W("stereo camera: \"%s\" claims CALIBRATED but returned no calibration — RAW only", + info->display_name); + cam->info.flags &= ~XRT_PLUGIN_STEREO_CAMERA_CALIBRATED; + } + } + U_LOG_W("stereo camera %llu: \"%s\" %ux%u per eye @ %.1f Hz, flags 0x%x, native format %u", + (unsigned long long)cam->camera_id, info->display_name, info->eye_width, info->eye_height, + info->max_frame_rate, info->flags, info->native_format); + m->camera_count++; + } + return m; +} + +void +ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr) +{ + struct ipc_server_stereo_camera *m = *mgr_ptr; + if (m == NULL) { + return; + } + os_mutex_lock(&m->lock); + m->shutting_down = true; + for (uint32_t i = 0; i < m->camera_count; i++) { + os_cond_broadcast(&m->cams[i].cond); + } + os_mutex_unlock(&m->lock); + for (uint32_t i = 0; i < m->camera_count; i++) { + if (m->cams[i].thread_started) { + os_thread_join(&m->cams[i].thread); + os_thread_destroy(&m->cams[i].thread); + } + os_cond_destroy(&m->cams[i].cond); + } + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + if (m->streams[i].used) { + stream_free_storage(&m->streams[i]); + } + } + os_mutex_destroy(&m->lock); + free(m); + *mgr_ptr = NULL; +} + +static void +stream_stop_locked(struct scam_stream *s, struct scam_camera *cam) +{ + if (!s->started) { + return; + } + s->started = false; + u_stereo_camera_ring_clear(&s->ring); + if (cam->started_count > 0) { + cam->started_count--; + } + if (cam->started_count == 0) { + cam->linger_deadline_ns = os_monotonic_get_ns() + LINGER_NS; + } + U_LOG_W("stereo camera %llu: stream %llu stopped (%s); %u stream(s) still started", + (unsigned long long)cam->camera_id, (unsigned long long)s->id, s->consumer, cam->started_count); +} + +static void +stream_destroy_locked(struct ipc_server_stereo_camera *m, struct scam_stream *s) +{ + stream_stop_locked(s, &m->cams[s->camera]); + stream_free_storage(s); + memset(s, 0, sizeof(*s)); + s->section = XRT_SHMEM_HANDLE_INVALID; +#ifndef XRT_OS_WINDOWS + s->wake.fds[0] = s->wake.fds[1] = -1; +#endif +} + +void +ipc_server_client_stereo_camera_release(volatile struct ipc_client_state *ics) +{ + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL || ics->stereo_camera_owner == 0) { + return; + } + os_mutex_lock(&m->lock); + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (s->used && s->owner == ics->stereo_camera_owner) { + stream_destroy_locked(m, s); + } + } + os_mutex_unlock(&m->lock); + ics->stereo_camera_owner = 0; +} + + +/* + * + * IPC handlers. + * + */ + +xrt_result_t +ipc_handle_stereo_camera_count(volatile struct ipc_client_state *ics, uint32_t *out_count) +{ + *out_count = 0; + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (require_camera_client(ics, "stereo_camera_count") != XRT_SUCCESS || m == NULL) { + return XRT_SUCCESS; // zero cameras, indistinguishable from "none" on purpose + } + os_mutex_lock(&m->lock); + *out_count = m->camera_count; + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_get_properties(volatile struct ipc_client_state *ics, + uint32_t index, + struct xrt_stereo_camera_properties *out_props) +{ + U_ZERO(out_props); + struct ipc_server_stereo_camera *m = mgr_of(ics); + xrt_result_t auth = require_camera_client(ics, "stereo_camera_get_properties"); + if (auth != XRT_SUCCESS) { + return auth; + } + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + char exe[260] = {0}; + ipc_server_peer_exe_path(ics->peer_pid, exe, sizeof(exe)); + + os_mutex_lock(&m->lock); + if (index >= m->camera_count) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + const struct scam_camera *cam = &m->cams[index]; + out_props->camera_id = cam->camera_id; + u_stereo_camera_persistent_id(cam->info.device_identity, exe, out_props->persistent_id); + snprintf(out_props->display_name, sizeof(out_props->display_name), "%s", cam->info.display_name); + snprintf(out_props->platform_device_hint, sizeof(out_props->platform_device_hint), "%s", + cam->info.platform_device_hint); + out_props->flags = cam->info.flags; + out_props->state = cam->state; + out_props->view_count = 2; + out_props->eye_width = cam->info.eye_width; + out_props->eye_height = cam->info.eye_height; + out_props->max_frame_rate = cam->info.max_frame_rate; + if (cam->have_calib) { + out_props->baseline_mm = (float)baseline_mm(&cam->calib); + double fx = cam->calib.k[0][0]; + if (fx > 0.0) { + out_props->horizontal_fov_deg = + (float)(2.0 * atan(cam->info.eye_width / (2.0 * fx)) * 180.0 / 3.14159265358979323846); + } + } + out_props->supported_formats = XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_GRAY8) | + XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_NV12) | + XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_BGRA8); + out_props->supported_transports = XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY); + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_get_calibration(volatile struct ipc_client_state *ics, + uint64_t camera_id, + uint32_t output, + struct xrt_stereo_camera_calibration *out_calib) +{ + U_ZERO(out_calib); + struct ipc_server_stereo_camera *m = mgr_of(ics); + xrt_result_t auth = require_camera_client(ics, "stereo_camera_get_calibration"); + if (auth != XRT_SUCCESS) { + return auth; + } + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_camera *cam = camera_by_id_locked(m, camera_id); + if (cam == NULL || (output != XRT_STEREO_CAMERA_OUTPUT_RAW && output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED)) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + // Spec §7.5: calibration identifies the device — only after §7.1 passes. + xrt_result_t xret = authorise_locked(ics, cam, "stereo_camera_get_calibration"); + if (xret == XRT_SUCCESS && !output_allowed(ics, output)) { + xret = XRT_ERROR_NOT_AUTHORIZED; + } + if (xret == XRT_SUCCESS && !cam->have_calib) { + xret = XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + if (xret == XRT_SUCCESS && output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !camera_can_rectify(cam)) { + xret = XRT_ERROR_FEATURE_NOT_SUPPORTED; // R2 computes post-rectification numbers + } + if (xret != XRT_SUCCESS) { + os_mutex_unlock(&m->lock); + return xret; + } + const struct xrt_plugin_stereo_camera_calibration *c = &cam->calib; + out_calib->output = output; + for (int e = 0; e < 2; e++) { + struct xrt_stereo_camera_intrinsics *in = &out_calib->eye[e]; + in->width = c->image_width; + in->height = c->image_height; + in->fx = (float)c->k[e][0]; + in->fy = (float)c->k[e][1]; + in->cx = (float)c->k[e][2]; + in->cy = (float)c->k[e][3]; + in->model = output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? XRT_PLUGIN_STEREO_CAMERA_DISTORTION_NONE + : c->distortion_model; + if (in->model != XRT_PLUGIN_STEREO_CAMERA_DISTORTION_NONE) { + for (int k = 0; k < 8; k++) { + in->coefficients[k] = (float)c->distortion[e][k]; + } + } + } + // Plug-in: OpenCV x_R = R x_L + T. Client: the pose of the right camera in + // the left camera's frame = (R^T, -R^T T), metres. + double rt[3][3]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + rt[i][j] = c->rotation_right_from_left[j][i]; + } + } + mat3_to_quat(rt, out_calib->orientation); + for (int i = 0; i < 3; i++) { + double p = 0.0; + for (int j = 0; j < 3; j++) { + p -= rt[i][j] * c->translation_right_from_left_mm[j]; + } + out_calib->position[i] = (float)(p / 1000.0); + } + out_calib->baseline_mm = (float)baseline_mm(c); + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, + const struct xrt_stereo_camera_stream_request *req, + uint64_t *out_stream_id) +{ + *out_stream_id = 0; + struct ipc_server_stereo_camera *m = mgr_of(ics); + xrt_result_t auth = require_camera_client(ics, "stereo_camera_stream_create"); + if (auth != XRT_SUCCESS) { + return auth; + } + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + if (req->transport != XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; // GPU transports: see the file comment + } + if (req->format < XRT_STEREO_CAMERA_FORMAT_GRAY8 || req->format > XRT_STEREO_CAMERA_FORMAT_BGRA8 || + (req->output != XRT_STEREO_CAMERA_OUTPUT_RAW && req->output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED)) { + return XRT_ERROR_INPUT_UNSUPPORTED; + } + if (!output_allowed(ics, req->output)) { + U_LOG_W("stereo_camera_stream_create: RAW refused for pid %ld (PRESENT_OWNER: rectified only)", + ics->peer_pid); + return XRT_ERROR_NOT_AUTHORIZED; + } + char exe[260] = {0}; + ipc_server_peer_exe_path(ics->peer_pid, exe, sizeof(exe)); + + os_mutex_lock(&m->lock); + struct scam_camera *cam = camera_by_id_locked(m, req->camera_id); + if (cam == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + bool calibrated = (cam->info.flags & (XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | + XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED)) != 0; + if (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !calibrated) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + uint32_t per_camera = 0; + struct scam_stream *slot = NULL; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (s->used && s->camera == cam->index) { + per_camera++; + } else if (!s->used && slot == NULL) { + slot = s; + } + } + if (per_camera >= XRT_STEREO_CAMERA_MAX_STREAMS_PER_CAMERA || slot == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_CLIENT_LIMIT_REACHED; + } + memset(slot, 0, sizeof(*slot)); + slot->section = XRT_SHMEM_HANDLE_INVALID; +#ifndef XRT_OS_WINDOWS + slot->wake.fds[0] = slot->wake.fds[1] = -1; +#endif + slot->used = true; + slot->id = ++m->next_stream_id; + slot->owner = owner_of(m, ics); + slot->ics = ics; + slot->camera = cam->index; + slot->req = *req; + // R1: RECTIFIED only from a natively rectified source; a calibrated one is + // delivered RAW and FLAGGED (frame.output) until u_stereo_rectify lands (R2). + slot->output = (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && camera_can_rectify(cam)) + ? XRT_STEREO_CAMERA_OUTPUT_RECTIFIED + : XRT_STEREO_CAMERA_OUTPUT_RAW; + snprintf(slot->consumer, sizeof(slot->consumer), "%s (pid %ld)", exe[0] ? exe : "?", ics->peer_pid); + *out_stream_id = slot->id; + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_destroy(volatile struct ipc_client_state *ics, uint64_t stream_id) +{ + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL) { + return XRT_SUCCESS; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s != NULL) { + stream_destroy_locked(m, s); + } + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_start(volatile struct ipc_client_state *ics, uint64_t stream_id) +{ + struct ipc_server_stereo_camera *m = mgr_of(ics); + xrt_result_t auth = require_camera_client(ics, "stereo_camera_stream_start"); + if (auth != XRT_SUCCESS) { + return auth; + } + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + if (s->started) { + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; + } + struct scam_camera *cam = &m->cams[s->camera]; + // Spec §7.1: start is THE authorisation point. + xrt_result_t xret = authorise_locked(ics, cam, "stereo_camera_stream_start"); + if (xret == XRT_SUCCESS) { + xret = stream_allocate_locked(s, cam); + } + if (xret != XRT_SUCCESS) { + os_mutex_unlock(&m->lock); + return xret; + } + u_stereo_camera_decimator_init(&s->dec, s->req.max_frame_rate, cam->info.max_frame_rate); + u_stereo_camera_ring_clear(&s->ring); + s->started = true; + cam->started_count++; + if (!cam->thread_started) { + os_thread_init(&cam->thread); + if (os_thread_start(&cam->thread, camera_thread, cam) != 0) { + s->started = false; + cam->started_count--; + os_mutex_unlock(&m->lock); + return XRT_ERROR_THREADING_INIT_FAILURE; + } + cam->thread_started = true; + } + os_cond_signal(&cam->cond); + // TODO(R3): runtime-owned in-use indicator naming s->consumer. + U_LOG_W("stereo camera %llu: stream %llu started by %s — output %s, format %u, max %.1f Hz; %u started", + (unsigned long long)cam->camera_id, (unsigned long long)s->id, s->consumer, + s->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", s->req.format, + s->req.max_frame_rate, cam->started_count); + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_stop(volatile struct ipc_client_state *ics, uint64_t stream_id) +{ + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL) { + return XRT_SUCCESS; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s != NULL) { + stream_stop_locked(s, &m->cams[s->camera]); + } + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_get_section(volatile struct ipc_client_state *ics, + uint64_t stream_id, + struct xrt_stereo_camera_stream_layout *out_layout, + uint32_t max_handle_count, + xrt_shmem_handle_t *out_handles, + uint32_t *out_handle_count) +{ + U_ZERO(out_layout); + *out_handle_count = 0; + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL || max_handle_count < 1) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL || !s->allocated) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; // not started yet + } + out_layout->width = s->width; + out_layout->height = s->height; + out_layout->format = s->req.format; + out_layout->output = s->output; + out_layout->transport = s->req.transport; + out_layout->max_frame_rate = s->dec.period_ns > 0 ? (float)(1e9 / (double)s->dec.period_ns) + : m->cams[s->camera].info.max_frame_rate; + out_layout->section_size = s->section_size; + out_layout->slot_stride = s->slot_stride; + out_layout->slot_count = U_STEREO_CAMERA_RING_SLOTS; +#ifdef XRT_OS_WINDOWS + // A READ-ONLY duplicate: the consumer can map it but never write the ring. + if (s->sent_section_dup != NULL) { + CloseHandle(s->sent_section_dup); + s->sent_section_dup = NULL; + } + if (!DuplicateHandle(GetCurrentProcess(), s->section, GetCurrentProcess(), &s->sent_section_dup, + FILE_MAP_READ, FALSE, 0)) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_IPC_FAILURE; + } + out_handles[0] = s->sent_section_dup; +#else + out_handles[0] = s->section; // SCM_RIGHTS installs a copy; ours stays open +#endif + *out_handle_count = 1; + ics->handles_sent = true; // browser#103 RC-1 + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_get_wake(volatile struct ipc_client_state *ics, + uint64_t stream_id, + uint32_t max_handle_count, + xrt_graphics_sync_handle_t *out_handles, + uint32_t *out_handle_count) +{ + *out_handle_count = 0; + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL || max_handle_count < 1) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL || !s->allocated) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } +#ifdef XRT_OS_WINDOWS + // SYNCHRONIZE only: the consumer can wait, never signal / reset it. + if (s->sent_wake_dup != NULL) { + CloseHandle(s->sent_wake_dup); + s->sent_wake_dup = NULL; + } + if (!DuplicateHandle(GetCurrentProcess(), s->wake.event, GetCurrentProcess(), &s->sent_wake_dup, SYNCHRONIZE, + FALSE, 0)) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_IPC_FAILURE; + } + out_handles[0] = s->sent_wake_dup; +#else + out_handles[0] = s->wake.fds[0]; +#endif + *out_handle_count = 1; + ics->handles_sent = true; + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_acquire(volatile struct ipc_client_state *ics, + uint64_t stream_id, + bool *out_ready, + struct xrt_stereo_camera_frame_info *out_frame) +{ + *out_ready = false; + U_ZERO(out_frame); + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + int32_t slot = -1; + uint64_t seq = 0; + if (s->started && s->allocated && u_stereo_camera_ring_acquire(&s->ring, &slot, &seq)) { + *out_frame = s->slot_info[slot]; + *out_ready = true; + } + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +xrt_result_t +ipc_handle_stereo_camera_stream_stats(volatile struct ipc_client_state *ics, + uint64_t stream_id, + struct xrt_stereo_camera_stream_stats *out_stats) +{ + U_ZERO(out_stats); + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + os_mutex_lock(&m->lock); + struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + out_stats->source_frame_rate = m->cams[s->camera].source_rate; + out_stats->delivered_frame_rate = s->delivered_rate; + out_stats->frames_published = s->ring.published; + out_stats->frames_skipped = s->ring.skipped; + out_stats->frames_acquired = s->ring.acquired; + out_stats->mean_latency_ns = s->latency_count > 0 ? s->latency_sum_ns / s->latency_count : 0; + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.h b/src/xrt/ipc/server/ipc_server_stereo_camera.h new file mode 100644 index 0000000000..6d979be1c1 --- /dev/null +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.h @@ -0,0 +1,50 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043): the service's camera manager. + * + * The service is the single owner of each plug-in-provided stereo camera: one + * runtime-owned thread per open camera pulls frames from the plug-in's + * wait_frame slot, and fans each frame out to every started, authorised stream + * — per stream a 3-slot latest-wins shared-memory ring (u_stereo_camera_ring), + * format conversion, decimation, and a per-stream wake handle. The source is + * opened on the first start and closed a linger (2 s) after the last stop. + * + * Platform-neutral (Windows D3D11 service, macOS / Linux service, Android + * runtime service): the only OS-specific pieces are the wake handle (event / + * pipe) and the read-only section duplicate, both in ipc_server_stereo_camera.c. + * + * @ingroup ipc_server + */ + +#pragma once + +#include "xrt/xrt_instance.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct ipc_server_stereo_camera; +struct ipc_client_state; + +/*! + * Create the manager over the active plug-in of @p xinst (the service's native + * instance). Always returns a manager; with no plug-in, no camera slots, or the + * kill switch (DXR_STEREO_CAMERA=0) it simply exposes zero cameras. + */ +struct ipc_server_stereo_camera * +ipc_server_stereo_camera_create(struct xrt_instance *xinst); + +//! Stop every camera thread, close every source, free every stream. +void +ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr); + +//! Destroy every stream @p ics created. Called once at client teardown. +void +ipc_server_client_stereo_camera_release(volatile struct ipc_client_state *ics); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/ipc/shared/ipc_protocol.h b/src/xrt/ipc/shared/ipc_protocol.h index 710ea4579f..799fd74733 100644 --- a/src/xrt/ipc/shared/ipc_protocol.h +++ b/src/xrt/ipc/shared/ipc_protocol.h @@ -25,6 +25,7 @@ #include "xrt/xrt_space.h" #include "xrt/xrt_tracking.h" #include "xrt/xrt_display_metrics.h" +#include "xrt/xrt_stereo_camera.h" #include "xrt/xrt_config_build.h" #include diff --git a/src/xrt/ipc/shared/proto.json b/src/xrt/ipc/shared/proto.json index 67c40f1784..bcc283c338 100644 --- a/src/xrt/ipc/shared/proto.json +++ b/src/xrt/ipc/shared/proto.json @@ -1194,5 +1194,93 @@ "out": [ {"name": "accepted", "type": "uint32_t"} ] + }, + + "stereo_camera_count": { + "out": [ + {"name": "count", "type": "uint32_t"} + ] + }, + + "stereo_camera_get_properties": { + "in": [ + {"name": "index", "type": "uint32_t"} + ], + "out": [ + {"name": "props", "type": "struct xrt_stereo_camera_properties"} + ] + }, + + "stereo_camera_get_calibration": { + "in": [ + {"name": "camera_id", "type": "uint64_t"}, + {"name": "output", "type": "uint32_t"} + ], + "out": [ + {"name": "calib", "type": "struct xrt_stereo_camera_calibration"} + ] + }, + + "stereo_camera_stream_create": { + "in": [ + {"name": "req", "type": "struct xrt_stereo_camera_stream_request"} + ], + "out": [ + {"name": "stream_id", "type": "uint64_t"} + ] + }, + + "stereo_camera_stream_destroy": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ] + }, + + "stereo_camera_stream_start": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ] + }, + + "stereo_camera_stream_stop": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ] + }, + + "stereo_camera_stream_get_section": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ], + "out": [ + {"name": "layout", "type": "struct xrt_stereo_camera_stream_layout"} + ], + "out_handles": {"type": "xrt_shmem_handle_t"} + }, + + "stereo_camera_stream_get_wake": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ], + "out_handles": {"type": "xrt_graphics_sync_handle_t"} + }, + + "stereo_camera_acquire": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ], + "out": [ + {"name": "ready", "type": "bool"}, + {"name": "frame", "type": "struct xrt_stereo_camera_frame_info"} + ] + }, + + "stereo_camera_stream_stats": { + "in": [ + {"name": "stream_id", "type": "uint64_t"} + ], + "out": [ + {"name": "stats", "type": "struct xrt_stereo_camera_stream_stats"} + ] } } From 5c043784babce23955f8372697ed873548929717 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:44:12 -0700 Subject: [PATCH 05/37] =?UTF-8?q?fix(sim=5Fdisplay):=20fake=20stereo=20cam?= =?UTF-8?q?era=20=E2=80=94=20keep=20the=20suspend=20square=20wave=20on=20i?= =?UTF-8?q?ts=20open=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-anchoring the pacing clock after a pause also moved the suspend phase, so every SUSPENDED half-period stretched (36 frames in 5 s instead of ~75 at a 1 s period). Separate origins; now 92 frames in 6 s, 3 suspensions. Co-Authored-By: Claude Opus 5.5 --- src/xrt/drivers/sim_display/sim_display_stereo_camera.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index 74933342f3..77e4d31d4b 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -48,7 +48,8 @@ struct xrt_plugin_stereo_camera uint32_t gray_pitch; uint8_t *converted; struct u_stereo_camera_planes conv_layout; - int64_t t0_ns; + int64_t t0_ns; //!< pacing anchor (re-anchored after a pause) + int64_t open_ns; //!< suspend square-wave origin (never moves) int64_t period_ns; uint64_t seq; }; @@ -202,6 +203,7 @@ sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, } cam->period_ns = (int64_t)(1e9 / cfg->fps); cam->t0_ns = os_monotonic_get_ns(); + cam->open_ns = cam->t0_ns; U_LOG_W("sim_display: FAKE stereo camera opened (bg disparity %u px, bar disparity %u px)", cam->scene.bg_disparity, cam->scene.bar_disparity); *out_cam = cam; @@ -227,7 +229,7 @@ sim_display_stereo_camera_wait_frame(struct xrt_plugin_stereo_camera *cam, int64_t now = os_monotonic_get_ns(); // SUSPEND square wave: odd half-periods are "tracker stopped". - if (cam->cfg.suspend_period_ns > 0 && ((now - cam->t0_ns) / cam->cfg.suspend_period_ns) % 2 == 1) { + if (cam->cfg.suspend_period_ns > 0 && ((now - cam->open_ns) / cam->cfg.suspend_period_ns) % 2 == 1) { sleep_ns(timeout_ns < 20000000 ? timeout_ns : 20000000); return XRT_PLUGIN_STEREO_CAMERA_WAIT_SUSPENDED; } From ec63d8f44dbe4ef0a7d4eeb365cc50912ee89b6e Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:44:12 -0700 Subject: [PATCH 06/37] feat(stereo-camera): displayxr-cli camera list | calib | probe A real DIAG consumer over IPC: probe creates + starts a stream (through the service's authorisation), maps the ring READ-ONLY, waits on the per-stream wake handle, acquires, and reports the measured delivery rate, frame-index gaps, the SBS layout, block-matched disparities of the last frame and the service's stream stats; --out writes the last frame as a PNG. Co-Authored-By: Claude Opus 5.5 --- src/xrt/targets/cli/CMakeLists.txt | 4 + src/xrt/targets/cli/cli_cmd_camera.c | 590 +++++++++++++++++++++++++++ src/xrt/targets/cli/cli_common.h | 4 + src/xrt/targets/cli/cli_main.c | 7 + 4 files changed, 605 insertions(+) create mode 100644 src/xrt/targets/cli/cli_cmd_camera.c diff --git a/src/xrt/targets/cli/CMakeLists.txt b/src/xrt/targets/cli/CMakeLists.txt index 6778feaeed..49578e1733 100644 --- a/src/xrt/targets/cli/CMakeLists.txt +++ b/src/xrt/targets/cli/CMakeLists.txt @@ -43,6 +43,7 @@ add_executable( cli_cmd_test.c cli_cmd_clients.c cli_cmd_lift.cpp + cli_cmd_camera.c cli_common.h cli_main.c ) @@ -95,5 +96,8 @@ target_include_directories(cli PRIVATE ${PROJECT_SOURCE_DIR}/src/external/stb) if(WIN32) target_link_libraries(cli PRIVATE d3d11 dxgi) endif() +# ADR-043: `displayxr-cli camera probe --out` writes PNGs (stb, header-only, +# compiled static into cli_cmd_camera.c). +target_include_directories(cli PRIVATE ${PROJECT_SOURCE_DIR}/src/external/stb) install(TARGETS cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c new file mode 100644 index 0000000000..3a578dc611 --- /dev/null +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -0,0 +1,590 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief `displayxr-cli camera` — XR_DXR_stereo_camera (ADR-043) over IPC. + * + * camera list [--json] properties + state of every camera + * camera calib [--raw|--rectified] [--json] + * camera probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] + * [--frames N] [--seconds S] [--out DIR] + * + * `probe` is a real consumer: it creates and starts a stream (so it goes + * through the service's authorisation — in R1, DXR_STEREO_CAMERA_DEV_ALLOW=1 + * on the SERVICE), maps the ring READ-ONLY, waits on the per-stream wake + * handle, acquires, and reports the measured delivery rate, frame-index gaps, + * the SBS layout, block-matched disparities of the last frame (the sim fake's + * scene is 12 px background / 40 px bar), and the service's stream stats. + * With --out it writes the last frame as cam_.png. + * + * Connects as a DIAG client, like `clients`: on Windows run it NON-elevated. + */ + +#include "cli_common.h" + +#ifdef CLI_HAVE_IPC + +#include "xrt/xrt_instance.h" +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_results.h" +#include "xrt/xrt_stereo_camera.h" +#include "os/os_time.h" +#include "util/u_logging.h" +#include "util/u_stereo_camera.h" + +#include "client/ipc_client_connection.h" +#include "client/ipc_client.h" +#include "client/ipc_client_stereo_camera.h" + +#define STB_IMAGE_WRITE_STATIC +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" + +#include +#include +#include + +#ifdef XRT_OS_WINDOWS +#include +#else +#include +#include +#include +#endif + +static const char * +state_str(uint32_t s) +{ + switch (s) { + case XRT_STEREO_CAMERA_STATE_AVAILABLE: return "AVAILABLE"; + case XRT_STEREO_CAMERA_STATE_WAITING: return "WAITING"; + case XRT_STEREO_CAMERA_STATE_SUSPENDED: return "SUSPENDED"; + case XRT_STEREO_CAMERA_STATE_UNAVAILABLE: return "UNAVAILABLE"; + default: return "?"; + } +} + +static const char * +format_str(uint32_t f) +{ + switch (f) { + case XRT_STEREO_CAMERA_FORMAT_GRAY8: return "GRAY8"; + case XRT_STEREO_CAMERA_FORMAT_NV12: return "NV12"; + case XRT_STEREO_CAMERA_FORMAT_BGRA8: return "BGRA8"; + default: return "?"; + } +} + +static void +flags_str(uint64_t f, char *out, size_t cap) +{ + out[0] = '\0'; + static const struct + { + uint64_t bit; + const char *name; + } k[] = { + {XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING, "SHARED_WITH_EYE_TRACKING"}, + {XRT_PLUGIN_STEREO_CAMERA_USER_FACING, "USER_FACING"}, + {XRT_PLUGIN_STEREO_CAMERA_CALIBRATED, "CALIBRATED"}, + {XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED, "NATIVELY_RECTIFIED"}, + {XRT_PLUGIN_STEREO_CAMERA_MONOCHROME, "MONOCHROME"}, + }; + for (size_t i = 0; i < sizeof(k) / sizeof(k[0]); i++) { + if (f & k[i].bit) { + size_t n = strlen(out); + snprintf(out + n, cap - n, "%s%s", n ? "|" : "", k[i].name); + } + } +} + +static const char * +result_hint(xrt_result_t x) +{ + switch (x) { + case XRT_ERROR_NOT_AUTHORIZED: + return " = NOT_AUTHORIZED (R1: set DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment)"; + case XRT_ERROR_FEATURE_NOT_SUPPORTED: return " = FEATURE_NOT_SUPPORTED"; + case XRT_ERROR_INPUT_UNSUPPORTED: return " = INPUT_UNSUPPORTED (bad id / format / output)"; + case XRT_ERROR_CLIENT_LIMIT_REACHED: return " = LIMIT_REACHED (8 streams per camera)"; + case XRT_ERROR_IPC_FAILURE: return " = IPC_FAILURE"; + default: return ""; + } +} + +static bool +cam_connect(struct ipc_connection *ipc_c) +{ + struct xrt_instance_info ii = {0}; + snprintf(ii.app_info.application_name, sizeof(ii.app_info.application_name), "%s", "displayxr-cli"); + ii.app_info.declared_client_class = XRT_CLIENT_CLASS_DIAG; + xrt_result_t xret = ipc_client_connection_init(ipc_c, U_LOGGING_ERROR, &ii); + if (xret != XRT_SUCCESS) { + printf("displayxr-cli camera: not connected to the service (xrt_result=%d).\n", (int)xret); + printf(" Is displayxr-service running? On Windows, run from a NON-elevated prompt.\n"); + return false; + } + return true; +} + + +/* + * + * list / calib. + * + */ + +static int +cmd_list(struct ipc_connection *ipc_c, bool json) +{ + uint32_t n = 0; + xrt_result_t xret = ipc_client_stereo_camera_count(ipc_c, &n); + if (xret != XRT_SUCCESS) { + printf("stereo_camera_count failed: %d%s\n", (int)xret, result_hint(xret)); + return 2; + } + if (json) { + printf("{\"cameras\": ["); + } else { + printf("%u stereo camera(s)\n", n); + } + for (uint32_t i = 0; i < n; i++) { + struct xrt_stereo_camera_properties p; + xret = ipc_client_stereo_camera_get_properties(ipc_c, i, &p); + if (xret != XRT_SUCCESS) { + continue; + } + char flags[160]; + flags_str(p.flags, flags, sizeof(flags)); + if (json) { + printf("%s{\"id\": %llu, \"name\": \"%s\", \"persistentId\": \"%s\", \"state\": \"%s\", " + "\"flags\": \"%s\", \"viewCount\": %u, \"eyeWidth\": %u, \"eyeHeight\": %u, " + "\"maxFrameRate\": %.2f, \"baselineMm\": %.2f, \"horizontalFovDeg\": %.2f, " + "\"formats\": \"0x%llx\", \"transports\": \"0x%llx\", \"platformDeviceHint\": \"%s\"}", + i ? ", " : "", (unsigned long long)p.camera_id, p.display_name, p.persistent_id, + state_str(p.state), flags, p.view_count, p.eye_width, p.eye_height, p.max_frame_rate, + p.baseline_mm, p.horizontal_fov_deg, (unsigned long long)p.supported_formats, + (unsigned long long)p.supported_transports, p.platform_device_hint); + } else { + printf(" [%llu] \"%s\" %s\n", (unsigned long long)p.camera_id, p.display_name, state_str(p.state)); + printf(" persistentId %s\n", p.persistent_id); + printf(" flags %s\n", flags); + printf(" eye %ux%u (SBS %ux%u), %u views, up to %.1f Hz\n", p.eye_width, + p.eye_height, 2 * p.eye_width, p.eye_height, p.view_count, p.max_frame_rate); + printf(" baseline %.2f mm, HFOV %.2f deg per eye\n", p.baseline_mm, p.horizontal_fov_deg); + printf(" formats 0x%llx transports 0x%llx platform hint \"%s\"\n", + (unsigned long long)p.supported_formats, (unsigned long long)p.supported_transports, + p.platform_device_hint); + } + } + if (json) { + printf("]}\n"); + } + return 0; +} + +static int +cmd_calib(struct ipc_connection *ipc_c, uint64_t id, uint32_t output, bool json) +{ + struct xrt_stereo_camera_calibration c; + xrt_result_t xret = ipc_client_stereo_camera_get_calibration(ipc_c, id, output, &c); + if (xret != XRT_SUCCESS) { + printf("stereo_camera_get_calibration(%llu) failed: %d%s\n", (unsigned long long)id, (int)xret, + result_hint(xret)); + return 2; + } + if (json) { + printf("{\"output\": \"%s\", \"baselineMm\": %.3f, \"eyes\": [", + output == XRT_STEREO_CAMERA_OUTPUT_RAW ? "RAW" : "RECTIFIED", c.baseline_mm); + for (int e = 0; e < 2; e++) { + printf("%s{\"w\": %u, \"h\": %u, \"fx\": %.3f, \"fy\": %.3f, \"cx\": %.3f, \"cy\": %.3f, " + "\"model\": %u}", + e ? ", " : "", c.eye[e].width, c.eye[e].height, c.eye[e].fx, c.eye[e].fy, c.eye[e].cx, + c.eye[e].cy, c.eye[e].model); + } + printf("], \"rightFromLeft\": {\"orientation\": [%.6f, %.6f, %.6f, %.6f], \"position\": [%.6f, " + "%.6f, %.6f]}}\n", + c.orientation[0], c.orientation[1], c.orientation[2], c.orientation[3], c.position[0], + c.position[1], c.position[2]); + return 0; + } + printf("calibration (%s), baseline %.3f mm\n", output == XRT_STEREO_CAMERA_OUTPUT_RAW ? "RAW" : "RECTIFIED", + c.baseline_mm); + for (int e = 0; e < 2; e++) { + printf(" %s eye: %ux%u fx %.3f fy %.3f cx %.3f cy %.3f model %u\n", e ? "right" : "left ", + c.eye[e].width, c.eye[e].height, c.eye[e].fx, c.eye[e].fy, c.eye[e].cx, c.eye[e].cy, + c.eye[e].model); + } + printf(" right camera in left frame: q (%.5f %.5f %.5f %.5f) p (%.5f %.5f %.5f) m\n", c.orientation[0], + c.orientation[1], c.orientation[2], c.orientation[3], c.position[0], c.position[1], c.position[2]); + return 0; +} + + +/* + * + * probe. + * + */ + +struct ro_map +{ + const uint8_t *ptr; + uint64_t size; +#ifdef XRT_OS_WINDOWS + HANDLE section; +#else + int section; +#endif +}; + +static bool +map_readonly(struct ro_map *m, xrt_shmem_handle_t h, uint64_t size) +{ + m->size = size; + m->section = h; +#ifdef XRT_OS_WINDOWS + m->ptr = (const uint8_t *)MapViewOfFile(h, FILE_MAP_READ, 0, 0, (SIZE_T)size); + return m->ptr != NULL; +#else + void *p = mmap(NULL, (size_t)size, PROT_READ, MAP_SHARED, h, 0); + m->ptr = p == MAP_FAILED ? NULL : (const uint8_t *)p; + return m->ptr != NULL; +#endif +} + +static void +unmap_readonly(struct ro_map *m) +{ +#ifdef XRT_OS_WINDOWS + if (m->ptr != NULL) { + UnmapViewOfFile(m->ptr); + } + if (m->section != NULL) { + CloseHandle(m->section); + } +#else + if (m->ptr != NULL) { + munmap((void *)m->ptr, (size_t)m->size); + } + if (m->section >= 0) { + close(m->section); + } +#endif + memset(m, 0, sizeof(*m)); +} + +//! Wait for the stream's wake handle, then drain it. true = signalled. +static bool +wait_wake(xrt_graphics_sync_handle_t w, int timeout_ms) +{ +#ifdef XRT_OS_WINDOWS + return WaitForSingleObject(w, (DWORD)timeout_ms) == WAIT_OBJECT_0; +#else + struct pollfd pfd = {.fd = w, .events = POLLIN, .revents = 0}; + if (poll(&pfd, 1, timeout_ms) <= 0) { + return false; + } + char buf[64]; + while (read(w, buf, sizeof(buf)) > 0) { + } + return true; +#endif +} + +static void +close_wake(xrt_graphics_sync_handle_t w) +{ +#ifdef XRT_OS_WINDOWS + if (w != NULL) { + CloseHandle(w); + } +#else + if (w >= 0) { + close(w); + } +#endif +} + +static int +cmp_int(const void *a, const void *b) +{ + return *(const int *)a - *(const int *)b; +} + +//! Block-match the last frame on a 64x64 grid and print the two dominant +//! disparities (the sim fake: background 12 px, bar 40 px). +static void +report_disparity(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, char *out, size_t cap) +{ + int vals[512]; + int n = 0; + for (uint32_t y = 48; y + 64 <= h && n < 512; y += 64) { + for (uint32_t x = 64; x + 64 <= eye_w && n < 512; x += 64) { + float d = 0.0f; + if (u_stereo_camera_estimate_disparity(gray, pitch, eye_w, h, x, y, 64, 64, 64, &d)) { + vals[n++] = (int)(d + 0.5f); + } + } + } + if (n == 0) { + snprintf(out, cap, "n/a (frame too small)"); + return; + } + qsort(vals, (size_t)n, sizeof(int), cmp_int); + // Histogram modes. + int best_v[2] = {-1, -1}, best_c[2] = {0, 0}; + for (int i = 0; i < n;) { + int j = i; + while (j < n && vals[j] == vals[i]) { + j++; + } + int c = j - i; + if (c > best_c[0]) { + best_v[1] = best_v[0]; + best_c[1] = best_c[0]; + best_v[0] = vals[i]; + best_c[0] = c; + } else if (c > best_c[1]) { + best_v[1] = vals[i]; + best_c[1] = c; + } + i = j; + } + snprintf(out, cap, "%d px (%d/%d blocks), %d px (%d/%d blocks), range %d..%d px", best_v[0], best_c[0], n, + best_v[1], best_c[1], n, vals[0], vals[n - 1]); +} + +static int +cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) +{ + uint64_t id = 0; + uint32_t output = XRT_STEREO_CAMERA_OUTPUT_RECTIFIED; + uint32_t format = XRT_STEREO_CAMERA_FORMAT_NV12; + float fps = 0.0f; + int frames = 90; + float seconds = 0.0f; + const char *out_dir = NULL; + for (int i = 3; i < argc; i++) { + if (strcmp(argv[i], "--raw") == 0) { + output = XRT_STEREO_CAMERA_OUTPUT_RAW; + } else if (strcmp(argv[i], "--format") == 0 && i + 1 < argc) { + const char *f = argv[++i]; + format = strcmp(f, "gray8") == 0 ? XRT_STEREO_CAMERA_FORMAT_GRAY8 + : strcmp(f, "bgra8") == 0 ? XRT_STEREO_CAMERA_FORMAT_BGRA8 + : XRT_STEREO_CAMERA_FORMAT_NV12; + } else if (strcmp(argv[i], "--fps") == 0 && i + 1 < argc) { + fps = (float)atof(argv[++i]); + } else if (strcmp(argv[i], "--frames") == 0 && i + 1 < argc) { + frames = atoi(argv[++i]); + } else if (strcmp(argv[i], "--seconds") == 0 && i + 1 < argc) { + seconds = (float)atof(argv[++i]); + } else if (strcmp(argv[i], "--out") == 0 && i + 1 < argc) { + out_dir = argv[++i]; + } else if (argv[i][0] != '-') { + id = strtoull(argv[i], NULL, 10); + } + } + if (id == 0) { + struct xrt_stereo_camera_properties p; + uint32_t n = 0; + if (ipc_client_stereo_camera_count(ipc_c, &n) != XRT_SUCCESS || n == 0 || + ipc_client_stereo_camera_get_properties(ipc_c, 0, &p) != XRT_SUCCESS) { + printf("camera probe: the service exposes no stereo camera.\n"); + return 3; + } + id = p.camera_id; + } + + struct xrt_stereo_camera_stream_request req = { + .camera_id = id, + .output = output, + .format = format, + .transport = XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY, + .max_frame_rate = fps, + }; + uint64_t sid = 0; + xrt_result_t xret = ipc_client_stereo_camera_stream_create(ipc_c, &req, &sid); + if (xret != XRT_SUCCESS) { + printf("stream create failed: %d%s\n", (int)xret, result_hint(xret)); + return 2; + } + xret = ipc_client_stereo_camera_stream_start(ipc_c, sid); + if (xret != XRT_SUCCESS) { + printf("stream start failed: %d%s\n", (int)xret, result_hint(xret)); + ipc_client_stereo_camera_stream_destroy(ipc_c, sid); + return 2; + } + struct xrt_stereo_camera_stream_layout lay; + xrt_shmem_handle_t section; + xrt_graphics_sync_handle_t wake; + xret = ipc_client_stereo_camera_stream_get_transport(ipc_c, sid, &lay, §ion, &wake); + if (xret != XRT_SUCCESS) { + printf("stream transport failed: %d%s\n", (int)xret, result_hint(xret)); + ipc_client_stereo_camera_stream_destroy(ipc_c, sid); + return 2; + } + struct ro_map map; + memset(&map, 0, sizeof(map)); + if (!map_readonly(&map, section, lay.section_size)) { + printf("could not map the ring read-only\n"); + close_wake(wake); + ipc_client_stereo_camera_stream_destroy(ipc_c, sid); + return 2; + } + printf("stream %llu on camera %llu: %ux%u %s SBS (%s), cap %.1f Hz, ring %u x %llu B (section %llu B)\n", + (unsigned long long)sid, (unsigned long long)id, lay.width, lay.height, format_str(lay.format), + lay.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW (flagged)", lay.max_frame_rate, + lay.slot_count, (unsigned long long)lay.slot_stride, (unsigned long long)lay.section_size); + + int64_t t_start = os_monotonic_get_ns(); + int64_t deadline = seconds > 0.0f ? t_start + (int64_t)(seconds * 1e9) : t_start + 60ll * 1000000000; + int got = 0, wakes = 0, not_ready = 0; + uint64_t first_index = 0, last_index = 0, gaps = 0; + int64_t t_first = 0, t_last = 0; + struct xrt_stereo_camera_frame_info last; + memset(&last, 0, sizeof(last)); + while ((seconds > 0.0f || got < frames) && os_monotonic_get_ns() < deadline) { + if (!wait_wake(wake, 1000)) { + continue; + } + wakes++; + bool ready = false; + struct xrt_stereo_camera_frame_info fi; + xret = ipc_client_stereo_camera_acquire(ipc_c, sid, &ready, &fi); + if (xret != XRT_SUCCESS) { + printf("acquire failed: %d%s\n", (int)xret, result_hint(xret)); + break; + } + if (!ready) { + not_ready++; + continue; + } + int64_t now = os_monotonic_get_ns(); + if (got == 0) { + first_index = fi.frame_index; + t_first = now; + } else if (fi.frame_index > last_index + 1) { + gaps += fi.frame_index - last_index - 1; + } + last_index = fi.frame_index; + t_last = now; + last = fi; + got++; + } + double span = (double)(t_last - t_first) * 1e-9; + double rate = (got > 1 && span > 0.0) ? (got - 1) / span : 0.0; + printf("received %d frames (index %llu..%llu, %llu source frames not delivered to this stream), %d wakes, " + "%d not-ready\n", + got, (unsigned long long)first_index, (unsigned long long)last_index, (unsigned long long)gaps, wakes, + not_ready); + printf("measured delivery rate: %.2f Hz over %.2f s\n", rate, span); + + if (got > 0) { + const uint8_t *slot = map.ptr + (uint64_t)last.slot * lay.slot_stride; + printf("last frame: index %llu, slot %u, %ux%u %s, pitch %u/%u, planes @%llu/%llu, %s time, output %s, " + "calibration gen %u\n", + (unsigned long long)last.frame_index, last.slot, last.width, last.height, format_str(last.format), + last.row_pitch[0], last.row_pitch[1], (unsigned long long)last.plane_offset[0], + (unsigned long long)last.plane_offset[1], last.time_is_exposure ? "exposure" : "arrival", + last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", last.calibration_generation); + + // Luma of the last frame, for the disparity probe and a GRAY8 PNG. + struct u_stereo_camera_planes gl; + u_stereo_camera_layout(XRT_STEREO_CAMERA_FORMAT_GRAY8, last.width, last.height, &gl); + uint8_t *gray = calloc((size_t)gl.size, 1); + const uint8_t *src[2] = {slot + last.plane_offset[0], slot + last.plane_offset[1]}; + if (gray != NULL && u_stereo_camera_convert(last.format, src, last.row_pitch, XRT_STEREO_CAMERA_FORMAT_GRAY8, + gray, &gl, last.width, last.height)) { + char disp[160]; + report_disparity(gray, gl.pitch[0], last.width / 2, last.height, disp, sizeof(disp)); + printf("block disparity (left x - right x): %s\n", disp); + } + if (out_dir != NULL) { + char path[1024]; + snprintf(path, sizeof(path), "%s/cam_%llu.png", out_dir, (unsigned long long)last.frame_index); + int ok = 0; + if (last.format == XRT_STEREO_CAMERA_FORMAT_GRAY8 && gray != NULL) { + ok = stbi_write_png(path, (int)last.width, (int)last.height, 1, gray, (int)gl.pitch[0]); + } else { + struct u_stereo_camera_planes bl; + u_stereo_camera_layout(XRT_STEREO_CAMERA_FORMAT_BGRA8, last.width, last.height, &bl); + uint8_t *rgba = calloc((size_t)bl.size, 1); + if (rgba != NULL && u_stereo_camera_convert(last.format, src, last.row_pitch, + XRT_STEREO_CAMERA_FORMAT_BGRA8, rgba, &bl, + last.width, last.height)) { + for (uint32_t y = 0; y < last.height; y++) { + uint8_t *r = rgba + (size_t)y * bl.pitch[0]; + for (uint32_t x = 0; x < last.width; x++) { + uint8_t t = r[4 * x]; + r[4 * x] = r[4 * x + 2]; + r[4 * x + 2] = t; + } + } + ok = stbi_write_png(path, (int)last.width, (int)last.height, 4, rgba, (int)bl.pitch[0]); + } + free(rgba); + } + printf("%s %s\n", ok ? "wrote" : "FAILED to write", path); + } + free(gray); + } + + struct xrt_stereo_camera_stream_stats st; + if (ipc_client_stereo_camera_stats(ipc_c, sid, &st) == XRT_SUCCESS) { + printf("service stats: source %.2f Hz, delivered %.2f Hz, published %llu, skipped %llu, acquired %llu, " + "mean latency %.3f ms\n", + st.source_frame_rate, st.delivered_frame_rate, (unsigned long long)st.frames_published, + (unsigned long long)st.frames_skipped, (unsigned long long)st.frames_acquired, + (double)st.mean_latency_ns * 1e-6); + } + + unmap_readonly(&map); + close_wake(wake); + ipc_client_stereo_camera_stream_stop(ipc_c, sid); + ipc_client_stereo_camera_stream_destroy(ipc_c, sid); + return got > 0 ? 0 : 4; +} + +int +cli_cmd_camera(int argc, const char **argv) +{ + const char *sub = argc >= 3 ? argv[2] : "list"; + bool json = cli_has_flag(argc, argv, "--json"); + struct ipc_connection ipc_c = {0}; + if (!cam_connect(&ipc_c)) { + return 2; + } + int ret; + if (strcmp(sub, "list") == 0) { + ret = cmd_list(&ipc_c, json); + } else if (strcmp(sub, "calib") == 0 && argc >= 4) { + uint32_t output = cli_has_flag(argc, argv, "--rectified") ? XRT_STEREO_CAMERA_OUTPUT_RECTIFIED + : XRT_STEREO_CAMERA_OUTPUT_RAW; + ret = cmd_calib(&ipc_c, strtoull(argv[3], NULL, 10), output, json); + } else if (strcmp(sub, "probe") == 0) { + ret = cmd_probe(&ipc_c, argc, argv); + } else { + printf("usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" + " probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] [--frames N] [--seconds S] " + "[--out DIR]\n"); + ret = 1; + } + ipc_client_connection_fini(&ipc_c); + return ret; +} + +#else // !CLI_HAVE_IPC + +#include + +int +cli_cmd_camera(int argc, const char **argv) +{ + (void)argc; + (void)argv; + printf("displayxr-cli camera: this build has no IPC client (XRT_FEATURE_SERVICE off).\n"); + return 1; +} + +#endif diff --git a/src/xrt/targets/cli/cli_common.h b/src/xrt/targets/cli/cli_common.h index 9d32e5ab8f..9c6ec7d4b9 100644 --- a/src/xrt/targets/cli/cli_common.h +++ b/src/xrt/targets/cli/cli_common.h @@ -52,6 +52,10 @@ cli_cmd_test(int argc, const char **argv); int cli_cmd_lift(int argc, const char **argv); +//! `camera` — XR_DXR_stereo_camera (ADR-043): `camera list|calib|probe`, over IPC (DIAG). +int +cli_cmd_camera(int argc, const char **argv); + /*! * True if @p flag appears anywhere in argv[2..]. Commands take raw * argc/argv; this is the one shared option-scan (e.g. `--json`). diff --git a/src/xrt/targets/cli/cli_main.c b/src/xrt/targets/cli/cli_main.c index eb017a351a..34412613a7 100644 --- a/src/xrt/targets/cli/cli_main.c +++ b/src/xrt/targets/cli/cli_main.c @@ -51,6 +51,10 @@ cli_print_help(int argc, const char **argv) P(" lift <...> - 2D->3D conversion module (XR_DXR_lift, ADR-042), over IPC (DIAG).\n"); P(" 'lift caps [--json]', 'lift probe [--mode depth|sbs|\n"); P(" nview|gaussians] [--n N]' (Windows) — writes lift_out_.png/.ply.\n"); + P(" camera <...> - Stereo camera sources (XR_DXR_stereo_camera, ADR-043), over IPC (DIAG).\n"); + P(" 'camera list [--json]', 'camera calib [--raw|--rectified]',\n"); + P(" 'camera probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F]\n"); + P(" [--frames N] [--seconds S] [--out DIR]' — rate, layout, disparity, PNG.\n"); P(" test - List found devices and role assignments, for prober testing.\n"); P(" probe - Just probe and then exit.\n"); @@ -106,5 +110,8 @@ main(int argc, const char **argv) if (strcmp(argv[1], "lift") == 0) { return cli_cmd_lift(argc, argv); } + if (strcmp(argv[1], "camera") == 0) { + return cli_cmd_camera(argc, argv); + } return cli_print_help(argc, argv); } From fe9dd3e32abf3e360a1df4e2db848330e06649cc Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:47:53 -0700 Subject: [PATCH 07/37] feat(stereo-camera): XR_DXR_stereo_camera OpenXR entry points (instance-level, service-only) Nine entry points over the instance's xrt_stereo_camera_client aspect. An in-process instance enumerates zero (maintainer decision: service clients only in R1). Consent refusal maps to XR_ERROR_PERMISSION_INSUFFICIENT (retryable), a dead pipe to XR_ERROR_INSTANCE_LOST; capture times are converted to XrTime; handles returned through XrStereoCameraStreamTransportDXR are the caller's (closed here when nothing is chained to receive them). Co-Authored-By: Claude Opus 5.5 --- src/xrt/state_trackers/oxr/CMakeLists.txt | 1 + src/xrt/state_trackers/oxr/oxr_api_funcs.h | 46 ++ .../state_trackers/oxr/oxr_api_negotiate.c | 12 + src/xrt/state_trackers/oxr/oxr_api_verify.h | 3 + src/xrt/state_trackers/oxr/oxr_defines.h | 2 + .../oxr/oxr_extension_support.h | 19 + src/xrt/state_trackers/oxr/oxr_objects.h | 22 + .../state_trackers/oxr/oxr_stereo_camera.c | 451 ++++++++++++++++++ 8 files changed, 556 insertions(+) create mode 100644 src/xrt/state_trackers/oxr/oxr_stereo_camera.c diff --git a/src/xrt/state_trackers/oxr/CMakeLists.txt b/src/xrt/state_trackers/oxr/CMakeLists.txt index a141f7efa5..6c21fc979e 100644 --- a/src/xrt/state_trackers/oxr/CMakeLists.txt +++ b/src/xrt/state_trackers/oxr/CMakeLists.txt @@ -61,6 +61,7 @@ add_library( oxr_views_change.h oxr_weave.c oxr_lift.c + oxr_stereo_camera.c oxr_workspace.c oxr_workspace_modal_win32.c oxr_workspace_file_dialog.c diff --git a/src/xrt/state_trackers/oxr/oxr_api_funcs.h b/src/xrt/state_trackers/oxr/oxr_api_funcs.h index 7774d11efa..fc48e55d10 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_funcs.h +++ b/src/xrt/state_trackers/oxr/oxr_api_funcs.h @@ -1112,6 +1112,52 @@ XRAPI_ATTR XrResult XRAPI_CALL oxr_xrGetLiftStreamStatsDXR(XrLiftStreamDXR stream, XrLiftStreamStatsDXR *stats); #endif +#ifdef OXR_HAVE_DXR_stereo_camera +//! OpenXR API function @ep{xrEnumerateStereoCamerasDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrEnumerateStereoCamerasDXR(XrInstance instance, + XrSystemId systemId, + uint32_t capacityInput, + uint32_t *countOutput, + XrStereoCameraPropertiesDXR *cameras); + +//! OpenXR API function @ep{xrGetStereoCameraCalibrationDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraCalibrationDXR(XrInstance instance, + uint64_t cameraId, + XrStereoCameraCalibrationDXR *calibration); + +//! OpenXR API function @ep{xrCreateStereoCameraStreamDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrCreateStereoCameraStreamDXR(XrInstance instance, + const XrStereoCameraStreamCreateInfoDXR *createInfo, + XrStereoCameraStreamDXR *stream); + +//! OpenXR API function @ep{xrDestroyStereoCameraStreamDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrDestroyStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +//! OpenXR API function @ep{xrStartStereoCameraStreamDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrStartStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +//! OpenXR API function @ep{xrStopStereoCameraStreamDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrStopStereoCameraStreamDXR(XrStereoCameraStreamDXR stream); + +//! OpenXR API function @ep{xrGetStereoCameraStreamInfoDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraStreamInfoDXR(XrStereoCameraStreamDXR stream, XrStereoCameraStreamInfoDXR *info); + +//! OpenXR API function @ep{xrAcquireStereoCameraFrameDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrAcquireStereoCameraFrameDXR(XrStereoCameraStreamDXR stream, XrStereoCameraFrameDXR *frame); + +//! OpenXR API function @ep{xrGetStereoCameraStreamStatsDXR} +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraStreamStatsDXR(XrStereoCameraStreamDXR stream, XrStereoCameraStreamStatsDXR *stats); +#endif + #ifdef OXR_HAVE_EXT_conformance_automation //! OpenXR API function @ep{xrSetInputDeviceActiveEXT} XRAPI_ATTR XrResult XRAPI_CALL diff --git a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c index 0a8fe96344..9481242809 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_negotiate.c +++ b/src/xrt/state_trackers/oxr/oxr_api_negotiate.c @@ -527,6 +527,18 @@ handle_non_null(struct oxr_instance *inst, struct oxr_logger *log, const char *n ENTRY_IF_EXT(xrGetLiftStreamStatsDXR, DXR_lift); #endif +#ifdef OXR_HAVE_DXR_stereo_camera + ENTRY_IF_EXT(xrEnumerateStereoCamerasDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrGetStereoCameraCalibrationDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrCreateStereoCameraStreamDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrDestroyStereoCameraStreamDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrStartStereoCameraStreamDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrStopStereoCameraStreamDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrGetStereoCameraStreamInfoDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrAcquireStereoCameraFrameDXR, DXR_stereo_camera); + ENTRY_IF_EXT(xrGetStereoCameraStreamStatsDXR, DXR_stereo_camera); +#endif + #ifdef OXR_HAVE_DXR_workspace_file_dialog ENTRY_IF_EXT(xrRequestFilePickerDXR, DXR_workspace_file_dialog); ENTRY_IF_EXT(xrGetFilePickerRequestDXR, DXR_workspace_file_dialog); diff --git a/src/xrt/state_trackers/oxr/oxr_api_verify.h b/src/xrt/state_trackers/oxr/oxr_api_verify.h index 8ea5037cbb..f27222b22e 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_verify.h +++ b/src/xrt/state_trackers/oxr/oxr_api_verify.h @@ -96,6 +96,9 @@ struct oxr_subaction_paths; OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_local_3d_zone_ext, LOCAL3DZONE, name, new_thing->sess->sys->inst) #define OXR_VERIFY_LIFT_STREAM_AND_INIT_LOG(log, thing, new_thing, name) \ OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_lift_stream_dxr, LIFTSTREAM, name, new_thing->sess->sys->inst) + +#define OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(log, thing, new_thing, name) \ + OXR_VERIFY_AND_SET_AND_INIT(log, thing, new_thing, oxr_stereo_camera_stream_dxr, STEREOCAMSTREAM, name, new_thing->inst) // clang-format on #define OXR_VERIFY_INSTANCE_NOT_NULL(log, arg, new_arg) OXR_VERIFY_SET(log, arg, new_arg, oxr_instance, INSTANCE); diff --git a/src/xrt/state_trackers/oxr/oxr_defines.h b/src/xrt/state_trackers/oxr/oxr_defines.h index dbb77e62df..25b8fe2af9 100644 --- a/src/xrt/state_trackers/oxr/oxr_defines.h +++ b/src/xrt/state_trackers/oxr/oxr_defines.h @@ -38,6 +38,8 @@ #define OXR_XR_DEBUG_LOCAL3DZONE (*(uint64_t *)"oxrl3dz\0") // XR_DXR_lift stream (ADR-042) #define OXR_XR_DEBUG_LIFTSTREAM (*(uint64_t *)"oxrlift\0") +// XR_DXR_stereo_camera stream (ADR-043) +#define OXR_XR_DEBUG_STEREOCAMSTREAM (*(uint64_t *)"oxrscam\0") // clang-format on /*! diff --git a/src/xrt/state_trackers/oxr/oxr_extension_support.h b/src/xrt/state_trackers/oxr/oxr_extension_support.h index 3574034b8f..ad72ecc781 100644 --- a/src/xrt/state_trackers/oxr/oxr_extension_support.h +++ b/src/xrt/state_trackers/oxr/oxr_extension_support.h @@ -768,6 +768,24 @@ #define OXR_EXTENSION_SUPPORT_DXR_lift(_) #endif +/* + * XR_DXR_stereo_camera + * + * Hand-added DisplayXR extension (generate_oxr_ext_support.py knows nothing of + * the DisplayXR blocks — keep them when regenerating). A display's stereo + * camera as a plug-in-provided, service-owned source (ADR-043). Advertised on + * every platform: the camera manager is platform-neutral service code, and an + * instance with no camera (in-process, no plug-in source, kill switch) simply + * enumerates zero. + */ +#if defined(XR_DXR_stereo_camera) +#define OXR_HAVE_DXR_stereo_camera +#define OXR_EXTENSION_SUPPORT_DXR_stereo_camera(_) \ + _(DXR_stereo_camera, DXR_STEREO_CAMERA) +#else +#define OXR_EXTENSION_SUPPORT_DXR_stereo_camera(_) +#endif + /* * XR_DXR_workspace_file_dialog @@ -1282,6 +1300,7 @@ OXR_EXTENSION_SUPPORT_DXR_display_zones(_) \ OXR_EXTENSION_SUPPORT_DXR_weave(_) \ OXR_EXTENSION_SUPPORT_DXR_lift(_) \ + OXR_EXTENSION_SUPPORT_DXR_stereo_camera(_) \ OXR_EXTENSION_SUPPORT_DXR_workspace_file_dialog(_) \ OXR_EXTENSION_SUPPORT_DXR_mcp_tools(_) \ OXR_EXTENSION_SUPPORT_BD_controller_interaction(_) \ diff --git a/src/xrt/state_trackers/oxr/oxr_objects.h b/src/xrt/state_trackers/oxr/oxr_objects.h index 745f1b71b1..7753ab179b 100644 --- a/src/xrt/state_trackers/oxr/oxr_objects.h +++ b/src/xrt/state_trackers/oxr/oxr_objects.h @@ -135,6 +135,7 @@ struct oxr_xdev_list; struct oxr_plane_detector_ext; struct oxr_local_3d_zone_ext; struct oxr_lift_stream_dxr; +struct oxr_stereo_camera_stream_dxr; #define XRT_MAX_HANDLE_CHILDREN 256 #define OXR_MAX_BINDINGS_PER_ACTION 32 @@ -4355,6 +4356,27 @@ struct oxr_lift_stream_dxr }; #endif // OXR_HAVE_DXR_lift +#ifdef OXR_HAVE_DXR_stereo_camera +/*! + * One XR_DXR_stereo_camera stream (ADR-043). The camera and its ring live in + * the service (ipc_server_stereo_camera.c); this is the handle and the + * service-side stream id, owned by the instance's IPC connection. + * + * Parent type/handle is @ref oxr_instance + * + * @obj{XrStereoCameraStreamDXR} + * @extends oxr_handle_base + */ +struct oxr_stereo_camera_stream_dxr +{ + struct oxr_handle_base handle; + struct oxr_instance *inst; + uint64_t id; //!< service-side stream id + uint64_t camera_id; //!< service-side camera id + bool started; +}; +#endif // OXR_HAVE_DXR_stereo_camera + #ifdef OXR_HAVE_EXT_user_presence XrResult diff --git a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c new file mode 100644 index 0000000000..1be6693a96 --- /dev/null +++ b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c @@ -0,0 +1,451 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera API entry points (ADR-043). + * @author David Fattal + * @ingroup oxr_api + * + * Instance-level: a capture component has no session. Every call goes through + * the instance's @ref xrt_stereo_camera_client aspect, which only the IPC + * client instance carries — an in-process instance enumerates zero cameras + * (the service is each camera's single owner; maintainer decision for R1, + * roadmap §G.3). + * + * Error contract: + * - dead pipe (XRT_ERROR_IPC_FAILURE) -> XR_ERROR_INSTANCE_LOST + * - refused by consent (XRT_ERROR_NOT_AUTHORIZED) -> XR_ERROR_PERMISSION_INSUFFICIENT + * (retryable: the stream stays created) + * - no such camera / bad format / output -> XR_ERROR_VALIDATION_FAILURE + * - transport or output the runtime lacks -> XR_ERROR_FEATURE_UNSUPPORTED + * - 8 streams on the camera already -> XR_ERROR_LIMIT_REACHED + * - anything else on a healthy pipe -> XR_ERROR_RUNTIME_FAILURE + */ + +#include "oxr_objects.h" +#include "oxr_logger.h" +#include "oxr_handle.h" +#include "oxr_api_funcs.h" +#include "oxr_api_verify.h" +#include "oxr_chain.h" + +#include "util/u_misc.h" +#include "util/u_time.h" +#include "util/u_trace_marker.h" + +#include "xrt/xrt_instance.h" +#include "xrt/xrt_plugin.h" +#include "xrt/xrt_stereo_camera.h" + +#include "xrt/xrt_config_os.h" + +#include +#if defined(XRT_OS_WINDOWS) +#include +#else +#include +#endif + +#ifdef OXR_HAVE_DXR_stereo_camera + +// The wire carries the xrt encodings; they equal the XR ones. +_Static_assert(XR_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING, + "flag mismatch"); +_Static_assert(XR_STEREO_CAMERA_USER_FACING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_USER_FACING, "flag mismatch"); +_Static_assert(XR_STEREO_CAMERA_CALIBRATED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_CALIBRATED, "flag mismatch"); +_Static_assert(XR_STEREO_CAMERA_NATIVELY_RECTIFIED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED, + "flag mismatch"); +_Static_assert(XR_STEREO_CAMERA_MONOCHROME_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_MONOCHROME, "flag mismatch"); +_Static_assert((int)XR_STEREO_CAMERA_STATE_AVAILABLE_DXR == (int)XRT_STEREO_CAMERA_STATE_AVAILABLE, "state"); +_Static_assert((int)XR_STEREO_CAMERA_STATE_WAITING_DXR == (int)XRT_STEREO_CAMERA_STATE_WAITING, "state"); +_Static_assert((int)XR_STEREO_CAMERA_STATE_SUSPENDED_DXR == (int)XRT_STEREO_CAMERA_STATE_SUSPENDED, "state"); +_Static_assert((int)XR_STEREO_CAMERA_STATE_UNAVAILABLE_DXR == (int)XRT_STEREO_CAMERA_STATE_UNAVAILABLE, "state"); +_Static_assert((int)XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR == (int)XRT_STEREO_CAMERA_OUTPUT_RECTIFIED, "output"); +_Static_assert((int)XR_STEREO_CAMERA_OUTPUT_RAW_DXR == (int)XRT_STEREO_CAMERA_OUTPUT_RAW, "output"); +_Static_assert((int)XR_STEREO_CAMERA_FORMAT_GRAY8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_GRAY8, "format"); +_Static_assert((int)XR_STEREO_CAMERA_FORMAT_NV12_DXR == (int)XRT_STEREO_CAMERA_FORMAT_NV12, "format"); +_Static_assert((int)XR_STEREO_CAMERA_FORMAT_BGRA8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_BGRA8, "format"); +_Static_assert(XR_STEREO_CAMERA_FORMAT_NV12_BIT_DXR == XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_NV12), + "format bit"); +_Static_assert(XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_BIT_DXR == + XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY), + "transport bit"); +_Static_assert((int)XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR == (int)XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY, + "transport"); +_Static_assert(XR_STEREO_CAMERA_RING_SLOTS_DXR == XRT_STEREO_CAMERA_RING_SLOTS, "ring slots"); +_Static_assert(XR_STEREO_CAMERA_PERSISTENT_ID_MAX_SIZE_DXR == + sizeof(((struct xrt_stereo_camera_properties *)0)->persistent_id), + "id size"); +_Static_assert(XR_STEREO_CAMERA_DISPLAY_NAME_MAX_SIZE_DXR == + sizeof(((struct xrt_stereo_camera_properties *)0)->display_name), + "name size"); +_Static_assert(XR_STEREO_CAMERA_PLATFORM_HINT_MAX_SIZE_DXR == + sizeof(((struct xrt_stereo_camera_properties *)0)->platform_device_hint), + "hint size"); + +static struct xrt_stereo_camera_client * +client_of(struct oxr_instance *inst) +{ + return inst->xinst != NULL ? inst->xinst->stereo_camera : NULL; +} + +static XrResult +scam_xret(struct oxr_logger *log, xrt_result_t xret, const char *what) +{ + switch (xret) { + case XRT_SUCCESS: return XR_SUCCESS; + case XRT_ERROR_IPC_FAILURE: + return oxr_error(log, XR_ERROR_INSTANCE_LOST, "%s: the runtime service connection is gone", what); + case XRT_ERROR_NOT_AUTHORIZED: + return oxr_error(log, XR_ERROR_PERMISSION_INSUFFICIENT, + "%s: the runtime refused camera access (consent; see XR_DXR_stereo_camera §7)", what); + case XRT_ERROR_INPUT_UNSUPPORTED: + return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "%s: unknown camera/stream or unsupported request", + what); + case XRT_ERROR_FEATURE_NOT_SUPPORTED: + return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "%s: not supported by this runtime / camera", what); + case XRT_ERROR_CLIENT_LIMIT_REACHED: + return oxr_error(log, XR_ERROR_LIMIT_REACHED, "%s: too many streams on this camera", what); + default: return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, "%s: service refused (xrt_result=%d)", what, (int)xret); + } +} + +static XrResult +stream_destroy_cb(struct oxr_logger *log, struct oxr_handle_base *hb) +{ + struct oxr_stereo_camera_stream_dxr *s = (struct oxr_stereo_camera_stream_dxr *)hb; + struct xrt_stereo_camera_client *c = client_of(s->inst); + if (c != NULL && s->id != 0) { + // Best effort: a dead connection already released every stream. + (void)c->stream_destroy(c, s->id); + } + free(s); + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrEnumerateStereoCamerasDXR(XrInstance instance, + XrSystemId systemId, + uint32_t capacityInput, + uint32_t *countOutput, + XrStereoCameraPropertiesDXR *cameras) +{ + OXR_TRACE_MARKER(); + struct oxr_instance *inst = NULL; + struct oxr_logger log; + OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrEnumerateStereoCamerasDXR"); + OXR_VERIFY_EXTENSION(&log, inst, DXR_stereo_camera); + OXR_VERIFY_ARG_NOT_NULL(&log, countOutput); + XrResult sr = oxr_system_verify_id(&log, inst, systemId); + if (sr != XR_SUCCESS) { + return sr; + } + if (capacityInput > 0 && cameras == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "(cameras == NULL) with capacityInput > 0"); + } + + *countOutput = 0; + struct xrt_stereo_camera_client *c = client_of(inst); + if (c == NULL) { + return XR_SUCCESS; // in-process instance: zero cameras, on purpose + } + uint32_t n = 0; + xrt_result_t xret = c->count(c, &n); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrEnumerateStereoCamerasDXR"); + } + *countOutput = n; + if (capacityInput == 0) { + return XR_SUCCESS; + } + if (capacityInput < n) { + return oxr_error(&log, XR_ERROR_SIZE_INSUFFICIENT, "(capacityInput == %u) < %u cameras", capacityInput, n); + } + for (uint32_t i = 0; i < n; i++) { + if (cameras[i].type != XR_TYPE_STEREO_CAMERA_PROPERTIES_DXR) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "cameras[%u].type", i); + } + } + for (uint32_t i = 0; i < n; i++) { + struct xrt_stereo_camera_properties p; + xret = c->get_properties(c, i, &p); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrEnumerateStereoCamerasDXR"); + } + XrStereoCameraPropertiesDXR *o = &cameras[i]; + o->cameraId = p.camera_id; + memcpy(o->persistentId, p.persistent_id, sizeof(o->persistentId)); + memcpy(o->displayName, p.display_name, sizeof(o->displayName)); + memcpy(o->platformDeviceHint, p.platform_device_hint, sizeof(o->platformDeviceHint)); + o->flags = p.flags; + o->state = (XrStereoCameraStateDXR)p.state; + o->viewCount = p.view_count; + o->eyeExtent.width = (int32_t)p.eye_width; + o->eyeExtent.height = (int32_t)p.eye_height; + o->maxFrameRate = p.max_frame_rate; + o->baselineMm = p.baseline_mm; + o->horizontalFovDeg = p.horizontal_fov_deg; + o->supportedFormats = p.supported_formats; + o->supportedTransports = p.supported_transports; + } + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraCalibrationDXR(XrInstance instance, uint64_t cameraId, XrStereoCameraCalibrationDXR *calibration) +{ + OXR_TRACE_MARKER(); + struct oxr_instance *inst = NULL; + struct oxr_logger log; + OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrGetStereoCameraCalibrationDXR"); + OXR_VERIFY_EXTENSION(&log, inst, DXR_stereo_camera); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, calibration, XR_TYPE_STEREO_CAMERA_CALIBRATION_DXR); + if (calibration->output != XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR && + calibration->output != XR_STEREO_CAMERA_OUTPUT_RAW_DXR) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "calibration->output"); + } + struct xrt_stereo_camera_client *c = client_of(inst); + if (c == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "no stereo camera %llu (in-process instance)", + (unsigned long long)cameraId); + } + struct xrt_stereo_camera_calibration k; + xrt_result_t xret = c->get_calibration(c, cameraId, (uint32_t)calibration->output, &k); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrGetStereoCameraCalibrationDXR"); + } + for (int e = 0; e < 2; e++) { + XrStereoCameraIntrinsicsDXR *o = &calibration->eye[e]; + o->imageExtent.width = (int32_t)k.eye[e].width; + o->imageExtent.height = (int32_t)k.eye[e].height; + o->fx = k.eye[e].fx; + o->fy = k.eye[e].fy; + o->cx = k.eye[e].cx; + o->cy = k.eye[e].cy; + o->model = (XrStereoCameraDistortionModelDXR)k.eye[e].model; + memcpy(o->coefficients, k.eye[e].coefficients, sizeof(o->coefficients)); + } + calibration->rightFromLeft.orientation.x = k.orientation[0]; + calibration->rightFromLeft.orientation.y = k.orientation[1]; + calibration->rightFromLeft.orientation.z = k.orientation[2]; + calibration->rightFromLeft.orientation.w = k.orientation[3]; + calibration->rightFromLeft.position.x = k.position[0]; + calibration->rightFromLeft.position.y = k.position[1]; + calibration->rightFromLeft.position.z = k.position[2]; + calibration->baselineMm = k.baseline_mm; + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrCreateStereoCameraStreamDXR(XrInstance instance, + const XrStereoCameraStreamCreateInfoDXR *createInfo, + XrStereoCameraStreamDXR *stream) +{ + OXR_TRACE_MARKER(); + struct oxr_instance *inst = NULL; + struct oxr_logger log; + OXR_VERIFY_INSTANCE_AND_INIT_LOG(&log, instance, inst, "xrCreateStereoCameraStreamDXR"); + OXR_VERIFY_EXTENSION(&log, inst, DXR_stereo_camera); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, createInfo, XR_TYPE_STEREO_CAMERA_STREAM_CREATE_INFO_DXR); + OXR_VERIFY_ARG_NOT_NULL(&log, stream); + + if (createInfo->format < XR_STEREO_CAMERA_FORMAT_GRAY8_DXR || + createInfo->format > XR_STEREO_CAMERA_FORMAT_BGRA8_DXR) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "createInfo->format"); + } + if (createInfo->output != XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR && + createInfo->output != XR_STEREO_CAMERA_OUTPUT_RAW_DXR) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "createInfo->output"); + } + if (createInfo->transport != XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR) { + return oxr_error(&log, XR_ERROR_FEATURE_UNSUPPORTED, + "createInfo->transport: only SHARED_MEMORY is implemented (spec v1)"); + } + struct xrt_stereo_camera_client *c = client_of(inst); + if (c == NULL) { + return oxr_error(&log, XR_ERROR_VALIDATION_FAILURE, "no stereo camera %llu (in-process instance)", + (unsigned long long)createInfo->cameraId); + } + + struct xrt_stereo_camera_stream_request req = { + .camera_id = createInfo->cameraId, + .output = (uint32_t)createInfo->output, + .format = (uint32_t)createInfo->format, + .transport = (uint32_t)createInfo->transport, + .max_frame_rate = createInfo->maxFrameRate, + }; + uint64_t id = 0; + xrt_result_t xret = c->stream_create(c, &req, &id); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrCreateStereoCameraStreamDXR"); + } + + struct oxr_stereo_camera_stream_dxr *s = NULL; + XrResult ret = OXR_ALLOCATE_HANDLE(&log, s, OXR_XR_DEBUG_STEREOCAMSTREAM, stream_destroy_cb, &inst->handle); + if (ret != XR_SUCCESS) { + (void)c->stream_destroy(c, id); + return ret; + } + s->inst = inst; + s->id = id; + s->camera_id = createInfo->cameraId; + *stream = XRT_CAST_PTR_TO_OXR_HANDLE(XrStereoCameraStreamDXR, s); + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrDestroyStereoCameraStreamDXR(XrStereoCameraStreamDXR stream) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrDestroyStereoCameraStreamDXR"); + return oxr_handle_destroy(&log, &s->handle); +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrStartStereoCameraStreamDXR(XrStereoCameraStreamDXR stream) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrStartStereoCameraStreamDXR"); + struct xrt_stereo_camera_client *c = client_of(s->inst); + xrt_result_t xret = c->stream_start(c, s->id); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrStartStereoCameraStreamDXR"); + } + s->started = true; + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrStopStereoCameraStreamDXR(XrStereoCameraStreamDXR stream) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrStopStereoCameraStreamDXR"); + struct xrt_stereo_camera_client *c = client_of(s->inst); + xrt_result_t xret = c->stream_stop(c, s->id); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrStopStereoCameraStreamDXR"); + } + s->started = false; + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraStreamInfoDXR(XrStereoCameraStreamDXR stream, XrStereoCameraStreamInfoDXR *info) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrGetStereoCameraStreamInfoDXR"); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, info, XR_TYPE_STEREO_CAMERA_STREAM_INFO_DXR); + if (!s->started) { + return oxr_error(&log, XR_ERROR_CALL_ORDER_INVALID, "the stream has not been started"); + } + struct xrt_stereo_camera_client *c = client_of(s->inst); + struct xrt_stereo_camera_stream_layout lay; + xrt_shmem_handle_t section = XRT_SHMEM_HANDLE_INVALID; + xrt_graphics_sync_handle_t wake = XRT_GRAPHICS_SYNC_HANDLE_INVALID; + xrt_result_t xret = c->stream_get_transport(c, s->id, &lay, §ion, &wake); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrGetStereoCameraStreamInfoDXR"); + } + info->extent.width = (int32_t)lay.width; + info->extent.height = (int32_t)lay.height; + info->format = (XrStereoCameraFormatDXR)lay.format; + info->output = (XrStereoCameraOutputDXR)lay.output; + info->transport = (XrStereoCameraTransportDXR)lay.transport; + info->maxFrameRate = lay.max_frame_rate; + + XrStereoCameraStreamTransportDXR *t = OXR_GET_OUTPUT_FROM_CHAIN(info, XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR, + XrStereoCameraStreamTransportDXR); + if (t != NULL) { + // NEW handles, now the caller's to close. + t->sectionHandle = (uint64_t)(uintptr_t)section; + t->sectionSize = lay.section_size; + t->slotCount = lay.slot_count; + t->slotStride = lay.slot_stride; + t->wakeHandle = (uint64_t)(uintptr_t)wake; + } else { + // Nobody to hand them to. +#if defined(XRT_OS_WINDOWS) + if (section != NULL) { + CloseHandle(section); + } + if (wake != NULL) { + CloseHandle(wake); + } +#else + if (section >= 0) { + close(section); + } + if (wake >= 0) { + close(wake); + } +#endif + } + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrAcquireStereoCameraFrameDXR(XrStereoCameraStreamDXR stream, XrStereoCameraFrameDXR *frame) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrAcquireStereoCameraFrameDXR"); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, frame, XR_TYPE_STEREO_CAMERA_FRAME_DXR); + struct xrt_stereo_camera_client *c = client_of(s->inst); + bool ready = false; + struct xrt_stereo_camera_frame_info fi; + xrt_result_t xret = c->acquire(c, s->id, &ready, &fi); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrAcquireStereoCameraFrameDXR"); + } + if (!ready) { + return XR_STEREO_CAMERA_FRAME_NOT_READY_DXR; + } + frame->frameIndex = fi.frame_index; + frame->slot = fi.slot; + frame->extent.width = (int32_t)fi.width; + frame->extent.height = (int32_t)fi.height; + frame->format = (XrStereoCameraFormatDXR)fi.format; + frame->rowPitch[0] = fi.row_pitch[0]; + frame->rowPitch[1] = fi.row_pitch[1]; + frame->planeOffset[0] = fi.plane_offset[0]; + frame->planeOffset[1] = fi.plane_offset[1]; + frame->captureTime = time_state_monotonic_to_ts_ns(s->inst->timekeeping, fi.capture_time_ns); + frame->captureTimeIsExposure = fi.time_is_exposure ? XR_TRUE : XR_FALSE; + frame->output = (XrStereoCameraOutputDXR)fi.output; + frame->calibrationGeneration = fi.calibration_generation; + return XR_SUCCESS; +} + +XRAPI_ATTR XrResult XRAPI_CALL +oxr_xrGetStereoCameraStreamStatsDXR(XrStereoCameraStreamDXR stream, XrStereoCameraStreamStatsDXR *stats) +{ + OXR_TRACE_MARKER(); + struct oxr_stereo_camera_stream_dxr *s = NULL; + struct oxr_logger log; + OXR_VERIFY_STEREO_CAMERA_STREAM_AND_INIT_LOG(&log, stream, s, "xrGetStereoCameraStreamStatsDXR"); + OXR_VERIFY_ARG_TYPE_AND_NOT_NULL(&log, stats, XR_TYPE_STEREO_CAMERA_STREAM_STATS_DXR); + struct xrt_stereo_camera_client *c = client_of(s->inst); + struct xrt_stereo_camera_stream_stats st; + xrt_result_t xret = c->stats(c, s->id, &st); + if (xret != XRT_SUCCESS) { + return scam_xret(&log, xret, "xrGetStereoCameraStreamStatsDXR"); + } + stats->sourceFrameRate = st.source_frame_rate; + stats->deliveredFrameRate = st.delivered_frame_rate; + stats->framesPublished = st.frames_published; + stats->framesSkipped = st.frames_skipped; + stats->framesAcquired = st.frames_acquired; + stats->meanLatencyNs = st.mean_latency_ns; + return XR_SUCCESS; +} + +#endif // OXR_HAVE_DXR_stereo_camera From 62752159b16167e62955750c353f744e853e70da Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:49:15 -0700 Subject: [PATCH 08/37] feat(stereo-camera): selftest stereo_camera_caps check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enumerates the active plug-in's cameras through the iface and validates each description (even non-zero eye extent, known format, positive rate, CALIBRATED backed by a real calibration). Absence never fails. sim_display: 'PASS: stereo_camera_caps — 0 cameras (OK)'; with the fake: '1 camera(s); [0] "Sim 3D camera (fake)" 640x480/eye @ 30.0 Hz, flags 0x1f'. Co-Authored-By: Claude Opus 5.5 --- src/xrt/targets/cli/cli_query.c | 74 +++++++++++++++++++++++++++++++++ src/xrt/targets/cli/cli_query.h | 15 +++++++ 2 files changed, 89 insertions(+) diff --git a/src/xrt/targets/cli/cli_query.c b/src/xrt/targets/cli/cli_query.c index 29d325bdf0..8359f071d8 100644 --- a/src/xrt/targets/cli/cli_query.c +++ b/src/xrt/targets/cli/cli_query.c @@ -71,7 +71,67 @@ or_q(const char *s) return (s != NULL && s[0] != '\0') ? s : "?"; } +/*! + * ADR-043 `stereo_camera_caps`: enumerate the plug-in's stereo cameras straight + * through the iface (this is the in-process no-comp instance — the service's + * camera manager is not involved) and validate each description. + */ +static void +probe_stereo_camera(struct cli_query_result *r, const struct xrt_plugin_iface *iface) +{ + r->stereo_camera_count = 0; + r->stereo_camera_malformed = false; + if (!xrt_plugin_iface_has_stereo_camera(iface)) { + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "plug-in has no stereo camera slots (OK)"); + return; + } + struct xrt_plugin_instance *inst = target_plugin_get_active_instance(); + struct xrt_plugin_stereo_camera_info infos[4]; + memset(infos, 0, sizeof(infos)); + for (int i = 0; i < 4; i++) { + infos[i].struct_size = (uint32_t)sizeof(infos[i]); + } + uint32_t n = iface->stereo_camera_enumerate(inst, 4, infos); + r->stereo_camera_count = n; + if (n == 0) { + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "0 cameras (OK)"); + return; + } + for (uint32_t i = 0; i < n && i < 4; i++) { + const struct xrt_plugin_stereo_camera_info *c = &infos[i]; + const char *why = NULL; + if (c->eye_width == 0 || c->eye_height == 0 || (c->eye_width & 1u) || (c->eye_height & 1u)) { + why = "zero or odd eye extent"; + } else if (c->native_format < 1 || c->native_format > 3) { + why = "unknown native format"; + } else if (!(c->max_frame_rate > 0.0f)) { + why = "max_frame_rate <= 0"; + } else if (c->flags & XRT_PLUGIN_STEREO_CAMERA_CALIBRATED) { + struct xrt_plugin_stereo_camera_calibration k; + memset(&k, 0, sizeof(k)); + k.struct_size = (uint32_t)sizeof(k); + const double *t = k.translation_right_from_left_mm; + if (iface->stereo_camera_get_calibration(inst, i, &k) != XRT_SUCCESS) { + why = "CALIBRATED but get_calibration failed"; + } else if (!(k.k[0][0] > 0.0) || !(k.k[1][0] > 0.0) || + !(t[0] * t[0] + t[1] * t[1] + t[2] * t[2] > 0.0)) { + why = "CALIBRATED with a non-positive focal length or zero baseline"; + } + } + if (why != NULL) { + r->stereo_camera_malformed = true; + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "camera %u \"%.60s\": %s", i, + c->display_name, why); + return; + } + } + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "%u camera(s); [0] \"%.60s\" %ux%u/eye @ %.1f Hz, flags 0x%x", + n, infos[0].display_name, infos[0].eye_width, infos[0].eye_height, (double)infos[0].max_frame_rate, + infos[0].flags); +} + #ifdef XRT_OS_WINDOWS + /*! * #224 / ADR-027 P4 — headless zone-caps probe. Creates a D3D11 WARP device * (no GPU / display required), asks the active plug-in's D3D11 DP factory @@ -1697,6 +1757,13 @@ cli_query_fill(struct cli_query_result *r, struct cli_query_handles *h, const st "not probed: the lift module is a D3D11-service feature (Windows-only); modes 0 (OK)"); #endif + // ADR-043 — stereo camera slots. Absence never fails; only a camera the + // plug-in describes out of contract flips the verdict. + probe_stereo_camera(r, iface); + if (r->stereo_camera_malformed && r->result_code == CLI_SELFTEST_PASS) { + r->result_code = CLI_SELFTEST_BAD_STEREO_CAMERA; + } + // #1234 / #902 — can the Vulkan loader still reach the queue-lock layer? // Pure reporting: never touches result_code. See the field comment in // cli_query.h for why this must not be fatal. @@ -2653,6 +2720,13 @@ build_checks(const struct cli_query_result *r, struct check *out) // #224 / ADR-027 P4 — zone-caps probe. ABSENCE NEVER FAILS: ok stays // true for legacy plug-ins / no factory / non-Windows; only a // present-but-malformed caps struct fails (BAD_ZONE_CAPS). + // ADR-043 — stereo camera slots. ABSENCE NEVER FAILS. + c = &out[n++]; + c->name = "stereo_camera_caps"; + c->ok = !r->stereo_camera_malformed; + snprintf(c->detail, sizeof(c->detail), "%s", + r->stereo_camera_note[0] != '\0' ? r->stereo_camera_note : "not evaluated"); + c = &out[n++]; c->name = "zone_caps"; c->ok = !r->zone_caps_malformed; diff --git a/src/xrt/targets/cli/cli_query.h b/src/xrt/targets/cli/cli_query.h index 44565dd558..2a5c1f7524 100644 --- a/src/xrt/targets/cli/cli_query.h +++ b/src/xrt/targets/cli/cli_query.h @@ -119,6 +119,15 @@ enum cli_selftest_result * one, and of sim_display — passes. */ CLI_SELFTEST_BAD_LIFT_CAPS = 11, + + /*! + * The active plug-in fills the stereo camera slots (ADR-043) and one of + * its cameras is described out of contract: zero / odd eye extent, an + * unknown native format, a non-positive frame rate, or CALIBRATED with no + * (or a degenerate) calibration. ABSENCE NEVER FAILS: no slots or zero + * cameras is the state of almost every box. + */ + CLI_SELFTEST_BAD_STEREO_CAMERA = 12, }; //! Hardware adapters reported by the GPU-topology probe (#918). @@ -283,6 +292,12 @@ struct cli_query_result bool zone_caps_probed; bool zone_caps_malformed; char zone_probe_note[128]; + + /* Stereo camera slots (ADR-043, `stereo_camera_caps`). Absence never + * fails; only an out-of-contract camera description does. */ + uint32_t stereo_camera_count; + bool stereo_camera_malformed; + char stereo_camera_note[200]; struct xrt_dp_local_zone_caps zone_caps; /* ADR-042 lift-caps probe (Windows-only: WARP D3D11 device + the plug-in's From 2668dfc8413f405a6a3aec83a81efc7382d1664d Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:50:11 -0700 Subject: [PATCH 09/37] style(stereo-camera): clang-format 14 the new stereo camera files Co-Authored-By: Claude Opus 5.5 --- src/xrt/auxiliary/util/u_stereo_camera.c | 3 +- .../sim_display/sim_display_stereo_camera.c | 4 +- .../sim_display_stereo_camera_pattern.c | 5 +- src/xrt/include/xrt/xrt_stereo_camera.h | 2 +- src/xrt/ipc/server/ipc_server_stereo_camera.c | 35 ++++---- src/xrt/state_trackers/oxr/oxr_api_funcs.h | 4 +- .../state_trackers/oxr/oxr_stereo_camera.c | 10 ++- src/xrt/targets/cli/cli_cmd_camera.c | 90 +++++++++++-------- src/xrt/targets/cli/cli_query.c | 9 +- tests/tests_stereo_camera.cpp | 2 +- 10 files changed, 91 insertions(+), 73 deletions(-) diff --git a/src/xrt/auxiliary/util/u_stereo_camera.c b/src/xrt/auxiliary/util/u_stereo_camera.c index c811125694..10f9d35faf 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.c +++ b/src/xrt/auxiliary/util/u_stereo_camera.c @@ -276,7 +276,8 @@ u_stereo_camera_convert(uint32_t src_format, uint8_t *o = d0 + (size_t)y * dl->pitch[0]; for (uint32_t x = 0; x < width; x++) { // Full-range luma: a GRAY8 consumer wants intensity, not video levels. - o[x] = (uint8_t)((29 * s[4 * x + 0] + 150 * s[4 * x + 1] + 77 * s[4 * x + 2] + 128) >> 8); + o[x] = + (uint8_t)((29 * s[4 * x + 0] + 150 * s[4 * x + 1] + 77 * s[4 * x + 2] + 128) >> 8); } } return true; diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index 77e4d31d4b..6b1d4f5382 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -48,8 +48,8 @@ struct xrt_plugin_stereo_camera uint32_t gray_pitch; uint8_t *converted; struct u_stereo_camera_planes conv_layout; - int64_t t0_ns; //!< pacing anchor (re-anchored after a pause) - int64_t open_ns; //!< suspend square-wave origin (never moves) + int64_t t0_ns; //!< pacing anchor (re-anchored after a pause) + int64_t open_ns; //!< suspend square-wave origin (never moves) int64_t period_ns; uint64_t seq; }; diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c index 5fc21237e3..b0d52b7102 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c @@ -114,8 +114,9 @@ render_counter(uint8_t *dst, uint32_t pitch, uint32_t x_off, uint32_t eye_w, uin continue; } for (uint32_t sy = 0; sy < COUNTER_SCALE; sy++) { - uint8_t *p = dst + (size_t)(COUNTER_Y + COUNTER_SCALE + r * COUNTER_SCALE + sy) * pitch + - x_off + gx + c * COUNTER_SCALE; + uint8_t *p = + dst + (size_t)(COUNTER_Y + COUNTER_SCALE + r * COUNTER_SCALE + sy) * pitch + + x_off + gx + c * COUNTER_SCALE; memset(p, 255, COUNTER_SCALE); } } diff --git a/src/xrt/include/xrt/xrt_stereo_camera.h b/src/xrt/include/xrt/xrt_stereo_camera.h index 40b05c3537..73fdcd7b57 100644 --- a/src/xrt/include/xrt/xrt_stereo_camera.h +++ b/src/xrt/include/xrt/xrt_stereo_camera.h @@ -125,7 +125,7 @@ struct xrt_stereo_camera_stream_request */ struct xrt_stereo_camera_stream_layout { - uint32_t width; //!< full SBS + uint32_t width; //!< full SBS uint32_t height; uint32_t format; uint32_t output; diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index b7f3282d3a..7e3e31fd25 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -326,9 +326,10 @@ authorise_locked(volatile struct ipc_client_state *ics, const struct scam_camera if (debug_get_bool_option_stereo_camera_dev_allow()) { return XRT_SUCCESS; } - U_LOG_W("%s: refused for pid %ld — stereo camera consent is not implemented yet (R3); set " - "DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment for development.", - what, ics->peer_pid); + U_LOG_W( + "%s: refused for pid %ld — stereo camera consent is not implemented yet (R3); set " + "DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment for development.", + what, ics->peer_pid); return XRT_ERROR_NOT_AUTHORIZED; } @@ -732,12 +733,13 @@ ipc_server_stereo_camera_create(struct xrt_instance *xinst) info->display_name[sizeof(info->display_name) - 1] = '\0'; info->device_identity[sizeof(info->device_identity) - 1] = '\0'; info->platform_device_hint[sizeof(info->platform_device_hint) - 1] = '\0'; - if (info->eye_width == 0 || info->eye_height == 0 || (info->eye_width & 1u) || (info->eye_height & 1u) || - info->native_format < XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 || + if (info->eye_width == 0 || info->eye_height == 0 || (info->eye_width & 1u) || + (info->eye_height & 1u) || info->native_format < XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 || info->native_format > XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8) { - U_LOG_W("stereo camera: plug-in camera %u has a malformed description (%ux%u, format %u) — " - "skipped", - i, info->eye_width, info->eye_height, info->native_format); + U_LOG_W( + "stereo camera: plug-in camera %u has a malformed description (%ux%u, format %u) — " + "skipped", + i, info->eye_width, info->eye_height, info->native_format); continue; } cam->index = i; @@ -751,8 +753,9 @@ ipc_server_stereo_camera_create(struct xrt_instance *xinst) if (iface->stereo_camera_get_calibration(inst, i, &cam->calib) == XRT_SUCCESS) { cam->have_calib = true; } else { - U_LOG_W("stereo camera: \"%s\" claims CALIBRATED but returned no calibration — RAW only", - info->display_name); + U_LOG_W( + "stereo camera: \"%s\" claims CALIBRATED but returned no calibration — RAW only", + info->display_name); cam->info.flags &= ~XRT_PLUGIN_STEREO_CAMERA_CALIBRATED; } } @@ -1023,8 +1026,8 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, os_mutex_unlock(&m->lock); return XRT_ERROR_INPUT_UNSUPPORTED; } - bool calibrated = (cam->info.flags & (XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | - XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED)) != 0; + bool calibrated = (cam->info.flags & + (XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED)) != 0; if (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !calibrated) { os_mutex_unlock(&m->lock); return XRT_ERROR_INPUT_UNSUPPORTED; @@ -1177,8 +1180,8 @@ ipc_handle_stereo_camera_stream_get_section(volatile struct ipc_client_state *ic out_layout->format = s->req.format; out_layout->output = s->output; out_layout->transport = s->req.transport; - out_layout->max_frame_rate = s->dec.period_ns > 0 ? (float)(1e9 / (double)s->dec.period_ns) - : m->cams[s->camera].info.max_frame_rate; + out_layout->max_frame_rate = + s->dec.period_ns > 0 ? (float)(1e9 / (double)s->dec.period_ns) : m->cams[s->camera].info.max_frame_rate; out_layout->section_size = s->section_size; out_layout->slot_stride = s->slot_stride; out_layout->slot_count = U_STEREO_CAMERA_RING_SLOTS; @@ -1188,8 +1191,8 @@ ipc_handle_stereo_camera_stream_get_section(volatile struct ipc_client_state *ic CloseHandle(s->sent_section_dup); s->sent_section_dup = NULL; } - if (!DuplicateHandle(GetCurrentProcess(), s->section, GetCurrentProcess(), &s->sent_section_dup, - FILE_MAP_READ, FALSE, 0)) { + if (!DuplicateHandle(GetCurrentProcess(), s->section, GetCurrentProcess(), &s->sent_section_dup, FILE_MAP_READ, + FALSE, 0)) { os_mutex_unlock(&m->lock); return XRT_ERROR_IPC_FAILURE; } diff --git a/src/xrt/state_trackers/oxr/oxr_api_funcs.h b/src/xrt/state_trackers/oxr/oxr_api_funcs.h index fc48e55d10..051679369f 100644 --- a/src/xrt/state_trackers/oxr/oxr_api_funcs.h +++ b/src/xrt/state_trackers/oxr/oxr_api_funcs.h @@ -1123,9 +1123,7 @@ oxr_xrEnumerateStereoCamerasDXR(XrInstance instance, //! OpenXR API function @ep{xrGetStereoCameraCalibrationDXR} XRAPI_ATTR XrResult XRAPI_CALL -oxr_xrGetStereoCameraCalibrationDXR(XrInstance instance, - uint64_t cameraId, - XrStereoCameraCalibrationDXR *calibration); +oxr_xrGetStereoCameraCalibrationDXR(XrInstance instance, uint64_t cameraId, XrStereoCameraCalibrationDXR *calibration); //! OpenXR API function @ep{xrCreateStereoCameraStreamDXR} XRAPI_ATTR XrResult XRAPI_CALL diff --git a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c index 1be6693a96..655ef3a8cb 100644 --- a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c +++ b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c @@ -106,7 +106,8 @@ scam_xret(struct oxr_logger *log, xrt_result_t xret, const char *what) return oxr_error(log, XR_ERROR_FEATURE_UNSUPPORTED, "%s: not supported by this runtime / camera", what); case XRT_ERROR_CLIENT_LIMIT_REACHED: return oxr_error(log, XR_ERROR_LIMIT_REACHED, "%s: too many streams on this camera", what); - default: return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, "%s: service refused (xrt_result=%d)", what, (int)xret); + default: + return oxr_error(log, XR_ERROR_RUNTIME_FAILURE, "%s: service refused (xrt_result=%d)", what, (int)xret); } } @@ -159,7 +160,8 @@ oxr_xrEnumerateStereoCamerasDXR(XrInstance instance, return XR_SUCCESS; } if (capacityInput < n) { - return oxr_error(&log, XR_ERROR_SIZE_INSUFFICIENT, "(capacityInput == %u) < %u cameras", capacityInput, n); + return oxr_error(&log, XR_ERROR_SIZE_INSUFFICIENT, "(capacityInput == %u) < %u cameras", capacityInput, + n); } for (uint32_t i = 0; i < n; i++) { if (cameras[i].type != XR_TYPE_STEREO_CAMERA_PROPERTIES_DXR) { @@ -361,8 +363,8 @@ oxr_xrGetStereoCameraStreamInfoDXR(XrStereoCameraStreamDXR stream, XrStereoCamer info->transport = (XrStereoCameraTransportDXR)lay.transport; info->maxFrameRate = lay.max_frame_rate; - XrStereoCameraStreamTransportDXR *t = OXR_GET_OUTPUT_FROM_CHAIN(info, XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR, - XrStereoCameraStreamTransportDXR); + XrStereoCameraStreamTransportDXR *t = OXR_GET_OUTPUT_FROM_CHAIN( + info, XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR, XrStereoCameraStreamTransportDXR); if (t != NULL) { // NEW handles, now the caller's to close. t->sectionHandle = (uint64_t)(uintptr_t)section; diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 3a578dc611..86e1d5f22b 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -157,21 +157,24 @@ cmd_list(struct ipc_connection *ipc_c, bool json) char flags[160]; flags_str(p.flags, flags, sizeof(flags)); if (json) { - printf("%s{\"id\": %llu, \"name\": \"%s\", \"persistentId\": \"%s\", \"state\": \"%s\", " - "\"flags\": \"%s\", \"viewCount\": %u, \"eyeWidth\": %u, \"eyeHeight\": %u, " - "\"maxFrameRate\": %.2f, \"baselineMm\": %.2f, \"horizontalFovDeg\": %.2f, " - "\"formats\": \"0x%llx\", \"transports\": \"0x%llx\", \"platformDeviceHint\": \"%s\"}", - i ? ", " : "", (unsigned long long)p.camera_id, p.display_name, p.persistent_id, - state_str(p.state), flags, p.view_count, p.eye_width, p.eye_height, p.max_frame_rate, - p.baseline_mm, p.horizontal_fov_deg, (unsigned long long)p.supported_formats, - (unsigned long long)p.supported_transports, p.platform_device_hint); + printf( + "%s{\"id\": %llu, \"name\": \"%s\", \"persistentId\": \"%s\", \"state\": \"%s\", " + "\"flags\": \"%s\", \"viewCount\": %u, \"eyeWidth\": %u, \"eyeHeight\": %u, " + "\"maxFrameRate\": %.2f, \"baselineMm\": %.2f, \"horizontalFovDeg\": %.2f, " + "\"formats\": \"0x%llx\", \"transports\": \"0x%llx\", \"platformDeviceHint\": \"%s\"}", + i ? ", " : "", (unsigned long long)p.camera_id, p.display_name, p.persistent_id, + state_str(p.state), flags, p.view_count, p.eye_width, p.eye_height, p.max_frame_rate, + p.baseline_mm, p.horizontal_fov_deg, (unsigned long long)p.supported_formats, + (unsigned long long)p.supported_transports, p.platform_device_hint); } else { - printf(" [%llu] \"%s\" %s\n", (unsigned long long)p.camera_id, p.display_name, state_str(p.state)); + printf(" [%llu] \"%s\" %s\n", (unsigned long long)p.camera_id, p.display_name, + state_str(p.state)); printf(" persistentId %s\n", p.persistent_id); printf(" flags %s\n", flags); printf(" eye %ux%u (SBS %ux%u), %u views, up to %.1f Hz\n", p.eye_width, p.eye_height, 2 * p.eye_width, p.eye_height, p.view_count, p.max_frame_rate); - printf(" baseline %.2f mm, HFOV %.2f deg per eye\n", p.baseline_mm, p.horizontal_fov_deg); + printf(" baseline %.2f mm, HFOV %.2f deg per eye\n", p.baseline_mm, + p.horizontal_fov_deg); printf(" formats 0x%llx transports 0x%llx platform hint \"%s\"\n", (unsigned long long)p.supported_formats, (unsigned long long)p.supported_transports, p.platform_device_hint); @@ -197,15 +200,17 @@ cmd_calib(struct ipc_connection *ipc_c, uint64_t id, uint32_t output, bool json) printf("{\"output\": \"%s\", \"baselineMm\": %.3f, \"eyes\": [", output == XRT_STEREO_CAMERA_OUTPUT_RAW ? "RAW" : "RECTIFIED", c.baseline_mm); for (int e = 0; e < 2; e++) { - printf("%s{\"w\": %u, \"h\": %u, \"fx\": %.3f, \"fy\": %.3f, \"cx\": %.3f, \"cy\": %.3f, " - "\"model\": %u}", - e ? ", " : "", c.eye[e].width, c.eye[e].height, c.eye[e].fx, c.eye[e].fy, c.eye[e].cx, - c.eye[e].cy, c.eye[e].model); + printf( + "%s{\"w\": %u, \"h\": %u, \"fx\": %.3f, \"fy\": %.3f, \"cx\": %.3f, \"cy\": %.3f, " + "\"model\": %u}", + e ? ", " : "", c.eye[e].width, c.eye[e].height, c.eye[e].fx, c.eye[e].fy, c.eye[e].cx, + c.eye[e].cy, c.eye[e].model); } - printf("], \"rightFromLeft\": {\"orientation\": [%.6f, %.6f, %.6f, %.6f], \"position\": [%.6f, " - "%.6f, %.6f]}}\n", - c.orientation[0], c.orientation[1], c.orientation[2], c.orientation[3], c.position[0], - c.position[1], c.position[2]); + printf( + "], \"rightFromLeft\": {\"orientation\": [%.6f, %.6f, %.6f, %.6f], \"position\": [%.6f, " + "%.6f, %.6f]}}\n", + c.orientation[0], c.orientation[1], c.orientation[2], c.orientation[3], c.position[0], + c.position[1], c.position[2]); return 0; } printf("calibration (%s), baseline %.3f mm\n", output == XRT_STEREO_CAMERA_OUTPUT_RAW ? "RAW" : "RECTIFIED", @@ -474,28 +479,32 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) } double span = (double)(t_last - t_first) * 1e-9; double rate = (got > 1 && span > 0.0) ? (got - 1) / span : 0.0; - printf("received %d frames (index %llu..%llu, %llu source frames not delivered to this stream), %d wakes, " - "%d not-ready\n", - got, (unsigned long long)first_index, (unsigned long long)last_index, (unsigned long long)gaps, wakes, - not_ready); + printf( + "received %d frames (index %llu..%llu, %llu source frames not delivered to this stream), %d wakes, " + "%d not-ready\n", + got, (unsigned long long)first_index, (unsigned long long)last_index, (unsigned long long)gaps, wakes, + not_ready); printf("measured delivery rate: %.2f Hz over %.2f s\n", rate, span); if (got > 0) { const uint8_t *slot = map.ptr + (uint64_t)last.slot * lay.slot_stride; - printf("last frame: index %llu, slot %u, %ux%u %s, pitch %u/%u, planes @%llu/%llu, %s time, output %s, " - "calibration gen %u\n", - (unsigned long long)last.frame_index, last.slot, last.width, last.height, format_str(last.format), - last.row_pitch[0], last.row_pitch[1], (unsigned long long)last.plane_offset[0], - (unsigned long long)last.plane_offset[1], last.time_is_exposure ? "exposure" : "arrival", - last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", last.calibration_generation); + printf( + "last frame: index %llu, slot %u, %ux%u %s, pitch %u/%u, planes @%llu/%llu, %s time, output %s, " + "calibration gen %u\n", + (unsigned long long)last.frame_index, last.slot, last.width, last.height, format_str(last.format), + last.row_pitch[0], last.row_pitch[1], (unsigned long long)last.plane_offset[0], + (unsigned long long)last.plane_offset[1], last.time_is_exposure ? "exposure" : "arrival", + last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", + last.calibration_generation); // Luma of the last frame, for the disparity probe and a GRAY8 PNG. struct u_stereo_camera_planes gl; u_stereo_camera_layout(XRT_STEREO_CAMERA_FORMAT_GRAY8, last.width, last.height, &gl); uint8_t *gray = calloc((size_t)gl.size, 1); const uint8_t *src[2] = {slot + last.plane_offset[0], slot + last.plane_offset[1]}; - if (gray != NULL && u_stereo_camera_convert(last.format, src, last.row_pitch, XRT_STEREO_CAMERA_FORMAT_GRAY8, - gray, &gl, last.width, last.height)) { + if (gray != NULL && + u_stereo_camera_convert(last.format, src, last.row_pitch, XRT_STEREO_CAMERA_FORMAT_GRAY8, gray, &gl, + last.width, last.height)) { char disp[160]; report_disparity(gray, gl.pitch[0], last.width / 2, last.height, disp, sizeof(disp)); printf("block disparity (left x - right x): %s\n", disp); @@ -521,7 +530,8 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) r[4 * x + 2] = t; } } - ok = stbi_write_png(path, (int)last.width, (int)last.height, 4, rgba, (int)bl.pitch[0]); + ok = stbi_write_png(path, (int)last.width, (int)last.height, 4, rgba, + (int)bl.pitch[0]); } free(rgba); } @@ -532,11 +542,12 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) struct xrt_stereo_camera_stream_stats st; if (ipc_client_stereo_camera_stats(ipc_c, sid, &st) == XRT_SUCCESS) { - printf("service stats: source %.2f Hz, delivered %.2f Hz, published %llu, skipped %llu, acquired %llu, " - "mean latency %.3f ms\n", - st.source_frame_rate, st.delivered_frame_rate, (unsigned long long)st.frames_published, - (unsigned long long)st.frames_skipped, (unsigned long long)st.frames_acquired, - (double)st.mean_latency_ns * 1e-6); + printf( + "service stats: source %.2f Hz, delivered %.2f Hz, published %llu, skipped %llu, acquired %llu, " + "mean latency %.3f ms\n", + st.source_frame_rate, st.delivered_frame_rate, (unsigned long long)st.frames_published, + (unsigned long long)st.frames_skipped, (unsigned long long)st.frames_acquired, + (double)st.mean_latency_ns * 1e-6); } unmap_readonly(&map); @@ -565,9 +576,10 @@ cli_cmd_camera(int argc, const char **argv) } else if (strcmp(sub, "probe") == 0) { ret = cmd_probe(&ipc_c, argc, argv); } else { - printf("usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" - " probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] [--frames N] [--seconds S] " - "[--out DIR]\n"); + printf( + "usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" + " probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] [--frames N] [--seconds S] " + "[--out DIR]\n"); ret = 1; } ipc_client_connection_fini(&ipc_c); diff --git a/src/xrt/targets/cli/cli_query.c b/src/xrt/targets/cli/cli_query.c index 8359f071d8..7df10f988b 100644 --- a/src/xrt/targets/cli/cli_query.c +++ b/src/xrt/targets/cli/cli_query.c @@ -82,7 +82,8 @@ probe_stereo_camera(struct cli_query_result *r, const struct xrt_plugin_iface *i r->stereo_camera_count = 0; r->stereo_camera_malformed = false; if (!xrt_plugin_iface_has_stereo_camera(iface)) { - snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "plug-in has no stereo camera slots (OK)"); + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), + "plug-in has no stereo camera slots (OK)"); return; } struct xrt_plugin_instance *inst = target_plugin_get_active_instance(); @@ -125,9 +126,9 @@ probe_stereo_camera(struct cli_query_result *r, const struct xrt_plugin_iface *i return; } } - snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), "%u camera(s); [0] \"%.60s\" %ux%u/eye @ %.1f Hz, flags 0x%x", - n, infos[0].display_name, infos[0].eye_width, infos[0].eye_height, (double)infos[0].max_frame_rate, - infos[0].flags); + snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), + "%u camera(s); [0] \"%.60s\" %ux%u/eye @ %.1f Hz, flags 0x%x", n, infos[0].display_name, + infos[0].eye_width, infos[0].eye_height, (double)infos[0].max_frame_rate, infos[0].flags); } #ifdef XRT_OS_WINDOWS diff --git a/tests/tests_stereo_camera.cpp b/tests/tests_stereo_camera.cpp index b530ec45cf..689a4690ad 100644 --- a/tests/tests_stereo_camera.cpp +++ b/tests/tests_stereo_camera.cpp @@ -139,7 +139,7 @@ TEST_CASE("stereo camera layout", "[stereo_camera]") CHECK(l.offset[1] == 1280ull * 480); CHECK(l.size == 1280ull * 480 * 3 / 2); REQUIRE(u_stereo_camera_layout(3, 1000, 10, &l)); - CHECK(l.pitch[0] == 4032); // 4000 rounded up to 64 + CHECK(l.pitch[0] == 4032); // 4000 rounded up to 64 CHECK_FALSE(u_stereo_camera_layout(2, 1281, 480, &l)); // odd NV12 CHECK_FALSE(u_stereo_camera_layout(9, 64, 64, &l)); } From a769549577b3b5ba739ced892bdc8ef8d09ac0a6 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:51:56 -0700 Subject: [PATCH 10/37] fix(stereo-camera): no _Static_assert on XrFlags64 constants (not constant expressions for MSVC/GCC) Caught by the MinGW syntax check: the XR flag bits are static const XrFlags64, which only clang accepts in _Static_assert. Pin the xrt mirrors to the header's literal values at compile time and cross-check the real constants once at runtime. Co-Authored-By: Claude Opus 5.5 --- .../state_trackers/oxr/oxr_stereo_camera.c | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c index 655ef3a8cb..d1dfea8160 100644 --- a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c +++ b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c @@ -48,14 +48,16 @@ #ifdef OXR_HAVE_DXR_stereo_camera -// The wire carries the xrt encodings; they equal the XR ones. -_Static_assert(XR_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING, - "flag mismatch"); -_Static_assert(XR_STEREO_CAMERA_USER_FACING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_USER_FACING, "flag mismatch"); -_Static_assert(XR_STEREO_CAMERA_CALIBRATED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_CALIBRATED, "flag mismatch"); -_Static_assert(XR_STEREO_CAMERA_NATIVELY_RECTIFIED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED, - "flag mismatch"); -_Static_assert(XR_STEREO_CAMERA_MONOCHROME_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_MONOCHROME, "flag mismatch"); +// The wire carries the xrt encodings; they equal the XR ones. The XR flag bits +// are `static const XrFlags64` — not constant expressions in C (MSVC and GCC +// reject them in _Static_assert; clang only accepts them as an extension) — so +// the flag / format-bit mirrors are pinned against the header's literal values +// here and cross-checked against the real constants in verify_encodings(). +_Static_assert(XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING == 0x1, "flag mirror"); +_Static_assert(XRT_PLUGIN_STEREO_CAMERA_USER_FACING == 0x2, "flag mirror"); +_Static_assert(XRT_PLUGIN_STEREO_CAMERA_CALIBRATED == 0x4, "flag mirror"); +_Static_assert(XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED == 0x8, "flag mirror"); +_Static_assert(XRT_PLUGIN_STEREO_CAMERA_MONOCHROME == 0x10, "flag mirror"); _Static_assert((int)XR_STEREO_CAMERA_STATE_AVAILABLE_DXR == (int)XRT_STEREO_CAMERA_STATE_AVAILABLE, "state"); _Static_assert((int)XR_STEREO_CAMERA_STATE_WAITING_DXR == (int)XRT_STEREO_CAMERA_STATE_WAITING, "state"); _Static_assert((int)XR_STEREO_CAMERA_STATE_SUSPENDED_DXR == (int)XRT_STEREO_CAMERA_STATE_SUSPENDED, "state"); @@ -65,11 +67,6 @@ _Static_assert((int)XR_STEREO_CAMERA_OUTPUT_RAW_DXR == (int)XRT_STEREO_CAMERA_OU _Static_assert((int)XR_STEREO_CAMERA_FORMAT_GRAY8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_GRAY8, "format"); _Static_assert((int)XR_STEREO_CAMERA_FORMAT_NV12_DXR == (int)XRT_STEREO_CAMERA_FORMAT_NV12, "format"); _Static_assert((int)XR_STEREO_CAMERA_FORMAT_BGRA8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_BGRA8, "format"); -_Static_assert(XR_STEREO_CAMERA_FORMAT_NV12_BIT_DXR == XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_NV12), - "format bit"); -_Static_assert(XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_BIT_DXR == - XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY), - "transport bit"); _Static_assert((int)XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR == (int)XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY, "transport"); _Static_assert(XR_STEREO_CAMERA_RING_SLOTS_DXR == XRT_STEREO_CAMERA_RING_SLOTS, "ring slots"); @@ -83,6 +80,22 @@ _Static_assert(XR_STEREO_CAMERA_PLATFORM_HINT_MAX_SIZE_DXR == sizeof(((struct xrt_stereo_camera_properties *)0)->platform_device_hint), "hint size"); +//! Runtime half of the encoding check (see the _Static_assert block above). +static bool +verify_encodings(void) +{ + return XR_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING && + XR_STEREO_CAMERA_USER_FACING_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_USER_FACING && + XR_STEREO_CAMERA_CALIBRATED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_CALIBRATED && + XR_STEREO_CAMERA_NATIVELY_RECTIFIED_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED && + XR_STEREO_CAMERA_MONOCHROME_BIT_DXR == XRT_PLUGIN_STEREO_CAMERA_MONOCHROME && + XR_STEREO_CAMERA_FORMAT_GRAY8_BIT_DXR == XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_GRAY8) && + XR_STEREO_CAMERA_FORMAT_NV12_BIT_DXR == XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_NV12) && + XR_STEREO_CAMERA_FORMAT_BGRA8_BIT_DXR == XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_FORMAT_BGRA8) && + XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_BIT_DXR == + XRT_STEREO_CAMERA_BIT(XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY); +} + static struct xrt_stereo_camera_client * client_of(struct oxr_instance *inst) { @@ -150,6 +163,9 @@ oxr_xrEnumerateStereoCamerasDXR(XrInstance instance, if (c == NULL) { return XR_SUCCESS; // in-process instance: zero cameras, on purpose } + if (!verify_encodings()) { + return oxr_error(&log, XR_ERROR_RUNTIME_FAILURE, "XR_DXR_stereo_camera encoding mismatch (build bug)"); + } uint32_t n = 0; xrt_result_t xret = c->count(c, &n); if (xret != XRT_SUCCESS) { From 8e589a50724be655a3e43bf1283686455b576018 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 02:54:46 -0700 Subject: [PATCH 11/37] docs(stereo-camera): R1 as built, maintainer decisions, measured Android facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - spec: status R1, the DXR_STEREO_CAMERA_DEV_ALLOW dev gate, privacy hook status, slot placement behind the lift placeholder, the OpenCV R,T plug-in convention and the wait_frame result enum, the fake as built, CLI output. - roadmap: §C Android rewritten from the NP02J/K68 measurements (logical cameras 4/5, two raw 1280x720 NV12 streams, LeiaCameraBuilder is not CNSDK, opening the front pair evicts the tracker for good), design consequence: the runtime owns the pair and runs in-app tracking from the same capture; §E R1 as built + merge order; §F Android risk + CNSDK asks; §G the four R1 maintainer decisions. - ADR-043: R1 status, Android context/consequence updated. - xrt_plugin_iface reference: the stereo_camera_* slot contract. - env census: DXR_STEREO_CAMERA, DXR_STEREO_CAMERA_DEV_ALLOW. - ADR index regenerated (it was stale on the docs branch). Co-Authored-By: Claude Opus 5.5 --- docs/adr/ADR-043-stereo-camera-source.md | 16 +- docs/reference/xrt_plugin_iface.md | 40 +++++ .../control-panel-performance-settings.md | 4 +- docs/roadmap/stereo-camera-source.md | 165 ++++++++++++++---- docs/specs/extensions/XR_DXR_stereo_camera.md | 61 ++++++- 5 files changed, 244 insertions(+), 42 deletions(-) diff --git a/docs/adr/ADR-043-stereo-camera-source.md b/docs/adr/ADR-043-stereo-camera-source.md index 90df2a3fa4..ce4cb62736 100644 --- a/docs/adr/ADR-043-stereo-camera-source.md +++ b/docs/adr/ADR-043-stereo-camera-source.md @@ -1,6 +1,8 @@ # ADR-043: A display's stereo camera is a plug-in-provided source, owned by the service and privacy-gated by the runtime -**Status:** Proposed (2026-09-25) — design only, nothing implemented · introduces +**Status:** Proposed (2026-09-25) · **R1 implemented** 2026-09-26 (runtime, hardware-free; maintainer +defaults recorded in the roadmap §G: stereo-only, iface slots, service clients only, no raw frames +to pages) · introduces [`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md) · appends optional camera slots to `xrt_plugin_iface` under the [ADR-020](ADR-020-plugin-abi-compatibility-policy.md) append-at-end rule · related: [ADR-019](ADR-019-vendor-plugin-aux-boundary.md), @@ -30,7 +32,11 @@ machines): correct design must key calibration by the **active** device; and the SDK's new-frame event is **auto-reset**, so two readers waiting on it steal each other's wake-ups. - 3D **tablets** carry a front stereo pair (15 mm and 25 mm baselines on two models, parallel) - owned by the vendor's camera/tracking service on Android. + owned by the vendor's camera/tracking service on Android. Measured 2026-09-26 (roadmap §C.0): + the pair is a hidden Camera2 **logical** camera (front 5, back 4) that delivers two raw 1280×720 + NV12 streams at 30 fps. Opening the front one **evicts the vendor tracker, which never + reacquires**. The vendor camera SDK (CNSDK) supplies only calibration and in-app face tracking, + not frames. The DisplayXR Browser cannot see the pair at all (browser-pvt#175). Everything a correct stereo camera needs — the private frame channel, the device's calibration, keeping the tracker alive — is vendor knowledge. The runtime's plug-in model (ADR-019) exists @@ -87,5 +93,7 @@ exactly so that knowledge lives in one vendor DLL and nowhere else. defensively until the vendor SDK formalises it (roadmap open questions). - **−** Frame quality is the tracker's: greyscale, 640×480, tracker-chosen rate/exposure. - **−** Another service thread, and a CPU copy per frame (≈ 27 MB/s at 1280×480 NV12 30 Hz). -- **−** Android depends on the vendor camera SDK sharing frames at all (open question); until then - Android documents a platform difference rather than breaking isolation. +- **−** On Android the source cannot be a passenger on the tracker: the pair and the tracker cannot + both hold Camera2. So the runtime (the vendor plug-in in the runtime APK, ADR-038) must own the + front pair and run the vendor's in-app tracking from the same capture, on a worker thread + (roadmap §C.1). That is more vendor code inside the runtime APK, and more CPU, than on Windows. diff --git a/docs/reference/xrt_plugin_iface.md b/docs/reference/xrt_plugin_iface.md index 8a87f30823..4355bebc8c 100644 --- a/docs/reference/xrt_plugin_iface.md +++ b/docs/reference/xrt_plugin_iface.md @@ -606,6 +606,46 @@ the runtime logs one warning and runs unpaced. Either way nothing fails — the difference is accuracy, not correctness. Application-side guidance: [INV-5.9](../guides/displayxr-app-rules.md). +## A stereo camera: the `stereo_camera_*` slots (ADR-043) + +Optional, appended per ADR-020, announced by `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA`; the runtime +uses them only when `struct_size` covers `stereo_camera_close` **and** all six are non-NULL +(`xrt_plugin_iface_has_stereo_camera()`). They let a plug-in expose the display's own stereo +camera — usually the one its eye tracker looks through — as a runtime source +([`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md)). The service opens each +camera once and fans frames out to every authorised client; the plug-in only produces frames. + +```c +uint32_t (*stereo_camera_enumerate)(inst, capacity, struct xrt_plugin_stereo_camera_info *out); +xrt_result_t (*stereo_camera_get_calibration)(inst, index, struct xrt_plugin_stereo_camera_calibration *out); +xrt_result_t (*stereo_camera_open)(inst, index, struct xrt_plugin_stereo_camera **out_cam); +uint32_t (*stereo_camera_wait_frame)(cam, int64_t timeout_ns, struct xrt_plugin_stereo_camera_frame *out); +void (*stereo_camera_release_frame)(cam); +void (*stereo_camera_close)(cam); +``` + +| Rule | Why | +|---|---| +| **Never take the device from the tracker.** Read the frames the tracker already publishes (shared memory, a vendor camera service); never open, reconfigure or re-time the device. | The whole point: while tracking runs the device is exclusive (Windows `NotReadableError`), and on Android opening the front pair evicts the tracker (roadmap §C) | +| `stereo_camera_open` is a **tracker keep-alive**: while any camera is open, keep the tracker running. | Frames only exist while tracking runs | +| `wait_frame` is called from **one runtime-owned thread per open camera**, may block up to `timeout_ns`, and returns an `enum xrt_plugin_stereo_camera_wait` (`OK` / `TIMEOUT` / `SUSPENDED` / `ERROR`). Planes stay valid until `release_frame`. | The service maps `TIMEOUT` → `WAITING` (then `SUSPENDED` after 1 s without frames), `SUSPENDED` → `SUSPENDED`, `ERROR` → close + retry in 1 s | +| **Never wait on a wake object shared with other readers** of the vendor channel — poll it. | An auto-reset event wakes exactly one waiter: two readers steal each other's frames (ADR-043) | +| Decode your transport (JPEG, YUY2, …) to `GRAY8`, `NV12` or `BGRA8` yourself; always deliver **one side-by-side image**, `2·eye_width × eye_height`, left eye left, unmirrored, even extents. | The runtime owns format conversion, decimation, rectification (R2) and transport; one SBS layout for every vendor | +| Calibration is the **ACTIVE** device's — the unit you weave for — keyed by its own identity (serial). RAW, OpenCV convention: `k[eye] = fx fy cx cy`, `distortion_model` + coefficients, and `R`, `T` with **`x_R = R·x_L + T`** (T in mm; a right camera at +x of the left has `T = (−B, 0, 0)`). The runtime converts it to the client's pose-of-the-right-camera form. Return `XRT_ERROR_FEATURE_NOT_SUPPORTED` and do **not** set `CALIBRATED` if you cannot resolve it. | One test box carried eleven calibration folders; "the first one" is the wrong camera | +| `device_identity` is private: the service hashes it per consumer (`persistentId`) and never exports it. | Fingerprinting (spec §7.5) | +| Nothing throws across the boundary. | C ABI | + +Flags (`XRT_PLUGIN_STEREO_CAMERA_*`) mirror the XR bits: `SHARED_WITH_EYE_TRACKING`, +`USER_FACING`, `CALIBRATED`, `NATIVELY_RECTIFIED`, `MONOCHROME`. The reference implementation is +`sim_display`'s fake (`drivers/sim_display/sim_display_stereo_camera.c`, enabled with +`SIM_DISPLAY_FAKE_STEREO_CAMERA=1` in the **service** environment). + +**Slot order.** The six slots follow a one-pointer placeholder, +`reserved_adr042_create_dp_d3d11_lift`, held for ADR-042's `create_dp_d3d11_lift`, which is not on +`main` yet but claims the offset right after `vk_bundle_fn_table_offset`. The placeholder keeps the +camera slots at their final offsets whichever PR merges first; the lift PR replaces it in place +(same size). `tests_stereo_camera` asserts the adjacency. + ## Aux surface — separate from the iface Logging, debug-variable tracking, frame metrics, Perfetto tracing, and unique-ID generation are **not** plumbed through this iface. Plug-ins reach them by linking the runtime DLL's import library (`DisplayXRClient.lib`) and getting `__declspec(dllimport)`'d symbols. See [ADR-019](../adr/ADR-019-vendor-plugin-aux-boundary.md) for the rationale. diff --git a/docs/roadmap/control-panel-performance-settings.md b/docs/roadmap/control-panel-performance-settings.md index 419749ffb3..e5b942bd89 100644 --- a/docs/roadmap/control-panel-performance-settings.md +++ b/docs/roadmap/control-panel-performance-settings.md @@ -324,7 +324,7 @@ never claim a mode the runtime is not in, and "Custom" falls out for free. ## Appendix A — census Every `DXR_*` name read at runtime under `src/xrt`, with its read site, mechanism, default -and tier. **100 distinct names**; the two `DXR_BG2D_*` knobs reach the environment through +and tier. **102 distinct names**; the two `DXR_BG2D_*` knobs reach the environment through `bg2d_int_knob()` rather than a literal `getenv` at the listed line. Process column: **App** = the runtime DLL, loaded into the OpenXR app's process · @@ -547,6 +547,7 @@ as `docs/specs/vendor/oem-android-platform-requirements.md` §R6. | `DXR_DEVICE_REMOVED_EXIT_MS` | `compositor/d3d11_service/comp_d3d11_service.cpp:10279` | `getenv`, `static` cached | 2000 | Svc | 2 | Delay before the service exits on an unrecoverable `DEVICE_REMOVED` | | `DXR_HEALTH_MS` | `ipc/server/ipc_server_process.c:82` | `DEBUG_GET_ONCE_NUM` | 10000 | Svc | 2 | #951 `[HEALTH]` telemetry cadence; 0 disables | | `DXR_MAX_CLIENTS` | `ipc/server/ipc_server_process.c:87` | `DEBUG_GET_ONCE_NUM`, consumed `:541-551` | 0 = auto | Svc | 1 | #959 admitted-client cap. Read once at service start | +| `DXR_STEREO_CAMERA` | `ipc/server/ipc_server_stereo_camera.c` (`DEBUG_GET_ONCE_BOOL_OPTION`, read at camera-manager create) | on | Svc | 1 | ADR-043 admin/dev kill switch: `0` = the service enumerates zero stereo cameras (indistinguishable from "none", on purpose). The user-facing twin is the planned R3 "Share the 3D camera with apps" toggle — a privacy control, not a performance lever | | `DXR_ULTRALEAP_SETTLE_MS` | `drivers/ultraleap/ultraleap_provider.cpp:1023` | `getenv` at provider init | `UL_SETTLE_DEFAULT_MS` | Both | 1 | How long to wait for a definitive Leap "attached" answer before role arbitration | ### Test / dev — never exposed @@ -571,6 +572,7 @@ as `docs/specs/vendor/oem-android-platform-requirements.md` §R6. | `DXR_VIEW_CONFIG_LEGACY` | `state_trackers/oxr/oxr_system.c` (`oxr_system_fill_in`, `DEBUG_GET_ONCE_BOOL_OPTION`) | off | App | #1486 kill switch: restores the pre-#1486 single view configuration — `PRIMARY_STEREO` for any `view_count >= 2`, reporting the max across modes (4 on sim-display), and no `PRIMARY_MULTIVIEW_DXR`. Reintroduces the spec deviation and the CTS red on purpose, for one round of field bisection. Logs one `U_LOG_W` when active. 22 characters, so `debug.xrt.DXR_VIEW_CONFIG_LEGACY` is **stranded on Android** by the `PROP_NAME_MAX` trap above — env-only in practice. See [View-Configuration Model](../reference/view-configuration-model.md) | | `DXR_MODE_FLOOR` | `state_trackers/oxr/oxr_session.c` (`DEBUG_GET_ONCE_BOOL_OPTION`, one TU; `oxr_api_session.c` reads it through `oxr_session_mode_floor_enabled()`) | on | App | #1499 kill switch. `0` restores the pre-#1499 behaviour for EXTENSION sessions: `xrBeginSession` leaves the display in whatever rendering mode it found, and `xrRequestDisplayRenderingModeDXR` honours a request whose `viewCount` exceeds the session's view configuration — so the compositor's under-submit clamp paints the first tiles and the rest stay at the clear colour, which is exactly the state the issue objected to. It also silences the two #1499 log lines (the begin-time `UNFILLABLE` warning and the observation warning on a display-global mode change), because `0` promises pre-#1499 behaviour and a line the runtime never used to print is part of that promise. **One switch for BOTH halves on purpose**: a floor without the denial lets an app walk straight back into the mode the floor removed it from, and a denial without the floor refuses a mode the session is already in — half of this feature is not a state anyone should be able to bisect into. #1510's LEGACY floor is deliberately NOT affected (a legacy app was sized for its floored mode at `xrGetSystem`, so leaving it in a wider one would make the compromise view scale and the mode disagree). See [View-Configuration Model](../reference/view-configuration-model.md) | | `DXR_UNDER_SUBMIT` | `state_trackers/oxr/oxr_session_frame_end.c` (`DEBUG_GET_ONCE_NUM_OPTION`, one TU; the value→enum map is the pure `oxr_under_submit_from_setting()` in `oxr_view_config_rule.h`) | `1` | App | ADR-041 staging switch for the fixed-view-count submission contract. `0` = strict: every view configuration must submit exactly the count `xrLocateViews` returned. `1` (default) = strict **plus** the deprecated `PRIMARY_STEREO` 1-view arm (an `XR_DXR_display_info` instance submitting 1 while the active rendering mode is 1-view), which is accepted and logged **once per session** with a line naming the fix — released demos submit one view in 2D mode and need the compat window. `2` = kill switch: the pre-ADR-041 rules, `PRIMARY_MULTIVIEW_DXR` under-submit included. Out-of-range values **clamp to an end** rather than falling back to the default, so a typo cannot silently land on the default. The default flips to `0` in the first runtime release after `displayxr-common` and the five `displayxr-demo-*` demos ship the alias submission — the trigger is that shipment, not a date. Not a user-facing setting: it is a compat bisect knob with no performance dimension. Unreachable from a CTS run at any value (a conformance session never enables `XR_DXR_display_info`, which the compat arm requires). Distinct from `DXR_VIEW_CONFIG_LEGACY`, which restores the pre-#1486 *reporting*. See [ADR-041](../adr/ADR-041-fixed-view-count-with-per-frame-activity.md) and [View-Configuration Model](../reference/view-configuration-model.md) | +| `DXR_STEREO_CAMERA_DEV_ALLOW` | `ipc/server/ipc_server_stereo_camera.c` (`authorise_locked`, `DEBUG_GET_ONCE_BOOL_OPTION`) | off | Svc | ADR-043 **R1-only** stand-in for camera consent: stream start and calibration reads are REFUSED (`XR_ERROR_PERMISSION_INSUFFICIENT`) unless it is set in the SERVICE environment. Deleted when R3 lands the OS + DisplayXR consent path. Never a setting | | `DXR_IPC_FD` | `auxiliary/util/u_sandbox.c:181`; `ipc/client/ipc_client_connection.c:211, 234` | unset | Both / App | #1056 adopts an embedder-supplied service socket. Not a setting | | `DXR_IPC_HANDLE` | `ipc/client/ipc_client_connection.c:437, 456` | unset | App | Windows analogue of the above | | `DXR_WL_ORIGIN_STAMP` | `compositor/vk_native/comp_vk_native_compositor.c` (`vk_debug_origin_stamp`, Linux + Wayland) | off | App | #1748 measurement hook: writes a barcode of each frame's present origin into its top-left 352x8 device px — it **overwrites content**. Read back by the extension's `DISPLAYXR_STAMP_AUDIT=1` (shell environment) in `scripts/linux/move_sync/`. Never for a real session | diff --git a/docs/roadmap/stereo-camera-source.md b/docs/roadmap/stereo-camera-source.md index 5026f11648..d1854004a3 100644 --- a/docs/roadmap/stereo-camera-source.md +++ b/docs/roadmap/stereo-camera-source.md @@ -1,6 +1,7 @@ # Stereo camera source — browser integration, vendor plug-in notes, phased plan -**Status:** design (2026-09-25). Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). +**Status:** design (2026-09-25); **R1 implemented** on `feat/stereo-camera-r1` (2026-09-26) — +see §E; maintainer decisions for R1 in §G. Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). Extension: [`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md). Consumer driving it: the web SDK's 3D call module (`@displayxr/inline3d/call`, RFC 0002 on `displayxr-web` branch `feat/call-p1`, §4 *Capture*). @@ -94,32 +95,82 @@ process across milestones): ## C. Android -### C.1 Runtime + plug-in - -On the tablets the front stereo pair (15 mm and 25 mm baselines on the two current models, -parallel) belongs to the vendor's head-tracking service; the plug-in (`drv_leia_android`) talks -to it through the vendor camera SDK (CNSDK), which today gives it **face/eye data, not images**. - -- **If CNSDK can share frames** (an AHardwareBuffer or buffer stream of the pair the tracker - already captures, with the pair's calibration): the Android plug-in implements the same slots; - the runtime service's camera thread, rings (memfd), consent (CAMERA permission of the peer uid + - runtime-app consent) and visibility rule (`android_package_is_visible`) are the platform-neutral - code of spec §5–7. This is the target, and the **first open question to the vendor**. -- **If it cannot yet — documented platform difference:** Android enumerates no runtime stereo - camera. We do **not** have the plug-in or the browser open the front cameras through Camera2 - while tracking runs: Camera2 evicts the lower-priority client, which is the tracker, and the - weave would lose viewer tracking for the whole call. Pages see the tablet's ordinary cameras via - `getUserMedia`; the call module sends mono (lifted on the receiver). -- Frames on this path are likely **colour and larger** (per-eye 1280×720 class), which is where - the optional `AHARDWAREBUFFER` transport earns its keep. +### C.0 Field facts (NP02J / K68 tablet, measured 2026-09-26) + +**Camera topology.** Camera2 reports six ids on the tablet: 0 and 2 are back lenses, 1 and 3 front +lenses, and **4 = back and 5 = front are LOGICAL multi-cameras — the stereo pairs**. The logical +ids are hidden from `getCameraIdList()` and are opened by hard-coded id. Physical lens per eye: +front L = 3, R = 1; back L = 2, R = 0. + +**The browser cannot reach them** (DisplayXR/displayxr-browser-pvt#175): `enumerateDevices()` +lists no video inputs, `getUserMedia` only opens the front single camera, and +`facingMode: 'environment'` fails with `NotFoundError`. + +**How the vendor's own app gets stereo — and what CNSDK is, and is not.** The vendor app uses +`LeiaCameraBuilder(ctx).setStereoMode(true).setFront(b).setConvergenceMode(AUTO) +.setPreviewSurfaceView(surfaceTexture)` and receives a ready SBS image (2560×720) in a +`SurfaceTexture`. **`LeiaCameraBuilder` is not part of CNSDK:** it is a separate, private camera +wrapper over plain Camera2. It opens the one logical camera (5 front / 4 back) with an output per +physical lens, and its own shader does the SBS pack and the rectification. **CNSDK supplies only +the calibration** (per-camera rotations Rx/Ry/Rz from its display-config service, config v4.0; +the `/sdcard/rect.js` file the vendor app also reads is absent on the unit) **and in-app face +tracking.** + +**What the pair delivers.** Two separate 1280×720 NV12/YUV streams, one per lens, at exactly +30.00 fps, hardware-synchronised (identical timestamps per pair), about 97–100 ms from exposure to +the app. The frames arrive **raw**: not side-by-side, not rectified, not mirrored. For viewer +left/right the front eyes need mirroring. The residual vertical misalignment is about −2 px +(median), and the roll about 0.09°. + +**Exclusivity — the constraint that decides the design.** Opening the front pair (camera 5) +**evicts the head-tracking service's camera** (it held camera 3 at 10 fps). The tracker then stays +wedged ("Camera has been invalidated") and never reacquires it. The rear pair does not conflict. +The vendor's answer is **in-app tracking**: the capturing process feeds a third stream (640×480, +left lens) to CNSDK in-app mode. CNSDK's frame hand-off runs face detection **on the caller's +thread** (~119 ms). Called on the camera thread, that dragged the pair to 24 fps, so it must run +on a worker behind a latest-frame slot. + +**Windows, same week:** on two SR machines the eye tracker holds the stereo camera exclusively +(`getUserMedia` → `NotReadableError`). The design (plug-in source, service owner) follows from +that on both platforms. + +### C.1 Runtime + plug-in — design consequence + +On Android the provider lives **inside the runtime APK** (the vendor plug-in ships there, ADR-038). +It is not Chromium, and it is not a separate app. Because the front pair and the tracker cannot +coexist as two Camera2 clients: + +- **The runtime service is the ONE owner of the front pair, and it runs tracking in-process from + that same capture.** Three outputs of the logical camera: the two 1280×720 NV12 eye streams, and a + 640×480 left-lens stream that goes to CNSDK in-app tracking on a **worker thread behind a + latest-frame slot**, never on the camera thread. The runtime's eye tracking on the tablet then + comes from this capture, and camera streams to clients are passengers on it (the Windows model, + reversed: there the tracker owns the device and the plug-in reads its frames). +- **Nothing else may open the pair behind the tracker's back**: not a page, not another plug-in, not + an app through Camera2. Doing so evicts the capture that tracking depends on, and the display + loses viewer tracking for the whole call. +- **Transport:** AHardwareBuffer per eye (NV12). A 2×1280×720 pair at 30 Hz is ≈ 83 MB/s through a + CPU ring, so this is where the optional `AHARDWAREBUFFER` transport (spec §5.3) becomes the + default. Rectification runs in the runtime (R2's `u_stereo_rectify`, fed with the CNSDK + rotations) or in the consumer from the calibration. +- **Contract gaps this exposes (A1):** the R1 plug-in frame is ONE side-by-side image in CPU + memory. The tablet produces two per-lens hardware buffers that need mirroring. A1 either packs + them in the plug-in (a GPU blit into one SBS AHB, un-mirroring on the way), or appends a + per-eye layout to `xrt_plugin_stereo_camera_frame` (ADR-020 append-only: a layout field + per-eye + AHB handles). The second is preferred. The calibration slot stays as is: CNSDK's per-camera + rotations become `R`, and the baseline comes from the display config. +- Consent: CAMERA permission of the peer uid + the runtime app's consent, and + `android_package_is_visible` for the foreground rule (spec §7), are the platform-neutral code R3 + adds. ### C.2 Browser -The browser's Android arm is in the patch series (runtime fd brokered by the browser-process Java -via `DXR_IPC_FD`, patches 0083+). The camera lands on top of it the same way as Windows: a -DisplayXR device factory composed in front of `VideoCaptureDeviceFactoryAndroid`, fed over the -existing runtime connection, NV12 from the shared-memory ring. Chromium's own Android camera -permission request covers the uid check the runtime makes. +The browser's Android arm is in the patch series (the runtime fd is brokered by the +browser-process Java via `DXR_IPC_FD`, patches 0083+). The camera lands on top of it the same way +as Windows: a DisplayXR device factory composed in front of `VideoCaptureDeviceFactoryAndroid`, fed +over the existing runtime connection (AHB transport). Chromium's own Android camera permission +request covers the uid check the runtime makes. It fixes browser-pvt#175 for the stereo pair +without the browser ever opening camera 5. ## D. Leia SR plug-in (Windows) — sourcing notes @@ -142,17 +193,43 @@ The implementation belongs in `displayxr-leia-plugin` (`src/drv_leia/`), per ADR | Phase | Repo | Scope | Exit criterion | |---|---|---|---| | **R0** | runtime | This design: ADR-043, spec, this page | Maintainer sign-off on the open questions below | -| **R1** | runtime | Header `XR_DXR_stereo_camera.h` + `index.json` note (together — the catalog lint requires both); `xrt_plugin_iface` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (appended after `create_dp_d3d11_lift`, so **lands after ADR-042's PR**); platform-neutral camera manager (thread, refcount + linger, 3-slot pinned ring, per-stream wake handles, decimation, NV12/BGRA conversion); IPC + OpenXR entry points; sim_display fake; `displayxr-cli camera list/calib/probe`; selftest check | `camera probe` on the sim fake writes frames on Windows + Linux CI; ring/pinning unit tests (mirror `tests_lift_mailbox`) | +| **R1** ✅ | runtime | **Implemented 2026-09-26 (see "R1 as built" below).** Header `XR_DXR_stereo_camera.h` + `index.json` note (together — the catalog lint requires both); `xrt_plugin_iface` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (appended after `create_dp_d3d11_lift`, so **lands after ADR-042's PR**); platform-neutral camera manager (thread, refcount + linger, 3-slot pinned ring, per-stream wake handles, decimation, NV12/BGRA conversion); IPC + OpenXR entry points; sim_display fake; `displayxr-cli camera list/calib/probe`; selftest check | `camera probe` on the sim fake writes frames on Windows + Linux CI; ring/pinning unit tests (mirror `tests_lift_mailbox`) | | **R2** | runtime | `u_stereo_rectify` (Bouguet, valid-region crop, LUT remap) with golden tests against OpenCV-generated fixtures; RECTIFIED output | sim fake: row error < 0.5 px, disparity = f·B/Z within 0.5 px | | **R3** | runtime | Privacy: OS consent check, consent store + tray prompt, delegating-client registration, visibility/lock suspension, indicator, kill switches | Manual matrix: allow/deny/revoke, OS switch off, lock screen, background app — each blocks frames | | **L1** | leia-plugin (Windows) | Slots over the SR raw-camera channel per §D; calibration by active serial; keep-alive; repin runtime (feature-macro repin, `downstream-pins.json` `features` track) | On a panel box, incl. the multi-folder box: `camera probe --rectified` rows aligned; the vendor call app running at the same time still gets every frame; tracking/weave unaffected (frame-time + tracking-state logs) | | **B1** | browser (Windows) | §B device factory + duplicate hiding + hint + delegating registration in the installer; web SDK: prefer the hint in `camera:'auto'`, fill `hello` from it | RFC 0002 P1 call between two panel laptops sends rectified SBS from the tracker camera **while both are tracking** | -| **A1** | leia-plugin (Android) + runtime | Only if CNSDK shares frames (§C.1): Android slots, memfd/AHB transport, Android consent + visibility | `camera probe` on the tablet via the runtime app | +| **A1** | leia-plugin (Android) + runtime | §C.1: the runtime owns the front pair and runs CNSDK in-app tracking from the same capture (worker thread, latest-frame slot); per-eye AHB transport + per-eye frame layout; Android consent + visibility | `camera probe` on the tablet via the runtime app **while the weave tracks from the same capture**; pair at 30.00 fps (not 24) | | **B2** | browser (Android) | §C.2 | Tablet ↔ laptop 3D call, both directions stereo | R1–R3 need no hardware; L1 and B1 can proceed in parallel once R1's header exists (B1 develops against the sim fake). +### R1 as built + +- **Header + catalog:** `openxr/XR_DXR_stereo_camera.h`, type values `1004999290–300` (the registry + holds a reserved row for lift's `270–289`); `index.json` note in the `capture` group. +- **Plug-in slots:** six `stereo_camera_*` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA`, after a + one-pointer placeholder for lift's `create_dp_d3d11_lift`. **Merge order: after the lift PR** + (`feat/lift-ext`); the rebase replaces the placeholder with lift's field in place. The offsets are + identical either way, and `tests_stereo_camera` asserts the adjacency. +- **Service:** `ipc/server/ipc_server_stereo_camera.c`. It is platform-neutral (Windows D3D11 + service, macOS/Linux service, Android service). One thread per open camera, open on the first + start, 2 s linger, per-stream 3-slot rings in shared memory, conversion + decimation, and a + per-stream wake (Windows: an auto-reset event, handed out SYNCHRONIZE-only; POSIX: a pipe). + Windows hands the section out read-only (`FILE_MAP_READ` duplicate). +- **IPC:** 12 `stereo_camera_*` calls in `proto.json`. Streams are owned by the connection. +- **OpenXR:** nine entry points. An in-process instance enumerates zero. +- **sim_display fake:** `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` (spec §10). +- **CLI:** `displayxr-cli camera list | calib | probe`; `selftest` gains `stereo_camera_caps`. +- **Privacy hooks (deny by default):** `DXR_STEREO_CAMERA_DEV_ALLOW=1` in the service environment + is the only way to start a stream or read calibration in R1. `DXR_STEREO_CAMERA=0` is the kill + switch. RAW is refused to `PRESENT_OWNER` (the browser). +- **Deferred beyond R1:** state-change events (structs defined, not delivered — they need a + server→client instance-event channel), the GPU transports (designed in the manager's file + comment: D3D11 texture ring + fence as the `XR_DXR_weave` output export; AHB / dma-buf ring + + sync_file as the #1699 weave path), a read-only POSIX section, and a distorted/misaligned fake + (all with R2/R3/A1). + ## F. Risks - **Undocumented vendor channel.** The raw-camera shared memory is a reader-defined contract with @@ -170,10 +247,38 @@ against the sim fake). hunk, and the device + factory are new files, to keep rebase conflicts small. - **Latency/sync.** Arrival timestamps hide tracker-internal latency; A/V sync relies on WebRTC's own jitter handling. Fine for calls, not for measurement. -- **ABI merge order.** The iface slots append after ADR-042's `create_dp_d3d11_lift`; merging out - of order would put two branches on the same offset. - -## G. Open questions for the maintainer +- **ABI merge order.** The iface slots append after ADR-042's `create_dp_d3d11_lift`. R1 holds + lift's offset with a same-size placeholder, so an out-of-order merge still yields the right + offsets, but the intended order is lift first and the camera PR rebased onto it. +- **Android: the pair and the tracker cannot share Camera2.** Opening the front logical camera + evicts the vendor tracker, which then never reacquires (§C.0). The mitigation is architectural: + the runtime owns both, and tracking runs in-process from the same capture (§C.1). It adds + CNSDK's in-app face detection (~119 ms per hand-off) to the runtime's CPU budget on a worker. + **Vendor asks (CNSDK / head-tracking service):** (1) the head-tracking service must reacquire its + camera when another client releases it; today it stays wedged ("Camera has been invalidated") + until reboot. (2) Can the tracker run from the pair's own frames (the 1280×720 eye streams, or a + downscaled copy), so that no third stream is needed? (3) Expose the pair's calibration through a + documented CNSDK call, rather than the display-config service plus an absent `/sdcard/rect.js`. + +## G. Open questions & maintainer decisions + +**Decided for R1 (maintainer defaults, 2026-09-26):** + +- **Q1 → stereo-only for now.** Names and struct shapes stay open for other camera kinds + (`XrStereoCameraPropertiesDXR::viewCount`, always 2 in v1, and next chains). +- **Q2 → plug-in iface slots** behind `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (ADR-020 append-only). + A separate camera-provider plug-in type waits for a camera vendor that is not a display vendor. +- **Q3 → service clients only.** In-process instances enumerate zero; there is no side IPC + connection. +- **Q9 → raw frames are not exposed to pages in R1.** Pages get RECTIFIED, or a RAW frame that + says so (`frame.output`) until R2's rectifier lands. The service refuses a RAW stream to + `PRESENT_OWNER` clients (the browser). +- **Q6 is answered by the field facts (§C.0):** CNSDK has no frames to share. The pair is plain + Camera2 and the vendor's camera wrapper is a separate library, so on Android the runtime owns + the pair and runs tracking from it (§C.1). The remaining CNSDK asks are in §F. +- Q4, Q5, Q7 and Q8 remain open (consent UX ownership, SR SDK asks, browser topology, hint shape). + +**Original questions:** 1. **Name and scope.** `XR_DXR_stereo_camera` (stereo only, SBS output) — or a broader `XR_DXR_camera_source` that also carries mono tracker cameras? The design is stereo-only; a diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index a9192ddd77..44d6418fae 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -3,10 +3,11 @@ | Field | Value | |---|---| | **Extension Name** | `XR_DXR_stereo_camera` | -| **Spec Version** | 1 (design) | +| **Spec Version** | 1 | | **Extension Type** | Instance extension, service path only (an in-process instance enumerates zero cameras) | -| **Header** | `src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h` — **not written yet.** It lands with the first implementation PR, together with its `docs/specs/extensions/index.json` note (the catalog lint joins notes to headers, so neither can land alone) | -| **Status** | **Design** — not implemented. Provisional type-value block proposed: `1004999290–299` (after `XR_DXR_lift`'s `1004999270–280`), pending Khronos registry | +| **Header** | [`src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h`](../../../src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h) (+ its `index.json` catalog note) | +| **Status** | **R1 implemented** (runtime, hardware-free): header, plug-in slots, service camera manager, IPC, OpenXR entry points, sim_display fake, `displayxr-cli camera`, selftest check. **Not yet:** rectification (R2), consent / indicator / foreground rule (R3 — deny-by-default hooks in place), GPU transports, state-change events, the Leia provider (L1), the browser (B1). Provisional type values `1004999290–300` (after `XR_DXR_lift`'s `1004999270–289`), pending Khronos registry | +| **R1 decisions** | stereo-only (names/structs kept open: `viewCount`, next chains) · camera provider = `xrt_plugin_iface` slots · service clients only (in-process enumerates zero) · raw frames never reach web pages (`RAW` refused to `PRESENT_OWNER` clients) — see [roadmap §G](../../roadmap/stereo-camera-source.md#g-open-questions--maintainer-decisions) | | **Decision record** | [ADR-043](../../adr/ADR-043-stereo-camera-source.md) | | **Plug-in contract** | appended `xrt_plugin_iface` camera slots, `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (§9) | | **Browser / Android integration + plan** | [roadmap/stereo-camera-source.md](../../roadmap/stereo-camera-source.md) | @@ -61,6 +62,12 @@ if (xrAcquireStereoCameraFrameDXR(stream, &f) == XR_SUCCESS) { /* f.slot is pinn | Service with a camera source | one entry per camera, with its `state` (§3) | | `DXR_STEREO_CAMERA=0` in the **service's** environment, or the user kill switch (§7.4) | count 0 — indistinguishable from "no camera", on purpose | +**R1 dev gate.** Until the consent path (§7.1, R3) exists, the service refuses every +`xrStartStereoCameraStreamDXR` and `xrGetStereoCameraCalibrationDXR` with +`XR_ERROR_PERMISSION_INSUFFICIENT` **unless `DXR_STEREO_CAMERA_DEV_ALLOW=1` is set in the +service's environment** — deny by default. Enumeration works without it (it reveals no +calibration and no serial). + Instance-level, not session-level: a capture component (a browser's video-capture service) has no compositor session and needs none. Calls travel on the instance's IPC connection; streams belong to that connection and die with it. @@ -289,6 +296,15 @@ When `SHARED_WITH_EYE_TRACKING` is set: ## 7. Privacy +> **R1 status.** Only the hook points exist, all deny-by-default, in +> `src/xrt/ipc/server/ipc_server_stereo_camera.c`: `authorise_locked()` (§7.1 — R1 allows only +> with `DXR_STEREO_CAMERA_DEV_ALLOW=1` in the service environment), `client_visible_locked()` +> (§7.2 — always true in R1), `output_allowed()` (RAW refused to `PRESENT_OWNER`, i.e. the +> browser, per the R1 decision), the `DXR_STEREO_CAMERA=0` kill switch (§7.4), and one WARN per +> stream start/stop naming the peer executable in place of the indicator (§7.3). `persistentId` +> (§7.5) is an unkeyed 128-bit hash of (device identity, peer executable) in R1; R3 keys it with a +> persisted per-user secret. + The frames bypass the OS camera stack (the tracker, not the consumer, opened the device), so OS camera permissions and in-use indicators do not see this consumer. The runtime therefore enforces the equivalent itself. All checks are made on the **OS-derived peer** of the connection @@ -350,7 +366,17 @@ passed §7.1 for that camera. A browser must coarsen what it gives pages (roadma ## 9. Plug-in contract (ADR-020 append-only) Appended to `struct xrt_plugin_iface` (after `create_dp_d3d11_lift`, ADR-042), gated by -`struct_size`, announced by `#define XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA 1`, no ABI bump. On the +`struct_size`, announced by `#define XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA 1`, no ABI bump. +**As implemented:** lift is not on `main` yet, so the slots follow a one-pointer placeholder, +`reserved_adr042_create_dp_d3d11_lift`, at lift's offset; the lift PR replaces it in place. +Either merge order yields the same offsets (`tests_stereo_camera` asserts it). The authoritative +header is `src/xrt/include/xrt/xrt_plugin.h`; the plug-in-facing rules are in +[`docs/reference/xrt_plugin_iface.md`](../../reference/xrt_plugin_iface.md#a-stereo-camera-the-stereo_camera_-slots-adr-043). +Two refinements over the sketch below: `wait_frame` returns an `enum xrt_plugin_stereo_camera_wait` +(`OK` / `TIMEOUT` / `SUSPENDED` / `ERROR`) rather than an `xrt_result_t`, so no new +`xrt_result_t` values were needed; and the extrinsics are **OpenCV's `x_R = R·x_L + T`** (T in +mm, `T = (−B, 0, 0)` for a right camera at +x) — the service converts them to the client's +`rightFromLeft` pose (the right camera in the left camera's frame, metres). On the **plug-in iface, not a display-processor vtable**: a camera is a sensor, not a weaver; it must be graphics-API-neutral (Windows D3D11 service, Android Vulkan runtime) and must not share a lifetime with a DP that is recreated on presenter changes. @@ -416,6 +442,15 @@ owns threads, fan-out, rectification, format conversion, transport, consent and ## 10. sim_display fake +> **R1 as built** (`drivers/sim_display/sim_display_stereo_camera{,_pattern}.c`): the pair is +> **undistorted and aligned** and the camera reports `NATIVELY_RECTIFIED` (+ `CALIBRATED`, +> `SHARED_WITH_EYE_TRACKING`, `USER_FACING`, and `MONOCHROME` for gray8), so the RECTIFIED output +> path runs end to end today. The scene is a random-dot stereogram: background at 2.0 m and a bar +> at 0.6 m, which at the defaults (640 px per eye, 68° HFOV → fx 474.4 px, 50 mm) are **12 px and +> 40 px** of disparity, plus an 8-digit frame counter in both halves. Knobs, read by the SERVICE: +> `SIM_DISPLAY_FAKE_STEREO_CAMERA=1`, `_SIZE=WxH` (per eye), `_FPS=N`, `_FORMAT=gray8|nv12|bgra`, +> `_SUSPEND_PERIOD_MS=N`. The distorted + misaligned variant below is R2's, with the rectifier. + `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` makes sim_display advertise one camera (`SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | MONOCHROME`, 640×480 per eye, 30 Hz, baseline 50 mm) producing a **synthetic SBS pattern**: a textured plane and a bar at known depths, @@ -435,8 +470,20 @@ The whole path (enumerate → consent → start → acquire → rectify) then ru ``` displayxr-cli camera list [--json] # properties + state of every camera displayxr-cli camera calib [--raw|--rectified] [--json] -displayxr-cli camera probe [--raw] [--format gray8|nv12|bgra8] [--seconds S] [--out DIR] - # writes cam_.png, prints rate, drops, latency +displayxr-cli camera probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] + [--frames N] [--seconds S] [--out DIR] + # rate, index gaps, SBS layout, block + # disparities, service stats; --out writes cam_.png +``` + +As built in R1, e.g. against the sim fake on macOS: + +``` +stream 1 on camera 1: 1280x480 NV12 SBS (RECTIFIED), cap 30.0 Hz, ring 3 x 921600 B (section 2764800 B) +received 150 frames (index 1..150, 0 source frames not delivered to this stream), 150 wakes, 0 not-ready +measured delivery rate: 29.95 Hz over 4.98 s +block disparity (left x - right x): 12 px (38/54 blocks), 40 px (16/54 blocks), range 12..40 px +service stats: source 30.49 Hz, delivered 30.49 Hz, published 155, skipped 4, acquired 150, mean latency 12.417 ms ``` Runs as a DIAG IPC client, non-elevated. `probe` goes through the full consent path (so its @@ -457,4 +504,4 @@ camera state change and per stream start/stop with the peer executable; INFO sta | Version | Change | |---|---| -| 1 (design) | Enumerate + state events, calibration (raw / rectified), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, optional GPU transports, runtime-enforced consent / foreground / indicator, plug-in iface slots, sim_display fake, CLI. | +| 1 | R1 implements: enumerate, calibration (RAW; RECTIFIED for natively rectified sources), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, stream stats, plug-in slots, sim_display fake, CLI, selftest. Deferred: state-change events (structs defined, not delivered), GPU transports, rectifier (R2), consent/indicator (R3). Design scope: Enumerate + state events, calibration (raw / rectified), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, optional GPU transports, runtime-enforced consent / foreground / indicator, plug-in iface slots, sim_display fake, CLI. | From 250a2f0c19855b119be0098008c1a60ee93541e2 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 10:59:59 -0700 Subject: [PATCH 12/37] fix(stereo-camera): no cross-enum comparisons in the encoding pins (MSVC C5287) MSVC /W4 raised C5287 ("operands are different enum types") on the XrStereoCamera*DXR vs xrt_stereo_camera_* _Static_asserts, even through the (int) casts. Pin each side against the header's literal instead: two enum-vs-literal checks carry the same proof with no cross-enum operator. Co-Authored-By: Claude Opus 5.5 --- .../state_trackers/oxr/oxr_stereo_camera.c | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c index d1dfea8160..5664d170c6 100644 --- a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c +++ b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c @@ -58,17 +58,22 @@ _Static_assert(XRT_PLUGIN_STEREO_CAMERA_USER_FACING == 0x2, "flag mirror"); _Static_assert(XRT_PLUGIN_STEREO_CAMERA_CALIBRATED == 0x4, "flag mirror"); _Static_assert(XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED == 0x8, "flag mirror"); _Static_assert(XRT_PLUGIN_STEREO_CAMERA_MONOCHROME == 0x10, "flag mirror"); -_Static_assert((int)XR_STEREO_CAMERA_STATE_AVAILABLE_DXR == (int)XRT_STEREO_CAMERA_STATE_AVAILABLE, "state"); -_Static_assert((int)XR_STEREO_CAMERA_STATE_WAITING_DXR == (int)XRT_STEREO_CAMERA_STATE_WAITING, "state"); -_Static_assert((int)XR_STEREO_CAMERA_STATE_SUSPENDED_DXR == (int)XRT_STEREO_CAMERA_STATE_SUSPENDED, "state"); -_Static_assert((int)XR_STEREO_CAMERA_STATE_UNAVAILABLE_DXR == (int)XRT_STEREO_CAMERA_STATE_UNAVAILABLE, "state"); -_Static_assert((int)XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR == (int)XRT_STEREO_CAMERA_OUTPUT_RECTIFIED, "output"); -_Static_assert((int)XR_STEREO_CAMERA_OUTPUT_RAW_DXR == (int)XRT_STEREO_CAMERA_OUTPUT_RAW, "output"); -_Static_assert((int)XR_STEREO_CAMERA_FORMAT_GRAY8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_GRAY8, "format"); -_Static_assert((int)XR_STEREO_CAMERA_FORMAT_NV12_DXR == (int)XRT_STEREO_CAMERA_FORMAT_NV12, "format"); -_Static_assert((int)XR_STEREO_CAMERA_FORMAT_BGRA8_DXR == (int)XRT_STEREO_CAMERA_FORMAT_BGRA8, "format"); -_Static_assert((int)XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR == (int)XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY, - "transport"); +// The enum mirrors are pinned the same way, each side against the literal on +// its own: a direct XR-vs-xrt comparison is "operands are different enum types" +// (MSVC C5287 at /W4), and MSVC's C front end raises it even through (int) +// casts. Two enum-vs-literal checks carry the same proof with no cross-enum op. +#define SCAM_PIN(xr, xrt, v) _Static_assert((xr) == (v) && (xrt) == (v), #xr " != " #xrt) +SCAM_PIN(XR_STEREO_CAMERA_STATE_AVAILABLE_DXR, XRT_STEREO_CAMERA_STATE_AVAILABLE, 1); +SCAM_PIN(XR_STEREO_CAMERA_STATE_WAITING_DXR, XRT_STEREO_CAMERA_STATE_WAITING, 2); +SCAM_PIN(XR_STEREO_CAMERA_STATE_SUSPENDED_DXR, XRT_STEREO_CAMERA_STATE_SUSPENDED, 3); +SCAM_PIN(XR_STEREO_CAMERA_STATE_UNAVAILABLE_DXR, XRT_STEREO_CAMERA_STATE_UNAVAILABLE, 4); +SCAM_PIN(XR_STEREO_CAMERA_OUTPUT_RECTIFIED_DXR, XRT_STEREO_CAMERA_OUTPUT_RECTIFIED, 1); +SCAM_PIN(XR_STEREO_CAMERA_OUTPUT_RAW_DXR, XRT_STEREO_CAMERA_OUTPUT_RAW, 2); +SCAM_PIN(XR_STEREO_CAMERA_FORMAT_GRAY8_DXR, XRT_STEREO_CAMERA_FORMAT_GRAY8, 1); +SCAM_PIN(XR_STEREO_CAMERA_FORMAT_NV12_DXR, XRT_STEREO_CAMERA_FORMAT_NV12, 2); +SCAM_PIN(XR_STEREO_CAMERA_FORMAT_BGRA8_DXR, XRT_STEREO_CAMERA_FORMAT_BGRA8, 3); +SCAM_PIN(XR_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY_DXR, XRT_STEREO_CAMERA_TRANSPORT_SHARED_MEMORY, 1); +#undef SCAM_PIN _Static_assert(XR_STEREO_CAMERA_RING_SLOTS_DXR == XRT_STEREO_CAMERA_RING_SLOTS, "ring slots"); _Static_assert(XR_STEREO_CAMERA_PERSISTENT_ID_MAX_SIZE_DXR == sizeof(((struct xrt_stereo_camera_properties *)0)->persistent_id), From 56bd0c42b38ff84e059b87292c970b91feaed6f9 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 10:59:59 -0700 Subject: [PATCH 13/37] feat(stereo-camera): report the MEASURED source rate, 0 = unknown, never a guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Field test (Leia SR tracking camera): the descriptor said "up to 30.0 Hz" while the source ran 58-64 Hz — the plug-in's enumerate-time rate was a hard-coded placeholder. The service now measures every source over the first 60 frames after each open (u_stereo_camera_rate_meter; a >0.5 s gap restarts the window) and the descriptor, the decimator and the stream info use that; until a window completes they use the plug-in's value, where 0 now means "unknown" (xrt_plugin doc, XR header doc, spec). One WARN on the first measurement, INFO after. - selftest: max_frame_rate 0 is legal; negative / NaN / > 1000 is malformed. - camera list prints "rate unknown (measured once a stream runs)"; probe prints "cap = source rate" instead of "cap 0.0 Hz". - sim fake: SIM_DISPLAY_FAKE_STEREO_CAMERA_ADVERTISED_FPS (0 = unknown) to exercise the measured path; tests_stereo_camera covers the meter (jitter, suspension gap, restart keeps the last value). Co-Authored-By: Claude Opus 5.5 --- docs/specs/extensions/XR_DXR_stereo_camera.md | 11 ++-- .../openxr/XR_DXR_stereo_camera.h | 3 +- src/xrt/auxiliary/util/u_stereo_camera.c | 34 ++++++++++++ src/xrt/auxiliary/util/u_stereo_camera.h | 36 +++++++++++++ .../sim_display/sim_display_stereo_camera.c | 8 ++- src/xrt/include/xrt/xrt_plugin.h | 7 ++- src/xrt/ipc/server/ipc_server_stereo_camera.c | 54 ++++++++++++++++--- src/xrt/targets/cli/cli_cmd_camera.c | 22 ++++++-- src/xrt/targets/cli/cli_query.c | 10 ++-- tests/tests_stereo_camera.cpp | 37 +++++++++++++ 10 files changed, 201 insertions(+), 21 deletions(-) diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index 44d6418fae..da8f8cc2ab 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -101,7 +101,8 @@ typedef struct XrStereoCameraPropertiesDXR { XrStereoCameraFlagsDXR flags; XrStereoCameraStateDXR state; XrExtent2Di eyeExtent; // native per-eye size (e.g. 640x480) - float maxFrameRate; // what the source can deliver, Hz + float maxFrameRate; // Hz the source delivers: MEASURED by the service after its + // first open, else the vendor's value; 0 = not known yet float baselineMm; // |T| of the pair; 0 if uncalibrated float horizontalFovDeg; // per eye, rectified; 0 if uncalibrated XrStereoCameraFormatFlagsDXR supportedFormats; // GRAY8 / NV12 / BGRA8 bits the service will produce @@ -389,7 +390,7 @@ struct xrt_plugin_stereo_camera_info { char platform_device_hint[256]; // OS id of the physical device read, or "" uint32_t flags; // XRT_PLUGIN_STEREO_CAMERA_* (shared-with-tracking, user-facing, calibrated, rectified, mono) uint32_t eye_width, eye_height; - float max_frame_rate; + float max_frame_rate; // only if KNOWN (vendor value / measured earlier); 0 = unknown, never a guess uint32_t native_format; // GRAY8 / NV12 / BGRA8 — the plug-in decodes its transport (e.g. JPEG) itself }; @@ -449,7 +450,8 @@ owns threads, fan-out, rectification, format conversion, transport, consent and > at 0.6 m, which at the defaults (640 px per eye, 68° HFOV → fx 474.4 px, 50 mm) are **12 px and > 40 px** of disparity, plus an 8-digit frame counter in both halves. Knobs, read by the SERVICE: > `SIM_DISPLAY_FAKE_STEREO_CAMERA=1`, `_SIZE=WxH` (per eye), `_FPS=N`, `_FORMAT=gray8|nv12|bgra`, -> `_SUSPEND_PERIOD_MS=N`. The distorted + misaligned variant below is R2's, with the rectifier. +> `_SUSPEND_PERIOD_MS=N`, `_ADVERTISED_FPS=N` (what enumerate claims, default = `_FPS`; `0` = unknown — +> exercises the service's measured-rate path). The distorted + misaligned variant below is R2's, with the rectifier. `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` makes sim_display advertise one camera (`SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | MONOCHROME`, 640×480 per eye, 30 Hz, @@ -457,6 +459,9 @@ baseline 50 mm) producing a **synthetic SBS pattern**: a textured plane and a ba rendered with a known synthetic calibration that includes radial distortion and a small vertical misalignment + roll between the eyes, plus a frame counter burned into each half. So: +- the descriptor's rate is testable: with `_ADVERTISED_FPS=0` the camera lists "rate unknown" until a + stream has run ~60 frames, then reports the measured rate (field finding: the Leia SR tracking + camera was hard-coded to advertise 30 Hz and actually ran 58–64 Hz); - rectification is testable exactly — after `RECTIFIED`, feature rows must align (|Δy| < 0.5 px) and the bar's disparity must equal `fx·B/Z`; - `SIM_DISPLAY_FAKE_STEREO_CAMERA_SUSPEND_PERIOD_MS=N` square-waves SUSPENDED/AVAILABLE, like the diff --git a/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h index bb635fd762..740871487c 100644 --- a/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h +++ b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h @@ -188,7 +188,8 @@ typedef struct XrStereoCameraPropertiesDXR { uint32_t viewCount; //! Native per-eye size (e.g. 640x480). XrExtent2Di eyeExtent; - //! What the source can deliver, Hz. + //! What the source delivers, Hz: measured by the runtime once the source + //! has run, else the vendor's value; 0 = not known yet. float maxFrameRate; //! |T| of the pair, millimetres; 0 if uncalibrated. float baselineMm; diff --git a/src/xrt/auxiliary/util/u_stereo_camera.c b/src/xrt/auxiliary/util/u_stereo_camera.c index 10f9d35faf..d11af289f3 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.c +++ b/src/xrt/auxiliary/util/u_stereo_camera.c @@ -89,6 +89,40 @@ u_stereo_camera_ring_clear(struct u_stereo_camera_ring *r) } +/* + * + * Source-rate meter. + * + */ + +void +u_stereo_camera_rate_meter_restart(struct u_stereo_camera_rate_meter *m) +{ + m->first_ns = 0; + m->last_ns = 0; + m->intervals = 0; +} + +bool +u_stereo_camera_rate_meter_push(struct u_stereo_camera_rate_meter *m, int64_t t_ns) +{ + if (m->first_ns == 0 || t_ns <= m->last_ns || t_ns - m->last_ns > U_STEREO_CAMERA_RATE_GAP_NS) { + m->first_ns = t_ns; + m->last_ns = t_ns; + m->intervals = 0; + return false; + } + m->last_ns = t_ns; + if (++m->intervals < U_STEREO_CAMERA_RATE_WINDOW) { + return false; + } + m->rate = (float)((double)m->intervals * 1e9 / (double)(m->last_ns - m->first_ns)); + m->first_ns = t_ns; // the next window starts here + m->intervals = 0; + return true; +} + + /* * * Decimator. diff --git a/src/xrt/auxiliary/util/u_stereo_camera.h b/src/xrt/auxiliary/util/u_stereo_camera.h index 95ee0c3692..ab17c84ad2 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.h +++ b/src/xrt/auxiliary/util/u_stereo_camera.h @@ -109,6 +109,42 @@ bool u_stereo_camera_decimator_accept(struct u_stereo_camera_decimator *d, int64_t t_ns); +/* + * + * Source-rate meter: what the source ACTUALLY delivers, not what it advertises. + * + * A plug-in often cannot know its rate at enumerate time (the Leia SR tracking + * camera advertised 30 Hz and ran 58-64 Hz). The service therefore measures + * the first @ref U_STEREO_CAMERA_RATE_WINDOW frames after every open and + * reports that as the descriptor's rate; until a window completes the rate is + * the plug-in's own value, where 0 means "unknown". + * + */ + +#define U_STEREO_CAMERA_RATE_WINDOW 60 //!< intervals per measurement +#define U_STEREO_CAMERA_RATE_GAP_NS (500ll * 1000 * 1000) //!< a longer gap restarts the window + +struct u_stereo_camera_rate_meter +{ + int64_t first_ns; + int64_t last_ns; + uint32_t intervals; + float rate; //!< last completed measurement, Hz; 0 = none yet +}; + +//! Forget the window in progress (a source re-open); keeps the last @ref rate. +void +u_stereo_camera_rate_meter_restart(struct u_stereo_camera_rate_meter *m); + +/*! + * Feed one frame's timestamp. A gap over @ref U_STEREO_CAMERA_RATE_GAP_NS (a + * suspension) restarts the window rather than diluting it. + * @return true exactly when this frame completed a window (@ref rate updated). + */ +bool +u_stereo_camera_rate_meter_push(struct u_stereo_camera_rate_meter *m, int64_t t_ns); + + /* * * Plane layout + conversion (GRAY8 = 1, NV12 = 2, BGRA8 = 3; the diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index 6b1d4f5382..d69a4e916d 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -35,6 +35,7 @@ struct fake_config bool enabled; uint32_t eye_w, eye_h; float fps; + float advertised_fps; //!< what enumerate claims; 0 = "unknown" (the service measures) uint32_t format; int64_t suspend_period_ns; }; @@ -79,6 +80,11 @@ fake_config(void) if (fps != NULL && atof(fps) >= 1.0 && atof(fps) <= 240.0) { cfg.fps = (float)atof(fps); } + cfg.advertised_fps = cfg.fps; + const char *afps = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_ADVERTISED_FPS"); + if (afps != NULL && afps[0] != '\0' && atof(afps) >= 0.0 && atof(afps) <= 240.0) { + cfg.advertised_fps = (float)atof(afps); + } const char *fmt = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_FORMAT"); if (fmt != NULL) { if (strcmp(fmt, "nv12") == 0) { @@ -128,7 +134,7 @@ sim_display_stereo_camera_enumerate(struct xrt_plugin_instance *inst, } info.eye_width = cfg->eye_w; info.eye_height = cfg->eye_h; - info.max_frame_rate = cfg->fps; + info.max_frame_rate = cfg->advertised_fps; info.native_format = cfg->format; // Never write past the runtime's struct_size (ADR-020). memcpy(out, &info, sz < sizeof(info) ? sz : sizeof(info)); diff --git a/src/xrt/include/xrt/xrt_plugin.h b/src/xrt/include/xrt/xrt_plugin.h index ceb4268518..7804744823 100644 --- a/src/xrt/include/xrt/xrt_plugin.h +++ b/src/xrt/include/xrt/xrt_plugin.h @@ -701,7 +701,12 @@ struct xrt_plugin_stereo_camera_info //! Native per-eye size. uint32_t eye_width; uint32_t eye_height; - //! What the source can deliver, Hz. + /*! + * What the source delivers, Hz — ONLY if the plug-in actually knows it + * (a vendor-provided value, or one it measured on an earlier open). 0 = + * unknown; never a placeholder guess. The service measures the real rate + * over the first frames after each open and reports that to apps. + */ float max_frame_rate; //! @ref xrt_plugin_stereo_camera_format the plug-in decodes to. uint32_t native_format; diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index 7e3e31fd25..7c343449c1 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -217,6 +217,11 @@ struct scam_camera int64_t last_frame_ns; int64_t retry_after_ns; float source_rate; + //! Measured over the first frames after each open; what the descriptor + //! reports once known (the plug-in's enumerate-time rate is a guess, and + //! was 2x wrong on the Leia SR tracking camera). + struct u_stereo_camera_rate_meter rate_meter; + bool rate_logged; int64_t last_stats_log_ns; bool thread_started; @@ -492,6 +497,14 @@ stream_allocate_locked(struct scam_stream *s, const struct scam_camera *cam) } +//! The descriptor's rate: measured once known, else the plug-in's (0 = unknown). +static float +source_rate_hz(const struct scam_camera *cam) +{ + return cam->rate_meter.rate > 0.0f ? cam->rate_meter.rate : cam->info.max_frame_rate; +} + + /* * * Camera thread. @@ -514,6 +527,23 @@ publish_locked(struct ipc_server_stereo_camera *m, cam->last_frame_ns = now; set_state_locked(cam, XRT_STEREO_CAMERA_STATE_AVAILABLE); + if (u_stereo_camera_rate_meter_push(&cam->rate_meter, f->time_ns > 0 ? f->time_ns : now)) { + float adv = cam->info.max_frame_rate; + bool off = adv <= 0.0f || fabsf(cam->rate_meter.rate - adv) > 0.1f * adv; + if (!cam->rate_logged) { + cam->rate_logged = true; + U_LOG_W( + "stereo camera %llu: measured source rate %.1f Hz over %d frames (plug-in advertised %.1f " + "Hz%s)%s", + (unsigned long long)cam->camera_id, cam->rate_meter.rate, U_STEREO_CAMERA_RATE_WINDOW, adv, + adv > 0.0f ? "" : " = unknown", + off ? " — the descriptor now reports the measured rate" : ""); + } else { + U_LOG_I("stereo camera %llu: measured source rate %.1f Hz", (unsigned long long)cam->camera_id, + cam->rate_meter.rate); + } + } + if (f->width != 2 * cam->info.eye_width || f->height != cam->info.eye_height) { U_LOG_W("stereo camera %llu: frame %ux%u does not match the advertised %ux%u SBS — dropped", (unsigned long long)cam->camera_id, f->width, f->height, 2 * cam->info.eye_width, @@ -637,6 +667,7 @@ camera_thread(void *ptr) } cam->src = src; cam->last_frame_ns = 0; + u_stereo_camera_rate_meter_restart(&cam->rate_meter); U_LOG_W("stereo camera %llu: source opened", (unsigned long long)cam->camera_id); set_state_locked(cam, XRT_STEREO_CAMERA_STATE_WAITING); } @@ -759,9 +790,20 @@ ipc_server_stereo_camera_create(struct xrt_instance *xinst) cam->info.flags &= ~XRT_PLUGIN_STEREO_CAMERA_CALIBRATED; } } - U_LOG_W("stereo camera %llu: \"%s\" %ux%u per eye @ %.1f Hz, flags 0x%x, native format %u", - (unsigned long long)cam->camera_id, info->display_name, info->eye_width, info->eye_height, - info->max_frame_rate, info->flags, info->native_format); + if (!(info->max_frame_rate >= 0.0f && info->max_frame_rate <= 1000.0f)) { + cam->info.max_frame_rate = 0.0f; // NaN / negative / absurd = unknown + } + char rate[48]; + if (cam->info.max_frame_rate > 0.0f) { + snprintf(rate, sizeof(rate), "advertised %.1f Hz", cam->info.max_frame_rate); + } else { + snprintf(rate, sizeof(rate), "rate unknown"); + } + U_LOG_W( + "stereo camera %llu: \"%s\" %ux%u per eye, %s (measured on first open), flags 0x%x, native " + "format %u", + (unsigned long long)cam->camera_id, info->display_name, info->eye_width, info->eye_height, rate, + info->flags, info->native_format); m->camera_count++; } return m; @@ -899,7 +941,7 @@ ipc_handle_stereo_camera_get_properties(volatile struct ipc_client_state *ics, out_props->view_count = 2; out_props->eye_width = cam->info.eye_width; out_props->eye_height = cam->info.eye_height; - out_props->max_frame_rate = cam->info.max_frame_rate; + out_props->max_frame_rate = source_rate_hz(cam); if (cam->have_calib) { out_props->baseline_mm = (float)baseline_mm(&cam->calib); double fx = cam->calib.k[0][0]; @@ -1115,7 +1157,7 @@ ipc_handle_stereo_camera_stream_start(volatile struct ipc_client_state *ics, uin os_mutex_unlock(&m->lock); return xret; } - u_stereo_camera_decimator_init(&s->dec, s->req.max_frame_rate, cam->info.max_frame_rate); + u_stereo_camera_decimator_init(&s->dec, s->req.max_frame_rate, source_rate_hz(cam)); u_stereo_camera_ring_clear(&s->ring); s->started = true; cam->started_count++; @@ -1181,7 +1223,7 @@ ipc_handle_stereo_camera_stream_get_section(volatile struct ipc_client_state *ic out_layout->output = s->output; out_layout->transport = s->req.transport; out_layout->max_frame_rate = - s->dec.period_ns > 0 ? (float)(1e9 / (double)s->dec.period_ns) : m->cams[s->camera].info.max_frame_rate; + s->dec.period_ns > 0 ? (float)(1e9 / (double)s->dec.period_ns) : source_rate_hz(&m->cams[s->camera]); out_layout->section_size = s->section_size; out_layout->slot_stride = s->slot_stride; out_layout->slot_count = U_STEREO_CAMERA_RING_SLOTS; diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 86e1d5f22b..9e60024c23 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -171,8 +171,14 @@ cmd_list(struct ipc_connection *ipc_c, bool json) state_str(p.state)); printf(" persistentId %s\n", p.persistent_id); printf(" flags %s\n", flags); - printf(" eye %ux%u (SBS %ux%u), %u views, up to %.1f Hz\n", p.eye_width, - p.eye_height, 2 * p.eye_width, p.eye_height, p.view_count, p.max_frame_rate); + char rate[64]; + if (p.max_frame_rate > 0.0f) { + snprintf(rate, sizeof(rate), "%.1f Hz", p.max_frame_rate); + } else { + snprintf(rate, sizeof(rate), "rate unknown (measured once a stream runs)"); + } + printf(" eye %ux%u (SBS %ux%u), %u views, %s\n", p.eye_width, p.eye_height, + 2 * p.eye_width, p.eye_height, p.view_count, rate); printf(" baseline %.2f mm, HFOV %.2f deg per eye\n", p.baseline_mm, p.horizontal_fov_deg); printf(" formats 0x%llx transports 0x%llx platform hint \"%s\"\n", @@ -437,10 +443,16 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) ipc_client_stereo_camera_stream_destroy(ipc_c, sid); return 2; } - printf("stream %llu on camera %llu: %ux%u %s SBS (%s), cap %.1f Hz, ring %u x %llu B (section %llu B)\n", + char cap[48]; + if (lay.max_frame_rate > 0.0f) { + snprintf(cap, sizeof(cap), "cap %.1f Hz", lay.max_frame_rate); + } else { + snprintf(cap, sizeof(cap), "cap = source rate (not measured yet)"); + } + printf("stream %llu on camera %llu: %ux%u %s SBS (%s), %s, ring %u x %llu B (section %llu B)\n", (unsigned long long)sid, (unsigned long long)id, lay.width, lay.height, format_str(lay.format), - lay.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW (flagged)", lay.max_frame_rate, - lay.slot_count, (unsigned long long)lay.slot_stride, (unsigned long long)lay.section_size); + lay.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW (flagged)", cap, lay.slot_count, + (unsigned long long)lay.slot_stride, (unsigned long long)lay.section_size); int64_t t_start = os_monotonic_get_ns(); int64_t deadline = seconds > 0.0f ? t_start + (int64_t)(seconds * 1e9) : t_start + 60ll * 1000000000; diff --git a/src/xrt/targets/cli/cli_query.c b/src/xrt/targets/cli/cli_query.c index 7df10f988b..e3fc04875c 100644 --- a/src/xrt/targets/cli/cli_query.c +++ b/src/xrt/targets/cli/cli_query.c @@ -105,8 +105,9 @@ probe_stereo_camera(struct cli_query_result *r, const struct xrt_plugin_iface *i why = "zero or odd eye extent"; } else if (c->native_format < 1 || c->native_format > 3) { why = "unknown native format"; - } else if (!(c->max_frame_rate > 0.0f)) { - why = "max_frame_rate <= 0"; + } else if (!(c->max_frame_rate >= 0.0f && c->max_frame_rate <= 1000.0f)) { + // 0 is legal: "unknown until measured" (the service measures it). + why = "max_frame_rate negative, NaN or > 1000"; } else if (c->flags & XRT_PLUGIN_STEREO_CAMERA_CALIBRATED) { struct xrt_plugin_stereo_camera_calibration k; memset(&k, 0, sizeof(k)); @@ -127,8 +128,9 @@ probe_stereo_camera(struct cli_query_result *r, const struct xrt_plugin_iface *i } } snprintf(r->stereo_camera_note, sizeof(r->stereo_camera_note), - "%u camera(s); [0] \"%.60s\" %ux%u/eye @ %.1f Hz, flags 0x%x", n, infos[0].display_name, - infos[0].eye_width, infos[0].eye_height, (double)infos[0].max_frame_rate, infos[0].flags); + "%u camera(s); [0] \"%.60s\" %ux%u/eye, advertised %.1f Hz (0 = unknown), flags 0x%x", n, + infos[0].display_name, infos[0].eye_width, infos[0].eye_height, (double)infos[0].max_frame_rate, + infos[0].flags); } #ifdef XRT_OS_WINDOWS diff --git a/tests/tests_stereo_camera.cpp b/tests/tests_stereo_camera.cpp index 689a4690ad..da923e355d 100644 --- a/tests/tests_stereo_camera.cpp +++ b/tests/tests_stereo_camera.cpp @@ -127,6 +127,43 @@ TEST_CASE("stereo camera decimator", "[stereo_camera]") CHECK(with_pause <= 151); } +TEST_CASE("stereo camera rate meter measures, never trusts the advert", "[stereo_camera]") +{ + u_stereo_camera_rate_meter m = {}; + CHECK(m.rate == 0.0f); // unknown until a window completes + // A 60 Hz source with +-2 ms arrival jitter (the SR tracker's shape). + int64_t t = 1000000000; + int completed = 0; + for (int i = 0; i <= U_STEREO_CAMERA_RATE_WINDOW; i++) { + int64_t jitter = (i % 2 == 0 ? 1 : -1) * 2000000; + completed += + u_stereo_camera_rate_meter_push(&m, t + (i == 0 || i == U_STEREO_CAMERA_RATE_WINDOW ? 0 : jitter)) + ? 1 + : 0; + t += 16666667; + } + CHECK(completed == 1); + CHECK(m.rate == Catch::Approx(60.0f).margin(0.05f)); + + // A suspension gap restarts the window instead of dragging the rate down. + u_stereo_camera_rate_meter g = {}; + t = 1000000000; + for (int i = 0; i < 30; i++, t += 33333333) { + u_stereo_camera_rate_meter_push(&g, t); + } + t += 3000000000ll; + bool done = false; + for (int i = 0; i <= U_STEREO_CAMERA_RATE_WINDOW; i++, t += 33333333) { + done = u_stereo_camera_rate_meter_push(&g, t) || done; + } + CHECK(done); + CHECK(g.rate == Catch::Approx(30.0f).margin(0.05f)); + + // Restart (source re-opened) keeps the last measurement. + u_stereo_camera_rate_meter_restart(&g); + CHECK(g.rate == Catch::Approx(30.0f).margin(0.05f)); +} + TEST_CASE("stereo camera layout", "[stereo_camera]") { u_stereo_camera_planes l; From 0163e69b19396896bca2265ee6f6eea32f884e9c Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 11:03:01 -0700 Subject: [PATCH 14/37] fix(stereo-camera): probe disparity search from the camera's geometry, not a fixed 64 px On the Leia SR tracking camera (baseline 120 mm, f ~513 px) a face at 0.6 m is ~95 px of disparity, so every block of the fixed 0..64 px search clamped at 64 and the probe reported "d = 64". The bound is now f*B/0.4 m from the camera's own baseline + HFOV (quarter eye width when uncalibrated; clamped to [64, 0.6*eye]), and a block whose best match sits ON the bound is counted and excluded ("AT THE SEARCH BOUND") instead of silently reported as the bound. sim fake: SIM_DISPLAY_FAKE_STEREO_CAMERA_BASELINE_MM (default 50). At 120 mm the probe reads 28 px (background) / 95 px (bar at 0.6 m), search 0..143. Co-Authored-By: Claude Opus 5.5 --- docs/specs/extensions/XR_DXR_stereo_camera.md | 3 +- .../sim_display/sim_display_stereo_camera.c | 15 +++- src/xrt/targets/cli/cli_cmd_camera.c | 77 +++++++++++++++---- 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index da8f8cc2ab..6671883ca1 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -451,7 +451,8 @@ owns threads, fan-out, rectification, format conversion, transport, consent and > 40 px** of disparity, plus an 8-digit frame counter in both halves. Knobs, read by the SERVICE: > `SIM_DISPLAY_FAKE_STEREO_CAMERA=1`, `_SIZE=WxH` (per eye), `_FPS=N`, `_FORMAT=gray8|nv12|bgra`, > `_SUSPEND_PERIOD_MS=N`, `_ADVERTISED_FPS=N` (what enumerate claims, default = `_FPS`; `0` = unknown — -> exercises the service's measured-rate path). The distorted + misaligned variant below is R2's, with the rectifier. +> exercises the service's measured-rate path), `_BASELINE_MM=N` (default 50; `120` reproduces the Leia SR +> tracking camera: the bar at 0.6 m is ~95 px). The distorted + misaligned variant below is R2's, with the rectifier. `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` makes sim_display advertise one camera (`SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | MONOCHROME`, 640×480 per eye, 30 Hz, diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index d69a4e916d..9908a35015 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -36,6 +36,7 @@ struct fake_config uint32_t eye_w, eye_h; float fps; float advertised_fps; //!< what enumerate claims; 0 = "unknown" (the service measures) + double baseline_mm; //!< 50 mm default; 120 mm reproduces the Leia SR tracking camera's disparities uint32_t format; int64_t suspend_period_ns; }; @@ -81,6 +82,11 @@ fake_config(void) cfg.fps = (float)atof(fps); } cfg.advertised_fps = cfg.fps; + cfg.baseline_mm = FAKE_BASELINE_MM; + const char *bl = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_BASELINE_MM"); + if (bl != NULL && atof(bl) >= 10.0 && atof(bl) <= 300.0) { + cfg.baseline_mm = atof(bl); + } const char *afps = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_ADVERTISED_FPS"); if (afps != NULL && afps[0] != '\0' && atof(afps) >= 0.0 && atof(afps) <= 240.0) { cfg.advertised_fps = (float)atof(afps); @@ -98,8 +104,9 @@ fake_config(void) cfg.suspend_period_ns = (int64_t)atol(sp) * 1000000; } if (cfg.enabled) { - U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, format %u%s", cfg.eye_w, - cfg.eye_h, cfg.fps, cfg.format, cfg.suspend_period_ns > 0 ? ", suspend square-wave" : ""); + U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, baseline %.1f mm, format %u%s", + cfg.eye_w, cfg.eye_h, cfg.fps, cfg.baseline_mm, cfg.format, + cfg.suspend_period_ns > 0 ? ", suspend square-wave" : ""); } return &cfg; } @@ -172,7 +179,7 @@ sim_display_stereo_camera_get_calibration(struct xrt_plugin_instance *inst, c.rotation_right_from_left[2][2] = 1.0; // OpenCV convention (x_R = R x_L + T): a right camera at +x of the left // one has T = (-B, 0, 0). - c.translation_right_from_left_mm[0] = -FAKE_BASELINE_MM; + c.translation_right_from_left_mm[0] = -cfg->baseline_mm; memcpy(out, &c, sz < sizeof(c) ? sz : sizeof(c)); out->struct_size = sz; return XRT_SUCCESS; @@ -193,7 +200,7 @@ sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, return XRT_ERROR_ALLOCATION; } cam->cfg = *cfg; - sim_stereo_camera_scene_init(&cam->scene, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), FAKE_BASELINE_MM, + sim_stereo_camera_scene_init(&cam->scene, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), cfg->baseline_mm, FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M); cam->gray_pitch = 2 * cfg->eye_w; cam->gray = calloc((size_t)cam->gray_pitch * cfg->eye_h, 1); diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 9e60024c23..1b2c20a492 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -40,6 +40,7 @@ #define STB_IMAGE_WRITE_IMPLEMENTATION #include "stb_image_write.h" +#include #include #include #include @@ -323,23 +324,55 @@ cmp_int(const void *a, const void *b) return *(const int *)a - *(const int *)b; } +//! Nearest subject the probe expects: its disparity bounds the search. +#define PROBE_NEAREST_M 0.4 + +/*! + * Disparity search bound from the camera's own geometry: the disparity of a + * subject at PROBE_NEAREST_M, f * B / Z, with f from the per-eye HFOV. A fixed + * 64 px was fine for the 50 mm sim fake and useless on the 120 mm Leia SR + * tracking camera, where a face at 0.6 m is ~95 px and every block clamped at + * 64. Uncalibrated: a quarter of the eye width. Always within [64, 0.6 * eye]. + */ +static uint32_t +probe_max_disparity(const struct xrt_stereo_camera_properties *p, uint32_t eye_w) +{ + double d = eye_w / 4.0; + if (p != NULL && p->baseline_mm > 0.0f && p->horizontal_fov_deg > 1.0f && p->horizontal_fov_deg < 179.0f) { + double f = (eye_w / 2.0) / tan(p->horizontal_fov_deg * 0.5 * 3.14159265358979323846 / 180.0); + d = f * (p->baseline_mm / 1000.0) / PROBE_NEAREST_M; + } + double cap = eye_w * 0.6; + if (d > cap) { + d = cap; + } + return d < 64.0 ? 64u : (uint32_t)ceil(d); +} + //! Block-match the last frame on a 64x64 grid and print the two dominant -//! disparities (the sim fake: background 12 px, bar 40 px). +//! disparities (the sim fake: background 12 px, bar 40 px). Blocks whose best +//! match sits ON the search bound are a clamp, not a measurement: counted and +//! reported, never folded into the modes. static void -report_disparity(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, char *out, size_t cap) +report_disparity(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, uint32_t max_d, char *out, size_t cap) { int vals[512]; - int n = 0; + int n = 0, at_edge = 0; for (uint32_t y = 48; y + 64 <= h && n < 512; y += 64) { for (uint32_t x = 64; x + 64 <= eye_w && n < 512; x += 64) { float d = 0.0f; - if (u_stereo_camera_estimate_disparity(gray, pitch, eye_w, h, x, y, 64, 64, 64, &d)) { + if (u_stereo_camera_estimate_disparity(gray, pitch, eye_w, h, x, y, 64, 64, max_d, &d)) { + uint32_t lim = max_d < x ? max_d : x; + if (d >= (float)lim - 0.5f) { + at_edge++; + continue; + } vals[n++] = (int)(d + 0.5f); } } } if (n == 0) { - snprintf(out, cap, "n/a (frame too small)"); + snprintf(out, cap, "n/a (%s)", at_edge > 0 ? "every block hit the search bound" : "frame too small"); return; } qsort(vals, (size_t)n, sizeof(int), cmp_int); @@ -362,8 +395,11 @@ report_disparity(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h } i = j; } - snprintf(out, cap, "%d px (%d/%d blocks), %d px (%d/%d blocks), range %d..%d px", best_v[0], best_c[0], n, - best_v[1], best_c[1], n, vals[0], vals[n - 1]); + int k = snprintf(out, cap, "%d px (%d/%d blocks), %d px (%d/%d blocks), range %d..%d px (search 0..%u)", + best_v[0], best_c[0], n, best_v[1], best_c[1], n, vals[0], vals[n - 1], max_d); + if (at_edge > 0 && k > 0 && (size_t)k < cap) { + snprintf(out + k, cap - (size_t)k, "; %d block(s) AT THE SEARCH BOUND, excluded", at_edge); + } } static int @@ -396,15 +432,27 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) id = strtoull(argv[i], NULL, 10); } } - if (id == 0) { - struct xrt_stereo_camera_properties p; + struct xrt_stereo_camera_properties props; + memset(&props, 0, sizeof(props)); + { uint32_t n = 0; - if (ipc_client_stereo_camera_count(ipc_c, &n) != XRT_SUCCESS || n == 0 || - ipc_client_stereo_camera_get_properties(ipc_c, 0, &p) != XRT_SUCCESS) { + bool found = false; + if (ipc_client_stereo_camera_count(ipc_c, &n) == XRT_SUCCESS) { + for (uint32_t i = 0; i < n && !found; i++) { + if (ipc_client_stereo_camera_get_properties(ipc_c, i, &props) == XRT_SUCCESS && + (id == 0 || props.camera_id == id)) { + found = true; + } + } + } + if (!found && id == 0) { printf("camera probe: the service exposes no stereo camera.\n"); return 3; } - id = p.camera_id; + if (!found) { + memset(&props, 0, sizeof(props)); // stream_create reports the bad id + } + id = found ? props.camera_id : id; } struct xrt_stereo_camera_stream_request req = { @@ -517,8 +565,9 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) if (gray != NULL && u_stereo_camera_convert(last.format, src, last.row_pitch, XRT_STEREO_CAMERA_FORMAT_GRAY8, gray, &gl, last.width, last.height)) { - char disp[160]; - report_disparity(gray, gl.pitch[0], last.width / 2, last.height, disp, sizeof(disp)); + char disp[256]; + report_disparity(gray, gl.pitch[0], last.width / 2, last.height, + probe_max_disparity(&props, last.width / 2), disp, sizeof(disp)); printf("block disparity (left x - right x): %s\n", disp); } if (out_dir != NULL) { From 51712a2f0f6d148d0aefb3c0a41108dd4fadc29e Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 19:57:33 -0700 Subject: [PATCH 15/37] =?UTF-8?q?chore(stereo-camera):=20rebase=20onto=20p?= =?UTF-8?q?ost-lift=20main=20=E2=80=94=20placeholder=20gone,=20slots=20aft?= =?UTF-8?q?er=20platform-state,=20values=20300-310?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical follow-through of the rebase onto main (lift v2.22.0 + ADR-045): - docs no longer describe the reserved_adr042_create_dp_d3d11_lift placeholder; the camera slots append after create_dp_d3d11_lift AND get_platform_state. - type values quoted as 1004999300-310 everywhere (290 is weave v14's). - cli CMake: one stb include for lift + camera, not two. Co-Authored-By: Claude Fable 5.1 --- docs/adr/ADR-043-stereo-camera-source.md | 2 +- docs/reference/xrt_plugin_iface.md | 11 ++++++----- docs/roadmap/stereo-camera-source.md | 18 +++++++++--------- docs/specs/extensions/XR_DXR_stereo_camera.md | 8 ++++---- src/xrt/targets/cli/CMakeLists.txt | 5 ++--- tests/tests_stereo_camera.cpp | 5 +++-- 6 files changed, 25 insertions(+), 24 deletions(-) diff --git a/docs/adr/ADR-043-stereo-camera-source.md b/docs/adr/ADR-043-stereo-camera-source.md index ce4cb62736..9495750153 100644 --- a/docs/adr/ADR-043-stereo-camera-source.md +++ b/docs/adr/ADR-043-stereo-camera-source.md @@ -7,7 +7,7 @@ to pages) · introduces camera slots to `xrt_plugin_iface` under the [ADR-020](ADR-020-plugin-abi-compatibility-policy.md) append-at-end rule · related: [ADR-019](ADR-019-vendor-plugin-aux-boundary.md), [ADR-034](ADR-034-input-provider-plugins.md), [ADR-035](ADR-035-service-owned-arbitration-single-pipeline-isolated-satellites.md), -ADR-042 (vendor 2D→3D conversion; in flight on branch `feat/lift-ext`, not yet on `main`) (the same shape in reverse) · +[ADR-042](ADR-042-vendor-2d3d-conversion-supersedes-default.md) (vendor 2D→3D conversion — the same shape in reverse) · integration + plan: [roadmap/stereo-camera-source.md](../roadmap/stereo-camera-source.md) ## Context diff --git a/docs/reference/xrt_plugin_iface.md b/docs/reference/xrt_plugin_iface.md index 4355bebc8c..40f5e18bb7 100644 --- a/docs/reference/xrt_plugin_iface.md +++ b/docs/reference/xrt_plugin_iface.md @@ -640,11 +640,12 @@ Flags (`XRT_PLUGIN_STEREO_CAMERA_*`) mirror the XR bits: `SHARED_WITH_EYE_TRACKI `sim_display`'s fake (`drivers/sim_display/sim_display_stereo_camera.c`, enabled with `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` in the **service** environment). -**Slot order.** The six slots follow a one-pointer placeholder, -`reserved_adr042_create_dp_d3d11_lift`, held for ADR-042's `create_dp_d3d11_lift`, which is not on -`main` yet but claims the offset right after `vk_bundle_fn_table_offset`. The placeholder keeps the -camera slots at their final offsets whichever PR merges first; the lift PR replaces it in place -(same size). `tests_stereo_camera` asserts the adjacency. +**Slot order.** The six slots are the LAST members of the iface, appended (ADR-020) after +everything that was on `main` before them: `… vk_bundle_fn_table_offset` → `create_dp_d3d11_lift` +(ADR-042) → `get_platform_state` (ADR-045) → `stereo_camera_enumerate … stereo_camera_close`. +`tests_stereo_camera` pins that order (lift right after the vk fingerprint, platform-state right +after lift, the camera block right after platform-state and ending the struct), so a reorder or a +slot squeezed in between fails on the host, not on a vendor box. ## Aux surface — separate from the iface diff --git a/docs/roadmap/stereo-camera-source.md b/docs/roadmap/stereo-camera-source.md index d1854004a3..575344dfc6 100644 --- a/docs/roadmap/stereo-camera-source.md +++ b/docs/roadmap/stereo-camera-source.md @@ -206,12 +206,12 @@ against the sim fake). ### R1 as built -- **Header + catalog:** `openxr/XR_DXR_stereo_camera.h`, type values `1004999290–300` (the registry - holds a reserved row for lift's `270–289`); `index.json` note in the `capture` group. -- **Plug-in slots:** six `stereo_camera_*` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA`, after a - one-pointer placeholder for lift's `create_dp_d3d11_lift`. **Merge order: after the lift PR** - (`feat/lift-ext`); the rebase replaces the placeholder with lift's field in place. The offsets are - identical either way, and `tests_stereo_camera` asserts the adjacency. +- **Header + catalog:** `openxr/XR_DXR_stereo_camera.h`, type values `1004999300–310` (relocated + from `290–300` on the rebase onto post-lift `main`: `290` had been taken by `XR_DXR_weave` v14; + lift holds `270–289`); `index.json` note in the `capture` group. +- **Plug-in slots:** six `stereo_camera_*` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA`, appended + after lift's `create_dp_d3d11_lift` (ADR-042, merged in v2.22.0) and ADR-045's + `get_platform_state` — the last members of the iface. `tests_stereo_camera` pins the order. - **Service:** `ipc/server/ipc_server_stereo_camera.c`. It is platform-neutral (Windows D3D11 service, macOS/Linux service, Android service). One thread per open camera, open on the first start, 2 s linger, per-stream 3-slot rings in shared memory, conversion + decimation, and a @@ -247,9 +247,9 @@ against the sim fake). hunk, and the device + factory are new files, to keep rebase conflicts small. - **Latency/sync.** Arrival timestamps hide tracker-internal latency; A/V sync relies on WebRTC's own jitter handling. Fine for calls, not for measurement. -- **ABI merge order.** The iface slots append after ADR-042's `create_dp_d3d11_lift`. R1 holds - lift's offset with a same-size placeholder, so an out-of-order merge still yields the right - offsets, but the intended order is lift first and the camera PR rebased onto it. +- **ABI merge order.** Lift (ADR-042) merged first (v2.22.0), then ADR-045's `get_platform_state`; + the camera slots were rebased to append after both. R1 originally held lift's offset with a + same-size placeholder so either order would have been ABI-safe; the placeholder is gone. - **Android: the pair and the tracker cannot share Camera2.** Opening the front logical camera evicts the vendor tracker, which then never reacquires (§C.0). The mitigation is architectural: the runtime owns both, and tracking runs in-process from the same capture (§C.1). It adds diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index 6671883ca1..8ee1034f5a 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -6,7 +6,7 @@ | **Spec Version** | 1 | | **Extension Type** | Instance extension, service path only (an in-process instance enumerates zero cameras) | | **Header** | [`src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h`](../../../src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h) (+ its `index.json` catalog note) | -| **Status** | **R1 implemented** (runtime, hardware-free): header, plug-in slots, service camera manager, IPC, OpenXR entry points, sim_display fake, `displayxr-cli camera`, selftest check. **Not yet:** rectification (R2), consent / indicator / foreground rule (R3 — deny-by-default hooks in place), GPU transports, state-change events, the Leia provider (L1), the browser (B1). Provisional type values `1004999290–300` (after `XR_DXR_lift`'s `1004999270–289`), pending Khronos registry | +| **Status** | **R1 implemented** (runtime, hardware-free): header, plug-in slots, service camera manager, IPC, OpenXR entry points, sim_display fake, `displayxr-cli camera`, selftest check. **Not yet:** rectification (R2), consent / indicator / foreground rule (R3 — deny-by-default hooks in place), GPU transports, state-change events, the Leia provider (L1), the browser (B1). Provisional type values `1004999300–310` (relocated from `290–300` when `XR_DXR_weave` v14 took `290`; `XR_DXR_lift` holds `270–289`), pending Khronos registry | | **R1 decisions** | stereo-only (names/structs kept open: `viewCount`, next chains) · camera provider = `xrt_plugin_iface` slots · service clients only (in-process enumerates zero) · raw frames never reach web pages (`RAW` refused to `PRESENT_OWNER` clients) — see [roadmap §G](../../roadmap/stereo-camera-source.md#g-open-questions--maintainer-decisions) | | **Decision record** | [ADR-043](../../adr/ADR-043-stereo-camera-source.md) | | **Plug-in contract** | appended `xrt_plugin_iface` camera slots, `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (§9) | @@ -368,9 +368,9 @@ passed §7.1 for that camera. A browser must coarsen what it gives pages (roadma Appended to `struct xrt_plugin_iface` (after `create_dp_d3d11_lift`, ADR-042), gated by `struct_size`, announced by `#define XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA 1`, no ABI bump. -**As implemented:** lift is not on `main` yet, so the slots follow a one-pointer placeholder, -`reserved_adr042_create_dp_d3d11_lift`, at lift's offset; the lift PR replaces it in place. -Either merge order yields the same offsets (`tests_stereo_camera` asserts it). The authoritative +**As implemented:** the six slots are the last members of the iface, after `create_dp_d3d11_lift` +(ADR-042) and `get_platform_state` (ADR-045), both of which were on `main` first; +`tests_stereo_camera` pins the order. The authoritative header is `src/xrt/include/xrt/xrt_plugin.h`; the plug-in-facing rules are in [`docs/reference/xrt_plugin_iface.md`](../../reference/xrt_plugin_iface.md#a-stereo-camera-the-stereo_camera_-slots-adr-043). Two refinements over the sketch below: `wait_frame` returns an `enum xrt_plugin_stereo_camera_wait` diff --git a/src/xrt/targets/cli/CMakeLists.txt b/src/xrt/targets/cli/CMakeLists.txt index 49578e1733..2e5d64859a 100644 --- a/src/xrt/targets/cli/CMakeLists.txt +++ b/src/xrt/targets/cli/CMakeLists.txt @@ -92,12 +92,11 @@ endif() # ADR-042: `displayxr-cli lift probe` loads / writes images (stb, header-only, # compiled static into cli_cmd_lift.cpp) and drives D3D11 shared textures. +# ADR-043: `displayxr-cli camera probe --out` writes PNGs with the same stb +# (compiled static into cli_cmd_camera.c). target_include_directories(cli PRIVATE ${PROJECT_SOURCE_DIR}/src/external/stb) if(WIN32) target_link_libraries(cli PRIVATE d3d11 dxgi) endif() -# ADR-043: `displayxr-cli camera probe --out` writes PNGs (stb, header-only, -# compiled static into cli_cmd_camera.c). -target_include_directories(cli PRIVATE ${PROJECT_SOURCE_DIR}/src/external/stb) install(TARGETS cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/tests/tests_stereo_camera.cpp b/tests/tests_stereo_camera.cpp index da923e355d..1829067169 100644 --- a/tests/tests_stereo_camera.cpp +++ b/tests/tests_stereo_camera.cpp @@ -14,8 +14,9 @@ * - the sim_display FAKE scene: the disparity probe measures exactly the * disparities the scene was built with, and the frame counter differs * between frames; - * - the plug-in iface: the camera slots sit right after the ADR-042 lift - * slot (or its placeholder), so the two branches cannot share an offset. + * - the plug-in iface: the camera slots are appended after the ADR-042 lift + * slot and the ADR-045 platform-state slot (ADR-020 append-only), and end + * the struct. */ #include "catch_amalgamated.hpp" From 00325bba78cb3b73e590860ce5e5aecdf6a5448a Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 16/37] =?UTF-8?q?feat(stereo-camera):=20u=5Fstereo=5Frecti?= =?UTF-8?q?fy=20=E2=80=94=20Bouguet=20rectification=20+=20LUT=20remap,=20n?= =?UTF-8?q?o=20OpenCV=20(R2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vendor-neutral rectifier for XR_DXR_stereo_camera (ADR-043 R2), in three layers so a GPU path can replace only the last one: - geometry: OpenCV's stereoRectify(CALIB_ZERO_DISPARITY, alpha = 0) in C (half rotations + baseline onto x, min-fy focal, corner-centroid principal point, 9x9 inner-rectangle zoom), plus a border check of the real maps that zooms further until no output pixel samples outside its raw image; - float maps (what a GPU backend uploads as an RG32F texture); - CPU backend: fixed-point bilinear taps over the whole SBS image, 1/2/4 channels (GRAY8, NV12 Y + half-res UV, BGRA8). RADTAN5 / RADTAN8 / KB4 lens models, calibration rescaled when it was taken at a different size than the frames. Also u_stereo_camera_estimate_offset: SSD block matching over dx AND dy, for row-alignment checks. Co-Authored-By: Claude Opus 5.5 --- src/xrt/auxiliary/util/CMakeLists.txt | 2 + src/xrt/auxiliary/util/u_stereo_camera.c | 76 +++ src/xrt/auxiliary/util/u_stereo_camera.h | 23 + src/xrt/auxiliary/util/u_stereo_rectify.c | 678 ++++++++++++++++++++++ src/xrt/auxiliary/util/u_stereo_rectify.h | 213 +++++++ 5 files changed, 992 insertions(+) create mode 100644 src/xrt/auxiliary/util/u_stereo_rectify.c create mode 100644 src/xrt/auxiliary/util/u_stereo_rectify.h diff --git a/src/xrt/auxiliary/util/CMakeLists.txt b/src/xrt/auxiliary/util/CMakeLists.txt index 4c6484c67d..be73fa7270 100644 --- a/src/xrt/auxiliary/util/CMakeLists.txt +++ b/src/xrt/auxiliary/util/CMakeLists.txt @@ -35,6 +35,8 @@ add_library( u_capture_dims.h u_stereo_camera.c u_stereo_camera.h + u_stereo_rectify.c + u_stereo_rectify.h u_color_encoding.h u_debug.c u_debug.h diff --git a/src/xrt/auxiliary/util/u_stereo_camera.c b/src/xrt/auxiliary/util/u_stereo_camera.c index d11af289f3..b08b5dca2d 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.c +++ b/src/xrt/auxiliary/util/u_stereo_camera.c @@ -9,6 +9,7 @@ #include "util/u_stereo_camera.h" #include +#include #include @@ -427,3 +428,78 @@ u_stereo_camera_estimate_disparity(const uint8_t *gray, *out_disparity = (float)best_d + sub; return true; } + +bool +u_stereo_camera_estimate_offset(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + uint32_t max_dy, + float *out_dx, + float *out_dy) +{ + if (gray == NULL || out_dx == NULL || out_dy == NULL || w == 0 || h == 0 || x0 + w > eye_width || + y0 + h > height || y0 < max_dy || y0 + h + max_dy > height || max_dy > 32) { + return false; + } + uint32_t dmax = max_disparity < x0 ? max_disparity : x0; + if (dmax > 255) { + dmax = 255; + } + const uint32_t ny = 2 * max_dy + 1; + // SSD surface over (d, dy); a quadratic cost makes the parabola fit unbiased. + static const uint64_t k_inf = UINT64_MAX; + uint64_t *ssd = (uint64_t *)malloc(sizeof(uint64_t) * (dmax + 1) * ny); + if (ssd == NULL) { + return false; + } + uint64_t best = k_inf; + uint32_t bd = 0, by = 0; + for (uint32_t j = 0; j < ny; j++) { + int dy = (int)j - (int)max_dy; + for (uint32_t d = 0; d <= dmax; d++) { + uint64_t s = 0; + for (uint32_t y = y0; y < y0 + h; y++) { + const uint8_t *l = gray + (size_t)y * pitch + x0; + const uint8_t *r = gray + (size_t)((int)y + dy) * pitch + eye_width + x0 - d; + for (uint32_t x = 0; x < w; x++) { + int diff = (int)l[x] - (int)r[x]; + s += (uint64_t)(diff * diff); + } + } + ssd[j * (dmax + 1) + d] = s; + if (s < best) { + best = s; + bd = d; + by = j; + } + } + } + float sx = 0.0f, sy = 0.0f; + if (bd > 0 && bd < dmax) { + double a = (double)ssd[by * (dmax + 1) + bd - 1], b = (double)ssd[by * (dmax + 1) + bd], + c = (double)ssd[by * (dmax + 1) + bd + 1]; + double den = a - 2.0 * b + c; + if (den > 0.0) { + sx = (float)(0.5 * (a - c) / den); + } + } + if (by > 0 && by + 1 < ny) { + double a = (double)ssd[(by - 1) * (dmax + 1) + bd], b = (double)ssd[by * (dmax + 1) + bd], + c = (double)ssd[(by + 1) * (dmax + 1) + bd]; + double den = a - 2.0 * b + c; + if (den > 0.0) { + sy = (float)(0.5 * (a - c) / den); + } + } + free(ssd); + *out_dx = (float)bd + sx; + // A left feature at row y is found at row y + dy in the right eye. + *out_dy = (float)((int)by - (int)max_dy) + sy; + return true; +} diff --git a/src/xrt/auxiliary/util/u_stereo_camera.h b/src/xrt/auxiliary/util/u_stereo_camera.h index ab17c84ad2..1e7243db38 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.h +++ b/src/xrt/auxiliary/util/u_stereo_camera.h @@ -220,6 +220,29 @@ u_stereo_camera_estimate_disparity(const uint8_t *gray, uint32_t max_disparity, float *out_disparity); +/*! + * Two-dimensional version of @ref u_stereo_camera_estimate_disparity (R2): + * SSD block matching over horizontal disparity 0..@p max_disparity AND a + * vertical offset -@p max_dy..@p max_dy, sub-pixel on both axes. @p out_dy is + * the row offset of the match in the RIGHT eye (right y - left y): the + * vertical disparity a rectified pair must bring to ~0. Used by the + * rectifier's tests and `displayxr-cli camera probe --rectified`. + * @return false if the region (plus the dy search band) does not fit. + */ +bool +u_stereo_camera_estimate_offset(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + uint32_t max_dy, + float *out_dx, + float *out_dy); + #ifdef __cplusplus } #endif diff --git a/src/xrt/auxiliary/util/u_stereo_rectify.c b/src/xrt/auxiliary/util/u_stereo_rectify.c new file mode 100644 index 0000000000..418e95ce35 --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_rectify.c @@ -0,0 +1,678 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: vendor-neutral stereo + * rectification. See u_stereo_rectify.h. + * + * The geometry is Bouguet's algorithm exactly as OpenCV's cvStereoRectify + * implements it (horizontal pair, CALIB_ZERO_DISPARITY, alpha = 0), so the + * numbers can be golden-tested against OpenCV without depending on it: + * + * 1. split the relative rotation in half (each camera turns by -om/2 / +om/2 + * so both look the same way): r_r = rodrigues(-rodrigues_inv(R) / 2); + * 2. rotate both about the axis t x e_x so the baseline becomes the x axis: + * R1 = wR * r_r^T, R2 = wR * r_r; + * 3. common focal = the smaller fy (reduced for barrel lenses, as OpenCV); + * common principal point = the mean of both eyes' rectified corner + * centroids (one cy for row alignment, one cx for zero disparity at + * infinity); + * 4. alpha = 0: zoom about the principal point until the inner rectangle of + * every eye's rectified raw-image border covers the whole output — then + * verified on every border pixel of the real maps and nudged up until no + * output pixel samples outside its raw image (no black corners). + * + * @ingroup aux_util + */ + +#include "util/u_stereo_rectify.h" + +#include +#include +#include +#include + + +/* + * + * Small 3x3 helpers. + * + */ + +/* + * Flat row-major 3x3 (m[i * 3 + j]): a `double (*)[3]` -> `const double (*)[3]` + * argument is a -Wpedantic warning before C23 (GCC/MinGW), a flat pointer is not. + */ +#define M3(m) (&(m)[0][0]) + +static void +mat3_mul(const double *a, const double *b, double *out) +{ + double r[9]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + r[i * 3 + j] = + a[i * 3 + 0] * b[0 * 3 + j] + a[i * 3 + 1] * b[1 * 3 + j] + a[i * 3 + 2] * b[2 * 3 + j]; + } + } + memcpy(out, r, sizeof(r)); +} + +static void +mat3_transpose(const double *a, double *out) +{ + double r[9]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + r[i * 3 + j] = a[j * 3 + i]; + } + } + memcpy(out, r, sizeof(r)); +} + +static void +mat3_vec(const double *a, const double v[3], double out[3]) +{ + double r[3]; + for (int i = 0; i < 3; i++) { + r[i] = a[i * 3 + 0] * v[0] + a[i * 3 + 1] * v[1] + a[i * 3 + 2] * v[2]; + } + memcpy(out, r, sizeof(r)); +} + +//! a^T * v +static void +mat3t_vec(const double *a, const double v[3], double out[3]) +{ + double r[3]; + for (int i = 0; i < 3; i++) { + r[i] = a[0 * 3 + i] * v[0] + a[1 * 3 + i] * v[1] + a[2 * 3 + i] * v[2]; + } + memcpy(out, r, sizeof(r)); +} + +void +u_stereo_rectify_rodrigues(const double rvec[3], double out[3][3]) +{ + double th = sqrt(rvec[0] * rvec[0] + rvec[1] * rvec[1] + rvec[2] * rvec[2]); + if (th < 1e-15) { + memset(out, 0, sizeof(double) * 9); + out[0][0] = out[1][1] = out[2][2] = 1.0; + return; + } + double x = rvec[0] / th, y = rvec[1] / th, z = rvec[2] / th; + double c = cos(th), s = sin(th), C = 1.0 - c; + out[0][0] = c + x * x * C; + out[0][1] = x * y * C - z * s; + out[0][2] = x * z * C + y * s; + out[1][0] = y * x * C + z * s; + out[1][1] = c + y * y * C; + out[1][2] = y * z * C - x * s; + out[2][0] = z * x * C - y * s; + out[2][1] = z * y * C + x * s; + out[2][2] = c + z * z * C; +} + +void +u_stereo_rectify_rodrigues_inv(const double m[3][3], double out_rvec[3]) +{ + double rx = m[2][1] - m[1][2]; + double ry = m[0][2] - m[2][0]; + double rz = m[1][0] - m[0][1]; + double s = sqrt(rx * rx + ry * ry + rz * rz) * 0.5; + double c = (m[0][0] + m[1][1] + m[2][2] - 1.0) * 0.5; + c = c > 1.0 ? 1.0 : c < -1.0 ? -1.0 : c; + double th = atan2(s, c); + if (s < 1e-5) { + if (c > 0.0) { + // Near identity: theta ~ s, vec = r / 2 (first order, exact to O(th^3)). + out_rvec[0] = rx * 0.5; + out_rvec[1] = ry * 0.5; + out_rvec[2] = rz * 0.5; + return; + } + // Near pi: the axis from the diagonal. + double t[3] = {(m[0][0] + 1.0) * 0.5, (m[1][1] + 1.0) * 0.5, (m[2][2] + 1.0) * 0.5}; + double ax = sqrt(t[0] > 0 ? t[0] : 0), ay = sqrt(t[1] > 0 ? t[1] : 0), az = sqrt(t[2] > 0 ? t[2] : 0); + if (m[0][1] < 0) { + ay = -ay; + } + if (m[0][2] < 0) { + az = -az; + } + if (fabs(ax) < fabs(ay) && fabs(ax) < fabs(az) && (m[1][2] > 0) != (ay * az > 0)) { + az = -az; + } + double n = sqrt(ax * ax + ay * ay + az * az); + n = n > 0 ? th / n : 0; + out_rvec[0] = ax * n; + out_rvec[1] = ay * n; + out_rvec[2] = az * n; + return; + } + double k = th / (2.0 * s); + out_rvec[0] = rx * k; + out_rvec[1] = ry * k; + out_rvec[2] = rz * k; +} + + +/* + * + * Lens models. + * + */ + +void +u_stereo_rectify_distort(const struct u_stereo_rectify_lens *l, double x, double y, double *out_x, double *out_y) +{ + switch (l->model) { + case U_STEREO_RECTIFY_MODEL_RADTAN5: + case U_STEREO_RECTIFY_MODEL_RADTAN8: { + const double *d = l->d; + double k4 = 0, k5 = 0, k6 = 0; + if (l->model == U_STEREO_RECTIFY_MODEL_RADTAN8) { + k4 = d[5]; + k5 = d[6]; + k6 = d[7]; + } + double r2 = x * x + y * y, r4 = r2 * r2, r6 = r4 * r2; + double cdist = 1.0 + d[0] * r2 + d[1] * r4 + d[4] * r6; + double icdist = 1.0 / (1.0 + k4 * r2 + k5 * r4 + k6 * r6); + double a1 = 2.0 * x * y, a2 = r2 + 2.0 * x * x, a3 = r2 + 2.0 * y * y; + *out_x = x * cdist * icdist + d[2] * a1 + d[3] * a2; + *out_y = y * cdist * icdist + d[2] * a3 + d[3] * a1; + return; + } + case U_STEREO_RECTIFY_MODEL_KB4: { + double r = sqrt(x * x + y * y); + if (r < 1e-12) { + *out_x = x; + *out_y = y; + return; + } + double th = atan(r), t2 = th * th, t4 = t2 * t2, t6 = t4 * t2, t8 = t4 * t4; + double thd = th * (1.0 + l->d[0] * t2 + l->d[1] * t4 + l->d[2] * t6 + l->d[3] * t8); + double s = thd / r; + *out_x = x * s; + *out_y = y * s; + return; + } + default: + *out_x = x; + *out_y = y; + return; + } +} + +void +u_stereo_rectify_undistort_pixel( + const struct u_stereo_rectify_lens *l, double u, double v, double *out_x, double *out_y) +{ + double x0 = (u - l->cx) / l->fx; + double y0 = (v - l->cy) / l->fy; + switch (l->model) { + case U_STEREO_RECTIFY_MODEL_RADTAN5: + case U_STEREO_RECTIFY_MODEL_RADTAN8: { + const double *d = l->d; + double k4 = 0, k5 = 0, k6 = 0; + if (l->model == U_STEREO_RECTIFY_MODEL_RADTAN8) { + k4 = d[5]; + k5 = d[6]; + k6 = d[7]; + } + // OpenCV's fixed-point iteration, run to convergence. + double x = x0, y = y0; + for (int it = 0; it < 100; it++) { + double r2 = x * x + y * y; + double icdist = + (1.0 + ((k6 * r2 + k5) * r2 + k4) * r2) / (1.0 + ((d[4] * r2 + d[1]) * r2 + d[0]) * r2); + if (!(icdist > 0.0)) { + break; // outside the model's invertible range: keep the last estimate + } + double dx = 2.0 * d[2] * x * y + d[3] * (r2 + 2.0 * x * x); + double dy = d[2] * (r2 + 2.0 * y * y) + 2.0 * d[3] * x * y; + double nx = (x0 - dx) * icdist, ny = (y0 - dy) * icdist; + double delta = fabs(nx - x) + fabs(ny - y); + x = nx; + y = ny; + if (delta < 1e-14) { + break; + } + } + *out_x = x; + *out_y = y; + return; + } + case U_STEREO_RECTIFY_MODEL_KB4: { + double thd = sqrt(x0 * x0 + y0 * y0); + if (thd < 1e-12) { + *out_x = x0; + *out_y = y0; + return; + } + // Newton on theta(1 + k1 th^2 + ...) = thd. + double th = thd; + for (int it = 0; it < 50; it++) { + double t2 = th * th, t4 = t2 * t2, t6 = t4 * t2, t8 = t4 * t4; + double f = th * (1.0 + l->d[0] * t2 + l->d[1] * t4 + l->d[2] * t6 + l->d[3] * t8) - thd; + double df = + 1.0 + 3.0 * l->d[0] * t2 + 5.0 * l->d[1] * t4 + 7.0 * l->d[2] * t6 + 9.0 * l->d[3] * t8; + double step = f / df; + th -= step; + if (fabs(step) < 1e-15) { + break; + } + } + double s = tan(th) / thd; + *out_x = x0 * s; + *out_y = y0 * s; + return; + } + default: + *out_x = x0; + *out_y = y0; + return; + } +} + + +/* + * + * Geometry. + * + */ + +//! RAW pixel of eye @p e -> rectified pixel under (f, cx, cy). false = behind. +static bool +raw_to_rect(const struct u_stereo_rectify_result *r, uint32_t e, double u, double v, double *ou, double *ov) +{ + double x, y; + u_stereo_rectify_undistort_pixel(&r->raw[e], u, v, &x, &y); + double p[3] = {x, y, 1.0}, q[3]; + mat3_vec(M3(r->rot[e]), p, q); + if (q[2] <= 1e-12) { + return false; + } + *ou = r->f * q[0] / q[2] + r->cx; + *ov = r->f * q[1] / q[2] + r->cy; + return true; +} + +bool +u_stereo_rectify_map_point( + const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_x, double *out_y) +{ + double p[3] = {(u - r->cx) / r->f, (v - r->cy) / r->f, 1.0}, q[3]; + mat3t_vec(M3(r->rot[eye]), p, q); // raw ray = R_e^T * rectified ray + if (q[2] <= 1e-12) { + return false; + } + double xd, yd; + u_stereo_rectify_distort(&r->raw[eye], q[0] / q[2], q[1] / q[2], &xd, &yd); + *out_x = r->raw[eye].fx * xd + r->raw[eye].cx; + *out_y = r->raw[eye].fy * yd + r->raw[eye].cy; + return true; +} + +//! Does every border pixel of both eyes' output sample inside its raw image? +static bool +border_valid(const struct u_stereo_rectify_result *r) +{ + const double wmax = (double)r->width - 1.0, hmax = (double)r->height - 1.0; + for (uint32_t e = 0; e < 2; e++) { + for (uint32_t i = 0; i < 2 * (r->width + r->height); i++) { + double u, v; + if (i < r->width) { + u = i, v = 0; + } else if (i < 2 * r->width) { + u = i - r->width, v = hmax; + } else if (i < 2 * r->width + r->height) { + u = 0, v = i - 2 * r->width; + } else { + u = wmax, v = i - 2 * r->width - r->height; + } + double x, y; + if (!u_stereo_rectify_map_point(r, e, u, v, &x, &y) || x < 0.0 || y < 0.0 || x > wmax || + y > hmax) { + return false; + } + } + } + return true; +} + +bool +u_stereo_rectify_compute(const struct u_stereo_rectify_input *in, struct u_stereo_rectify_result *out) +{ + memset(out, 0, sizeof(*out)); + if (in->width < 8 || in->height < 8) { + return false; + } + const double nx = in->width, ny = in->height; + out->width = in->width; + out->height = in->height; + + // 0. Intrinsics at the frame size (pixel centres: u' = (u + 0.5) * s - 0.5). + double sx = in->calib_width ? nx / (double)in->calib_width : 1.0; + double sy = in->calib_height ? ny / (double)in->calib_height : 1.0; + for (int e = 0; e < 2; e++) { + struct u_stereo_rectify_lens *l = &out->raw[e]; + *l = in->eye[e]; + l->fx = in->eye[e].fx * sx; + l->fy = in->eye[e].fy * sy; + l->cx = (in->eye[e].cx + 0.5) * sx - 0.5; + l->cy = (in->eye[e].cy + 0.5) * sy - 0.5; + if (!(l->fx > 0.0) || !(l->fy > 0.0) || l->model > U_STEREO_RECTIFY_MODEL_KB4) { + return false; + } + } + double nt = sqrt(in->T[0] * in->T[0] + in->T[1] * in->T[1] + in->T[2] * in->T[2]); + if (!(nt > 0.0)) { + return false; + } + out->baseline = nt; + + // 1. Half rotation each. + double om[3], r_r[3][3]; + u_stereo_rectify_rodrigues_inv(in->R, om); + for (int i = 0; i < 3; i++) { + om[i] *= -0.5; + } + u_stereo_rectify_rodrigues(om, r_r); + double t[3]; + mat3_vec(M3(r_r), in->T, t); + if (fabs(t[0]) < fabs(t[1])) { + return false; // a vertical pair: not a side-by-side camera + } + + // 2. Baseline onto the x axis. + double c = t[0]; + double uu[3] = {c > 0.0 ? 1.0 : -1.0, 0.0, 0.0}; + double ww[3] = {t[1] * uu[2] - t[2] * uu[1], t[2] * uu[0] - t[0] * uu[2], t[0] * uu[1] - t[1] * uu[0]}; + double nw = sqrt(ww[0] * ww[0] + ww[1] * ww[1] + ww[2] * ww[2]); + if (nw > 0.0) { + double k = acos(fabs(c) / nt) / nw; + for (int i = 0; i < 3; i++) { + ww[i] *= k; + } + } + double wR[3][3], r_rt[3][3]; + u_stereo_rectify_rodrigues(ww, wR); + mat3_transpose(M3(r_r), M3(r_rt)); + mat3_mul(M3(wR), M3(r_rt), M3(out->rot[0])); + mat3_mul(M3(wR), M3(r_r), M3(out->rot[1])); + mat3_vec(M3(out->rot[1]), in->T, out->t_rect); + + // 3. Common focal: the smaller fy, shrunk for barrel lenses (OpenCV). + double fc = DBL_MAX; + for (int e = 0; e < 2; e++) { + double f = out->raw[e].fy; + double dk1 = (out->raw[e].model == U_STEREO_RECTIFY_MODEL_RADTAN5 || + out->raw[e].model == U_STEREO_RECTIFY_MODEL_RADTAN8) + ? out->raw[e].d[0] + : 0.0; + if (dk1 < 0.0) { + f *= 1.0 + dk1 * (nx * nx + ny * ny) / (4.0 * f * f); + } + fc = f < fc ? f : fc; + } + if (!(fc > 0.0)) { + return false; + } + + // Common principal point: each eye's rectified corner centroid, averaged. + double ccx[2], ccy[2]; + for (uint32_t e = 0; e < 2; e++) { + const double corners[4][2] = {{0, 0}, {nx - 1, 0}, {0, ny - 1}, {nx - 1, ny - 1}}; + double ax = 0, ay = 0; + for (int k = 0; k < 4; k++) { + double x, y; + u_stereo_rectify_undistort_pixel(&out->raw[e], corners[k][0], corners[k][1], &x, &y); + double p[3] = {x, y, 1.0}, q[3]; + mat3_vec(M3(out->rot[e]), p, q); + ax += q[0] / q[2]; + ay += q[1] / q[2]; + } + ccx[e] = (nx - 1) * 0.5 - fc * ax * 0.25; + ccy[e] = (ny - 1) * 0.5 - fc * ay * 0.25; + } + out->cx = (ccx[0] + ccx[1]) * 0.5; + out->cy = (ccy[0] + ccy[1]) * 0.5; + out->f = fc; + + // 4. alpha = 0: the inner rectangle of each eye's rectified border (the 9x9 + // grid OpenCV's icvGetRectangles uses) must cover the whole output. + enum + { + N = 9 + }; + double s0 = 0.0; + for (uint32_t e = 0; e < 2; e++) { + double ix0 = -DBL_MAX, ix1 = DBL_MAX, iy0 = -DBL_MAX, iy1 = DBL_MAX; + for (int i = 0; i < N; i++) { + for (int j = 0; j < N; j++) { + double u, v; + if (!raw_to_rect(out, e, j * (nx - 1) / (N - 1), i * (ny - 1) / (N - 1), &u, &v)) { + return false; + } + if (j == 0 && u > ix0) { + ix0 = u; + } + if (j == N - 1 && u < ix1) { + ix1 = u; + } + if (i == 0 && v > iy0) { + iy0 = v; + } + if (i == N - 1 && v < iy1) { + iy1 = v; + } + } + } + if (!(ix0 < out->cx && out->cx < ix1 && iy0 < out->cy && out->cy < iy1)) { + return false; // the principal point is outside the valid region + } + double s = out->cx / (out->cx - ix0); + double s2 = out->cy / (out->cy - iy0); + double s3 = (nx - 1 - out->cx) / (ix1 - out->cx); + double s4 = (ny - 1 - out->cy) / (iy1 - out->cy); + s = s2 > s ? s2 : s; + s = s3 > s ? s3 : s; + s = s4 > s ? s4 : s; + s0 = s > s0 ? s : s0; + } + out->crop_scale = s0; + out->f = fc * s0; + + // The 9-point inner rectangle is an approximation of the valid region's + // inscribed rectangle: verify on every border pixel of the real maps + // and zoom a little further until no output pixel samples outside. + for (int it = 0; it < 200 && !border_valid(out); it++) { + out->crop_scale *= 1.001; + out->f = fc * out->crop_scale; + } + if (!border_valid(out)) { + return false; + } + + memset(out->P, 0, sizeof(out->P)); + for (int e = 0; e < 2; e++) { + out->P[e][0][0] = out->f; + out->P[e][0][2] = out->cx; + out->P[e][1][1] = out->f; + out->P[e][1][2] = out->cy; + out->P[e][2][2] = 1.0; + } + out->P[1][0][3] = out->f * out->t_rect[0]; + return true; +} + +uint32_t +u_stereo_rectify_build_map(const struct u_stereo_rectify_result *r, uint32_t eye, uint32_t sub, float *map_xy) +{ + if (sub == 0) { + sub = 1; + } + const uint32_t w = r->width / sub, h = r->height / sub; + const double half = (sub - 1) * 0.5; // sample centre offset in full-res pixels + uint32_t valid = 0; + for (uint32_t y = 0; y < h; y++) { + for (uint32_t x = 0; x < w; x++) { + double sxf, syf; + float *o = map_xy + 2 * ((size_t)y * w + x); + if (u_stereo_rectify_map_point(r, eye, x * (double)sub + half, y * (double)sub + half, &sxf, + &syf)) { + sxf = (sxf - half) / sub; + syf = (syf - half) / sub; + if (sxf >= 0.0 && syf >= 0.0 && sxf <= w - 1.0 && syf <= h - 1.0) { + o[0] = (float)sxf; + o[1] = (float)syf; + valid++; + continue; + } + } + o[0] = -1.0f; + o[1] = -1.0f; + } + } + return valid; +} + + +/* + * + * CPU LUT. + * + */ + +bool +u_stereo_rectify_lut_init(struct u_stereo_rectify_lut *lut, const struct u_stereo_rectify_result *r, uint32_t sub) +{ + memset(lut, 0, sizeof(*lut)); + if (sub == 0) { + sub = 1; + } + const uint32_t ew = r->width / sub, eh = r->height / sub; + if (ew < 2 || eh < 2 || 2 * ew > 32767 || eh > 32767) { + return false; + } + float *map = malloc(sizeof(float) * 2 * (size_t)ew * eh); + lut->taps = malloc(sizeof(*lut->taps) * 2 * (size_t)ew * eh); + if (map == NULL || lut->taps == NULL) { + free(map); + free(lut->taps); + lut->taps = NULL; + return false; + } + lut->width = 2 * ew; + lut->height = eh; + lut->sub = sub; + for (uint32_t e = 0; e < 2; e++) { + lut->valid += u_stereo_rectify_build_map(r, e, sub, map); + for (uint32_t y = 0; y < eh; y++) { + for (uint32_t x = 0; x < ew; x++) { + const float *m = map + 2 * ((size_t)y * ew + x); + struct u_stereo_rectify_tap *t = &lut->taps[(size_t)y * lut->width + e * ew + x]; + if (m[0] < 0.0f) { + t->sx = -1; + t->sy = -1; + t->wx = t->wy = 0; + continue; + } + int ix = (int)floorf(m[0]), iy = (int)floorf(m[1]); + int wx = (int)lrintf((m[0] - (float)ix) * 256.0f); + int wy = (int)lrintf((m[1] - (float)iy) * 256.0f); + if (wx >= 256) { + ix++, wx = 0; + } + if (wy >= 256) { + iy++, wy = 0; + } + // Keep both bilinear taps inside this eye's half. + if (ix >= (int)ew - 1) { + ix = (int)ew - 2, wx = 255; + } + if (iy >= (int)eh - 1) { + iy = (int)eh - 2, wy = 255; + } + t->sx = (int16_t)(ix + (int)(e * ew)); + t->sy = (int16_t)iy; + t->wx = (uint8_t)wx; + t->wy = (uint8_t)wy; + } + } + } + free(map); + return true; +} + +void +u_stereo_rectify_lut_fini(struct u_stereo_rectify_lut *lut) +{ + free(lut->taps); + memset(lut, 0, sizeof(*lut)); +} + +void +u_stereo_rectify_lut_apply_rows(const struct u_stereo_rectify_lut *lut, + uint32_t channels, + const uint8_t *src, + uint32_t src_pitch, + uint8_t *dst, + uint32_t dst_pitch, + uint32_t y0, + uint32_t y1) +{ + if (y1 > lut->height) { + y1 = lut->height; + } + const uint32_t W = lut->width; + for (uint32_t y = y0; y < y1; y++) { + const struct u_stereo_rectify_tap *t = lut->taps + (size_t)y * W; + uint8_t *d = dst + (size_t)y * dst_pitch; + if (channels == 1) { + for (uint32_t x = 0; x < W; x++, t++) { + if (t->sx < 0) { + d[x] = 0; + continue; + } + const uint8_t *p = src + (size_t)t->sy * src_pitch + t->sx; + uint32_t wx = t->wx, wy = t->wy; + uint32_t top = p[0] * (256 - wx) + p[1] * wx; + uint32_t bot = p[src_pitch] * (256 - wx) + p[src_pitch + 1] * wx; + d[x] = (uint8_t)((top * (256 - wy) + bot * wy + 32768u) >> 16); + } + continue; + } + for (uint32_t x = 0; x < W; x++, t++) { + uint8_t *o = d + (size_t)x * channels; + if (t->sx < 0) { + for (uint32_t ch = 0; ch < channels; ch++) { + // black: neutral chroma for NV12 UV, opaque for BGRA + o[ch] = channels == 2 ? 128 : (channels == 4 && ch == 3) ? 255 : 0; + } + continue; + } + const uint8_t *p = src + (size_t)t->sy * src_pitch + (size_t)t->sx * channels; + const uint8_t *q = p + src_pitch; + uint32_t wx = t->wx, wy = t->wy; + for (uint32_t ch = 0; ch < channels; ch++) { + uint32_t top = p[ch] * (256 - wx) + p[ch + channels] * wx; + uint32_t bot = q[ch] * (256 - wx) + q[ch + channels] * wx; + o[ch] = (uint8_t)((top * (256 - wy) + bot * wy + 32768u) >> 16); + } + } + } +} + +void +u_stereo_rectify_lut_apply(const struct u_stereo_rectify_lut *lut, + uint32_t channels, + const uint8_t *src, + uint32_t src_pitch, + uint8_t *dst, + uint32_t dst_pitch) +{ + u_stereo_rectify_lut_apply_rows(lut, channels, src, src_pitch, dst, dst_pitch, 0, lut->height); +} diff --git a/src/xrt/auxiliary/util/u_stereo_rectify.h b/src/xrt/auxiliary/util/u_stereo_rectify.h new file mode 100644 index 0000000000..c9b49a3133 --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_rectify.h @@ -0,0 +1,213 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: vendor-neutral stereo + * rectification — Bouguet rectification to a parallel pair with zero + * disparity at infinity, cropped to the valid region, applied as a + * per-pixel remap LUT. No OpenCV dependency; the numbers follow + * OpenCV's `stereoRectify(..., CALIB_ZERO_DISPARITY, alpha = 0)` + + * `initUndistortRectifyMap` (golden-tested against OpenCV-generated + * fixtures in tests_stereo_rectify). + * + * Three layers, so a GPU path can replace only the last one: + * + * 1. **Geometry** — @ref u_stereo_rectify_compute: from the RAW calibration + * (per-eye K + distortion, OpenCV extrinsics `x_R = R x_L + T`) to the + * rectified rotations R1/R2, the common focal and principal point, the + * projection matrices P1/P2 and the baseline. Pure math, no allocation. + * 2. **Maps** — @ref u_stereo_rectify_map_point / @ref + * u_stereo_rectify_build_map: for each OUTPUT pixel, the RAW source + * position (float). A float map is what a GPU backend uploads (RG32F + * texture sampled bilinearly in a compute/fragment pass). + * 3. **CPU backend** — @ref u_stereo_rectify_lut: the maps packed as + * fixed-point bilinear taps over the whole side-by-side image, applied per + * plane (GRAY8 = 1 channel, NV12 = a full-res Y LUT + a half-res 2-channel + * UV LUT, BGRA8 = 4 channels). + * + * Conventions: pixel centres at integer coordinates (OpenCV), intrinsics fx + * fy cx cy with zero skew, distortion coefficients in OpenCV order + * (k1 k2 p1 p2 k3 k4 k5 k6 for RADTAN5/8; k1..k4 for KB4 fisheye). + * + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! Lens models (values equal xrt_plugin_stereo_camera_distortion). +enum u_stereo_rectify_model +{ + U_STEREO_RECTIFY_MODEL_NONE = 0, + U_STEREO_RECTIFY_MODEL_RADTAN5 = 1, + U_STEREO_RECTIFY_MODEL_RADTAN8 = 2, + U_STEREO_RECTIFY_MODEL_KB4 = 3, +}; + +//! One eye's pinhole + lens model. +struct u_stereo_rectify_lens +{ + double fx, fy, cx, cy; + uint32_t model; //!< enum u_stereo_rectify_model + double d[8]; //!< OpenCV order +}; + +//! The RAW calibration of a pair. +struct u_stereo_rectify_input +{ + //! Per-eye size of the frames to rectify. + uint32_t width, height; + //! Per-eye size the intrinsics describe; 0 = same as the frames. When it + //! differs, K is rescaled (pixel-centre aware); distortion is unitless. + uint32_t calib_width, calib_height; + struct u_stereo_rectify_lens eye[2]; + //! OpenCV extrinsics: x_right = R * x_left + T (T in any unit, e.g. mm). + double R[3][3]; + double T[3]; +}; + +//! The rectified geometry. +struct u_stereo_rectify_result +{ + uint32_t width, height; //!< per eye, unchanged by rectification + //! RAW intrinsics rescaled to the frame size (what the maps sample). + struct u_stereo_rectify_lens raw[2]; + //! Rectified-from-raw rotation per eye (OpenCV R1, R2). + double rot[2][3][3]; + //! Rectified intrinsics shared by both eyes: fx = fy = f, one principal + //! point (CALIB_ZERO_DISPARITY: a point at infinity has zero disparity). + double f, cx, cy; + //! OpenCV P1, P2; P[1][0][3] = f * Tx (units of T; Tx < 0 when the right + //! camera is at +x). + double P[2][3][4]; + //! |T|, units of T. + double baseline; + //! Rectified translation R2 * T (≈ (-baseline, 0, 0)), units of T. + double t_rect[3]; + //! The valid-region zoom applied to f (alpha = 0): >= 1 when the lens and + //! rotations would otherwise leave black borders. + double crop_scale; +}; + +/*! + * Bouguet rectification of @p in, horizontal pair, zero disparity at + * infinity, cropped so every output pixel of both eyes samples inside its raw + * image (no black corners), per-eye size unchanged, no convergence shear. + * @return false on a degenerate calibration (zero baseline, non-positive + * focal, a vertical pair, or a rotation that does not converge). + */ +bool +u_stereo_rectify_compute(const struct u_stereo_rectify_input *in, struct u_stereo_rectify_result *out); + +/*! + * Map one rectified pixel (@p u, @p v) of eye @p eye to its RAW source pixel. + * @return false when the ray points behind the raw camera. + */ +bool +u_stereo_rectify_map_point( + const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_x, double *out_y); + +/*! + * Float map of eye @p eye on a grid subsampled by @p sub (1 = full + * resolution; 2 = NV12 chroma: output sample (x, y) sits at full-res + * (2x + 0.5, 2y + 0.5) and the result is in half-res source coordinates). + * @p map_xy gets 2 floats per output sample, row-major, + * (width/sub) x (height/sub) samples. Samples whose source falls outside the + * raw image are written as (-1, -1). + * @return the number of valid samples. + */ +uint32_t +u_stereo_rectify_build_map(const struct u_stereo_rectify_result *r, uint32_t eye, uint32_t sub, float *map_xy); + +/*! + * Forward lens model: undistorted normalized (x, y) -> distorted normalized. + */ +void +u_stereo_rectify_distort(const struct u_stereo_rectify_lens *lens, double x, double y, double *out_x, double *out_y); + +/*! + * Inverse lens model: RAW pixel -> undistorted normalized coordinates + * (iterative, converged to 1e-12). + */ +void +u_stereo_rectify_undistort_pixel( + const struct u_stereo_rectify_lens *lens, double u, double v, double *out_x, double *out_y); + +//! Rotation vector (axis * angle, radians) -> matrix. +void +u_stereo_rectify_rodrigues(const double rvec[3], double out[3][3]); + +//! Rotation matrix -> rotation vector (axis * angle, radians). +void +u_stereo_rectify_rodrigues_inv(const double m[3][3], double out_rvec[3]); + + +/* + * + * CPU backend: fixed-point bilinear LUT over the whole SBS image. + * + */ + +//! One output sample: top-left source tap + 8-bit fractions. sx < 0 = black. +struct u_stereo_rectify_tap +{ + int16_t sx, sy; + uint8_t wx, wy; //!< fraction * 256, 0..255 (the 256 case folds into the next tap) +}; + +struct u_stereo_rectify_lut +{ + uint32_t width; //!< SBS width of this plane (samples) + uint32_t height; //!< rows of this plane + uint32_t sub; //!< 1 = full-res, 2 = NV12 chroma + uint32_t valid; //!< taps that sample inside the raw image + struct u_stereo_rectify_tap *taps; +}; + +/*! + * Build the SBS LUT for one plane: the left eye fills x < width/2, the right + * eye x >= width/2 (sampling its own half of the source). + * @return false on allocation failure or an extent the tap format cannot hold. + */ +bool +u_stereo_rectify_lut_init(struct u_stereo_rectify_lut *lut, const struct u_stereo_rectify_result *r, uint32_t sub); + +void +u_stereo_rectify_lut_fini(struct u_stereo_rectify_lut *lut); + +/*! + * Remap one plane of @p channels interleaved 8-bit channels (1, 2 or 4). + * @p src / @p dst are SBS planes of lut->width x lut->height samples. + */ +void +u_stereo_rectify_lut_apply(const struct u_stereo_rectify_lut *lut, + uint32_t channels, + const uint8_t *src, + uint32_t src_pitch, + uint8_t *dst, + uint32_t dst_pitch); + +/*! + * Remap rows [@p y0, @p y1) only — the unit a threaded CPU path splits on. + */ +void +u_stereo_rectify_lut_apply_rows(const struct u_stereo_rectify_lut *lut, + uint32_t channels, + const uint8_t *src, + uint32_t src_pitch, + uint8_t *dst, + uint32_t dst_pitch, + uint32_t y0, + uint32_t y1); + +#ifdef __cplusplus +} +#endif From fd7d75c31a2f5ed66a37024b1d4a23bce5618706 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 17/37] feat(sim_display): distorted, misaligned fake stereo camera with ground truth (R2) SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1 renders the fake's scene through two RAW cameras: different per-eye intrinsics, RADTAN5 barrel lenses, and each camera turned by half of (pitch 0.35, yaw 0.25, roll 0.45 deg) in opposite senses (~6 px vertical misalignment, 0.9 deg relative roll). The symmetric split makes Bouguet recover exactly the virtual parallel pair the scene is defined in, so the ground truth after rectification is exact: aligned rows and disparity f_rect * B / Z. The camera drops NATIVELY_RECTIFIED and its calibration slot returns the truth, so RECTIFIED output exercises the service's rectifier. The scene is sampled continuously (bilinear value noise), so resampling is honest. Co-Authored-By: Claude Opus 5.5 --- .../sim_display/sim_display_stereo_camera.c | 56 +++++- .../sim_display/sim_display_stereo_camera.h | 10 + .../sim_display_stereo_camera_pattern.c | 176 ++++++++++++++++++ .../sim_display_stereo_camera_pattern.h | 88 ++++++++- 4 files changed, 321 insertions(+), 9 deletions(-) diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index 9908a35015..dbf03f3a2f 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -39,6 +39,9 @@ struct fake_config double baseline_mm; //!< 50 mm default; 120 mm reproduces the Leia SR tracking camera's disparities uint32_t format; int64_t suspend_period_ns; + //! R2: raw, distorted, misaligned pair with known ground truth (not + //! NATIVELY_RECTIFIED; the service's rectifier does the work). + bool distort; }; //! Completes the plug-in-owned opaque type. @@ -46,6 +49,7 @@ struct xrt_plugin_stereo_camera { struct fake_config cfg; struct sim_stereo_camera_scene scene; + struct sim_stereo_camera_distorted distorted; //!< cfg.distort only uint8_t *gray; uint32_t gray_pitch; uint8_t *converted; @@ -103,10 +107,13 @@ fake_config(void) if (sp != NULL && atol(sp) > 0) { cfg.suspend_period_ns = (int64_t)atol(sp) * 1000000; } + const char *dist = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT"); + cfg.distort = dist != NULL && dist[0] != '\0' && dist[0] != '0'; if (cfg.enabled) { - U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, baseline %.1f mm, format %u%s", + U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, baseline %.1f mm, format %u%s%s", cfg.eye_w, cfg.eye_h, cfg.fps, cfg.baseline_mm, cfg.format, - cfg.suspend_period_ns > 0 ? ", suspend square-wave" : ""); + cfg.suspend_period_ns > 0 ? ", suspend square-wave" : "", + cfg.distort ? ", DISTORTED raw pair (rectified by the service)" : ""); } return &cfg; } @@ -135,7 +142,10 @@ sim_display_stereo_camera_enumerate(struct xrt_plugin_instance *inst, snprintf(info.display_name, sizeof(info.display_name), "Sim 3D camera (fake)"); snprintf(info.device_identity, sizeof(info.device_identity), "sim-display-fake-stereo-camera-0001"); info.flags = XRT_PLUGIN_STEREO_CAMERA_SHARED_WITH_EYE_TRACKING | XRT_PLUGIN_STEREO_CAMERA_USER_FACING | - XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED; + XRT_PLUGIN_STEREO_CAMERA_CALIBRATED; + if (!cfg->distort) { + info.flags |= XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED; + } if (cfg->format == XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { info.flags |= XRT_PLUGIN_STEREO_CAMERA_MONOCHROME; } @@ -166,6 +176,27 @@ sim_display_stereo_camera_get_calibration(struct xrt_plugin_instance *inst, c.struct_size = (uint32_t)sizeof(c); c.image_width = cfg->eye_w; c.image_height = cfg->eye_h; + if (cfg->distort) { + // The raw pair's ground truth (sim_display_stereo_camera_pattern.h). + struct sim_stereo_camera_truth t; + sim_stereo_camera_truth_init(&t, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), FAKE_BASELINE_MM, + FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M); + for (int e = 0; e < 2; e++) { + c.k[e][0] = t.fx[e]; + c.k[e][1] = t.fy[e]; + c.k[e][2] = t.cx[e]; + c.k[e][3] = t.cy[e]; + for (int k = 0; k < 5; k++) { + c.distortion[e][k] = t.dist[e][k]; + } + } + c.distortion_model = XRT_PLUGIN_STEREO_CAMERA_DISTORTION_RADTAN5; + memcpy(c.rotation_right_from_left, t.R, sizeof(t.R)); + memcpy(c.translation_right_from_left_mm, t.T_mm, sizeof(t.T_mm)); + memcpy(out, &c, sz < sizeof(c) ? sz : sizeof(c)); + out->struct_size = sz; + return XRT_SUCCESS; + } double fx = fake_fx(cfg->eye_w); for (int e = 0; e < 2; e++) { c.k[e][0] = fx; @@ -202,6 +233,12 @@ sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, cam->cfg = *cfg; sim_stereo_camera_scene_init(&cam->scene, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), cfg->baseline_mm, FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M); + if (cfg->distort && + !sim_stereo_camera_distorted_init(&cam->distorted, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), + cfg->baseline_mm, FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M)) { + free(cam); + return XRT_ERROR_ALLOCATION; + } cam->gray_pitch = 2 * cfg->eye_w; cam->gray = calloc((size_t)cam->gray_pitch * cfg->eye_h, 1); if (cfg->format != XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { @@ -211,14 +248,16 @@ sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, if (cam->gray == NULL || (cfg->format != XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8 && cam->converted == NULL)) { free(cam->gray); free(cam->converted); + sim_stereo_camera_distorted_fini(&cam->distorted); free(cam); return XRT_ERROR_ALLOCATION; } cam->period_ns = (int64_t)(1e9 / cfg->fps); cam->t0_ns = os_monotonic_get_ns(); cam->open_ns = cam->t0_ns; - U_LOG_W("sim_display: FAKE stereo camera opened (bg disparity %u px, bar disparity %u px)", - cam->scene.bg_disparity, cam->scene.bar_disparity); + U_LOG_W("sim_display: FAKE stereo camera opened (%s; bg disparity %.2f px, bar disparity %.2f px in the %s)", + cfg->distort ? "DISTORTED raw pair" : "parallel pair", cam->scene.bg_disparity_f, + cam->scene.bar_disparity_f, cfg->distort ? "virtual parallel frame" : "frame"); *out_cam = cam; return XRT_SUCCESS; } @@ -260,7 +299,11 @@ sim_display_stereo_camera_wait_frame(struct xrt_plugin_stereo_camera *cam, sleep_ns(due - now); cam->seq++; - sim_stereo_camera_render_gray(&cam->scene, cam->seq, cam->gray, cam->gray_pitch); + if (cam->cfg.distort) { + sim_stereo_camera_render_gray_distorted(&cam->distorted, cam->seq, cam->gray, cam->gray_pitch); + } else { + sim_stereo_camera_render_gray(&cam->scene, cam->seq, cam->gray, cam->gray_pitch); + } memset(out, 0, sizeof(*out)); out->sequence = cam->seq; @@ -300,5 +343,6 @@ sim_display_stereo_camera_close(struct xrt_plugin_stereo_camera *cam) U_LOG_W("sim_display: FAKE stereo camera closed after %llu frames", (unsigned long long)cam->seq); free(cam->gray); free(cam->converted); + sim_stereo_camera_distorted_fini(&cam->distorted); free(cam); } diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.h b/src/xrt/drivers/sim_display/sim_display_stereo_camera.h index 94f1f8809e..b90b316bb4 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.h +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.h @@ -11,12 +11,22 @@ * SIM_DISPLAY_FAKE_STEREO_CAMERA_FPS=N source rate (default 30) * SIM_DISPLAY_FAKE_STEREO_CAMERA_FORMAT=gray8|nv12|bgra native format (default gray8) * SIM_DISPLAY_FAKE_STEREO_CAMERA_SUSPEND_PERIOD_MS=N square-wave SUSPENDED/AVAILABLE + * SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1 R2: a RAW distorted pair (see below) * * The camera: SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | * NATIVELY_RECTIFIED (+ MONOCHROME for gray8), 68 deg horizontal FOV per eye, * 50 mm baseline, background at 2.0 m and a bar at 0.6 m — at 640 px per eye * that is fx = 474.4 px and disparities of 12 px and 40 px. * + * With _DISTORT=1 the same scene is seen through two raw cameras with known + * ground truth (per-eye intrinsics, RADTAN5 barrel lenses, 0.7 deg relative + * pitch = ~6 px of vertical misalignment, 0.9 deg relative roll, 0.5 deg yaw; + * sim_stereo_camera_truth_init), the calibration slot returns that truth, and + * NATIVELY_RECTIFIED is dropped — so RECTIFIED output exercises the service's + * rectifier (u_stereo_rectify). After rectification rows align and the + * disparities are f_rect * B / Z (f_rect: `displayxr-cli camera calib + * --rectified`). + * * @ingroup drv_sim_display */ diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c index b0d52b7102..1d038f9141 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c @@ -8,8 +8,11 @@ #include "sim_display_stereo_camera_pattern.h" +#include "util/u_stereo_rectify.h" + #include #include +#include #include //! 5x7 digits, one byte per row, bit 4 = leftmost column. @@ -63,6 +66,8 @@ sim_stereo_camera_scene_init(struct sim_stereo_camera_scene *s, s->eye_height = eye_height; s->bg_disparity = (uint32_t)lround(fx_px * (baseline_mm / 1000.0) / bg_depth_m); s->bar_disparity = (uint32_t)lround(fx_px * (baseline_mm / 1000.0) / bar_depth_m); + s->bg_disparity_f = fx_px * (baseline_mm / 1000.0) / bg_depth_m; + s->bar_disparity_f = fx_px * (baseline_mm / 1000.0) / bar_depth_m; s->bar_x0 = eye_width * 3 / 8; s->bar_x1 = eye_width * 5 / 8; s->bar_y0 = eye_height / 4; @@ -135,3 +140,174 @@ sim_stereo_camera_render_gray(const struct sim_stereo_camera_scene *s, render_counter(dst, pitch, 0, s->eye_width, s->eye_height, frame_number); render_counter(dst, pitch, s->eye_width, s->eye_width, s->eye_height, frame_number); } + + +/* + * + * DISTORTED variant (R2): the same scene through two raw cameras with known + * ground truth. + * + */ + +#define DEG(x) ((x)*3.14159265358979323846 / 180.0) + +//! Bilinear value noise on 2-pixel cells: continuous, so a resampled image is +//! an honest sample of the scene (no pixel-locked texture). +static float +vnoise(double u, double v, uint32_t seed, float lo, float span) +{ + double gu = u * 0.5, gv = v * 0.5; + double fu = floor(gu), fv = floor(gv); + int32_t iu = (int32_t)fu, iv = (int32_t)fv; + float a = (float)(gu - fu), b = (float)(gv - fv); + uint32_t sp = (uint32_t)span; + float h00 = (float)(hash3((uint32_t)iu, (uint32_t)iv, seed) % sp); + float h10 = (float)(hash3((uint32_t)(iu + 1), (uint32_t)iv, seed) % sp); + float h01 = (float)(hash3((uint32_t)iu, (uint32_t)(iv + 1), seed) % sp); + float h11 = (float)(hash3((uint32_t)(iu + 1), (uint32_t)(iv + 1), seed) % sp); + float top = h00 + (h10 - h00) * a, bot = h01 + (h11 - h01) * a; + return lo + top + (bot - top) * b; +} + +float +sim_stereo_camera_scene_sample(const struct sim_stereo_camera_scene *s, int right, double u, double v) +{ + double ub = right ? u + s->bg_disparity_f : u; + double ubar = right ? u + s->bar_disparity_f : u; + if (v >= s->bar_y0 && v < s->bar_y1 && ubar >= s->bar_x0 && ubar < s->bar_x1) { + return vnoise(ubar, v, 0xBA5u, 128.0f, 127.0f); + } + return vnoise(ub, v, 0x0B6u, 0.0f, 111.0f); +} + +void +sim_stereo_camera_truth_init(struct sim_stereo_camera_truth *t, + uint32_t eye_width, + uint32_t eye_height, + double ideal_fx, + double baseline_mm, + double bg_depth_m, + double bar_depth_m) +{ + memset(t, 0, sizeof(*t)); + const double w = eye_width, h = eye_height; + t->eye_width = eye_width; + t->eye_height = eye_height; + t->ideal_fx = ideal_fx; + t->ideal_cx = (w - 1.0) * 0.5; + t->ideal_cy = (h - 1.0) * 0.5; + t->baseline_mm = baseline_mm; + t->bg_depth_m = bg_depth_m; + t->bar_depth_m = bar_depth_m; + + // Two different, off-centre, barrel lenses (scale with the eye size). + t->fx[0] = ideal_fx * 1.012; + t->fy[0] = ideal_fx * 1.008; + t->cx[0] = t->ideal_cx + w * 0.009; + t->cy[0] = t->ideal_cy - h * 0.008; + const double d0[5] = {-0.21, 0.06, 0.0008, -0.0006, -0.005}; + t->fx[1] = ideal_fx * 0.991; + t->fy[1] = ideal_fx * 0.994; + t->cx[1] = t->ideal_cx - w * 0.007; + t->cy[1] = t->ideal_cy + h * 0.007; + const double d1[5] = {-0.17, 0.04, -0.0005, 0.0007, 0.0}; + memcpy(t->dist[0], d0, sizeof(d0)); + memcpy(t->dist[1], d1, sizeof(d1)); + + // Each camera turned by half of (pitch 0.35 deg, yaw 0.25 deg, roll 0.45 + // deg), in opposite senses: 0.7 deg of relative pitch is the vertical + // misalignment, 0.9 deg of relative roll the tilt. + const double rv[3] = {DEG(0.35), DEG(0.25), DEG(0.45)}; + u_stereo_rectify_rodrigues(rv, t->S); + // R = S * S (right-from-left rotation); T = -S * C_right, C_right = (B, 0, 0). + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + t->R[i][j] = t->S[i][0] * t->S[0][j] + t->S[i][1] * t->S[1][j] + t->S[i][2] * t->S[2][j]; + } + t->T_mm[i] = -t->S[i][0] * baseline_mm; + } +} + +bool +sim_stereo_camera_distorted_init(struct sim_stereo_camera_distorted *d, + uint32_t eye_width, + uint32_t eye_height, + double ideal_fx, + double baseline_mm, + double bg_depth_m, + double bar_depth_m) +{ + memset(d, 0, sizeof(*d)); + sim_stereo_camera_scene_init(&d->scene, eye_width, eye_height, ideal_fx, baseline_mm, bg_depth_m, bar_depth_m); + sim_stereo_camera_truth_init(&d->truth, eye_width, eye_height, ideal_fx, baseline_mm, bg_depth_m, bar_depth_m); + d->ideal_uv = malloc(sizeof(float) * 4 * (size_t)eye_width * eye_height); + if (d->ideal_uv == NULL) { + return false; + } + const struct sim_stereo_camera_truth *t = &d->truth; + for (uint32_t e = 0; e < 2; e++) { + struct u_stereo_rectify_lens lens; + memset(&lens, 0, sizeof(lens)); + lens.fx = t->fx[e]; + lens.fy = t->fy[e]; + lens.cx = t->cx[e]; + lens.cy = t->cy[e]; + lens.model = U_STEREO_RECTIFY_MODEL_RADTAN5; + memcpy(lens.d, t->dist[e], sizeof(t->dist[e])); + float *uv = d->ideal_uv + (size_t)e * 2 * eye_width * eye_height; + for (uint32_t y = 0; y < eye_height; y++) { + for (uint32_t x = 0; x < eye_width; x++) { + double xn, yn; + u_stereo_rectify_undistort_pixel(&lens, x, y, &xn, &yn); + // Raw ray -> virtual parallel frame: left x_rect = S x_raw + // (x_raw = S^T x_rect), right x_rect = S^T x_raw. + double r[3] = {xn, yn, 1.0}, q[3]; + for (int i = 0; i < 3; i++) { + q[i] = e == 0 ? t->S[i][0] * r[0] + t->S[i][1] * r[1] + t->S[i][2] * r[2] + : t->S[0][i] * r[0] + t->S[1][i] * r[1] + t->S[2][i] * r[2]; + } + float *o = uv + 2 * ((size_t)y * eye_width + x); + if (q[2] <= 1e-9) { + o[0] = o[1] = -1e9f; + continue; + } + o[0] = (float)(t->ideal_fx * q[0] / q[2] + t->ideal_cx); + o[1] = (float)(t->ideal_fx * q[1] / q[2] + t->ideal_cy); + } + } + } + return true; +} + +void +sim_stereo_camera_distorted_fini(struct sim_stereo_camera_distorted *d) +{ + free(d->ideal_uv); + d->ideal_uv = NULL; +} + +void +sim_stereo_camera_render_gray_distorted(const struct sim_stereo_camera_distorted *d, + uint64_t frame_number, + uint8_t *dst, + uint32_t pitch) +{ + const uint32_t w = d->scene.eye_width, h = d->scene.eye_height; + for (uint32_t e = 0; e < 2; e++) { + const float *uv = d->ideal_uv + (size_t)e * 2 * w * h; + for (uint32_t y = 0; y < h; y++) { + uint8_t *row = dst + (size_t)y * pitch + e * w; + const float *p = uv + 2 * (size_t)y * w; + for (uint32_t x = 0; x < w; x++, p += 2) { + if (p[0] < -1e8f) { + row[x] = 0; + continue; + } + float val = sim_stereo_camera_scene_sample(&d->scene, (int)e, p[0], p[1]); + row[x] = (uint8_t)(val < 0.0f ? 0 : val > 255.0f ? 255 : (int)(val + 0.5f)); + } + } + } + render_counter(dst, pitch, 0, w, h, frame_number); + render_counter(dst, pitch, w, w, h, frame_number); +} diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h index 81b4953522..5a66133860 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h @@ -10,15 +10,24 @@ * The scene is a random-dot stereogram with two planes at KNOWN integer * disparities (left x - right x, pixels): a textured background and a nearer * textured bar in the middle of the frame. A frame counter is burned into the - * top-left of BOTH halves at zero disparity. The pair is parallel and - * undistorted (the R1 fake reports NATIVELY_RECTIFIED); a distorted, - * misaligned variant for the rectifier's tests is R2's. + * top-left of BOTH halves at zero disparity. The default pair is parallel and + * undistorted (the fake then reports NATIVELY_RECTIFIED). + * + * The DISTORTED variant (R2, SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1) renders + * the same scene through two raw cameras with known ground truth: per-eye + * intrinsics that differ, RADTAN5 barrel distortion, and a small rotation of + * each camera (pitch -> vertical misalignment, roll, yaw) — the kind of pair a + * real tracker camera delivers. The rotations are split symmetrically + * (left = S^T, right = S), which makes Bouguet rectification recover EXACTLY + * the virtual parallel pair the scene is defined in, so after rectification + * rows must align and the disparity must be f_rect * B / Z. * * @ingroup drv_sim_display */ #pragma once +#include #include #ifdef __cplusplus @@ -33,8 +42,81 @@ struct sim_stereo_camera_scene uint32_t bar_disparity; //!< near bar, pixels //! The bar's rectangle in LEFT-eye coordinates. uint32_t bar_x0, bar_y0, bar_x1, bar_y1; + //! Unrounded disparities (fx * B / Z), used by the distorted variant. + double bg_disparity_f; + double bar_disparity_f; +}; + +/*! + * Ground truth of the DISTORTED fake: the virtual parallel camera the scene is + * defined in, and the two raw cameras that look at it. + */ +struct sim_stereo_camera_truth +{ + uint32_t eye_width, eye_height; + //! The virtual rectified camera (both eyes): the scene's pixel frame. + double ideal_fx, ideal_cx, ideal_cy; + double baseline_mm; + double bg_depth_m, bar_depth_m; + //! Raw per-eye intrinsics (pixels) + RADTAN5 (k1 k2 p1 p2 k3). + double fx[2], fy[2], cx[2], cy[2]; + double dist[2][5]; + //! Half rotation: x_raw_left = S^T x_rect, x_raw_right = S (x_rect - C_right). + double S[3][3]; + //! OpenCV extrinsics of the raw pair: x_R = R x_L + T, T in mm. + double R[3][3]; + double T_mm[3]; +}; + +//! Fill the truth for a pair of @p eye_width x @p eye_height whose virtual +//! parallel camera has focal @p ideal_fx and baseline @p baseline_mm. +void +sim_stereo_camera_truth_init(struct sim_stereo_camera_truth *t, + uint32_t eye_width, + uint32_t eye_height, + double ideal_fx, + double baseline_mm, + double bg_depth_m, + double bar_depth_m); + +/*! + * The distorted renderer: the truth + a per-raw-pixel lookup of where each + * raw pixel looks in the virtual parallel frame (built once). + */ +struct sim_stereo_camera_distorted +{ + struct sim_stereo_camera_scene scene; + struct sim_stereo_camera_truth truth; + //! 2 floats per raw pixel, eye-major: virtual-frame (u, v); u < -1e8 = none. + float *ideal_uv; }; +bool +sim_stereo_camera_distorted_init(struct sim_stereo_camera_distorted *d, + uint32_t eye_width, + uint32_t eye_height, + double ideal_fx, + double baseline_mm, + double bg_depth_m, + double bar_depth_m); + +void +sim_stereo_camera_distorted_fini(struct sim_stereo_camera_distorted *d); + +//! Render frame @p frame_number of the distorted pair (GRAY8 SBS). +void +sim_stereo_camera_render_gray_distorted(const struct sim_stereo_camera_distorted *d, + uint64_t frame_number, + uint8_t *dst, + uint32_t pitch); + +/*! + * The continuous scene as the VIRTUAL parallel camera sees it: eye @p right at + * virtual pixel (@p u, @p v), bilinear value noise (so resampling is honest). + */ +float +sim_stereo_camera_scene_sample(const struct sim_stereo_camera_scene *s, int right, double u, double v); + /*! * Scene for a pinhole pair of focal @p fx_px and baseline @p baseline_mm, with * the background at @p bg_depth_m and the bar at @p bar_depth_m: disparities From ee2a0343bc53eee82ad07cfe5b611f645b18a0ae Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 18/37] test(stereo-camera): golden rectifier tests vs OpenCV + end to end on the distorted fake tests_stereo_rectify, with fixtures generated offline by tests/fixtures/gen_stereo_rectify_fixtures.py (OpenCV 4.10) for three calibrations (the sim fake, an SR-like pair with a non-axial baseline, a RADTAN8 pair calibrated at 2x the frame size): - R1/R2 equal OpenCV's to 1e-12; principal point equals the converged reference to 1e-4 px (OpenCV's own P1 within 0.75 px: it stops undistortPoints after 5 iterations); focal within 1 % (measured +0.05..0.13 %); - maps equal initUndistortRectifyMap to 3e-5 px; - alpha = 0 really leaves no black pixel (full and half resolution); - end to end: raw |dy| median 2.9 px -> rectified worst block 0.061 px; disparity 11.33 / 37.77 px ground truth, worst block error 0.063 px; - NV12 / BGRA8 planes agree with GRAY8; hidden [.perf] case. Co-Authored-By: Claude Opus 5.5 --- tests/CMakeLists.txt | 7 + tests/fixtures/gen_stereo_rectify_fixtures.py | 145 ++++++ tests/fixtures/stereo_rectify_fixtures.h | 50 ++ tests/tests_stereo_rectify.cpp | 482 ++++++++++++++++++ 4 files changed, 684 insertions(+) create mode 100644 tests/fixtures/gen_stereo_rectify_fixtures.py create mode 100644 tests/fixtures/stereo_rectify_fixtures.h create mode 100644 tests/tests_stereo_rectify.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 073006fb21..66f2e4ea92 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -68,6 +68,7 @@ set(tests tests_comp_color_policy tests_m_matrix_projection tests_stereo_camera + tests_stereo_rectify ) if(XRT_BUILD_DRIVER_QWERTY) list(APPEND tests tests_qwerty_camera_profile tests_qwerty_velocity) @@ -709,3 +710,9 @@ target_include_directories(tests_lift_mailbox PRIVATE ${PROJECT_SOURCE_DIR}/src/ # the ring / conversion / disparity-probe tests (it compiles into the plug-in). target_sources(tests_stereo_camera PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c) target_include_directories(tests_stereo_camera PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display) + +# ADR-043 R2: the rectifier (u_stereo_rectify) against OpenCV-generated golden +# fixtures, and end to end on the sim_display DISTORTED fake (its scene + +# synthetic lens live in the pattern file). +target_sources(tests_stereo_rectify PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c) +target_include_directories(tests_stereo_rectify PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display ${CMAKE_CURRENT_SOURCE_DIR}/fixtures) diff --git a/tests/fixtures/gen_stereo_rectify_fixtures.py b/tests/fixtures/gen_stereo_rectify_fixtures.py new file mode 100644 index 0000000000..26eb1535fc --- /dev/null +++ b/tests/fixtures/gen_stereo_rectify_fixtures.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# Copyright 2026, The DisplayXR Project +# SPDX-License-Identifier: BSL-1.0 +"""Generate stereo_rectify_fixtures.h: OpenCV's answers for u_stereo_rectify. + +ADR-043 R2. The runtime's rectifier (src/xrt/auxiliary/util/u_stereo_rectify.c) +has NO OpenCV dependency; these fixtures pin it to OpenCV's +stereoRectify(flags=CALIB_ZERO_DISPARITY, alpha=0) + initUndistortRectifyMap. +Run offline (needs opencv-python + numpy) and commit the header: + + python3 tests/fixtures/gen_stereo_rectify_fixtures.py > tests/fixtures/stereo_rectify_fixtures.h + +Case "sim_distorted" mirrors sim_stereo_camera_truth_init() in +src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c — keep the two +in sync (tests_stereo_rectify also checks the C truth against these inputs). +""" +import math + +import cv2 +import numpy as np + + +def sim_truth(w, h): + ideal_fx = (w / 2.0) / math.tan(math.radians(34.0)) + icx, icy = (w - 1) * 0.5, (h - 1) * 0.5 + K1 = np.array([[ideal_fx * 1.012, 0, icx + w * 0.009], [0, ideal_fx * 1.008, icy - h * 0.008], [0, 0, 1]]) + K2 = np.array([[ideal_fx * 0.991, 0, icx - w * 0.007], [0, ideal_fx * 0.994, icy + h * 0.007], [0, 0, 1]]) + D1 = np.array([-0.21, 0.06, 0.0008, -0.0006, -0.005, 0, 0, 0]) + D2 = np.array([-0.17, 0.04, -0.0005, 0.0007, 0.0, 0, 0, 0]) + S, _ = cv2.Rodrigues(np.radians(np.array([0.35, 0.25, 0.45]))) + R = S @ S + T = -S @ np.array([50.0, 0, 0]) + return dict(name="sim_distorted", w=w, h=h, cw=0, ch=0, model=1, K=[K1, K2], D=[D1, D2], R=R, T=T) + + +def sr_like(): + K1 = np.array([[421.7, 0, 318.2], [0, 422.9, 243.6], [0, 0, 1]]) + K2 = np.array([[424.1, 0, 322.9], [0, 425.0, 238.1], [0, 0, 1]]) + D1 = np.array([-0.082, 0.121, 0.0011, -0.0004, -0.061, 0, 0, 0]) + D2 = np.array([-0.075, 0.098, -0.0007, 0.0009, -0.044, 0, 0, 0]) + R, _ = cv2.Rodrigues(np.array([0.0021, -0.0043, 0.0032])) + T = np.array([-63.47, 0.41, -0.83]) + return dict(name="sr_like", w=640, h=480, cw=0, ch=0, model=1, K=[K1, K2], D=[D1, D2], R=R, T=T) + + +def radtan8_scaled(): + # Calibrated at 1280x960, frames at 640x480: the rectifier rescales K. + K1c = np.array([[905.0, 0, 641.5], [0, 903.0, 477.0], [0, 0, 1]]) + K2c = np.array([[899.0, 0, 636.0], [0, 900.5, 483.5], [0, 0, 1]]) + D1 = np.array([0.21, -0.05, 0.0006, 0.0003, 0.01, 0.43, 0.02, 0.004]) + D2 = np.array([0.19, -0.04, -0.0004, 0.0005, 0.008, 0.40, 0.015, 0.003]) + R, _ = cv2.Rodrigues(np.array([-0.0035, 0.0061, -0.0018])) + T = np.array([-40.2, -0.35, 0.52]) + + def scale(K, s): + K = K.copy() + K[0, 0] *= s + K[1, 1] *= s + K[0, 2] = (K[0, 2] + 0.5) * s - 0.5 + K[1, 2] = (K[1, 2] + 0.5) * s - 0.5 + return K + + return dict(name="radtan8_scaled", w=640, h=480, cw=1280, ch=960, model=2, K=[K1c, K2c], D=[D1, D2], R=R, + T=T, Kframe=[scale(K1c, 0.5), scale(K2c, 0.5)]) + + +def converged_cc(Kf, D, R1, R2, w, h): + """OpenCV's principal-point rule with undistortPoints run to convergence. + + cvStereoRectify undistorts the four corners with only 5 fixed-point + iterations, which leaves a strong lens tenths of a pixel (or more) short; + the runtime iterates to convergence. This is the converged reference. + """ + fc = 1e300 + for K, d in zip(Kf, D): + f = K[1, 1] + if d[0] < 0: + f *= 1 + d[0] * (w * w + h * h) / (4 * f * f) + fc = min(fc, f) + corners = np.array([[[0, 0]], [[w - 1, 0]], [[0, h - 1]], [[w - 1, h - 1]]], np.float64) + cc = [] + for K, d, Rr in zip(Kf, D, (R1, R2)): + u = cv2.undistortPointsIter(corners, K, d, None, None, + (cv2.TERM_CRITERIA_COUNT | cv2.TERM_CRITERIA_EPS, 500, 1e-15)).reshape(-1, 2) + q = (Rr @ np.hstack([u, np.ones((4, 1))]).T).T + avg = (q[:, :2] / q[:, 2:3]).mean(axis=0) + cc.append(((w - 1) / 2 - fc * avg[0], (h - 1) / 2 - fc * avg[1])) + return fc, (cc[0][0] + cc[1][0]) / 2, (cc[0][1] + cc[1][1]) / 2 + + +def arr(a): + return "{" + ", ".join("%.17g" % float(x) for x in np.asarray(a).ravel()) + "}" + + +GRID_X, GRID_Y = 7, 5 + + +def emit(c): + w, h = c["w"], c["h"] + Kf = c.get("Kframe", c["K"]) + D = [d if c["model"] == 2 else d[:5] for d in c["D"]] + R1, R2, P1, P2, Q, roi1, roi2 = cv2.stereoRectify(Kf[0], D[0], Kf[1], D[1], (w, h), c["R"], c["T"], + flags=cv2.CALIB_ZERO_DISPARITY, alpha=0) + fc, ccx, ccy = converged_cc(Kf, D, R1, R2, w, h) + samples = [] + for e, (Rr, P) in enumerate(((R1, P1), (R2, P2))): + mx, my = cv2.initUndistortRectifyMap(Kf[e], D[e], Rr, P, (w, h), cv2.CV_32FC1) + for j in range(GRID_Y): + for i in range(GRID_X): + u = round(i * (w - 1) / (GRID_X - 1)) + v = round(j * (h - 1) / (GRID_Y - 1)) + samples += [e, u, v, float(mx[v, u]), float(my[v, u])] + n = c["name"] + print("static const struct stereo_rectify_fixture k_fixture_%s = {" % n) + print('\t"%s", %d, %d, %d, %d, %d,' % (n, w, h, c["cw"], c["ch"], c["model"])) + print("\t{%s, %s}," % (arr(c["K"][0]), arr(c["K"][1]))) + print("\t{%s, %s}," % (arr(c["D"][0]), arr(c["D"][1]))) + print("\t%s, %s," % (arr(c["R"]), arr(c["T"]))) + print("\t%s, %s, %s, %s," % (arr(R1), arr(R2), arr(P1), arr(P2))) + print("\t%s," % arr([fc, ccx, ccy])) + print("\t%s," % arr(samples)) + print("};\n") + + +def main(): + print("// Copyright 2026, The DisplayXR Project") + print("// SPDX-License-Identifier: BSL-1.0") + print("// GENERATED by gen_stereo_rectify_fixtures.py with OpenCV %s — do not edit." % cv2.__version__) + print("// OpenCV stereoRectify(CALIB_ZERO_DISPARITY, alpha=0) + initUndistortRectifyMap samples.") + print("#pragma once\n") + print("#define STEREO_RECTIFY_FIXTURE_SAMPLES (2 * %d * %d)\n" % (GRID_X, GRID_Y)) + print("struct stereo_rectify_fixture\n{") + print("\tconst char *name;\n\tint width, height, calib_width, calib_height, model;") + print("\tdouble K[2][9]; //!< as calibrated (calib size when set)\n\tdouble D[2][8];") + print("\tdouble R[9], T[3];") + print("\tdouble R1[9], R2[9], P1[12], P2[12]; //!< OpenCV's answer") + print("\tdouble converged[3]; //!< pre-crop focal, cx, cy with converged undistortPoints") + print("\tdouble samples[STEREO_RECTIFY_FIXTURE_SAMPLES * 5]; //!< eye, u, v, map_x, map_y (OpenCV R/P)") + print("};\n") + for c in (sim_truth(640, 480), sr_like(), radtan8_scaled()): + emit(c) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/stereo_rectify_fixtures.h b/tests/fixtures/stereo_rectify_fixtures.h new file mode 100644 index 0000000000..58a1296487 --- /dev/null +++ b/tests/fixtures/stereo_rectify_fixtures.h @@ -0,0 +1,50 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +// GENERATED by gen_stereo_rectify_fixtures.py with OpenCV 4.10.0 — do not edit. +// OpenCV stereoRectify(CALIB_ZERO_DISPARITY, alpha=0) + initUndistortRectifyMap samples. +#pragma once + +#define STEREO_RECTIFY_FIXTURE_SAMPLES (2 * 7 * 5) + +struct stereo_rectify_fixture +{ + const char *name; + int width, height, calib_width, calib_height, model; + double K[2][9]; //!< as calibrated (calib size when set) + double D[2][8]; + double R[9], T[3]; + double R1[9], R2[9], P1[12], P2[12]; //!< OpenCV's answer + double converged[3]; //!< pre-crop focal, cx, cy with converged undistortPoints + double samples[STEREO_RECTIFY_FIXTURE_SAMPLES * 5]; //!< eye, u, v, map_x, map_y (OpenCV R/P) +}; + +static const struct stereo_rectify_fixture k_fixture_sim_distorted = { + "sim_distorted", 640, 480, 0, 0, 1, + {{480.11254404316571, 0, 325.25999999999999, 0, 478.21486600346941, 235.66, 0, 0, 1}, {470.14973433476013, 0, 315.01999999999998, 0, 471.57299286453235, 242.86000000000001, 0, 0, 1}}, + {{-0.20999999999999999, 0.059999999999999998, 0.00080000000000000004, -0.00059999999999999995, -0.0050000000000000001, 0, 0, 0}, {-0.17000000000000001, 0.040000000000000001, -0.00050000000000000001, 0.00069999999999999999, 0, 0, 0, 0}}, + {0.99983855911978303, -0.015653421241978178, 0.0088219102634899179, 0.015760033144008215, 0.9998020064676586, -0.012147807149594446, -0.0086300088398358569, 0.012284879595061634, 0.99988729598928239}, {-49.99798192944499, -0.39335770014464566, 0.21696244520423971}, + {0.9999596385888998, -0.0078405002408463032, 0.0043872256757703267, 0.0078671540028936084, 0.99995050015619791, -0.0060913976445827285, -0.0043392489040851804, 0.0061256667672149765, 0.99997182316583566}, {0.9999596385888998, 0.0078671540028936084, -0.0043392489040851804, -0.0078405002408463032, 0.99995050015619791, 0.0061256667672149765, 0.0043872256757703267, -0.0060913976445827285, 0.99997182316583566}, {452.68462339907495, 0, 320.57709884643555, 0, 0, 452.68462339907495, 239.3410758972168, 0, 0, 0, 1, 0}, {452.68462339907495, 0, 320.57709884643555, -22634.231169953749, 0, 452.68462339907495, 239.3410758972168, 0, 0, 0, 1, 0}, + {407.95356216571662, 320.42562266203367, 239.36410779466294}, + {0, 0, 0, 25.628753662109375, 20.80931282043457, 0, 106, 0, 114.78317260742188, 10.564011573791504, 0, 213, 0, 214.73896789550781, 3.1000354290008545, 0, 320, 0, 320.75457763671875, -0.008007747121155262, 0, 426, 0, 425.70660400390625, 1.7409622669219971, 0, 532, 0, 524.57373046875, 7.8479142189025879, 0, 639, 0, 614.207275390625, 17.065437316894531, 0, 0, 120, 16.38664436340332, 127.46588134765625, 0, 106, 120, 107.5118408203125, 121.24662017822266, 0, 213, 120, 211.09086608886719, 116.63072967529297, 0, 320, 120, 321.60733032226562, 114.47244262695312, 0, 426, 120, 431.00091552734375, 115.06279754638672, 0, 532, 120, 533.433349609375, 118.15943145751953, 0, 639, 120, 625.0633544921875, 123.12503051757812, 0, 0, 240, 13.342729568481445, 241.57992553710938, 0, 106, 240, 105.31760406494141, 240.92079162597656, 0, 213, 240, 210.32466125488281, 240.15034484863281, 0, 320, 240, 322.56997680664062, 239.2904052734375, 0, 426, 240, 433.653076171875, 238.39909362792969, 0, 532, 240, 537.4432373046875, 237.53233337402344, 0, 639, 240, 629.86065673828125, 236.74293518066406, 0, 0, 359, 17.423341751098633, 354.94357299804688, 0, 106, 359, 108.85330963134766, 359.81146240234375, 0, 213, 359, 212.74403381347656, 362.88214111328125, 0, 320, 359, 323.56402587890625, 363.33782958984375, 0, 426, 359, 433.21401977539062, 360.99703979492188, 0, 532, 359, 535.8306884765625, 356.21051025390625, 0, 639, 359, 627.58306884765625, 349.72073364257812, 0, 0, 479, 27.618436813354492, 462.25592041015625, 0, 106, 479, 117.3876953125, 471.15267944335938, 0, 213, 479, 217.93798828125, 477.12750244140625, 0, 320, 479, 324.51718139648438, 478.60397338867188, 0, 426, 479, 429.94891357421875, 475.16604614257812, 0, 532, 479, 529.18408203125, 467.41690063476562, 0, 639, 479, 619.11138916015625, 456.72235107421875, 1, 0, 0, 22.116100311279297, 15.563237190246582, 1, 106, 0, 112.76789093017578, 7.8436222076416016, 1, 213, 0, 212.92782592773438, 2.6479806900024414, 1, 320, 0, 318.38385009765625, 1.1816521883010864, 1, 426, 0, 423.07159423828125, 3.7653830051422119, 1, 532, 0, 522.976806640625, 9.9769821166992188, 1, 639, 0, 615.55633544921875, 18.768739700317383, 1, 0, 120, 13.51005744934082, 124.00925445556641, 1, 106, 120, 105.67232513427734, 119.92747497558594, 1, 213, 120, 208.59645080566406, 117.38086700439453, 1, 320, 120, 317.45404052734375, 116.96125793457031, 1, 426, 120, 425.53848266601562, 118.80167388916016, 1, 532, 120, 528.25469970703125, 122.67184448242188, 1, 639, 120, 622.42694091796875, 128.04299926757812, 1, 0, 240, 10.047764778137207, 238.30494689941406, 1, 106, 240, 102.69903564453125, 239.01350402832031, 1, 213, 240, 206.51715087890625, 239.81143188476562, 1, 320, 240, 316.47799682617188, 240.6690673828125, 1, 426, 240, 425.67752075195312, 241.53524780273438, 1, 532, 240, 529.34136962890625, 242.36903381347656, 1, 639, 240, 624.0872802734375, 243.13426208496094, 1, 0, 359, 12.477910041809082, 351.41976928710938, 1, 106, 359, 104.35537719726562, 356.85409545898438, 1, 213, 359, 206.97434997558594, 360.9371337890625, 1, 320, 359, 315.52447509765625, 363.03515625, 1, 426, 359, 423.33761596679688, 362.92056274414062, 1, 532, 359, 525.84564208984375, 360.74029541015625, 1, 639, 359, 619.86956787109375, 356.94107055664062, 1, 0, 479, 20.100147247314453, 459.20794677734375, 1, 106, 479, 110.1802978515625, 468.25103759765625, 1, 213, 479, 209.76011657714844, 474.91726684570312, 1, 320, 479, 314.64419555664062, 477.99530029296875, 1, 426, 479, 418.82501220703125, 477.0888671875, 1, 532, 479, 518.32244873046875, 472.53683471679688, 1, 639, 479, 610.56573486328125, 465.28729248046875}, +}; + +static const struct stereo_rectify_fixture k_fixture_sr_like = { + "sr_like", 640, 480, 0, 0, 1, + {{421.69999999999999, 0, 318.19999999999999, 0, 422.89999999999998, 243.59999999999999, 0, 0, 1}, {424.10000000000002, 0, 322.89999999999998, 0, 425, 238.09999999999999, 0, 0, 1}}, + {{-0.082000000000000003, 0.121, 0.0011000000000000001, -0.00040000000000000002, -0.060999999999999999, 0, 0, 0}, {-0.074999999999999997, 0.098000000000000004, -0.00069999999999999999, 0.00089999999999999998, -0.043999999999999997, 0, 0, 0}}, + {0.99998563503967131, -0.0032044973128937176, -0.0042966162589852044, 0.0031954673378315228, 0.99999267502022915, -0.0021068683820189863, 0.0043033362404268378, 0.0020931084200194522, 0.99998855003162102}, {-63.469999999999999, 0.40999999999999998, -0.82999999999999996}, + {0.99991491406671129, -0.0096357981924167398, 0.0087929528689994867, 0.0096451782567072188, 0.99995295886120927, -0.001024988345914208, -0.0087826626576323896, 0.0011097107316485149, 0.99996081592167119}, {0.99989364827379268, -0.006459057756292289, 0.01307565350664453, 0.0064450971160348661, 0.99997861459363835, 0.0011095397931037939, -0.013082540460087409, -0.001025147934925552, 0.99991389439632339}, {429.86495602842393, 0, 313.38402938842773, 0, 0, 429.86495602842393, 241.02614402770996, 0, 0, 0, 1, 0}, {429.86495602842393, 0, 313.38402938842773, -27286.430718132982, 0, 429.86495602842393, 241.02614402770996, 0, 0, 0, 1, 0}, + {391.87611728541026, 313.84673321042442, 240.91068135796641}, + {0, 0, 0, 9.2827739715576172, 13.777859687805176, 0, 106, 0, 112.01531982421875, 12.907480239868164, 0, 213, 0, 215.40269470214844, 11.963018417358398, 0, 320, 0, 318.64385986328125, 11.201150894165039, 0, 426, 0, 420.40866088867188, 11.097481727600098, 0, 532, 0, 521.31646728515625, 11.158949851989746, 0, 639, 0, 622.492431640625, 11.143622398376465, 0, 0, 120, 9.8415422439575195, 129.75579833984375, 0, 106, 120, 112.40514373779297, 128.54570007324219, 0, 213, 120, 215.66424560546875, 126.94230651855469, 0, 320, 120, 319.816650390625, 125.78262329101562, 0, 426, 120, 422.45095825195312, 125.53066253662109, 0, 532, 120, 523.2119140625, 125.69633483886719, 0, 639, 120, 624.20782470703125, 125.48140716552734, 0, 0, 240, 10.620198249816895, 246.25874328613281, 0, 106, 240, 112.88777160644531, 245.1260986328125, 0, 213, 240, 216.22691345214844, 244.03797912597656, 0, 320, 240, 320.97607421875, 242.9971923828125, 0, 426, 240, 424.17300415039062, 242.03355407714844, 0, 532, 240, 524.97601318359375, 241.15110778808594, 0, 639, 240, 625.6756591796875, 240.32371520996094, 0, 0, 359, 11.514342308044434, 362.07369995117188, 0, 106, 359, 114.25772857666016, 361.017822265625, 0, 213, 359, 217.71333312988281, 360.45199584960938, 0, 320, 359, 322.0709228515625, 359.54281616210938, 0, 426, 359, 424.89013671875, 357.87509155273438, 0, 532, 359, 525.81005859375, 355.93862915039062, 0, 639, 359, 626.96600341796875, 354.48715209960938, 0, 0, 479, 12.622738838195801, 478.9141845703125, 0, 106, 479, 115.73625183105469, 477.50299072265625, 0, 213, 479, 219.4949951171875, 476.28683471679688, 0, 320, 479, 323.11593627929688, 475.00119018554688, 0, 426, 479, 425.24160766601562, 473.18679809570312, 0, 532, 479, 526.49114990234375, 471.33016967773438, 0, 639, 479, 628.023193359375, 469.67559814453125, 1, 0, 0, 10.344363212585449, 3.6066198348999023, 1, 106, 0, 114.04583740234375, 3.4317293167114258, 1, 213, 0, 218.2841796875, 2.9229519367218018, 1, 320, 0, 322.4739990234375, 2.5816488265991211, 1, 426, 0, 425.1719970703125, 2.8780093193054199, 1, 532, 0, 526.87469482421875, 3.3557379245758057, 1, 639, 0, 628.93365478515625, 3.4970929622650146, 1, 0, 120, 11.003759384155273, 121.06476593017578, 1, 106, 120, 114.13439178466797, 120.36711120605469, 1, 213, 120, 218.17823791503906, 119.25482177734375, 1, 320, 120, 323.13818359375, 118.52081298828125, 1, 426, 120, 426.58740234375, 118.56651306152344, 1, 532, 120, 528.11944580078125, 118.98307800292969, 1, 639, 120, 629.578857421875, 119.02649688720703, 1, 0, 240, 11.82154369354248, 238.47807312011719, 1, 106, 240, 114.50177001953125, 237.90115356445312, 1, 213, 240, 218.50215148925781, 237.27389526367188, 1, 320, 240, 323.87161254882812, 236.60771179199219, 1, 426, 240, 427.711669921875, 235.92776489257812, 1, 532, 240, 529.1932373046875, 235.23455810546875, 1, 639, 240, 630.21673583984375, 234.50439453125, 1, 0, 359, 12.93834400177002, 354.7034912109375, 1, 106, 359, 115.89998626708984, 354.262451171875, 1, 213, 359, 219.80398559570312, 354.13876342773438, 1, 320, 359, 324.64675903320312, 353.55496215820312, 1, 426, 359, 427.9708251953125, 352.15399169921875, 1, 532, 359, 529.348876953125, 350.34158325195312, 1, 639, 359, 630.63555908203125, 348.82196044921875, 1, 0, 479, 14.214091300964355, 471.5322265625, 1, 106, 479, 117.6163330078125, 470.56671142578125, 1, 213, 479, 221.54965209960938, 469.88070678710938, 1, 320, 479, 325.46670532226562, 468.93719482421875, 1, 426, 479, 427.88900756835938, 467.2821044921875, 1, 532, 479, 529.28631591796875, 465.37380981445312, 1, 639, 479, 631.0482177734375, 463.74661254882812}, +}; + +static const struct stereo_rectify_fixture k_fixture_radtan8_scaled = { + "radtan8_scaled", 640, 480, 1280, 960, 2, + {{905, 0, 641.5, 0, 903, 477, 0, 0, 1}, {899, 0, 636, 0, 900.5, 483.5, 0, 0, 1}}, + {{0.20999999999999999, -0.050000000000000003, 0.00059999999999999995, 0.00029999999999999997, 0.01, 0.42999999999999999, 0.02, 0.0040000000000000001}, {0.19, -0.040000000000000001, -0.00040000000000000002, 0.00050000000000000001, 0.0080000000000000002, 0.40000000000000002, 0.014999999999999999, 0.0030000000000000001}}, + {0.9999797750888213, 0.0017893092369226367, 0.0061030964079741151, -0.0018106591431606817, 0.99999225503401346, 0.0034944792825245545, -0.0060967964356415764, -0.0035054592343041206, 0.99997527010860576}, {-40.200000000000003, -0.34999999999999998, 0.52000000000000002}, + {0.99992132904265318, 0.010539772551984196, -0.0068006558600212045, -0.010527985703966673, 0.99994301857135048, 0.0017666712860391006, 0.0068188886624635211, -0.0016949350926458114, 0.99997531467153766}, {0.99987845957661869, 0.0087054094739190593, -0.012933751218397103, -0.0087277833316827796, 0.99996051023646515, -0.0016744449060687062, 0.012918663739071023, 0.0017871243716259515, 0.99991495354038851}, {425.59643371580466, 0, 328.47037124633789, 0, 0, 425.59643371580466, 240.27606582641602, 0, 0, 0, 1, 0}, {425.59643371580466, 0, 328.47037124633789, -17111.056320405034, 0, 425.59643371580466, 240.27606582641602, 0, 0, 0, 1, 0}, + {450.25, 328.81033655905583, 240.34823533756631}, + {0, 0, 0, 39.806583404541016, 26.450933456420898, 0, 106, 0, 118.28645324707031, 12.094629287719727, 0, 213, 0, 213.35528564453125, 2.9100487232208252, 0, 320, 0, 317.53338623046875, -0.0073644723743200302, 0, 426, 0, 421.48922729492188, 3.6148638725280762, 0, 532, 0, 517.80120849609375, 13.458554267883301, 0, 639, 0, 599.892333984375, 28.825506210327148, 0, 0, 120, 23.845294952392578, 124.79419708251953, 0, 106, 120, 106.19843292236328, 117.49972534179688, 0, 213, 120, 206.2528076171875, 113.09403228759766, 0, 320, 120, 316.01846313476562, 112.08939361572266, 0, 426, 120, 425.62353515625, 114.57035064697266, 0, 532, 120, 527.20294189453125, 120.33719635009766, 0, 639, 120, 613.689697265625, 129.01432800292969, 0, 0, 240, 17.447366714477539, 234.26361083984375, 0, 106, 240, 101.13021850585938, 234.98895263671875, 0, 213, 240, 202.89291381835938, 235.96000671386719, 0, 320, 240, 314.58450317382812, 237.097412109375, 0, 426, 240, 426.1614990234375, 238.29718017578125, 0, 532, 240, 529.61895751953125, 239.47416687011719, 0, 639, 240, 617.7322998046875, 240.55113220214844, 0, 0, 359, 21.263168334960938, 343.102783203125, 0, 106, 359, 103.56227874755859, 351.74288940429688, 0, 213, 359, 203.59284973144531, 358.00244140625, 0, 320, 359, 313.385498046875, 361.22146606445312, 0, 426, 359, 423.08840942382812, 361.11367797851562, 0, 532, 359, 524.851806640625, 357.70999145507812, 0, 639, 359, 611.6148681640625, 351.23043823242188, 0, 0, 479, 34.816947937011719, 442.3594970703125, 0, 106, 479, 113.21434783935547, 457.9637451171875, 0, 213, 479, 208.26432800292969, 468.88372802734375, 0, 320, 479, 312.52322387695312, 473.8944091796875, 0, 426, 479, 416.69427490234375, 472.53567504882812, 0, 532, 479, 513.38165283203125, 464.96435546875, 0, 639, 479, 596.0145263671875, 451.72686767578125, 1, 0, 0, 37.449436187744141, 29.096076965332031, 1, 106, 0, 117.55707550048828, 15.905220031738281, 1, 213, 0, 212.98945617675781, 7.2534098625183105, 1, 320, 0, 316.95709228515625, 4.2553362846374512, 1, 426, 0, 420.79556274414062, 7.2715229988098145, 1, 532, 0, 517.6470947265625, 16.048103332519531, 1, 639, 0, 601.45654296875, 29.928749084472656, 1, 0, 120, 23.134662628173828, 129.13873291015625, 1, 106, 120, 106.65638732910156, 122.45317840576172, 1, 213, 120, 206.61662292480469, 118.25478363037109, 1, 320, 120, 315.77243041992188, 117.09933471679688, 1, 426, 120, 424.88827514648438, 119.14150238037109, 1, 532, 120, 526.606689453125, 124.21871948242188, 1, 639, 120, 614.3831787109375, 131.96627807617188, 1, 0, 240, 17.487483978271484, 239.18548583984375, 1, 106, 240, 102.14824676513672, 240.00784301757812, 1, 213, 240, 203.65617370605469, 240.94680786132812, 1, 320, 240, 314.61508178710938, 241.93450927734375, 1, 426, 240, 425.58609008789062, 242.88748168945312, 1, 532, 240, 529.03094482421875, 243.74014282226562, 1, 639, 240, 618.23284912109375, 244.43392944335938, 1, 0, 359, 21.163570404052734, 348.37802124023438, 1, 106, 359, 104.54096984863281, 356.64190673828125, 1, 213, 359, 204.43681335449219, 362.66021728515625, 1, 320, 359, 313.60433959960938, 365.74667358398438, 1, 426, 359, 422.789306640625, 365.58413696289062, 1, 532, 359, 524.60986328125, 362.20648193359375, 1, 639, 359, 612.49774169921875, 355.86325073242188, 1, 0, 479, 33.737724304199219, 448.59530639648438, 1, 106, 479, 113.56036376953125, 463.35128784179688, 1, 213, 479, 208.85313415527344, 473.77713012695312, 1, 320, 479, 312.8216552734375, 478.62850952148438, 1, 426, 479, 416.76837158203125, 477.38937377929688, 1, 532, 479, 513.79119873046875, 470.19485473632812, 1, 639, 479, 597.79248046875, 457.59927368164062}, +}; + diff --git a/tests/tests_stereo_rectify.cpp b/tests/tests_stereo_rectify.cpp new file mode 100644 index 0000000000..f3f3edce35 --- /dev/null +++ b/tests/tests_stereo_rectify.cpp @@ -0,0 +1,482 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: the vendor-neutral rectifier. + * + * - GOLDEN: u_stereo_rectify_compute against OpenCV 4.10's + * stereoRectify(CALIB_ZERO_DISPARITY, alpha = 0) on three calibrations + * (the sim_display distorted fake, an SR-like tracker pair with a + * non-axial baseline, a RADTAN8 pair calibrated at 2x the frame size), and + * the per-pixel maps against initUndistortRectifyMap samples + * (fixtures/stereo_rectify_fixtures.h, generated offline); + * - the lens models invert (RADTAN5/8, KB4) and Rodrigues round-trips; + * - alpha = 0 really holds: every output pixel of both eyes samples inside its + * raw image (no black corners); + * - END TO END on the sim_display DISTORTED fake: the raw pair is visibly + * misaligned; after the CPU LUT the vertical disparity on matched blocks is + * <= 0.5 px and the horizontal disparity equals f_rect * B / Z within + * 0.5 px, for the background and the bar; + * - the NV12 / BGRA8 plane paths agree with GRAY8; + * - [.perf] (hidden): CPU cost per 1280x480 frame. + */ + +#include "catch_amalgamated.hpp" + +#include "util/u_stereo_camera.h" +#include "util/u_stereo_rectify.h" + +#include "sim_display_stereo_camera_pattern.h" +#include "stereo_rectify_fixtures.h" + +#include +#include +#include +#include +#include +#include + +namespace { + +const stereo_rectify_fixture *const k_fixtures[] = { + &k_fixture_sim_distorted, + &k_fixture_sr_like, + &k_fixture_radtan8_scaled, +}; + +u_stereo_rectify_input +input_from(const stereo_rectify_fixture &f) +{ + u_stereo_rectify_input in; + std::memset(&in, 0, sizeof(in)); + in.width = (uint32_t)f.width; + in.height = (uint32_t)f.height; + in.calib_width = (uint32_t)f.calib_width; + in.calib_height = (uint32_t)f.calib_height; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = f.K[e][0]; + in.eye[e].fy = f.K[e][4]; + in.eye[e].cx = f.K[e][2]; + in.eye[e].cy = f.K[e][5]; + in.eye[e].model = (uint32_t)f.model; + for (int k = 0; k < 8; k++) { + in.eye[e].d[k] = f.D[e][k]; + } + } + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + in.R[i][j] = f.R[i * 3 + j]; + } + in.T[i] = f.T[i]; + } + return in; +} + +const double k_ideal_fx_640 = 320.0 / std::tan(34.0 * M_PI / 180.0); + +} // namespace + + +TEST_CASE("stereo rectify: Rodrigues and lens models invert", "[stereo_rectify]") +{ + const double rv[3] = {0.012, -0.031, 0.007}; + double m[3][3], back[3]; + u_stereo_rectify_rodrigues(rv, m); + u_stereo_rectify_rodrigues_inv(m, back); + for (int i = 0; i < 3; i++) { + CHECK(back[i] == Catch::Approx(rv[i]).margin(1e-14)); + } + const double big[3] = {1.1, -0.4, 2.2}; + u_stereo_rectify_rodrigues(big, m); + u_stereo_rectify_rodrigues_inv(m, back); + for (int i = 0; i < 3; i++) { + CHECK(back[i] == Catch::Approx(big[i]).margin(1e-12)); + } + + u_stereo_rectify_lens lenses[3]; + std::memset(lenses, 0, sizeof(lenses)); + const double r5[8] = {-0.21, 0.06, 0.0008, -0.0006, -0.005, 0, 0, 0}; + const double r8[8] = {0.31, -0.12, 0.0006, 0.0003, 0.02, 0.61, -0.05, 0.03}; + const double kb[8] = {0.021, -0.013, 0.004, -0.001, 0, 0, 0, 0}; + const uint32_t models[3] = {U_STEREO_RECTIFY_MODEL_RADTAN5, U_STEREO_RECTIFY_MODEL_RADTAN8, + U_STEREO_RECTIFY_MODEL_KB4}; + const double *coeffs[3] = {r5, r8, kb}; + for (int l = 0; l < 3; l++) { + lenses[l].fx = 480.0; + lenses[l].fy = 478.0; + lenses[l].cx = 325.0; + lenses[l].cy = 236.0; + lenses[l].model = models[l]; + std::memcpy(lenses[l].d, coeffs[l], sizeof(r5)); + for (double y = -0.45; y <= 0.45; y += 0.15) { + for (double x = -0.6; x <= 0.6; x += 0.15) { + double xd, yd, xu, yu; + u_stereo_rectify_distort(&lenses[l], x, y, &xd, &yd); + u_stereo_rectify_undistort_pixel(&lenses[l], lenses[l].fx * xd + lenses[l].cx, + lenses[l].fy * yd + lenses[l].cy, &xu, &yu); + CHECK(xu == Catch::Approx(x).margin(1e-9)); + CHECK(yu == Catch::Approx(y).margin(1e-9)); + } + } + } +} + +TEST_CASE("stereo rectify: GOLDEN geometry vs OpenCV stereoRectify", "[stereo_rectify]") +{ + for (const stereo_rectify_fixture *fx : k_fixtures) { + const stereo_rectify_fixture &f = *fx; + INFO("fixture " << f.name); + u_stereo_rectify_input in = input_from(f); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + + // R1 / R2: identical algorithm, so identical to double precision. + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + CHECK(r.rot[0][i][j] == Catch::Approx(f.R1[i * 3 + j]).margin(1e-12)); + CHECK(r.rot[1][i][j] == Catch::Approx(f.R2[i * 3 + j]).margin(1e-12)); + } + } + // Principal point: OpenCV's corner-centroid rule, one cx for both eyes + // (zero disparity at infinity). cvStereoRectify runs undistortPoints for + // only 5 iterations; we converge, so we match the CONVERGED reference + // exactly and OpenCV's own P1 to a fraction of a pixel. + CHECK(r.cx == Catch::Approx(f.converged[1]).margin(1e-4)); + CHECK(r.cy == Catch::Approx(f.converged[2]).margin(1e-4)); + CHECK(r.cx == Catch::Approx(f.P1[2]).margin(0.75)); + CHECK(r.cy == Catch::Approx(f.P1[6]).margin(0.75)); + CHECK(f.P2[2] == Catch::Approx(f.P1[2]).margin(1e-9)); + // Focal before the crop zoom: OpenCV's min-fy rule, exactly. + CHECK(r.f / r.crop_scale == Catch::Approx(f.converged[0]).epsilon(1e-9)); + // After the alpha = 0 zoom: OpenCV's 9x9 inner rectangle, plus our + // border verification, which may only zoom further (OpenCV itself leaves + // a few black pixels on strong lenses). Within 1 % of OpenCV. + double f_cv = f.P1[0]; + CHECK(r.f == Catch::Approx(f_cv).epsilon(0.01)); + std::printf( + "[golden] %-15s f %.4f (OpenCV %.4f, %+.3f%%) cx %.4f (OpenCV %.4f) cy %.4f (OpenCV %.4f) crop " + "x%.4f baseline %.3f\n", + f.name, r.f, f_cv, 100.0 * (r.f / f_cv - 1.0), r.cx, f.P1[2], r.cy, f.P1[6], r.crop_scale, + r.baseline); + // Rectified translation: OpenCV P2[0][3] = f * Tx. + CHECK(r.P[1][0][3] / r.f == Catch::Approx(f.P2[3] / f.P2[0]).margin(1e-9)); + CHECK(std::fabs(r.t_rect[1]) < 1e-7 * r.baseline); // baseline on the x axis + CHECK(std::fabs(r.t_rect[2]) < 1e-7 * r.baseline); + CHECK(std::fabs(r.t_rect[0]) == Catch::Approx(r.baseline).epsilon(1e-12)); + } +} + +TEST_CASE("stereo rectify: GOLDEN maps vs OpenCV initUndistortRectifyMap", "[stereo_rectify]") +{ + for (const stereo_rectify_fixture *fx : k_fixtures) { + const stereo_rectify_fixture &f = *fx; + INFO("fixture " << f.name); + u_stereo_rectify_input in = input_from(f); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + // Feed OpenCV's own R/P so the map math is compared in isolation. + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + r.rot[0][i][j] = f.R1[i * 3 + j]; + r.rot[1][i][j] = f.R2[i * 3 + j]; + } + } + r.f = f.P1[0]; + r.cx = f.P1[2]; + r.cy = f.P1[6]; + double worst = 0.0; + for (int s = 0; s < STEREO_RECTIFY_FIXTURE_SAMPLES; s++) { + const double *smp = &f.samples[s * 5]; + double x, y; + REQUIRE(u_stereo_rectify_map_point(&r, (uint32_t)smp[0], smp[1], smp[2], &x, &y)); + worst = std::max(worst, std::max(std::fabs(x - smp[3]), std::fabs(y - smp[4]))); + } + // OpenCV stores float32 maps: agreement to its precision. + CHECK(worst < 2e-3); + std::printf("[golden] %-15s map max |ours - OpenCV| = %.2e px over %d samples\n", f.name, worst, + STEREO_RECTIFY_FIXTURE_SAMPLES); + } +} + +TEST_CASE("stereo rectify: alpha = 0 leaves no black pixel", "[stereo_rectify]") +{ + for (const stereo_rectify_fixture *fx : k_fixtures) { + INFO("fixture " << fx->name); + u_stereo_rectify_input in = input_from(*fx); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + for (uint32_t sub = 1; sub <= 2; sub++) { + u_stereo_rectify_lut lut; + REQUIRE(u_stereo_rectify_lut_init(&lut, &r, sub)); + CHECK(lut.width == 2 * r.width / sub); + CHECK(lut.valid == lut.width * lut.height); + u_stereo_rectify_lut_fini(&lut); + } + } +} + +TEST_CASE("stereo rectify: the C fake's truth equals the fixture's inputs", "[stereo_rectify]") +{ + sim_stereo_camera_truth t; + sim_stereo_camera_truth_init(&t, 640, 480, k_ideal_fx_640, 50.0, 2.0, 0.6); + const stereo_rectify_fixture &f = k_fixture_sim_distorted; + for (int e = 0; e < 2; e++) { + CHECK(t.fx[e] == Catch::Approx(f.K[e][0]).margin(1e-9)); + CHECK(t.fy[e] == Catch::Approx(f.K[e][4]).margin(1e-9)); + CHECK(t.cx[e] == Catch::Approx(f.K[e][2]).margin(1e-9)); + CHECK(t.cy[e] == Catch::Approx(f.K[e][5]).margin(1e-9)); + for (int k = 0; k < 5; k++) { + CHECK(t.dist[e][k] == Catch::Approx(f.D[e][k]).margin(1e-15)); + } + } + for (int i = 0; i < 9; i++) { + CHECK(t.R[i / 3][i % 3] == Catch::Approx(f.R[i]).margin(1e-12)); + } + for (int i = 0; i < 3; i++) { + CHECK(t.T_mm[i] == Catch::Approx(f.T[i]).margin(1e-9)); + } +} + + +/* + * + * End to end on the distorted fake. + * + */ + +namespace { + +struct block_stats +{ + std::vector dy_bg, dy_bar, dx_bg, dx_bar; +}; + +float +median(std::vector v) +{ + if (v.empty()) { + return NAN; + } + std::sort(v.begin(), v.end()); + return v[v.size() / 2]; +} + +float +max_abs_dev(const std::vector &v, float ref) +{ + float m = 0.0f; + for (float x : v) { + m = std::max(m, std::fabs(x - ref)); + } + return m; +} + +/*! + * Block-match a GRAY8 SBS pair on a grid. @p to_ideal maps a LEFT-eye pixel of + * @p img to the virtual parallel frame the scene is defined in, so each block + * is classified as background or bar (and skipped when it straddles an edge, + * the occlusion band next to the bar, or the frame counter). + */ +template +block_stats +match_blocks( + const uint8_t *img, uint32_t pitch, uint32_t eye_w, uint32_t h, const sim_stereo_camera_scene &sc, ToIdeal to_ideal) +{ + block_stats st; + const uint32_t B = 32, max_dy = 10; + for (uint32_t y = 48; y + B + max_dy <= h; y += B / 2) { + for (uint32_t x = 72; x + B <= eye_w; x += B / 2) { + double u0, v0, u1, v1; + to_ideal(x, y, &u0, &v0); + to_ideal(x + B, y + B, &u1, &v1); + const double m = 4.0; + bool rows_in = v0 >= sc.bar_y0 + m && v1 <= sc.bar_y1 - m; + bool rows_out = v1 < sc.bar_y0 - m || v0 > sc.bar_y1 + m; + bool cols_in = u0 >= sc.bar_x0 + m && u1 <= sc.bar_x1 - m; + // Background blocks must also keep clear of where the bar occludes the + // right eye's view of them (left x in [x0 - d_bar, x1)). + bool cols_out = u1 < sc.bar_x0 - sc.bar_disparity_f - m || u0 > sc.bar_x1 + m; + bool is_bar = rows_in && cols_in; + bool is_bg = rows_out || cols_out; + if (!is_bar && !is_bg) { + continue; + } + float dx, dy; + if (!u_stereo_camera_estimate_offset(img, pitch, eye_w, h, x, y, B, B, 64, max_dy, &dx, &dy)) { + continue; + } + (is_bar ? st.dx_bar : st.dx_bg).push_back(dx); + (is_bar ? st.dy_bar : st.dy_bg).push_back(dy); + } + } + return st; +} + +} // namespace + +TEST_CASE("stereo rectify: END TO END on the distorted sim fake", "[stereo_rectify]") +{ + const uint32_t W = 640, H = 480; + sim_stereo_camera_distorted d; + REQUIRE(sim_stereo_camera_distorted_init(&d, W, H, k_ideal_fx_640, 50.0, 2.0, 0.6)); + std::vector raw(2 * W * H), rect(2 * W * H); + sim_stereo_camera_render_gray_distorted(&d, 1234, raw.data(), 2 * W); + + // The RAW pair really is misaligned (the thing the rectifier must fix): + // measured in raw pixels, the rows of the two eyes differ by several px. + { + block_stats st = + match_blocks(raw.data(), 2 * W, W, H, d.scene, [](uint32_t x, uint32_t y, double *u, double *v) { + *u = x; // roughly: classification only needs to be conservative + *v = y; + }); + std::vector all = st.dy_bg; + all.insert(all.end(), st.dy_bar.begin(), st.dy_bar.end()); + REQUIRE(all.size() > 20); + std::vector absdy; + for (float v : all) { + absdy.push_back(std::fabs(v)); + } + float med = median(absdy); + float mx = *std::max_element(absdy.begin(), absdy.end()); + std::printf("[e2e] RAW pair: |dy| median %.2f px, max %.2f px over %zu blocks\n", med, mx, all.size()); + CHECK(med > 2.0f); + } + + // Rectify with the calibration the fake's plug-in slot reports. + u_stereo_rectify_input in; + std::memset(&in, 0, sizeof(in)); + in.width = W; + in.height = H; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = d.truth.fx[e]; + in.eye[e].fy = d.truth.fy[e]; + in.eye[e].cx = d.truth.cx[e]; + in.eye[e].cy = d.truth.cy[e]; + in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + std::memcpy(in.eye[e].d, d.truth.dist[e], sizeof(d.truth.dist[e])); + } + std::memcpy(in.R, d.truth.R, sizeof(in.R)); + std::memcpy(in.T, d.truth.T_mm, sizeof(in.T)); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + u_stereo_rectify_lut lut; + REQUIRE(u_stereo_rectify_lut_init(&lut, &r, 1)); + u_stereo_rectify_lut_apply(&lut, 1, raw.data(), 2 * W, rect.data(), 2 * W); + + // The symmetric construction makes the rectified frame the virtual one: + // rectified (u, v) -> virtual pixel by the focal / principal point only. + const double ifx = d.truth.ideal_fx, icx = d.truth.ideal_cx, icy = d.truth.ideal_cy; + auto to_ideal = [&](uint32_t x, uint32_t y, double *u, double *v) { + *u = ifx * (x - r.cx) / r.f + icx; + *v = ifx * (y - r.cy) / r.f + icy; + }; + block_stats st = match_blocks(rect.data(), 2 * W, W, H, d.scene, to_ideal); + REQUIRE(st.dx_bg.size() > 20); + REQUIRE(st.dx_bar.size() > 5); + + const double baseline_m = 0.05; + const float gt_bg = (float)(r.f * baseline_m / 2.0); + const float gt_bar = (float)(r.f * baseline_m / 0.6); + std::vector dy_all = st.dy_bg; + dy_all.insert(dy_all.end(), st.dy_bar.begin(), st.dy_bar.end()); + float dy_max = max_abs_dev(dy_all, 0.0f); + float bg_med = median(st.dx_bg), bar_med = median(st.dx_bar); + float bg_dev = max_abs_dev(st.dx_bg, gt_bg), bar_dev = max_abs_dev(st.dx_bar, gt_bar); + std::printf( + "[e2e] RECTIFIED (f %.3f px, crop x%.4f): |dy| max %.3f px over %zu blocks (median %.3f)\n" + "[e2e] background: GT f*B/Z = %.3f px, measured median %.3f, max |err| %.3f px (%zu blocks)\n" + "[e2e] bar: GT f*B/Z = %.3f px, measured median %.3f, max |err| %.3f px (%zu blocks)\n", + r.f, r.crop_scale, dy_max, dy_all.size(), median(dy_all), gt_bg, bg_med, bg_dev, st.dx_bg.size(), gt_bar, + bar_med, bar_dev, st.dx_bar.size()); + CHECK(dy_max <= 0.5f); + CHECK(bg_dev <= 0.5f); + CHECK(bar_dev <= 0.5f); + + u_stereo_rectify_lut_fini(&lut); + sim_stereo_camera_distorted_fini(&d); +} + +TEST_CASE("stereo rectify: NV12 and BGRA8 planes agree with GRAY8", "[stereo_rectify]") +{ + const uint32_t W = 640, H = 480, SW = 2 * W; + sim_stereo_camera_distorted d; + REQUIRE(sim_stereo_camera_distorted_init(&d, W, H, k_ideal_fx_640, 50.0, 2.0, 0.6)); + std::vector gray(SW * H), gray_r(SW * H); + sim_stereo_camera_render_gray_distorted(&d, 7, gray.data(), SW); + + u_stereo_rectify_input in = input_from(k_fixture_sim_distorted); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + u_stereo_rectify_lut full, half; + REQUIRE(u_stereo_rectify_lut_init(&full, &r, 1)); + REQUIRE(u_stereo_rectify_lut_init(&half, &r, 2)); + u_stereo_rectify_lut_apply(&full, 1, gray.data(), SW, gray_r.data(), SW); + + // BGRA with B = G = R = gray: every colour channel equals the GRAY8 result. + std::vector bgra(SW * H * 4), bgra_r(SW * H * 4); + for (size_t i = 0; i < gray.size(); i++) { + bgra[4 * i] = bgra[4 * i + 1] = bgra[4 * i + 2] = gray[i]; + bgra[4 * i + 3] = 255; + } + u_stereo_rectify_lut_apply(&full, 4, bgra.data(), SW * 4, bgra_r.data(), SW * 4); + size_t mism = 0; + for (size_t i = 0; i < gray.size(); i++) { + mism += bgra_r[4 * i] != gray_r[i] || bgra_r[4 * i + 2] != gray_r[i] || bgra_r[4 * i + 3] != 255; + } + CHECK(mism == 0); + + // NV12 chroma: a constant UV plane stays constant (no black pull-in). + std::vector uv(SW * H / 2, 0), uv_r(SW * H / 2, 0); + for (size_t i = 0; i < uv.size(); i += 2) { + uv[i] = 90; + uv[i + 1] = 170; + } + REQUIRE(half.width == W); + u_stereo_rectify_lut_apply(&half, 2, uv.data(), SW, uv_r.data(), SW); + size_t off = 0; + for (size_t i = 0; i < uv_r.size(); i += 2) { + off += uv_r[i] != 90 || uv_r[i + 1] != 170; + } + CHECK(off == 0); + + u_stereo_rectify_lut_fini(&full); + u_stereo_rectify_lut_fini(&half); + sim_stereo_camera_distorted_fini(&d); +} + +TEST_CASE("stereo rectify: CPU cost per 1280x480 frame", "[.perf][stereo_rectify]") +{ + const uint32_t W = 640, H = 480, SW = 2 * W; + u_stereo_rectify_input in = input_from(k_fixture_sim_distorted); + u_stereo_rectify_result r; + auto t0 = std::chrono::steady_clock::now(); + REQUIRE(u_stereo_rectify_compute(&in, &r)); + u_stereo_rectify_lut full, half; + REQUIRE(u_stereo_rectify_lut_init(&full, &r, 1)); + REQUIRE(u_stereo_rectify_lut_init(&half, &r, 2)); + double setup_ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + + std::vector src(SW * H * 4, 77), dst(SW * H * 4); + for (size_t i = 0; i < src.size(); i++) { + src[i] = (uint8_t)(i * 2654435761u >> 24); + } + const int N = 200; + auto run = [&](const u_stereo_rectify_lut &lut, uint32_t ch) { + auto a = std::chrono::steady_clock::now(); + for (int i = 0; i < N; i++) { + u_stereo_rectify_lut_apply(&lut, ch, src.data(), lut.width * ch, dst.data(), lut.width * ch); + } + return std::chrono::duration(std::chrono::steady_clock::now() - a).count() / N; + }; + double g = run(full, 1); + double y = g; + double uvp = run(half, 2); + double b = run(full, 4); + std::printf( + "[perf] setup (geometry + both LUTs) %.1f ms | per 1280x480 frame: GRAY8 %.3f ms, NV12 %.3f ms " + "(Y %.3f + UV %.3f), BGRA8 %.3f ms\n", + setup_ms, g, y + uvp, y, uvp, b); + u_stereo_rectify_lut_fini(&full); + u_stereo_rectify_lut_fini(&half); +} From c918101e31bfbc4d9d7cb27e071024cc8dfd45a6 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 19/37] =?UTF-8?q?feat(stereo-camera):=20the=20service=20re?= =?UTF-8?q?ctifies=20=E2=80=94=20RECTIFIED=20frames=20+=20rectified=20cali?= =?UTF-8?q?bration=20(R2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A per-camera rectifier is built at manager create from the plug-in's RAW calibration when it is CALIBRATED and not NATIVELY_RECTIFIED. It runs once per source frame on the camera thread, outside the manager lock, and only while a started stream wants RECTIFIED; RAW streams read the original frame. A RECTIFIED stream never receives unrectified pixels (a frame produced before it started is skipped). get_calibration(RECTIFIED) returns the same geometry: one pinhole for both eyes (zero disparity at infinity), no distortion, rightFromLeft = identity + (+baseline, 0, 0). horizontalFovDeg is the rectified one. The browser (PRESENT_OWNER) is refused a camera the rectifier rejected instead of getting the RAW-flagged fallback native clients get. The seam is struct scam_rectifier (backend-neutral geometry + an apply() backend: CPU LUT today). Co-Authored-By: Claude Opus 5.5 --- src/xrt/ipc/server/ipc_server_stereo_camera.c | 294 +++++++++++++++++- 1 file changed, 281 insertions(+), 13 deletions(-) diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index 7c343449c1..d76bd2a115 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -25,6 +25,21 @@ * - the in-use indicator (tray badge naming consumers) is R3; R1 logs one * WARN per stream start/stop naming the peer executable. * + * Rectification (R2) — when the plug-in reports CALIBRATED but not + * NATIVELY_RECTIFIED, the manager builds a per-camera rectifier at create time + * from the plug-in's RAW calibration (u_stereo_rectify: Bouguet, zero + * disparity at infinity, alpha = 0 crop, no convergence shear). It runs ONCE + * per source frame, on the camera thread, with the manager lock RELEASED, and + * only while at least one started stream wants RECTIFIED; every RECTIFIED + * stream then converts from the rectified image, RAW streams from the + * original. The seam is `struct scam_rectifier`: the geometry and the float + * maps are backend-neutral; `apply` is the backend. CPU today (fixed-point + * bilinear LUT: ~0.9 ms GRAY8 / ~1.4 ms NV12 per 1280x480 frame on an M1 Pro, + * -O2). A GPU backend uploads u_stereo_rectify_build_map() as an RG32F texture + * and remaps straight into the GPU transports' slots below. Rectified + * calibration (common f / principal point, rightFromLeft = pure +x baseline) + * comes from the same geometry, so frames and numbers cannot disagree. + * * Transports — R1 implements SHARED_MEMORY only. The GPU transports are * designed to reuse this ring's state machine unchanged, only the slot storage * differs: @@ -61,9 +76,11 @@ #include "util/u_logging.h" #include "util/u_misc.h" #include "util/u_stereo_camera.h" +#include "util/u_stereo_rectify.h" #include #include +#include #include #ifdef XRT_OS_WINDOWS @@ -201,6 +218,29 @@ struct scam_stream uint64_t latency_count; }; +/*! + * The rectifier seam (R2). Geometry + maps are backend-neutral; @ref apply is + * the backend. Owned by one camera, used only on its camera thread. + */ +struct scam_rectifier +{ + struct u_stereo_rectify_result geo; + //! [0] full resolution (GRAY8, NV12 Y, BGRA8), [1] NV12 UV (half-res, 2 ch). + struct u_stereo_rectify_lut lut[2]; + //! Rectified output in the SOURCE frame's format (tight layout). + uint8_t *buf; + uint64_t buf_size; + uint32_t buf_format; + struct u_stereo_camera_planes buf_layout; + //! Backend: rectify @p in into @p out (planes point into buf). CPU LUT today. + bool (*apply)(struct scam_rectifier *r, + const struct xrt_plugin_stereo_camera_frame *in, + struct xrt_plugin_stereo_camera_frame *out); + // stats + uint64_t ns_sum; + uint64_t count; +}; + struct scam_camera { uint32_t index; //!< plug-in index @@ -210,6 +250,9 @@ struct scam_camera struct xrt_plugin_stereo_camera_calibration calib; uint32_t state; uint32_t calibration_generation; + //! R2: set when CALIBRATED and not NATIVELY_RECTIFIED and the geometry is + //! sane; NULL = no service-side rectification (RAW-flagged fallback). + struct scam_rectifier *rect; struct xrt_plugin_stereo_camera *src; //!< owned by the camera thread uint32_t started_count; @@ -363,9 +406,9 @@ output_allowed(volatile struct ipc_client_state *ics, uint32_t output) static bool camera_can_rectify(const struct scam_camera *cam) { - // R1 has no rectifier (u_stereo_rectify is R2): only a natively rectified - // source yields RECTIFIED frames; a CALIBRATED one is delivered RAW-flagged. - return (cam->info.flags & XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED) != 0; + // A natively rectified source, or a calibrated one the service rectifies + // (R2). A calibrated camera whose geometry was rejected stays RAW-only. + return (cam->info.flags & XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED) != 0 || cam->rect != NULL; } static void @@ -435,6 +478,154 @@ baseline_mm(const struct xrt_plugin_stereo_camera_calibration *c) } +/* + * + * Rectifier (R2). + * + */ + +//! CPU backend: LUT remap per plane into the rectifier's buffer. +static bool +rectify_apply_cpu(struct scam_rectifier *r, + const struct xrt_plugin_stereo_camera_frame *in, + struct xrt_plugin_stereo_camera_frame *out) +{ + if (in->width != 2 * r->geo.width || in->height != r->geo.height || in->planes[0] == NULL) { + return false; + } + if (r->buf == NULL || r->buf_format != in->format) { + struct u_stereo_camera_planes lay; + if (!u_stereo_camera_layout(in->format, in->width, in->height, &lay)) { + return false; + } + if (lay.size > r->buf_size) { + uint8_t *nb = realloc(r->buf, (size_t)lay.size); + if (nb == NULL) { + return false; + } + r->buf = nb; + r->buf_size = lay.size; + } + r->buf_layout = lay; + r->buf_format = in->format; + } + int64_t t0 = os_monotonic_get_ns(); + const struct u_stereo_camera_planes *lay = &r->buf_layout; + switch (in->format) { + case XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8: + u_stereo_rectify_lut_apply(&r->lut[0], 1, in->planes[0], in->pitches[0], r->buf + lay->offset[0], + lay->pitch[0]); + break; + case XRT_PLUGIN_STEREO_CAMERA_FORMAT_NV12: + if (in->planes[1] == NULL) { + return false; + } + u_stereo_rectify_lut_apply(&r->lut[0], 1, in->planes[0], in->pitches[0], r->buf + lay->offset[0], + lay->pitch[0]); + u_stereo_rectify_lut_apply(&r->lut[1], 2, in->planes[1], in->pitches[1], r->buf + lay->offset[1], + lay->pitch[1]); + break; + case XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8: + u_stereo_rectify_lut_apply(&r->lut[0], 4, in->planes[0], in->pitches[0], r->buf + lay->offset[0], + lay->pitch[0]); + break; + default: return false; + } + r->ns_sum += (uint64_t)(os_monotonic_get_ns() - t0); + r->count++; + *out = *in; + out->planes[0] = r->buf + lay->offset[0]; + out->pitches[0] = lay->pitch[0]; + out->planes[1] = lay->plane_count > 1 ? r->buf + lay->offset[1] : NULL; + out->pitches[1] = lay->plane_count > 1 ? lay->pitch[1] : 0; + return true; +} + +static void +rectifier_destroy(struct scam_rectifier **rp) +{ + struct scam_rectifier *r = *rp; + if (r == NULL) { + return; + } + u_stereo_rectify_lut_fini(&r->lut[0]); + u_stereo_rectify_lut_fini(&r->lut[1]); + free(r->buf); + free(r); + *rp = NULL; +} + +/*! + * Build the rectifier from the plug-in's RAW calibration: per-eye K + + * distortion at image_width x image_height (rescaled to the frame's eye size + * when they differ), OpenCV extrinsics x_R = R x_L + T. + */ +static struct scam_rectifier * +rectifier_create(const struct scam_camera *cam) +{ + const struct xrt_plugin_stereo_camera_calibration *c = &cam->calib; + struct u_stereo_rectify_input in; + memset(&in, 0, sizeof(in)); + in.width = cam->info.eye_width; + in.height = cam->info.eye_height; + in.calib_width = c->image_width; + in.calib_height = c->image_height; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = c->k[e][0]; + in.eye[e].fy = c->k[e][1]; + in.eye[e].cx = c->k[e][2]; + in.eye[e].cy = c->k[e][3]; + in.eye[e].model = c->distortion_model; + for (int k = 0; k < 8; k++) { + in.eye[e].d[k] = c->distortion[e][k]; + } + } + memcpy(in.R, c->rotation_right_from_left, sizeof(in.R)); + memcpy(in.T, c->translation_right_from_left_mm, sizeof(in.T)); + + int64_t t0 = os_monotonic_get_ns(); + struct scam_rectifier *r = U_TYPED_CALLOC(struct scam_rectifier); + if (!u_stereo_rectify_compute(&in, &r->geo)) { + U_LOG_W( + "stereo camera %llu: calibration rejected by the rectifier (degenerate or vertical pair) — " + "RAW only", + (unsigned long long)cam->camera_id); + free(r); + return NULL; + } + if (!u_stereo_rectify_lut_init(&r->lut[0], &r->geo, 1) || !u_stereo_rectify_lut_init(&r->lut[1], &r->geo, 2)) { + U_LOG_W("stereo camera %llu: rectifier LUT allocation failed — RAW only", + (unsigned long long)cam->camera_id); + rectifier_destroy(&r); + return NULL; + } + r->apply = rectify_apply_cpu; + U_LOG_W( + "stereo camera %llu: rectifier ready (CPU) — f %.2f px, principal point (%.2f, %.2f), baseline %.2f mm, " + "valid-region zoom x%.4f, %u/%u valid taps, built in %.1f ms", + (unsigned long long)cam->camera_id, r->geo.f, r->geo.cx, r->geo.cy, r->geo.baseline, r->geo.crop_scale, + r->lut[0].valid, r->lut[0].width * r->lut[0].height, (double)(os_monotonic_get_ns() - t0) * 1e-6); + return r; +} + +//! Does any started stream on @p cam want the service's rectified image? +static bool +camera_wants_rectified_locked(struct ipc_server_stereo_camera *m, const struct scam_camera *cam) +{ + if (cam->rect == NULL) { + return false; + } + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + const struct scam_stream *s = &m->streams[i]; + if (s->used && s->started && s->camera == cam->index && + s->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { + return true; + } + } + return false; +} + + /* * * Stream storage. @@ -511,10 +702,15 @@ source_rate_hz(const struct scam_camera *cam) * */ +/*! + * Fan one source frame out. @p rf is the service-rectified image of @p f (NULL + * when the camera has no rectifier or nobody wanted it this frame). + */ static void publish_locked(struct ipc_server_stereo_camera *m, struct scam_camera *cam, const struct xrt_plugin_stereo_camera_frame *f, + const struct xrt_plugin_stereo_camera_frame *rf, int64_t now) { if (cam->last_frame_ns > 0) { @@ -560,6 +756,17 @@ publish_locked(struct ipc_server_stereo_camera *m, u_stereo_camera_ring_clear(&s->ring); continue; } + // A RECTIFIED stream on a service-rectified camera takes the rectified + // image; if it was not produced this frame (the stream started while the + // camera thread was rectifying without it), skip — never RAW pixels + // labelled RECTIFIED. + const struct xrt_plugin_stereo_camera_frame *src = f; + if (s->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && cam->rect != NULL) { + if (rf == NULL) { + continue; + } + src = rf; + } if (!u_stereo_camera_decimator_accept(&s->dec, f->time_ns)) { continue; } @@ -568,8 +775,8 @@ publish_locked(struct ipc_server_stereo_camera *m, continue; } uint8_t *dst = s->map + (uint64_t)slot * s->slot_stride; - if (!u_stereo_camera_convert(f->format, f->planes, f->pitches, s->req.format, dst, &s->layout, f->width, - f->height)) { + if (!u_stereo_camera_convert(src->format, src->planes, src->pitches, s->req.format, dst, &s->layout, + src->width, src->height)) { u_stereo_camera_ring_abort_write(&s->ring); continue; } @@ -605,8 +812,12 @@ publish_locked(struct ipc_server_stereo_camera *m, if (now - cam->last_stats_log_ns >= STATS_LOG_NS) { cam->last_stats_log_ns = now; - U_LOG_I("stereo camera %llu: source %.1f Hz, %u started stream(s)", (unsigned long long)cam->camera_id, - cam->source_rate, cam->started_count); + double rect_ms = (cam->rect != NULL && cam->rect->count > 0) + ? (double)cam->rect->ns_sum / (double)cam->rect->count * 1e-6 + : 0.0; + U_LOG_I("stereo camera %llu: source %.1f Hz, %u started stream(s), rectify %.3f ms/frame (%llu frames)", + (unsigned long long)cam->camera_id, cam->source_rate, cam->started_count, rect_ms, + (unsigned long long)(cam->rect != NULL ? cam->rect->count : 0)); } } @@ -681,10 +892,23 @@ camera_thread(void *ptr) now = os_monotonic_get_ns(); switch (w) { - case XRT_PLUGIN_STEREO_CAMERA_WAIT_OK: - publish_locked(m, cam, &f, now); + case XRT_PLUGIN_STEREO_CAMERA_WAIT_OK: { + // R2: rectify once per source frame, outside the lock (only this + // thread touches cam->rect after create), only if someone wants it. + struct xrt_plugin_stereo_camera_frame rf; + const struct xrt_plugin_stereo_camera_frame *rfp = NULL; + if (camera_wants_rectified_locked(m, cam)) { + struct scam_rectifier *rect = cam->rect; + os_mutex_unlock(&m->lock); + bool ok = rect->apply(rect, &f, &rf); + os_mutex_lock(&m->lock); + now = os_monotonic_get_ns(); + rfp = ok ? &rf : NULL; + } + publish_locked(m, cam, &f, rfp, now); iface->stereo_camera_release_frame(src); break; + } case XRT_PLUGIN_STEREO_CAMERA_WAIT_TIMEOUT: if (cam->last_frame_ns == 0) { set_state_locked(cam, XRT_STEREO_CAMERA_STATE_WAITING); @@ -790,6 +1014,9 @@ ipc_server_stereo_camera_create(struct xrt_instance *xinst) cam->info.flags &= ~XRT_PLUGIN_STEREO_CAMERA_CALIBRATED; } } + if (cam->have_calib && (cam->info.flags & XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED) == 0) { + cam->rect = rectifier_create(cam); + } if (!(info->max_frame_rate >= 0.0f && info->max_frame_rate <= 1000.0f)) { cam->info.max_frame_rate = 0.0f; // NaN / negative / absurd = unknown } @@ -828,6 +1055,7 @@ ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr) os_thread_destroy(&m->cams[i].thread); } os_cond_destroy(&m->cams[i].cond); + rectifier_destroy(&m->cams[i].rect); } for (uint32_t i = 0; i < MAX_STREAMS; i++) { if (m->streams[i].used) { @@ -944,7 +1172,9 @@ ipc_handle_stereo_camera_get_properties(volatile struct ipc_client_state *ics, out_props->max_frame_rate = source_rate_hz(cam); if (cam->have_calib) { out_props->baseline_mm = (float)baseline_mm(&cam->calib); - double fx = cam->calib.k[0][0]; + // Spec §3: per eye, RECTIFIED (the service's rectified focal when it + // rectifies; the plug-in's own for a natively rectified source). + double fx = cam->rect != NULL ? cam->rect->geo.f : cam->calib.k[0][0]; if (fx > 0.0) { out_props->horizontal_fov_deg = (float)(2.0 * atan(cam->info.eye_width / (2.0 * fx)) * 180.0 / 3.14159265358979323846); @@ -988,12 +1218,38 @@ ipc_handle_stereo_camera_get_calibration(volatile struct ipc_client_state *ics, xret = XRT_ERROR_FEATURE_NOT_SUPPORTED; } if (xret == XRT_SUCCESS && output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !camera_can_rectify(cam)) { - xret = XRT_ERROR_FEATURE_NOT_SUPPORTED; // R2 computes post-rectification numbers + xret = XRT_ERROR_FEATURE_NOT_SUPPORTED; } if (xret != XRT_SUCCESS) { os_mutex_unlock(&m->lock); return xret; } + if (output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && cam->rect != NULL) { + // R2: the numbers of the frames the service rectifies — one pinhole for + // both eyes (fx = fy = f, one principal point: zero disparity at + // infinity), no distortion, and the right camera a pure +x translation + // of the baseline (OpenCV P1/P2 with P2[0][3] = f * Tx, Tx = -baseline). + const struct u_stereo_rectify_result *g = &cam->rect->geo; + out_calib->output = output; + for (int e = 0; e < 2; e++) { + struct xrt_stereo_camera_intrinsics *in = &out_calib->eye[e]; + in->width = g->width; + in->height = g->height; + in->fx = (float)g->f; + in->fy = (float)g->f; + in->cx = (float)g->cx; + in->cy = (float)g->cy; + in->model = XRT_PLUGIN_STEREO_CAMERA_DISTORTION_NONE; + } + out_calib->orientation[3] = 1.0f; // identity + // Right camera centre in the rectified left frame = -t_rect (mm -> m). + for (int i = 0; i < 3; i++) { + out_calib->position[i] = (float)(-g->t_rect[i] / 1000.0); + } + out_calib->baseline_mm = (float)g->baseline; + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; + } const struct xrt_plugin_stereo_camera_calibration *c = &cam->calib; out_calib->output = output; for (int e = 0; e < 2; e++) { @@ -1074,6 +1330,17 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, os_mutex_unlock(&m->lock); return XRT_ERROR_INPUT_UNSUPPORTED; } + // Raw frames never reach web pages: the browser gets truly rectified + // frames or nothing, never the RAW-flagged fallback. + if (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !camera_can_rectify(cam) && + !output_allowed(ics, XRT_STEREO_CAMERA_OUTPUT_RAW)) { + os_mutex_unlock(&m->lock); + U_LOG_W( + "stereo_camera_stream_create: camera %llu cannot be rectified; refused for pid %ld " + "(PRESENT_OWNER: rectified only)", + (unsigned long long)cam->camera_id, ics->peer_pid); + return XRT_ERROR_INPUT_UNSUPPORTED; + } uint32_t per_camera = 0; struct scam_stream *slot = NULL; for (uint32_t i = 0; i < MAX_STREAMS; i++) { @@ -1099,8 +1366,9 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, slot->ics = ics; slot->camera = cam->index; slot->req = *req; - // R1: RECTIFIED only from a natively rectified source; a calibrated one is - // delivered RAW and FLAGGED (frame.output) until u_stereo_rectify lands (R2). + // RECTIFIED from a natively rectified source or the service's rectifier + // (R2); a calibrated camera whose geometry the rectifier rejected is + // delivered RAW and FLAGGED (frame.output) — to native clients only. slot->output = (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && camera_can_rectify(cam)) ? XRT_STEREO_CAMERA_OUTPUT_RECTIFIED : XRT_STEREO_CAMERA_OUTPUT_RAW; From d1e06ee7c2a23b3e1e81076d032d663763624836 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 20/37] feat(cli): camera calib --rectified prints P1/P2; probe reports row alignment, --rectified gates it Every probe now 2-D block-matches the last frame (textured 32x32 blocks) and prints |dy| median / p90 / blocks above 1 px, plus the dominant disparities as depths Z = f*B/d from the RECTIFIED calibration. probe --rectified asks for RECTIFIED output and exits 5 when frames come back RAW or the median |dy| exceeds 0.5 px. Against the distorted fake: median 0.016 px, depths 2.004 m and 0.599 m (truth 2.0 / 0.6); --raw shows the 2.6 px it fixes. Co-Authored-By: Claude Opus 5.5 --- src/xrt/targets/cli/cli_cmd_camera.c | 171 ++++++++++++++++++++++++++- 1 file changed, 168 insertions(+), 3 deletions(-) diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 1b2c20a492..20647dadc3 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -6,7 +6,7 @@ * * camera list [--json] properties + state of every camera * camera calib [--raw|--rectified] [--json] - * camera probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] + * camera probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] * [--frames N] [--seconds S] [--out DIR] * * `probe` is a real consumer: it creates and starts a stream (so it goes @@ -17,6 +17,16 @@ * scene is 12 px background / 40 px bar), and the service's stream stats. * With --out it writes the last frame as cam_.png. * + * R2: every probe also reports ROW ALIGNMENT — 2-D block matching (dx AND dy) + * of the last frame on textured 32x32 blocks: the vertical disparity a + * rectified pair must bring to ~0 (median / p90 / max |dy|) and the dominant + * horizontal disparities, each converted to a depth Z = f * B / d with the + * RECTIFIED calibration. `--rectified` insists on RECTIFIED output and fails + * (exit 5) if the frames came back RAW-flagged or the rows do not align + * (median |dy| > 0.5 px). On the distorted sim fake + * (SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1) the depths read back 2.00 m + * (background) and 0.60 m (bar); `--raw` shows the misalignment it fixes. + * * Connects as a DIAG client, like `clients`: on Windows run it NON-elevated. */ @@ -229,6 +239,16 @@ cmd_calib(struct ipc_connection *ipc_c, uint64_t id, uint32_t output, bool json) } printf(" right camera in left frame: q (%.5f %.5f %.5f %.5f) p (%.5f %.5f %.5f) m\n", c.orientation[0], c.orientation[1], c.orientation[2], c.orientation[3], c.position[0], c.position[1], c.position[2]); + if (output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { + // OpenCV-style projection matrices of the rectified pair (Tx in mm). + double tx = -(double)c.position[0] * 1000.0; + printf(" P1 = [%.3f 0 %.3f 0; 0 %.3f %.3f 0; 0 0 1 0]\n", c.eye[0].fx, c.eye[0].cx, c.eye[0].fy, + c.eye[0].cy); + printf(" P2 = [%.3f 0 %.3f %.3f; 0 %.3f %.3f 0; 0 0 1 0] (P2[0][3] = f * Tx, Tx = %.3f mm)\n", + c.eye[1].fx, c.eye[1].cx, c.eye[1].fx * tx, c.eye[1].fy, c.eye[1].cy, tx); + printf(" depth from disparity: Z = f * B / d = %.1f / d(px) m\n", + c.eye[0].fx * c.baseline_mm / 1000.0); + } return 0; } @@ -402,6 +422,98 @@ report_disparity(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h } } +static int +cmp_float(const void *a, const void *b) +{ + float x = *(const float *)a, y = *(const float *)b; + return (x > y) - (x < y); +} + +struct row_alignment +{ + int blocks; + float dy_median, dy_p90; //!< |dy|, px + int outliers; //!< blocks with |dy| > 1 px (mismatches: occlusion edges, flat or repeated texture) + int modes; + float mode_dx[2]; + int mode_count[2]; +}; + +/*! + * 2-D block matching (u_stereo_camera_estimate_offset) of a GRAY8 SBS frame on + * a 32 px grid, textured blocks only (a flat wall matches anywhere): |dy| + * statistics and the two dominant horizontal disparities (1 px clusters). + */ +static void +measure_rows(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, struct row_alignment *out) +{ + memset(out, 0, sizeof(*out)); + enum + { + B = 32, + MAXB = 1024 + }; + static float dxs[MAXB], ady[MAXB]; + int n = 0; + for (uint32_t y = 48; y + B + 12 <= h && n < MAXB; y += B) { + for (uint32_t x = 72; x + B <= eye_w && n < MAXB; x += B) { + // Texture gate: standard deviation of the left block >= 6 levels. + double sum = 0, sq = 0; + for (uint32_t j = 0; j < B; j++) { + const uint8_t *r = gray + (size_t)(y + j) * pitch + x; + for (uint32_t i = 0; i < B; i++) { + sum += r[i]; + sq += (double)r[i] * r[i]; + } + } + double mean = sum / (B * B); + if (sq / (B * B) - mean * mean < 36.0) { + continue; + } + float dx, dy; + if (!u_stereo_camera_estimate_offset(gray, pitch, eye_w, h, x, y, B, B, 64, 10, &dx, &dy)) { + continue; + } + dxs[n] = dx; + ady[n] = dy < 0 ? -dy : dy; + n++; + } + } + out->blocks = n; + if (n == 0) { + return; + } + qsort(ady, (size_t)n, sizeof(float), cmp_float); + out->dy_median = ady[n / 2]; + out->dy_p90 = ady[(n * 9) / 10]; + for (int i = 0; i < n; i++) { + out->outliers += ady[i] > 1.0f; + } + qsort(dxs, (size_t)n, sizeof(float), cmp_float); + // Greedy 1 px clusters; keep the two most populated. + for (int i = 0; i < n;) { + int j = i; + double s = 0; + while (j < n && dxs[j] - dxs[i] <= 1.0f) { + s += dxs[j]; + j++; + } + int c = j - i; + float mean = (float)(s / c); + if (c > out->mode_count[0]) { + out->mode_dx[1] = out->mode_dx[0]; + out->mode_count[1] = out->mode_count[0]; + out->mode_dx[0] = mean; + out->mode_count[0] = c; + } else if (c > out->mode_count[1]) { + out->mode_dx[1] = mean; + out->mode_count[1] = c; + } + i = j; + } + out->modes = out->mode_count[1] > 0 ? 2 : 1; +} + static int cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) { @@ -412,9 +524,13 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) int frames = 90; float seconds = 0.0f; const char *out_dir = NULL; + bool require_rectified = false; for (int i = 3; i < argc; i++) { if (strcmp(argv[i], "--raw") == 0) { output = XRT_STEREO_CAMERA_OUTPUT_RAW; + } else if (strcmp(argv[i], "--rectified") == 0) { + output = XRT_STEREO_CAMERA_OUTPUT_RECTIFIED; + require_rectified = true; } else if (strcmp(argv[i], "--format") == 0 && i + 1 < argc) { const char *f = argv[++i]; format = strcmp(f, "gray8") == 0 ? XRT_STEREO_CAMERA_FORMAT_GRAY8 @@ -505,6 +621,7 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) int64_t t_start = os_monotonic_get_ns(); int64_t deadline = seconds > 0.0f ? t_start + (int64_t)(seconds * 1e9) : t_start + 60ll * 1000000000; int got = 0, wakes = 0, not_ready = 0; + bool rows_ok = true; uint64_t first_index = 0, last_index = 0, gaps = 0; int64_t t_first = 0, t_last = 0; struct xrt_stereo_camera_frame_info last; @@ -569,6 +686,39 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) report_disparity(gray, gl.pitch[0], last.width / 2, last.height, probe_max_disparity(&props, last.width / 2), disp, sizeof(disp)); printf("block disparity (left x - right x): %s\n", disp); + + // R2: row alignment + depth of the dominant disparities. + struct row_alignment ra; + measure_rows(gray, gl.pitch[0], last.width / 2, last.height, &ra); + struct xrt_stereo_camera_calibration rc; + double fb = 0.0; // f * B, px * m + if (last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && + ipc_client_stereo_camera_get_calibration(ipc_c, id, XRT_STEREO_CAMERA_OUTPUT_RECTIFIED, + &rc) == XRT_SUCCESS) { + fb = rc.eye[0].fx * rc.baseline_mm / 1000.0; + } + if (ra.blocks == 0) { + printf("row alignment: n/a (no textured block)\n"); + rows_ok = false; + } else { + printf( + "row alignment (%s, %d textured 32x32 blocks): |dy| median %.3f px, p90 %.3f px, " + "%d block(s) " + "> 1 px\n", + last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", ra.blocks, + ra.dy_median, ra.dy_p90, ra.outliers); + for (int k = 0; k < ra.modes; k++) { + if (fb > 0.0 && ra.mode_dx[k] > 0.25f) { + printf( + " disparity mode %d: %.2f px (%d blocks) -> Z = f*B/d = %.3f m\n", + k + 1, ra.mode_dx[k], ra.mode_count[k], fb / ra.mode_dx[k]); + } else { + printf(" disparity mode %d: %.2f px (%d blocks)\n", k + 1, + ra.mode_dx[k], ra.mode_count[k]); + } + } + rows_ok = ra.dy_median <= 0.5f; + } } if (out_dir != NULL) { char path[1024]; @@ -615,7 +765,21 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) close_wake(wake); ipc_client_stereo_camera_stream_stop(ipc_c, sid); ipc_client_stereo_camera_stream_destroy(ipc_c, sid); - return got > 0 ? 0 : 4; + if (got == 0) { + return 4; + } + if (require_rectified) { + if (last.output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { + printf("--rectified: FAIL — frames are RAW (the camera is not calibrated / not rectifiable)\n"); + return 5; + } + if (!rows_ok) { + printf("--rectified: FAIL — rows do not align (median |dy| > 0.5 px)\n"); + return 5; + } + printf("--rectified: PASS — rows aligned\n"); + } + return 0; } int @@ -639,7 +803,8 @@ cli_cmd_camera(int argc, const char **argv) } else { printf( "usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" - " probe [] [--raw] [--format gray8|nv12|bgra8] [--fps F] [--frames N] [--seconds S] " + " probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] [--frames N] " + "[--seconds S] " "[--out DIR]\n"); ret = 1; } From 2d57844c10e27e0b0cbfa1f79633704f817ad1ba Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:04:31 -0700 Subject: [PATCH 21/37] =?UTF-8?q?docs(stereo-camera):=20R2=20as=20built=20?= =?UTF-8?q?=E2=80=94=20rectifier,=20distorted=20fake,=20probe,=20measured?= =?UTF-8?q?=20numbers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- docs/adr/ADR-043-stereo-camera-source.md | 2 +- docs/reference/xrt_plugin_iface.md | 2 +- docs/roadmap/stereo-camera-source.md | 41 ++++++++++++- docs/specs/extensions/XR_DXR_stereo_camera.md | 58 ++++++++++++++++++- 4 files changed, 95 insertions(+), 8 deletions(-) diff --git a/docs/adr/ADR-043-stereo-camera-source.md b/docs/adr/ADR-043-stereo-camera-source.md index 9495750153..56b1823eee 100644 --- a/docs/adr/ADR-043-stereo-camera-source.md +++ b/docs/adr/ADR-043-stereo-camera-source.md @@ -2,7 +2,7 @@ **Status:** Proposed (2026-09-25) · **R1 implemented** 2026-09-26 (runtime, hardware-free; maintainer defaults recorded in the roadmap §G: stereo-only, iface slots, service clients only, no raw frames -to pages) · introduces +to pages) · **R2 implemented** 2026-09-26 (the service-side rectifier, decision 2 below) · introduces [`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md) · appends optional camera slots to `xrt_plugin_iface` under the [ADR-020](ADR-020-plugin-abi-compatibility-policy.md) append-at-end rule · related: [ADR-019](ADR-019-vendor-plugin-aux-boundary.md), diff --git a/docs/reference/xrt_plugin_iface.md b/docs/reference/xrt_plugin_iface.md index 40f5e18bb7..d6f00c39e5 100644 --- a/docs/reference/xrt_plugin_iface.md +++ b/docs/reference/xrt_plugin_iface.md @@ -631,7 +631,7 @@ void (*stereo_camera_close)(cam); | `wait_frame` is called from **one runtime-owned thread per open camera**, may block up to `timeout_ns`, and returns an `enum xrt_plugin_stereo_camera_wait` (`OK` / `TIMEOUT` / `SUSPENDED` / `ERROR`). Planes stay valid until `release_frame`. | The service maps `TIMEOUT` → `WAITING` (then `SUSPENDED` after 1 s without frames), `SUSPENDED` → `SUSPENDED`, `ERROR` → close + retry in 1 s | | **Never wait on a wake object shared with other readers** of the vendor channel — poll it. | An auto-reset event wakes exactly one waiter: two readers steal each other's frames (ADR-043) | | Decode your transport (JPEG, YUY2, …) to `GRAY8`, `NV12` or `BGRA8` yourself; always deliver **one side-by-side image**, `2·eye_width × eye_height`, left eye left, unmirrored, even extents. | The runtime owns format conversion, decimation, rectification (R2) and transport; one SBS layout for every vendor | -| Calibration is the **ACTIVE** device's — the unit you weave for — keyed by its own identity (serial). RAW, OpenCV convention: `k[eye] = fx fy cx cy`, `distortion_model` + coefficients, and `R`, `T` with **`x_R = R·x_L + T`** (T in mm; a right camera at +x of the left has `T = (−B, 0, 0)`). The runtime converts it to the client's pose-of-the-right-camera form. Return `XRT_ERROR_FEATURE_NOT_SUPPORTED` and do **not** set `CALIBRATED` if you cannot resolve it. | One test box carried eleven calibration folders; "the first one" is the wrong camera | +| Calibration is the **ACTIVE** device's — the unit you weave for — keyed by its own identity (serial). RAW, OpenCV convention: `k[eye] = fx fy cx cy`, `distortion_model` + coefficients, and `R`, `T` with **`x_R = R·x_L + T`** (T in mm; a right camera at +x of the left has `T = (−B, 0, 0)`). The runtime converts it to the client's pose-of-the-right-camera form. Return `XRT_ERROR_FEATURE_NOT_SUPPORTED` and do **not** set `CALIBRATED` if you cannot resolve it. `image_width/height` = the size the intrinsics were calibrated at; the service rescales them to the frame's eye size when the two differ. Unless you set `NATIVELY_RECTIFIED`, the service rectifies from exactly these numbers (R2, `u_stereo_rectify`), so a wrong calibration shows up as misaligned rows in `displayxr-cli camera probe --rectified`. | One test box carried eleven calibration folders; "the first one" is the wrong camera | | `device_identity` is private: the service hashes it per consumer (`persistentId`) and never exports it. | Fingerprinting (spec §7.5) | | Nothing throws across the boundary. | C ABI | diff --git a/docs/roadmap/stereo-camera-source.md b/docs/roadmap/stereo-camera-source.md index 575344dfc6..2b38111571 100644 --- a/docs/roadmap/stereo-camera-source.md +++ b/docs/roadmap/stereo-camera-source.md @@ -1,7 +1,8 @@ # Stereo camera source — browser integration, vendor plug-in notes, phased plan -**Status:** design (2026-09-25); **R1 implemented** on `feat/stereo-camera-r1` (2026-09-26) — -see §E; maintainer decisions for R1 in §G. Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). +**Status:** design (2026-09-25); **R1 implemented** on `feat/stereo-camera-r1` (2026-09-26), +**R2 implemented** on `feat/stereo-camera-r2` (2026-09-26, stacked) — see §E; maintainer decisions +for R1 in §G. **L1** (Leia provider) is on `displayxr-leia-plugin` `feat/stereo-camera-l1`. Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). Extension: [`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md). Consumer driving it: the web SDK's 3D call module (`@displayxr/inline3d/call`, RFC 0002 on `displayxr-web` branch `feat/call-p1`, §4 *Capture*). @@ -194,7 +195,7 @@ The implementation belongs in `displayxr-leia-plugin` (`src/drv_leia/`), per ADR |---|---|---|---| | **R0** | runtime | This design: ADR-043, spec, this page | Maintainer sign-off on the open questions below | | **R1** ✅ | runtime | **Implemented 2026-09-26 (see "R1 as built" below).** Header `XR_DXR_stereo_camera.h` + `index.json` note (together — the catalog lint requires both); `xrt_plugin_iface` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (appended after `create_dp_d3d11_lift`, so **lands after ADR-042's PR**); platform-neutral camera manager (thread, refcount + linger, 3-slot pinned ring, per-stream wake handles, decimation, NV12/BGRA conversion); IPC + OpenXR entry points; sim_display fake; `displayxr-cli camera list/calib/probe`; selftest check | `camera probe` on the sim fake writes frames on Windows + Linux CI; ring/pinning unit tests (mirror `tests_lift_mailbox`) | -| **R2** | runtime | `u_stereo_rectify` (Bouguet, valid-region crop, LUT remap) with golden tests against OpenCV-generated fixtures; RECTIFIED output | sim fake: row error < 0.5 px, disparity = f·B/Z within 0.5 px | +| **R2** ✅ | runtime | **Implemented 2026-09-26 (see "R2 as built" below).** `u_stereo_rectify` (Bouguet, valid-region crop, LUT remap) with golden tests against OpenCV-generated fixtures; RECTIFIED output | sim fake: row error < 0.5 px, disparity = f·B/Z within 0.5 px — **measured 0.061 px worst block, 0.063 px worst disparity error** | | **R3** | runtime | Privacy: OS consent check, consent store + tray prompt, delegating-client registration, visibility/lock suspension, indicator, kill switches | Manual matrix: allow/deny/revoke, OS switch off, lock screen, background app — each blocks frames | | **L1** | leia-plugin (Windows) | Slots over the SR raw-camera channel per §D; calibration by active serial; keep-alive; repin runtime (feature-macro repin, `downstream-pins.json` `features` track) | On a panel box, incl. the multi-folder box: `camera probe --rectified` rows aligned; the vendor call app running at the same time still gets every frame; tracking/weave unaffected (frame-time + tracking-state logs) | | **B1** | browser (Windows) | §B device factory + duplicate hiding + hint + delegating registration in the installer; web SDK: prefer the hint in `camera:'auto'`, fill `hello` from it | RFC 0002 P1 call between two panel laptops sends rectified SBS from the tracker camera **while both are tracking** | @@ -230,6 +231,40 @@ against the sim fake). sync_file as the #1699 weave path), a read-only POSIX section, and a distorted/misaligned fake (all with R2/R3/A1). +### R2 as built + +- **`auxiliary/util/u_stereo_rectify.{h,c}`**, no OpenCV. Three layers so a GPU path replaces only + the last: geometry (`u_stereo_rectify_compute`: OpenCV's `stereoRectify(CALIB_ZERO_DISPARITY, + alpha = 0)` in C), float maps (`u_stereo_rectify_build_map`, what a GPU backend uploads as an + RG32F texture), and the CPU backend (a fixed-point bilinear LUT over the whole SBS image; NV12 = + full-res Y LUT + half-res UV LUT). RADTAN5 / RADTAN8 / KB4. Calibration at a different size than + the frames is rescaled. The alpha = 0 crop is verified on every border pixel of the real maps. +- **Service:** a per-camera `struct scam_rectifier` built at manager create from the plug-in's + RAW calibration (WARN line with f, principal point, baseline, zoom, build time). It runs once + per source frame on the camera thread with the lock released, only while a started stream wants + RECTIFIED; RAW streams read the original. `get_calibration(RECTIFIED)` returns the same + geometry (one pinhole for both eyes, pure +x baseline); `horizontalFovDeg` is the rectified one. + The browser is refused (`INPUT_UNSUPPORTED`) a camera the rectifier rejected rather than being + given the RAW-flagged fallback. INFO stats add `rectify ms/frame`. +- **Fake:** `SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1` — the raw distorted, vertically misaligned, + rolled pair with exact ground truth (spec §10). +- **Tests** (`tests_stereo_rectify`, fixtures in `tests/fixtures/` from + `gen_stereo_rectify_fixtures.py`, OpenCV 4.10, offline): R1/R2 to 1e-12, principal point to the + converged reference within 1e-4 px (and OpenCV's own P1 within 0.75 px — OpenCV stops + `undistortPoints` at 5 iterations), maps to 3e-5 px, no black pixel, and end to end on the fake: + raw |Δy| median 2.9 px → rectified worst block 0.061 px; disparity 11.33 / 37.77 px ground truth + vs 11.31 / 37.79 measured (worst block 0.063 px off). +- **CLI:** `camera calib --rectified` prints P1/P2 and `Z = f·B/d`; every probe reports row + alignment and the dominant disparities as depths; `probe --rectified` exits 5 unless rows align. + Against the fake: |Δy| median 0.016 px, depths read back 2.004 m and 0.599 m. +- **Perf:** 0.94 ms GRAY8 / 1.36 ms NV12 / 2.25 ms BGRA8 per 1280×480 frame, M1 Pro, -O2, one + thread (3 ms GRAY8 in a Debug service). At 30 Hz that is ~3 % of one core for the Leia GRAY8 + source; **budget: ≤ 2 ms per frame on the camera thread**, above which the rows split across + threads (`u_stereo_rectify_lut_apply_rows`) or the GPU backend takes over. Setup (geometry + both + LUTs) 9–18 ms, once per calibration; LUT memory 6 bytes per output sample (3.7 MB + 0.9 MB). +- **Not in R2:** the GPU backend (seam in place), per-frame recalibration (`calibrationGeneration` + stays 0), and a rectifier for a camera whose calibration changes at runtime. + ## F. Risks - **Undocumented vendor channel.** The raw-camera shared memory is a reader-defined contract with diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index 8ee1034f5a..6fe939a185 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -6,7 +6,7 @@ | **Spec Version** | 1 | | **Extension Type** | Instance extension, service path only (an in-process instance enumerates zero cameras) | | **Header** | [`src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h`](../../../src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h) (+ its `index.json` catalog note) | -| **Status** | **R1 implemented** (runtime, hardware-free): header, plug-in slots, service camera manager, IPC, OpenXR entry points, sim_display fake, `displayxr-cli camera`, selftest check. **Not yet:** rectification (R2), consent / indicator / foreground rule (R3 — deny-by-default hooks in place), GPU transports, state-change events, the Leia provider (L1), the browser (B1). Provisional type values `1004999300–310` (relocated from `290–300` when `XR_DXR_weave` v14 took `290`; `XR_DXR_lift` holds `270–289`), pending Khronos registry | +| **Status** | **R1 implemented** (runtime, hardware-free): header, plug-in slots, service camera manager, IPC, OpenXR entry points, sim_display fake, `displayxr-cli camera`, selftest check. **R2 implemented**: the service-side rectifier (`u_stereo_rectify`, CPU) — RECTIFIED frames + rectified calibration from any CALIBRATED source, golden-tested against OpenCV, and a distorted sim fake with ground truth. **Not yet:** consent / indicator / foreground rule (R3 — deny-by-default hooks in place), GPU transports (the rectifier has the seam), state-change events, the Leia provider (L1), the browser (B1). Provisional type values `1004999300–310` (relocated from `290–300` when `XR_DXR_weave` v14 took `290`; `XR_DXR_lift` holds `270–289`), pending Khronos registry | | **R1 decisions** | stereo-only (names/structs kept open: `viewCount`, next chains) · camera provider = `xrt_plugin_iface` slots · service clients only (in-process enumerates zero) · raw frames never reach web pages (`RAW` refused to `PRESENT_OWNER` clients) — see [roadmap §G](../../roadmap/stereo-camera-source.md#g-open-questions--maintainer-decisions) | | **Decision record** | [ADR-043](../../adr/ADR-043-stereo-camera-source.md) | | **Plug-in contract** | appended `xrt_plugin_iface` camera slots, `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (§9) | @@ -155,13 +155,32 @@ one field box carried eleven. A plug-in that cannot resolve the active device's reports the camera without `CALIBRATED`, and the service offers RAW only. **Rectification** (when not `NATIVELY_RECTIFIED`) is the service's, vendor-neutral -(`u_stereo_rectify`, planned in `auxiliary/util`): Bouguet rectification to a **parallel** pair +(`auxiliary/util/u_stereo_rectify.{h,c}`, R2): Bouguet rectification to a **parallel** pair with zero disparity at infinity, cropped to the valid region (no black corners), per-eye size unchanged; applied as a per-pixel remap LUT built once per (calibration, size). **No convergence shear is baked in.** Placing the subject at the display plane is the receiver's decision (a per-eye crop offset of `f_px · baseline / (2 · subjectZ)`), so the source stays a physically honest parallel pair. +> **R2 as built.** The geometry is OpenCV's `stereoRectify(CALIB_ZERO_DISPARITY, alpha = 0)` +> re-implemented in C (no OpenCV dependency), golden-tested against OpenCV 4.10 fixtures +> (`tests/fixtures/`, generated offline): R1/R2 agree to 1e-12, the principal point matches the +> converged reference to 1e-4 px (OpenCV itself stops `undistortPoints` after 5 iterations and +> lands a fraction of a pixel off), the maps agree with `initUndistortRectifyMap` to 3e-5 px. Two +> deliberate differences: the alpha = 0 zoom is **verified on every border pixel** of the real +> maps and nudged up until no output pixel samples outside its raw image (OpenCV's 9-point +> inner rectangle leaves a few black pixels on strong lenses) — at most +0.13 % focal on the +> fixtures; and calibration given at a different size than the frames (`image_width/height` ≠ the +> eye size) is rescaled pixel-centre-aware. Lens models: RADTAN5, RADTAN8 (rational), KB4 +> (fisheye; same Bouguet geometry). The service rectifies **once per source frame** on the camera +> thread (outside its lock) and only while some started stream wants RECTIFIED; RAW streams read +> the original. `xrGetStereoCameraCalibrationDXR(RECTIFIED)` returns that same geometry: both +> eyes `fx = fy = f` with one principal point, model NONE, and `rightFromLeft` = identity +> rotation + `(+baseline, 0, 0)` m — so frames and numbers cannot disagree. The browser +> (`PRESENT_OWNER`) is refused a stream on a calibrated camera the rectifier rejected, instead of +> getting the RAW-flagged fallback native clients get. CPU cost per 1280×480 frame (M1 Pro, -O2): +> **0.94 ms GRAY8, 1.36 ms NV12, 2.25 ms BGRA8**; setup (geometry + both LUTs) 9–18 ms, once. + **Eye order and mirroring.** Left half = the lens on the camera's own left (as seen from behind the camera, looking at the scene) = the remote viewer's left eye. Frames are **never mirrored**; a self-view mirror is the consumer's job (mirror each half AND swap halves). @@ -452,7 +471,17 @@ owns threads, fan-out, rectification, format conversion, transport, consent and > `SIM_DISPLAY_FAKE_STEREO_CAMERA=1`, `_SIZE=WxH` (per eye), `_FPS=N`, `_FORMAT=gray8|nv12|bgra`, > `_SUSPEND_PERIOD_MS=N`, `_ADVERTISED_FPS=N` (what enumerate claims, default = `_FPS`; `0` = unknown — > exercises the service's measured-rate path), `_BASELINE_MM=N` (default 50; `120` reproduces the Leia SR -> tracking camera: the bar at 0.6 m is ~95 px). The distorted + misaligned variant below is R2's, with the rectifier. +> tracking camera: the bar at 0.6 m is ~95 px). +> +> **R2 as built:** `SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1` renders the same scene through two +> RAW cameras with known ground truth (`sim_stereo_camera_truth_init`): different per-eye +> intrinsics and principal points, RADTAN5 barrel lenses (k1 −0.21 / −0.17), and each camera +> turned by half of (pitch 0.35°, yaw 0.25°, roll 0.45°) in opposite senses — 0.7° of relative +> pitch, 0.9° of relative roll. The raw pair measures |Δy| median 2.6 px, p90 5.3 px. The +> rotation split is symmetric (left `Sᵀ`, right `S`), which makes Bouguet recover **exactly** the +> virtual parallel pair the scene is defined in, so the ground truth after rectification is +> exact: rows aligned and disparity `f_rect · B / Z`. The camera then drops `NATIVELY_RECTIFIED` +> and its calibration slot returns the truth (RADTAN5, `R = S²`, `T = −S·(B, 0, 0)`, B = `_BASELINE_MM`). `SIM_DISPLAY_FAKE_STEREO_CAMERA=1` makes sim_display advertise one camera (`SHARED_WITH_EYE_TRACKING | USER_FACING | CALIBRATED | MONOCHROME`, 640×480 per eye, 30 Hz, @@ -492,6 +521,28 @@ block disparity (left x - right x): 12 px (38/54 blocks), 40 px (16/54 blocks), service stats: source 30.49 Hz, delivered 30.49 Hz, published 155, skipped 4, acquired 150, mean latency 12.417 ms ``` +R2 adds row alignment to every probe (2-D block matching of the last frame, textured 32×32 +blocks: |Δy| median / p90 / count above 1 px, and the dominant disparities converted to depth +with the RECTIFIED calibration), `camera calib --rectified` prints P1/P2 and `Z = f·B/d`, +and `probe --rectified` insists on RECTIFIED output and exits 5 unless the median |Δy| ≤ 0.5 px. +Against the DISTORTED fake (macOS, Debug service): + +``` +$ displayxr-cli camera calib 1 --rectified + left eye: 640x480 fx 453.181 fy 453.181 cx 320.426 cy 239.364 model 0 + P2 = [453.181 0 320.426 -22659.030; 0 453.181 239.364 0; 0 0 1 0] (P2[0][3] = f * Tx, Tx = -50.000 mm) +$ displayxr-cli camera probe --rectified --frames 150 +row alignment (RECTIFIED, 221 textured 32x32 blocks): |dy| median 0.016 px, p90 0.064 px, 9 block(s) > 1 px + disparity mode 1: 11.30 px (165 blocks) -> Z = f*B/d = 2.004 m + disparity mode 2: 37.81 px (48 blocks) -> Z = f*B/d = 0.599 m +--rectified: PASS — rows aligned +$ displayxr-cli camera probe --raw --format gray8 +row alignment (RAW, 221 textured 32x32 blocks): |dy| median 2.589 px, p90 5.292 px, 183 block(s) > 1 px +``` + +(The blocks above 1 px on the rectified pair are mismatches at the bar's occlusion edges and on +the frame counter, which is burned in raw space.) + Runs as a DIAG IPC client, non-elevated. `probe` goes through the full consent path (so its first run on a box raises the prompt — which is how a developer checks the prompt). `selftest` gains `stereo_camera_caps`: zero cameras passes; malformed properties (zero extent, CALIBRATED @@ -510,4 +561,5 @@ camera state change and per stream start/stop with the peer executable; INFO sta | Version | Change | |---|---| +| 1 (R2) | No API change. RECTIFIED output and rectified calibration now come from the service's rectifier for any CALIBRATED, non-natively-rectified source; the browser is refused a camera it cannot rectify; sim_display `_DISTORT=1` fake; `camera probe` row alignment + `--rectified` | | 1 | R1 implements: enumerate, calibration (RAW; RECTIFIED for natively rectified sources), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, stream stats, plug-in slots, sim_display fake, CLI, selftest. Deferred: state-change events (structs defined, not delivered), GPU transports, rectifier (R2), consent/indicator (R3). Design scope: Enumerate + state events, calibration (raw / rectified), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, optional GPU transports, runtime-enforced consent / foreground / indicator, plug-in iface slots, sim_display fake, CLI. | From 6ba44ef7aa6caba1461dd3d10ba58271c185b10d Mon Sep 17 00:00:00 2001 From: David Date: Sat, 26 Sep 2026 11:14:05 -0700 Subject: [PATCH 22/37] feat(stereo-camera): WARN when a plug-in's calibration puts the right camera at -x A swapped SBS pair still rectifies (Bouguet does not care), but every disparity comes out negative and rightFromLeft points the wrong way. The L1 Leia provider decides eye order from the sign of T; this makes a wrong decision visible in the service log instead of only in a probe. Co-Authored-By: Claude Opus 5.5 --- src/xrt/ipc/server/ipc_server_stereo_camera.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index d76bd2a115..b2d39bfcac 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -600,6 +600,13 @@ rectifier_create(const struct scam_camera *cam) return NULL; } r->apply = rectify_apply_cpu; + if (r->geo.t_rect[0] > 0.0) { + // Contract: SBS left half = the camera's LEFT lens, T.x < 0 (xrt_plugin.h). + U_LOG_W( + "stereo camera %llu: calibration puts the right camera at -x (T.x > 0) — the plug-in's SBS halves " + "look swapped; rectified disparities will be NEGATIVE", + (unsigned long long)cam->camera_id); + } U_LOG_W( "stereo camera %llu: rectifier ready (CPU) — f %.2f px, principal point (%.2f, %.2f), baseline %.2f mm, " "valid-region zoom x%.4f, %u/%u valid taps, built in %.1f ms", From e4221f3ca432f48afe5876c8ee0a41a2d8e1a983 Mon Sep 17 00:00:00 2001 From: dfattal Date: Sun, 27 Sep 2026 00:13:49 -0700 Subject: [PATCH 23/37] fix(stereo-camera): tests_stereo_rectify builds on MSVC (no M_PI) MSVC only defines M_PI when _USE_MATH_DEFINES precedes the first , which Catch may already have pulled in. Use a local constant. First MSVC build of R2; the test passes (716 assertions, 7 cases). Co-Authored-By: Claude Opus 5.5 (1M context) --- tests/tests_stereo_rectify.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/tests_stereo_rectify.cpp b/tests/tests_stereo_rectify.cpp index f3f3edce35..98ed2b6b48 100644 --- a/tests/tests_stereo_rectify.cpp +++ b/tests/tests_stereo_rectify.cpp @@ -72,7 +72,10 @@ input_from(const stereo_rectify_fixture &f) return in; } -const double k_ideal_fx_640 = 320.0 / std::tan(34.0 * M_PI / 180.0); +// Not M_PI: MSVC only defines it with _USE_MATH_DEFINES before the first , +// which Catch may already have included. +constexpr double k_pi = 3.14159265358979323846; +const double k_ideal_fx_640 = 320.0 / std::tan(34.0 * k_pi / 180.0); } // namespace From 378d7117614efc7488f3300a4612ebfd8e913a40 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 11:11:33 -0700 Subject: [PATCH 24/37] fix(stereo-camera): probe row alignment = NCC, baseline-aware windows, signed dy, bound hits flagged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First SR-hardware run: a face at 0.6 m on the 120 mm tracker is ~95 px of disparity, past the probe's 0..64 search, and the SSD matcher's +-10 px dy window clamped too — it reported "d = 64, |dy| = 10.000" as if measured. - u_stereo_camera_match_block: zero-mean NCC (blind to the two sensors' gain/offset), coarse pass on every 2nd pixel + full-res hill-climb, parabolic sub-pixel on both axes, the searched window returned, and dx_at_edge / dy_at_edge when the integer peak sits ON a bound (a clamp, not a measurement). - camera probe: disparity window f*B/0.4 m from the camera (R1 helper), dy +-24 px, accept NCC >= 0.90; prints accepted / low-NCC / at-bound counts, warns when > 20 % of blocks hit a bound, reports the SIGNED dy median, and --rectified now also fails on a constant offset (|signed median| > 0.5 px). --max-disparity / --max-dy / --min-ncc override. - sim fake: the DISTORTED variant honours _BASELINE_MM too (truth + slot). - tests: large-baseline matcher (bar 94.9 px, signed dy +1.68 recovered to 0.07 px, gain/offset, flat refused, the old 64 px window is REJECTED not reported); END TO END at 120 mm (bar 90.64 px GT, measured 90.66); the [vshift] hypotheses for the field's +1.7 px residual (per-eye cy error = constant dy ~ -dcy, transposed R = large roll-shaped dy, calib-size rescale and the plug-in's half swap exact, identity = pure scale about cy). Co-Authored-By: Claude Opus 5.5 --- docs/specs/extensions/XR_DXR_stereo_camera.md | 50 ++- src/xrt/auxiliary/util/u_stereo_camera.c | 171 ++++++++ src/xrt/auxiliary/util/u_stereo_camera.h | 44 ++ .../sim_display/sim_display_stereo_camera.c | 2 +- src/xrt/targets/cli/cli_cmd_camera.c | 122 +++-- tests/tests_stereo_rectify.cpp | 415 +++++++++++++++++- 6 files changed, 763 insertions(+), 41 deletions(-) diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index 6fe939a185..7773f530d4 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -522,27 +522,57 @@ service stats: source 30.49 Hz, delivered 30.49 Hz, published 155, skipped 4, ac ``` R2 adds row alignment to every probe (2-D block matching of the last frame, textured 32×32 -blocks: |Δy| median / p90 / count above 1 px, and the dominant disparities converted to depth -with the RECTIFIED calibration), `camera calib --rectified` prints P1/P2 and `Z = f·B/d`, -and `probe --rectified` insists on RECTIFIED output and exits 5 unless the median |Δy| ≤ 0.5 px. -Against the DISTORTED fake (macOS, Debug service): +blocks: SIGNED Δy median, |Δy| median / p90 / count above 1 px, and the dominant disparities +converted to depth with the RECTIFIED calibration), `camera calib --rectified` prints P1/P2 +and `Z = f·B/d`, and `probe --rectified` insists on RECTIFIED output and exits 5 unless the median +|Δy| ≤ 0.5 px **and** the signed median is within ±0.5 px (a constant vertical offset fails even +when the scatter is small). + +**Matching (after the first SR-hardware run).** Zero-mean NCC with parabolic sub-pixel refinement +(coarse pass on every second pixel, full-resolution refinement), blind to the two sensors' +gain/offset differences. The disparity window is **baseline-aware**: `f·B / 0.4 m` from the +camera's own calibration (quarter eye width uncalibrated, clamped to [64, 0.6 · eye]); the +vertical window is ±24 px. Blocks with peak NCC < 0.90, or whose peak sits **on** either +window's bound, are counted and excluded — a bound hit is a clamp, not a measurement. (The first +SR run read "d = 64, |dy| = 10.000" because a face at 0.6 m on the 120 mm tracker is ~95 px and +both fixed windows clamped.) Knobs: `--max-disparity N`, `--max-dy N`, `--min-ncc C`. +Against the DISTORTED fake (macOS, Debug service), default 50 mm and the SR-like 120 mm: ``` $ displayxr-cli camera calib 1 --rectified left eye: 640x480 fx 453.181 fy 453.181 cx 320.426 cy 239.364 model 0 P2 = [453.181 0 320.426 -22659.030; 0 453.181 239.364 0; 0 0 1 0] (P2[0][3] = f * Tx, Tx = -50.000 mm) -$ displayxr-cli camera probe --rectified --frames 150 -row alignment (RECTIFIED, 221 textured 32x32 blocks): |dy| median 0.016 px, p90 0.064 px, 9 block(s) > 1 px - disparity mode 1: 11.30 px (165 blocks) -> Z = f*B/d = 2.004 m - disparity mode 2: 37.81 px (48 blocks) -> Z = f*B/d = 0.599 m +$ displayxr-cli camera probe --rectified --seconds 2 +row alignment (RECTIFIED): NCC 32x32 blocks, search d -8..64 px, dy +-24 px, accept NCC >= 0.90 + 266 textured block(s): 255 accepted, 11 below NCC 0.90, 0 AT the disparity bound, 0 AT the dy bound + dy (right - left) median +0.001 px SIGNED; |dy| median 0.016 px, p90 0.038 px, 1 block(s) > 1 px + disparity mode 1: 11.30 px (213 blocks) -> Z = f*B/d = 2.005 m + disparity mode 2: 37.81 px (40 blocks) -> Z = f*B/d = 0.599 m +--rectified: PASS — rows aligned +$ displayxr-cli camera probe --raw --seconds 2 + dy (right - left) median +0.019 px SIGNED; |dy| median 2.015 px, p90 4.026 px, 100 block(s) > 1 px +# service with SIM_DISPLAY_FAKE_STEREO_CAMERA_BASELINE_MM=120: +$ displayxr-cli camera probe --rectified --seconds 2 +row alignment (RECTIFIED): NCC 32x32 blocks, search d -8..136 px, dy +-24 px, accept NCC >= 0.90 + 266 textured block(s): 233 accepted, 33 below NCC 0.90, 0 AT the disparity bound, 0 AT the dy bound + dy (right - left) median +0.000 px SIGNED; |dy| median 0.011 px, p90 0.031 px, 1 block(s) > 1 px + disparity mode 1: 27.17 px (191 blocks) -> Z = f*B/d = 2.002 m + disparity mode 2: 90.68 px (40 blocks) -> Z = f*B/d = 0.600 m --rectified: PASS — rows aligned -$ displayxr-cli camera probe --raw --format gray8 -row alignment (RAW, 221 textured 32x32 blocks): |dy| median 2.589 px, p90 5.292 px, 183 block(s) > 1 px ``` (The blocks above 1 px on the rectified pair are mismatches at the bar's occlusion edges and on the frame counter, which is burned in raw space.) +**Open: a +1.7 px signed Δy on real SR data after rectification.** The `[vshift]` tests in +`tests_stereo_rectify` pin what each candidate cause looks like on the 120 mm distorted fake, so +the raw frames + calibration dumps can be matched to one: a per-eye principal-point error Δcy is +a *constant* Δy ≈ −(f_rect/fy)·Δcy with small spread (flipped row convention, a crop taken for a +scale, a one-eye ±0.5 px centre convention); a transposed R is a large, roll-shaped Δy (spread +> 1 px), so a clean constant offset is *not* that; the calibration-size rescale and the plug-in's +half swap (`R' = Rᵀ`, `T' = −RᵀT`) are exact; an identity calibration maps as a pure scale about +(cx, cy) — no half-pixel offset. + Runs as a DIAG IPC client, non-elevated. `probe` goes through the full consent path (so its first run on a box raises the prompt — which is how a developer checks the prompt). `selftest` gains `stereo_camera_caps`: zero cameras passes; malformed properties (zero extent, CALIBRATED diff --git a/src/xrt/auxiliary/util/u_stereo_camera.c b/src/xrt/auxiliary/util/u_stereo_camera.c index b08b5dca2d..3da7c9ee92 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.c +++ b/src/xrt/auxiliary/util/u_stereo_camera.c @@ -8,6 +8,7 @@ #include "util/u_stereo_camera.h" +#include #include #include #include @@ -503,3 +504,173 @@ u_stereo_camera_estimate_offset(const uint8_t *gray, *out_dy = (float)((int)by - (int)max_dy) + sy; return true; } + + +/* + * + * NCC block matcher (R2 follow-up: baseline-aware probe). + * + */ + +struct ncc_left +{ + double sum, norm; //!< sum and sqrt(n * var) of the left block's samples + uint32_t n; +}; + +static bool +ncc_left_init(const uint8_t *gray, + uint32_t pitch, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t step, + struct ncc_left *l) +{ + double s = 0.0, ss = 0.0; + uint32_t n = 0; + for (uint32_t j = 0; j < h; j += step) { + const uint8_t *row = gray + (size_t)(y0 + j) * pitch + x0; + for (uint32_t i = 0; i < w; i += step) { + s += row[i]; + ss += (double)row[i] * row[i]; + n++; + } + } + double var_n = ss - s * s / n; // n * variance + l->sum = s; + l->n = n; + l->norm = var_n > 0.0 ? sqrt(var_n) : 0.0; + // Flat: < 1 grey level of standard deviation matches anywhere. + return var_n >= (double)n; +} + +static double +ncc_at(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t step, + const struct ncc_left *l, + int32_t d, + int32_t dy) +{ + uint64_t cross = 0, sr = 0, srr = 0; + for (uint32_t j = 0; j < h; j += step) { + const uint8_t *lr = gray + (size_t)(y0 + j) * pitch + x0; + const uint8_t *rr = gray + (size_t)((int64_t)y0 + j + dy) * pitch + eye_width + (int64_t)x0 - d; + for (uint32_t i = 0; i < w; i += step) { + uint32_t r = rr[i]; + cross += (uint64_t)lr[i] * r; + sr += r; + srr += (uint64_t)r * r; + } + } + double var_r = (double)srr - (double)sr * (double)sr / l->n; + if (var_r <= 0.0 || l->norm <= 0.0) { + return -1.0; + } + double num = (double)cross - l->sum * (double)sr / l->n; + return num / (l->norm * sqrt(var_r)); +} + +//! Parabola vertex offset of three samples around a MAXIMUM; 0 if not one. +static float +peak_offset(double a, double b, double c) +{ + double den = a - 2.0 * b + c; + if (den >= 0.0) { + return 0.0f; + } + double o = 0.5 * (a - c) / den; + return (float)(o < -0.5 ? -0.5 : (o > 0.5 ? 0.5 : o)); +} + +bool +u_stereo_camera_match_block(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + uint32_t max_dy, + struct u_stereo_camera_block_match *out) +{ + if (gray == NULL || out == NULL || w < 4 || h < 4 || x0 + w > eye_width || y0 + h > height) { + return false; + } + memset(out, 0, sizeof(*out)); + // Right block x = x0 - d must stay inside the right eye. + const int32_t d_hi = (int32_t)(max_disparity < x0 ? max_disparity : x0); + const uint32_t room_r = eye_width - (x0 + w); + const int32_t d_lo = -(int32_t)(room_r < 8 ? room_r : 8); + const int32_t dy_lo = -(int32_t)(max_dy < y0 ? max_dy : y0); + const uint32_t room_b = height - (y0 + h); + const int32_t dy_hi = (int32_t)(max_dy < room_b ? max_dy : room_b); + out->d_lo = d_lo; + out->d_hi = d_hi; + out->dy_lo = dy_lo; + out->dy_hi = dy_hi; + + struct ncc_left lc, lf; + if (!ncc_left_init(gray, pitch, x0, y0, w, h, 2, &lc) || !ncc_left_init(gray, pitch, x0, y0, w, h, 1, &lf)) { + return false; + } + + // Coarse: every integer (d, dy), every second pixel of the block. + double best = -2.0; + int32_t bd = 0, by = 0; + for (int32_t dy = dy_lo; dy <= dy_hi; dy++) { + for (int32_t d = d_lo; d <= d_hi; d++) { + double c = ncc_at(gray, pitch, eye_width, x0, y0, w, h, 2, &lc, d, dy); + if (c > best) { + best = c; + bd = d; + by = dy; + } + } + } + + // Fine: full resolution, hill-climb on the 3x3 neighbourhood (a few steps at + // most — the coarse peak is within a pixel or two of the true one). + double f[3][3]; + for (int iter = 0;; iter++) { + int32_t md = 0, my = 0; + double fb = -2.0; + for (int32_t j = -1; j <= 1; j++) { + for (int32_t i = -1; i <= 1; i++) { + int32_t d = bd + i, dy = by + j; + double c = (d < d_lo || d > d_hi || dy < dy_lo || dy > dy_hi) + ? -2.0 + : ncc_at(gray, pitch, eye_width, x0, y0, w, h, 1, &lf, d, dy); + f[j + 1][i + 1] = c; + if (c > fb) { + fb = c; + md = i; + my = j; + } + } + } + if ((md == 0 && my == 0) || iter >= 8) { + break; // f[][] is centred on (bd, by) + } + bd += md; + by += my; + } + + out->ncc = (float)f[1][1]; + out->dx_at_edge = bd == d_hi || bd == d_lo; + out->dy_at_edge = by == dy_hi || by == dy_lo; + float sx = (bd > d_lo && bd < d_hi) ? peak_offset(f[1][0], f[1][1], f[1][2]) : 0.0f; + float sy = (by > dy_lo && by < dy_hi) ? peak_offset(f[0][1], f[1][1], f[2][1]) : 0.0f; + out->dx = (float)bd + sx; + out->dy = (float)by + sy; + return true; +} diff --git a/src/xrt/auxiliary/util/u_stereo_camera.h b/src/xrt/auxiliary/util/u_stereo_camera.h index 1e7243db38..f2d41db1e4 100644 --- a/src/xrt/auxiliary/util/u_stereo_camera.h +++ b/src/xrt/auxiliary/util/u_stereo_camera.h @@ -243,6 +243,50 @@ u_stereo_camera_estimate_offset(const uint8_t *gray, float *out_dx, float *out_dy); +/*! + * One block's 2-D match, @ref u_stereo_camera_match_block. + * + * The edge flags are the point: a peak ON the search boundary is a clamp, not a + * measurement (the R2 probe reported "d = 64, |dy| = 10.000" on the Leia SR + * camera because both windows were too small). Callers must exclude and count + * flagged blocks, never average them in. + */ +struct u_stereo_camera_block_match +{ + float dx; //!< left x - right x, px, sub-pixel + float dy; //!< right y - left y, px, sub-pixel, SIGNED + float ncc; //!< peak zero-mean normalised cross-correlation, -1..1 + bool dx_at_edge; //!< integer peak on the disparity window's bound + bool dy_at_edge; //!< integer peak on the vertical window's bound + int32_t d_lo, d_hi; //!< the disparity window actually searched (clipped by the image) + int32_t dy_lo, dy_hi; //!< the vertical window actually searched (clipped by the image) +}; + +/*! + * Zero-mean NCC block matching of the LEFT-eye block [x0, x0+w) x [y0, y0+h) + * against the right eye over disparity -8..@p max_disparity and vertical offset + * -@p max_dy..@p max_dy (both clipped to the image), with parabolic sub-pixel + * refinement on each axis. Coarse pass on every second pixel of the block, + * then a full-resolution pass around the coarse peak. NCC, unlike SSD, is + * blind to the gain/offset differences of two real sensors, and its peak value + * is a usable confidence: callers threshold it (the field reference used + * 0.93). + * + * @return false if the block does not fit or is flat (no texture to match). + */ +bool +u_stereo_camera_match_block(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + uint32_t x0, + uint32_t y0, + uint32_t w, + uint32_t h, + uint32_t max_disparity, + uint32_t max_dy, + struct u_stereo_camera_block_match *out); + #ifdef __cplusplus } #endif diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index dbf03f3a2f..b2254e1b54 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -179,7 +179,7 @@ sim_display_stereo_camera_get_calibration(struct xrt_plugin_instance *inst, if (cfg->distort) { // The raw pair's ground truth (sim_display_stereo_camera_pattern.h). struct sim_stereo_camera_truth t; - sim_stereo_camera_truth_init(&t, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), FAKE_BASELINE_MM, + sim_stereo_camera_truth_init(&t, cfg->eye_w, cfg->eye_h, fake_fx(cfg->eye_w), cfg->baseline_mm, FAKE_BG_DEPTH_M, FAKE_BAR_DEPTH_M); for (int e = 0; e < 2; e++) { c.k[e][0] = t.fx[e]; diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 20647dadc3..d10ac11c78 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -8,6 +8,7 @@ * camera calib [--raw|--rectified] [--json] * camera probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] * [--frames N] [--seconds S] [--out DIR] + * [--max-disparity N] [--max-dy N] [--min-ncc C] * * `probe` is a real consumer: it creates and starts a stream (so it goes * through the service's authorisation — in R1, DXR_STEREO_CAMERA_DEV_ALLOW=1 @@ -23,7 +24,13 @@ * horizontal disparities, each converted to a depth Z = f * B / d with the * RECTIFIED calibration. `--rectified` insists on RECTIFIED output and fails * (exit 5) if the frames came back RAW-flagged or the rows do not align - * (median |dy| > 0.5 px). On the distorted sim fake + * (median |dy| > 0.5 px, or a SIGNED median dy beyond +-0.5 px: a constant + * vertical offset). Matching is zero-mean NCC with sub-pixel refinement; the + * disparity window comes from the camera's own f * B (a subject at 0.4 m), + * the vertical one is +-24 px, blocks under NCC 0.90 or peaking ON either + * bound are counted and excluded (a bound hit is a clamp, not a measurement — + * the first Leia SR run read "d = 64, |dy| = 10.000" that way). On the + * distorted sim fake * (SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1) the depths read back 2.00 m * (background) and 0.60 m (bar); `--raw` shows the misalignment it fixes. * @@ -431,21 +438,40 @@ cmp_float(const void *a, const void *b) struct row_alignment { - int blocks; + int textured; //!< blocks that passed the texture gate and were matched + int blocks; //!< ...of which accepted (NCC >= min, peak inside both windows) + int low_corr; //!< rejected: peak NCC below the threshold + int dx_edge; //!< rejected: peak ON the disparity bound (a clamp, not a measurement) + int dy_edge; //!< rejected: peak ON the vertical bound + float dy_signed_median; //!< right y - left y, px (a constant vertical offset shows here) float dy_median, dy_p90; //!< |dy|, px - int outliers; //!< blocks with |dy| > 1 px (mismatches: occlusion edges, flat or repeated texture) + int outliers; //!< accepted blocks with |dy| > 1 px int modes; float mode_dx[2]; int mode_count[2]; }; +struct row_params +{ + uint32_t max_d; //!< disparity window (probe_max_disparity) + uint32_t max_dy; //!< vertical window, +-px + float min_ncc; //!< acceptance threshold on the peak correlation +}; + /*! - * 2-D block matching (u_stereo_camera_estimate_offset) of a GRAY8 SBS frame on - * a 32 px grid, textured blocks only (a flat wall matches anywhere): |dy| - * statistics and the two dominant horizontal disparities (1 px clusters). + * 2-D NCC block matching (u_stereo_camera_match_block) of a GRAY8 SBS frame on + * a 32 px grid, textured blocks only (a flat wall matches anywhere): signed and + * |dy| statistics and the two dominant horizontal disparities (1 px clusters). + * Blocks below the correlation threshold or whose peak sits on either window's + * bound are counted and excluded — never clamped into the statistics. */ static void -measure_rows(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, struct row_alignment *out) +measure_rows(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_w, + uint32_t h, + const struct row_params *rp, + struct row_alignment *out) { memset(out, 0, sizeof(*out)); enum @@ -453,10 +479,10 @@ measure_rows(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, st B = 32, MAXB = 1024 }; - static float dxs[MAXB], ady[MAXB]; + static float dxs[MAXB], ady[MAXB], sdy[MAXB]; int n = 0; - for (uint32_t y = 48; y + B + 12 <= h && n < MAXB; y += B) { - for (uint32_t x = 72; x + B <= eye_w && n < MAXB; x += B) { + for (uint32_t y = 16; y + B + 16 <= h && out->textured < MAXB; y += B) { + for (uint32_t x = 16; x + B + 8 <= eye_w && out->textured < MAXB; x += B) { // Texture gate: standard deviation of the left block >= 6 levels. double sum = 0, sq = 0; for (uint32_t j = 0; j < B; j++) { @@ -470,12 +496,27 @@ measure_rows(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, st if (sq / (B * B) - mean * mean < 36.0) { continue; } - float dx, dy; - if (!u_stereo_camera_estimate_offset(gray, pitch, eye_w, h, x, y, B, B, 64, 10, &dx, &dy)) { + struct u_stereo_camera_block_match m; + if (!u_stereo_camera_match_block(gray, pitch, eye_w, h, x, y, B, B, rp->max_d, rp->max_dy, + &m)) { + continue; + } + out->textured++; + if (m.ncc < rp->min_ncc) { + out->low_corr++; + continue; + } + if (m.dx_at_edge) { + out->dx_edge++; continue; } - dxs[n] = dx; - ady[n] = dy < 0 ? -dy : dy; + if (m.dy_at_edge) { + out->dy_edge++; + continue; + } + dxs[n] = m.dx; + sdy[n] = m.dy; + ady[n] = m.dy < 0 ? -m.dy : m.dy; n++; } } @@ -483,6 +524,8 @@ measure_rows(const uint8_t *gray, uint32_t pitch, uint32_t eye_w, uint32_t h, st if (n == 0) { return; } + qsort(sdy, (size_t)n, sizeof(float), cmp_float); + out->dy_signed_median = sdy[n / 2]; qsort(ady, (size_t)n, sizeof(float), cmp_float); out->dy_median = ady[n / 2]; out->dy_p90 = ady[(n * 9) / 10]; @@ -525,6 +568,8 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) float seconds = 0.0f; const char *out_dir = NULL; bool require_rectified = false; + uint32_t max_d_arg = 0; // 0 = from the camera's geometry + struct row_params rp = {.max_d = 0, .max_dy = 24, .min_ncc = 0.90f}; for (int i = 3; i < argc; i++) { if (strcmp(argv[i], "--raw") == 0) { output = XRT_STEREO_CAMERA_OUTPUT_RAW; @@ -542,6 +587,12 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) frames = atoi(argv[++i]); } else if (strcmp(argv[i], "--seconds") == 0 && i + 1 < argc) { seconds = (float)atof(argv[++i]); + } else if (strcmp(argv[i], "--max-disparity") == 0 && i + 1 < argc) { + max_d_arg = (uint32_t)strtoul(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--max-dy") == 0 && i + 1 < argc) { + rp.max_dy = (uint32_t)strtoul(argv[++i], NULL, 10); + } else if (strcmp(argv[i], "--min-ncc") == 0 && i + 1 < argc) { + rp.min_ncc = (float)atof(argv[++i]); } else if (strcmp(argv[i], "--out") == 0 && i + 1 < argc) { out_dir = argv[++i]; } else if (argv[i][0] != '-') { @@ -683,13 +734,13 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) u_stereo_camera_convert(last.format, src, last.row_pitch, XRT_STEREO_CAMERA_FORMAT_GRAY8, gray, &gl, last.width, last.height)) { char disp[256]; - report_disparity(gray, gl.pitch[0], last.width / 2, last.height, - probe_max_disparity(&props, last.width / 2), disp, sizeof(disp)); + rp.max_d = max_d_arg > 0 ? max_d_arg : probe_max_disparity(&props, last.width / 2); + report_disparity(gray, gl.pitch[0], last.width / 2, last.height, rp.max_d, disp, sizeof(disp)); printf("block disparity (left x - right x): %s\n", disp); // R2: row alignment + depth of the dominant disparities. struct row_alignment ra; - measure_rows(gray, gl.pitch[0], last.width / 2, last.height, &ra); + measure_rows(gray, gl.pitch[0], last.width / 2, last.height, &rp, &ra); struct xrt_stereo_camera_calibration rc; double fb = 0.0; // f * B, px * m if (last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && @@ -697,16 +748,28 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) &rc) == XRT_SUCCESS) { fb = rc.eye[0].fx * rc.baseline_mm / 1000.0; } - if (ra.blocks == 0) { - printf("row alignment: n/a (no textured block)\n"); + const char *out_name = last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW"; + printf( + "row alignment (%s): NCC 32x32 blocks, search d -8..%u px, dy +-%u px, accept NCC >= " + "%.2f\n", + out_name, rp.max_d, rp.max_dy, rp.min_ncc); + printf( + " %d textured block(s): %d accepted, %d below NCC %.2f, %d AT the disparity bound, %d AT " + "the dy bound\n", + ra.textured, ra.blocks, ra.low_corr, rp.min_ncc, ra.dx_edge, ra.dy_edge); + if (ra.dx_edge * 5 > ra.textured || ra.dy_edge * 5 > ra.textured) { + printf( + " WARNING: over 20%% of blocks peak ON a search bound — that is a clamp, not a " + "measurement; widen it (--max-disparity N / --max-dy N)\n"); + } + if (ra.blocks < 5) { + printf(" row alignment: n/a (fewer than 5 accepted blocks)\n"); rows_ok = false; } else { printf( - "row alignment (%s, %d textured 32x32 blocks): |dy| median %.3f px, p90 %.3f px, " - "%d block(s) " - "> 1 px\n", - last.output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", ra.blocks, - ra.dy_median, ra.dy_p90, ra.outliers); + " dy (right - left) median %+.3f px SIGNED; |dy| median %.3f px, p90 %.3f px, %d " + "block(s) > 1 px\n", + ra.dy_signed_median, ra.dy_median, ra.dy_p90, ra.outliers); for (int k = 0; k < ra.modes; k++) { if (fb > 0.0 && ra.mode_dx[k] > 0.25f) { printf( @@ -717,7 +780,9 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) ra.mode_dx[k], ra.mode_count[k]); } } - rows_ok = ra.dy_median <= 0.5f; + // A constant vertical offset fails even when |dy| scatter is small. + float sm = ra.dy_signed_median < 0 ? -ra.dy_signed_median : ra.dy_signed_median; + rows_ok = ra.dy_median <= 0.5f && sm <= 0.5f; } } if (out_dir != NULL) { @@ -774,7 +839,9 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) return 5; } if (!rows_ok) { - printf("--rectified: FAIL — rows do not align (median |dy| > 0.5 px)\n"); + printf( + "--rectified: FAIL — rows do not align (median |dy| or |signed median dy| > 0.5 px, or too " + "few matched blocks)\n"); return 5; } printf("--rectified: PASS — rows aligned\n"); @@ -805,7 +872,8 @@ cli_cmd_camera(int argc, const char **argv) "usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" " probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] [--frames N] " "[--seconds S] " - "[--out DIR]\n"); + "[--out DIR]\n" + " [--max-disparity N] [--max-dy N (24)] [--min-ncc C (0.90)]\n"); ret = 1; } ipc_client_connection_fini(&ipc_c); diff --git a/tests/tests_stereo_rectify.cpp b/tests/tests_stereo_rectify.cpp index 98ed2b6b48..8637968f7c 100644 --- a/tests/tests_stereo_rectify.cpp +++ b/tests/tests_stereo_rectify.cpp @@ -282,8 +282,14 @@ max_abs_dev(const std::vector &v, float ref) */ template block_stats -match_blocks( - const uint8_t *img, uint32_t pitch, uint32_t eye_w, uint32_t h, const sim_stereo_camera_scene &sc, ToIdeal to_ideal) +match_blocks(const uint8_t *img, + uint32_t pitch, + uint32_t eye_w, + uint32_t h, + const sim_stereo_camera_scene &sc, + ToIdeal to_ideal, + uint32_t max_d = 64, + bool ncc = false) { block_stats st; const uint32_t B = 32, max_dy = 10; @@ -305,7 +311,18 @@ match_blocks( continue; } float dx, dy; - if (!u_stereo_camera_estimate_offset(img, pitch, eye_w, h, x, y, B, B, 64, max_dy, &dx, &dy)) { + if (ncc) { + // The probe's matcher: flagged / weak blocks are excluded, never clamped. + u_stereo_camera_block_match bm; + if (!u_stereo_camera_match_block(img, pitch, eye_w, h, x, y, B, B, max_d, max_dy, + &bm) || + bm.dx_at_edge || bm.dy_at_edge || bm.ncc < 0.9f) { + continue; + } + dx = bm.dx; + dy = bm.dy; + } else if (!u_stereo_camera_estimate_offset(img, pitch, eye_w, h, x, y, B, B, max_d, max_dy, + &dx, &dy)) { continue; } (is_bar ? st.dx_bar : st.dx_bg).push_back(dx); @@ -400,6 +417,398 @@ TEST_CASE("stereo rectify: END TO END on the distorted sim fake", "[stereo_recti sim_stereo_camera_distorted_fini(&d); } +namespace { + +//! A parallel pair of the scene, the right eye shifted DOWN by @p dy_px rows +//! (a feature at left row v is at right row v + dy) and given a gain/offset. +std::vector +render_parallel(const sim_stereo_camera_scene &sc, double dy_px, double gain = 1.0, double offset = 0.0) +{ + const uint32_t W = sc.eye_width, H = sc.eye_height; + std::vector img(2 * W * H); + for (uint32_t y = 0; y < H; y++) { + for (uint32_t x = 0; x < W; x++) { + img[y * 2 * W + x] = (uint8_t)std::lround(sim_stereo_camera_scene_sample(&sc, 0, x, y)); + double v = gain * sim_stereo_camera_scene_sample(&sc, 1, x, y - dy_px) + offset; + img[y * 2 * W + W + x] = (uint8_t)std::lround(std::min(255.0, std::max(0.0, v))); + } + } + return img; +} + +} // namespace + +TEST_CASE("probe matcher: Leia-SR-sized disparities, SIGNED dy, bound hits flagged", "[stereo_rectify][probe]") +{ + // 120 mm at f 474 px: the bar at 0.6 m is ~95 px (the field case: a face at + // 0.6 m on the SR tracking camera, which the old fixed 0..64 search clamped). + const uint32_t W = 640, H = 480; + sim_stereo_camera_scene sc; + sim_stereo_camera_scene_init(&sc, W, H, k_ideal_fx_640, 120.0, 2.0, 0.6); + REQUIRE(sc.bar_disparity_f > 90.0); + const double dy0 = 1.68; // the field's signed residual + std::vector img = render_parallel(sc, dy0); + const uint32_t max_d = (uint32_t)std::ceil(k_ideal_fx_640 * 0.120 / 0.4); // what the probe computes + const uint32_t bx = 280, by = 200; // inside the bar + + u_stereo_camera_block_match m; + REQUIRE(u_stereo_camera_match_block(img.data(), 2 * W, W, H, bx, by, 32, 32, max_d, 24, &m)); + std::printf("[probe] bar block: dx %.3f (GT %.3f), dy %+.3f (GT %+.3f), ncc %.4f, window d %d..%d dy %d..%d\n", + m.dx, sc.bar_disparity_f, m.dy, dy0, m.ncc, m.d_lo, m.d_hi, m.dy_lo, m.dy_hi); + CHECK_FALSE(m.dx_at_edge); + CHECK_FALSE(m.dy_at_edge); + CHECK(m.ncc > 0.93f); + CHECK(m.dx == Catch::Approx(sc.bar_disparity_f).margin(0.25)); + CHECK(m.dy == Catch::Approx(dy0).margin(0.15)); // SIGNED, sub-pixel + + // Background block (28.5 px), same signed dy. + REQUIRE(u_stereo_camera_match_block(img.data(), 2 * W, W, H, 520, 40, 32, 32, max_d, 24, &m)); + CHECK(m.dx == Catch::Approx(sc.bg_disparity_f).margin(0.25)); + CHECK(m.dy == Catch::Approx(dy0).margin(0.15)); + + // The field failure, reproduced: with the old 64 px window the true match is + // out of reach. The block must then be REJECTED — a peak on the bound, or a + // correlation under the probe's 0.90 — never reported as a disparity. + REQUIRE(u_stereo_camera_match_block(img.data(), 2 * W, W, H, bx, by, 32, 32, 64, 24, &m)); + std::printf("[probe] same block, 0..64 window: dx %.2f, ncc %.3f, at_edge %d\n", m.dx, m.ncc, + (int)m.dx_at_edge); + CHECK((m.dx_at_edge || m.ncc < 0.90f)); + // Likewise a dy window narrower than the true offset. + std::vector img3 = render_parallel(sc, 3.0); + REQUIRE(u_stereo_camera_match_block(img3.data(), 2 * W, W, H, bx, by, 32, 32, max_d, 1, &m)); + CHECK(m.dy_at_edge); + REQUIRE(u_stereo_camera_match_block(img3.data(), 2 * W, W, H, bx, by, 32, 32, max_d, 24, &m)); + CHECK_FALSE(m.dy_at_edge); + CHECK(m.dy == Catch::Approx(3.0).margin(0.15)); + + // NCC is blind to a gain/offset mismatch between the two sensors. + std::vector img_g = render_parallel(sc, dy0, 0.75, 30.0); + REQUIRE(u_stereo_camera_match_block(img_g.data(), 2 * W, W, H, bx, by, 32, 32, max_d, 24, &m)); + CHECK(m.ncc > 0.93f); + CHECK(m.dx == Catch::Approx(sc.bar_disparity_f).margin(0.25)); + CHECK(m.dy == Catch::Approx(dy0).margin(0.15)); + + // A flat block is refused, not matched anywhere. + std::vector flat(2 * W * H, 128); + CHECK_FALSE(u_stereo_camera_match_block(flat.data(), 2 * W, W, H, bx, by, 32, 32, max_d, 24, &m)); +} + +TEST_CASE("stereo rectify: END TO END at the Leia SR baseline (120 mm)", "[stereo_rectify][probe]") +{ + const uint32_t W = 640, H = 480; + const double B_mm = 120.0; + sim_stereo_camera_distorted d; + REQUIRE(sim_stereo_camera_distorted_init(&d, W, H, k_ideal_fx_640, B_mm, 2.0, 0.6)); + std::vector raw(2 * W * H), rect(2 * W * H); + sim_stereo_camera_render_gray_distorted(&d, 77, raw.data(), 2 * W); + + u_stereo_rectify_input in; + std::memset(&in, 0, sizeof(in)); + in.width = W; + in.height = H; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = d.truth.fx[e]; + in.eye[e].fy = d.truth.fy[e]; + in.eye[e].cx = d.truth.cx[e]; + in.eye[e].cy = d.truth.cy[e]; + in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + std::memcpy(in.eye[e].d, d.truth.dist[e], sizeof(d.truth.dist[e])); + } + std::memcpy(in.R, d.truth.R, sizeof(in.R)); + std::memcpy(in.T, d.truth.T_mm, sizeof(in.T)); + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + u_stereo_rectify_lut lut; + REQUIRE(u_stereo_rectify_lut_init(&lut, &r, 1)); + u_stereo_rectify_lut_apply(&lut, 1, raw.data(), 2 * W, rect.data(), 2 * W); + + const double ifx = d.truth.ideal_fx, icx = d.truth.ideal_cx, icy = d.truth.ideal_cy; + auto to_ideal = [&](uint32_t x, uint32_t y, double *u, double *v) { + *u = ifx * (x - r.cx) / r.f + icx; + *v = ifx * (y - r.cy) / r.f + icy; + }; + const uint32_t max_d = (uint32_t)std::ceil(r.f * (B_mm / 1000.0) / 0.4); + block_stats st = match_blocks(rect.data(), 2 * W, W, H, d.scene, to_ideal, max_d, true); + REQUIRE(st.dx_bg.size() > 20); + REQUIRE(st.dx_bar.size() > 5); + const float gt_bg = (float)(r.f * (B_mm / 1000.0) / 2.0); + const float gt_bar = (float)(r.f * (B_mm / 1000.0) / 0.6); + std::vector dy_all = st.dy_bg; + dy_all.insert(dy_all.end(), st.dy_bar.begin(), st.dy_bar.end()); + std::printf( + "[e2e 120mm] f %.3f: bar GT %.3f px measured median %.3f (%zu blocks); bg GT %.3f median %.3f; " + "signed dy median %+.4f px, |dy| max %.3f\n", + r.f, gt_bar, median(st.dx_bar), st.dx_bar.size(), gt_bg, median(st.dx_bg), median(dy_all), + max_abs_dev(dy_all, 0.0f)); + CHECK(gt_bar > 64.0f); // the case the old fixed window could not measure + CHECK(max_abs_dev(st.dx_bar, gt_bar) <= 0.5f); + CHECK(max_abs_dev(st.dx_bg, gt_bg) <= 0.5f); + CHECK(max_abs_dev(dy_all, 0.0f) <= 0.5f); + CHECK(std::fabs(median(dy_all)) <= 0.1f); // no constant vertical offset + u_stereo_rectify_lut_fini(&lut); + sim_stereo_camera_distorted_fini(&d); +} + +/* + * CONSTANT VERTICAL OFFSET hypotheses (field: +1.7 px signed dy after + * rectification on the Leia SR laptop). Each case feeds the rectifier a + * calibration that differs from the truth in ONE specific way and measures the + * signed dy it leaves, so the field residual can be matched to a cause once the + * raw frames + calibration dumps arrive. The matched pair is always the + * distorted fake at the SR-like 120 mm baseline. + */ +namespace { + +struct vshift_rig +{ + sim_stereo_camera_distorted d; + std::vector raw; + u_stereo_rectify_input truth_in; + static constexpr uint32_t W = 640, H = 480; + + vshift_rig() + { + REQUIRE(sim_stereo_camera_distorted_init(&d, W, H, k_ideal_fx_640, 120.0, 2.0, 0.6)); + raw.resize(2 * W * H); + sim_stereo_camera_render_gray_distorted(&d, 5, raw.data(), 2 * W); + std::memset(&truth_in, 0, sizeof(truth_in)); + truth_in.width = W; + truth_in.height = H; + for (int e = 0; e < 2; e++) { + truth_in.eye[e].fx = d.truth.fx[e]; + truth_in.eye[e].fy = d.truth.fy[e]; + truth_in.eye[e].cx = d.truth.cx[e]; + truth_in.eye[e].cy = d.truth.cy[e]; + truth_in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + std::memcpy(truth_in.eye[e].d, d.truth.dist[e], sizeof(d.truth.dist[e])); + } + std::memcpy(truth_in.R, d.truth.R, sizeof(truth_in.R)); + std::memcpy(truth_in.T, d.truth.T_mm, sizeof(truth_in.T)); + } + ~vshift_rig() + { + sim_stereo_camera_distorted_fini(&d); + } + + //! Rectify @p img with @p in; signed dy median + spread over NCC-accepted blocks. + void + measure(const u_stereo_rectify_input &in, + const std::vector &img, + float *dy_median, + float *dy_spread, + float *dx_median, + double *f_rect = nullptr) + { + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + u_stereo_rectify_lut lut; + REQUIRE(u_stereo_rectify_lut_init(&lut, &r, 1)); + std::vector rect(2 * W * H); + u_stereo_rectify_lut_apply(&lut, 1, img.data(), 2 * W, rect.data(), 2 * W); + u_stereo_rectify_lut_fini(&lut); + std::vector dys, dxs; + const uint32_t max_d = (uint32_t)std::ceil(r.f * 0.120 / 0.4); + for (uint32_t y = 40; y + 72 <= H; y += 32) { + for (uint32_t x = 40; x + 40 <= W; x += 32) { + u_stereo_camera_block_match m; + if (u_stereo_camera_match_block(rect.data(), 2 * W, W, H, x, y, 32, 32, max_d, 24, + &m) && + !m.dx_at_edge && !m.dy_at_edge && m.ncc >= 0.9f) { + dys.push_back(m.dy); + dxs.push_back(m.dx); + } + } + } + REQUIRE(dys.size() > 30); + *dy_median = median(dys); + std::vector dev; + for (float v : dys) { + dev.push_back(std::fabs(v - *dy_median)); + } + std::sort(dev.begin(), dev.end()); + *dy_spread = dev[(dev.size() * 9) / 10]; // p90 |dy - median| + *dx_median = median(dxs); + if (f_rect != nullptr) { + *f_rect = r.f; + } + } +}; + +} // namespace + +TEST_CASE("vertical offset H0: the true calibration leaves no constant dy", "[stereo_rectify][vshift]") +{ + vshift_rig rig; + float med, spread, dx; + rig.measure(rig.truth_in, rig.raw, &med, &spread, &dx); + std::printf("[vshift] H0 truth: signed dy median %+.3f px, p90 spread %.3f\n", med, spread); + CHECK(std::fabs(med) <= 0.1f); +} + +TEST_CASE("vertical offset H1: a per-eye cy error is a CONSTANT dy of (f_rect/fy) * dcy", "[stereo_rectify][vshift]") +{ + // Candidate field causes that all reduce to "one eye's cy is off": a flipped + // (bottom-up) row convention (dcy = H-1-2cy per eye), a crop instead of a + // scale between calibration and frame size, a +-0.5 px centre convention + // applied to one eye only, an integer-rounded cy on one eye. + vshift_rig rig; + for (double dcy : {0.5, 1.0, -1.5}) { + u_stereo_rectify_input in = rig.truth_in; + in.eye[1].cy += dcy; // the calibration says the RIGHT principal point is dcy lower + float med, spread, dx; + double f; + rig.measure(in, rig.raw, &med, &spread, &dx, &f); + double pred = -f / rig.d.truth.fy[1] * dcy; // first order; barrel magnification adds a few % + std::printf( + "[vshift] H1 right cy %+.1f px: signed dy median %+.3f px (predicted %+.3f), p90 spread %.3f\n", + dcy, med, pred, spread); + CHECK(med == Catch::Approx(pred).epsilon(0.1).margin(0.05)); + CHECK(spread <= 0.35f); // constant across the frame: an offset, not a rotation + } +} + +TEST_CASE("vertical offset H2: R given in the wrong direction (R^T) is a large, roll-shaped dy", + "[stereo_rectify][vshift]") +{ + // If the vendor's R were x_left = R x_right (camera-to-camera the other way) + // the rectifier would double the relative rotation instead of cancelling it. + // Signature: a dy offset ~ f * 2 * relative pitch PLUS a roll gradient (large + // spread) — so a pure constant +1.7 px does NOT look like this. + vshift_rig rig; + u_stereo_rectify_input in = rig.truth_in; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + in.R[i][j] = rig.truth_in.R[j][i]; + } + } + float med, spread, dx; + rig.measure(in, rig.raw, &med, &spread, &dx); + std::printf("[vshift] H2 R^T: signed dy median %+.3f px, p90 spread %.3f\n", med, spread); + CHECK((std::fabs(med) > 2.0f || spread > 1.0f)); +} + +TEST_CASE("vertical offset H3: calibration size vs frame size (pixel-centre rescale)", "[stereo_rectify][vshift]") +{ + // Intrinsics expressed at 2x the frame (u2 = 2u + 0.5) must rectify + // identically; the WRONG convention (u2 = 2u) shifts both eyes' cy by 0.25 px + // at frame scale — equal in both eyes, so it must NOT create relative dy. + vshift_rig rig; + u_stereo_rectify_input in = rig.truth_in; + in.calib_width = 2 * rig.W; + in.calib_height = 2 * rig.H; + for (int e = 0; e < 2; e++) { + in.eye[e].fx *= 2.0; + in.eye[e].fy *= 2.0; + in.eye[e].cx = (in.eye[e].cx + 0.5) * 2.0 - 0.5; + in.eye[e].cy = (in.eye[e].cy + 0.5) * 2.0 - 0.5; + } + float med, spread, dx, med0, spread0, dx0; + rig.measure(rig.truth_in, rig.raw, &med0, &spread0, &dx0); + rig.measure(in, rig.raw, &med, &spread, &dx); + CHECK(med == Catch::Approx(med0).margin(0.02)); + CHECK(dx == Catch::Approx(dx0).margin(0.02)); + for (int e = 0; e < 2; e++) { + in.eye[e].cx = rig.truth_in.eye[e].cx * 2.0; // wrong (corner) convention + in.eye[e].cy = rig.truth_in.eye[e].cy * 2.0; + } + rig.measure(in, rig.raw, &med, &spread, &dx); + std::printf("[vshift] H3 corner-convention rescale: signed dy median %+.3f px (truth %+.3f)\n", med, med0); + CHECK(std::fabs(med - med0) <= 0.1f); // common-mode: no relative dy +} + +TEST_CASE("vertical offset H4: the plug-in's half swap + (R^T, -R^T T) is exact", "[stereo_rectify][vshift]") +{ + // L1 (Leia) swaps the SBS halves and re-expresses the calibration from camera + // 2's side: K/D swapped, R' = R^T, T' = -R^T T. Model exactly that here. + vshift_rig rig; + const uint32_t W = rig.W, H = rig.H; + std::vector swapped(2 * W * H); + for (uint32_t y = 0; y < H; y++) { + std::memcpy(&swapped[y * 2 * W], &rig.raw[y * 2 * W + W], W); + std::memcpy(&swapped[y * 2 * W + W], &rig.raw[y * 2 * W], W); + } + // The vendor's own calibration of the swapped pair (camera 1 = SBS RIGHT): + // x_2 = R_v x_1 + T_v with camera 1 the truth's RIGHT eye. + u_stereo_rectify_input v = rig.truth_in; + v.eye[0] = rig.truth_in.eye[1]; + v.eye[1] = rig.truth_in.eye[0]; + double Rv[3][3], Tv[3]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + Rv[i][j] = rig.truth_in.R[j][i]; + } + } + for (int i = 0; i < 3; i++) { + Tv[i] = -(Rv[i][0] * rig.truth_in.T[0] + Rv[i][1] * rig.truth_in.T[1] + Rv[i][2] * rig.truth_in.T[2]); + } + REQUIRE(Tv[0] > 0.0); // what makes L1 decide to swap + // L1's leia_scam_calibration_for_runtime(swap = true): + u_stereo_rectify_input in = v; + in.eye[0] = v.eye[1]; + in.eye[1] = v.eye[0]; + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 3; j++) { + in.R[i][j] = Rv[j][i]; + } + } + for (int i = 0; i < 3; i++) { + in.T[i] = -(in.R[i][0] * Tv[0] + in.R[i][1] * Tv[1] + in.R[i][2] * Tv[2]); + } + // ...applied to the halves swapped BACK, which is what L1 delivers. + std::vector back(2 * W * H); + for (uint32_t y = 0; y < H; y++) { + std::memcpy(&back[y * 2 * W], &swapped[y * 2 * W + W], W); + std::memcpy(&back[y * 2 * W + W], &swapped[y * 2 * W], W); + } + float med, spread, dx; + rig.measure(in, back, &med, &spread, &dx); + std::printf("[vshift] H4 swap round trip: signed dy median %+.3f px, dx median %.2f\n", med, dx); + CHECK(std::fabs(med) <= 0.1f); + CHECK(dx > 20.0f); // positive disparities + // Swapping the intrinsics but NOT the distortion (a plausible slip) is visible: + u_stereo_rectify_input bad = in; + std::memcpy(bad.eye[0].d, in.eye[1].d, sizeof(bad.eye[0].d)); + std::memcpy(bad.eye[1].d, in.eye[0].d, sizeof(bad.eye[1].d)); + rig.measure(bad, back, &med, &spread, &dx); + std::printf("[vshift] H4 distortion NOT swapped: signed dy median %+.3f px, p90 spread %.3f\n", med, spread); +} + +TEST_CASE("vertical offset H5: identity calibration is an identity LUT (no half-pixel shift)", + "[stereo_rectify][vshift]") +{ + u_stereo_rectify_input in; + std::memset(&in, 0, sizeof(in)); + in.width = 640; + in.height = 480; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = in.eye[e].fy = 500.0; + in.eye[e].cx = 319.5; + in.eye[e].cy = 239.5; + in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + } + in.R[0][0] = in.R[1][1] = in.R[2][2] = 1.0; + in.T[0] = -120.0; + u_stereo_rectify_result r; + REQUIRE(u_stereo_rectify_compute(&in, &r)); + CHECK(r.cy == Catch::Approx(239.5).margin(1e-6)); + // The border check may zoom a hair (x1.001 per step) even here, but the map + // must then be a PURE scale about (cx, cy) — identical in both eyes, with the + // principal row fixed: no half-pixel or rounding offset anywhere. + std::printf("[vshift] H5 identity: f %.4f (raw 500), crop x%.5f, cy %.4f\n", r.f, r.crop_scale, r.cy); + CHECK(r.crop_scale < 1.0025); + double worst = 0.0; + for (uint32_t e = 0; e < 2; e++) { + for (double v : {0.0, 100.0, 239.5, 479.0}) { + for (double u : {0.0, 320.0, 639.0}) { + double x, y; + REQUIRE(u_stereo_rectify_map_point(&r, e, u, v, &x, &y)); + worst = std::max(worst, std::fabs((y - 239.5) - (v - 239.5) * 500.0 / r.f)); + } + } + } + CHECK(worst < 1e-6); +} + TEST_CASE("stereo rectify: NV12 and BGRA8 planes agree with GRAY8", "[stereo_rectify]") { const uint32_t W = 640, H = 480, SW = 2 * W; From c86b8a6c3b4fb2c2f712cdf018ef947f195f7fc2 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 12:14:19 -0700 Subject: [PATCH 25/37] feat(sim_display): uncalibrated vertical misalignment knob on the distorted fake (R2) SIM_DISPLAY_FAKE_STEREO_CAMERA_DY (px) and _DY_SLOPE (px per 100 px) shift the right eye's rows on top of the DISTORTED pair without telling the reported calibration, like a device whose stored calibration is slightly off (the Leia SR laptop: +2.6 px, -0.47 px / 100 px). Drives the online refinement's tests. Co-Authored-By: Claude Opus 5.5 --- .../sim_display/sim_display_stereo_camera.c | 21 +++++++++++++++++++ .../sim_display_stereo_camera_pattern.c | 19 +++++++++++++++++ .../sim_display_stereo_camera_pattern.h | 12 +++++++++++ 3 files changed, 52 insertions(+) diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c index b2254e1b54..caf56718aa 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera.c @@ -42,6 +42,9 @@ struct fake_config //! R2: raw, distorted, misaligned pair with known ground truth (not //! NATIVELY_RECTIFIED; the service's rectifier does the work). bool distort; + //! R2 refinement: an UNCALIBRATED extra vertical offset (px) + slope (px + //! per 100 px) of the right eye on the distorted pair. + double extra_dy_px, extra_dy_slope_100; }; //! Completes the plug-in-owned opaque type. @@ -109,11 +112,25 @@ fake_config(void) } const char *dist = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT"); cfg.distort = dist != NULL && dist[0] != '\0' && dist[0] != '0'; + const char *edy = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_DY"); + if (edy != NULL && fabs(atof(edy)) <= 20.0) { + cfg.extra_dy_px = atof(edy); + } + const char *eds = getenv("SIM_DISPLAY_FAKE_STEREO_CAMERA_DY_SLOPE"); + if (eds != NULL && fabs(atof(eds)) <= 5.0) { + cfg.extra_dy_slope_100 = atof(eds); + } if (cfg.enabled) { U_LOG_W("sim_display: FAKE stereo camera ON — %ux%u per eye @ %.1f Hz, baseline %.1f mm, format %u%s%s", cfg.eye_w, cfg.eye_h, cfg.fps, cfg.baseline_mm, cfg.format, cfg.suspend_period_ns > 0 ? ", suspend square-wave" : "", cfg.distort ? ", DISTORTED raw pair (rectified by the service)" : ""); + if (cfg.distort && (cfg.extra_dy_px != 0.0 || cfg.extra_dy_slope_100 != 0.0)) { + U_LOG_W( + "sim_display: FAKE stereo camera — UNCALIBRATED vertical misalignment of the right eye: " + "%+.2f px %+.3f px/100 px (the service's refinement should remove it)", + cfg.extra_dy_px, cfg.extra_dy_slope_100); + } } return &cfg; } @@ -239,6 +256,10 @@ sim_display_stereo_camera_open(struct xrt_plugin_instance *inst, free(cam); return XRT_ERROR_ALLOCATION; } + if (cfg->distort) { + sim_stereo_camera_distorted_set_vmisalign(&cam->distorted, cfg->extra_dy_px, + cfg->extra_dy_slope_100 / 100.0); + } cam->gray_pitch = 2 * cfg->eye_w; cam->gray = calloc((size_t)cam->gray_pitch * cfg->eye_h, 1); if (cfg->format != XRT_PLUGIN_STEREO_CAMERA_FORMAT_GRAY8) { diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c index 1d038f9141..b50ebcf930 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c @@ -279,6 +279,25 @@ sim_stereo_camera_distorted_init(struct sim_stereo_camera_distorted *d, return true; } +void +sim_stereo_camera_distorted_set_vmisalign(struct sim_stereo_camera_distorted *d, double dy_px, double slope) +{ + if (d->ideal_uv == NULL || (dy_px == 0.0 && slope == 0.0) || !(slope > -0.5 && slope < 0.5)) { + return; + } + // The right raw pixel that looks at virtual row v sees the scene row whose + // image the misalignment moved there: v = vs + dy + slope (vs - cy). + const uint32_t w = d->scene.eye_width, h = d->scene.eye_height; + const double cy = d->truth.ideal_cy; + float *uv = d->ideal_uv + (size_t)2 * w * h; + for (size_t i = 0; i < (size_t)w * h; i++) { + if (uv[2 * i] < -1e8f) { + continue; + } + uv[2 * i + 1] = (float)(cy + ((double)uv[2 * i + 1] - cy - dy_px) / (1.0 + slope)); + } +} + void sim_stereo_camera_distorted_fini(struct sim_stereo_camera_distorted *d) { diff --git a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h index 5a66133860..247843a0d4 100644 --- a/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h +++ b/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.h @@ -103,6 +103,18 @@ sim_stereo_camera_distorted_init(struct sim_stereo_camera_distorted *d, void sim_stereo_camera_distorted_fini(struct sim_stereo_camera_distorted *d); +/*! + * An UNCALIBRATED vertical misalignment on top of the distorted pair (R2 + * refinement tests): the right eye shows the scene @p dy_px lower, plus + * @p slope px per px of distance from the centre row, in the virtual parallel + * frame — dy(v) = dy_px + slope * (v - ideal_cy). The calibration the fake + * reports does NOT know about it, like a real device whose stored calibration + * is slightly off (the Leia SR laptop: +2.6 px, -0.0047 px/px). Call once + * after init. + */ +void +sim_stereo_camera_distorted_set_vmisalign(struct sim_stereo_camera_distorted *d, double dy_px, double slope); + //! Render frame @p frame_number of the distorted pair (GRAY8 SBS). void sim_stereo_camera_render_gray_distorted(const struct sim_stereo_camera_distorted *d, From 1641d720002513ee707242f1c6cc9dd39900e467 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 12:14:19 -0700 Subject: [PATCH 26/37] feat(stereo-camera): online vertical-alignment refinement core (u_stereo_vrefine) On a Leia SR laptop the rectified rows stayed +1.71 px apart, and OpenCV's stereoRectify reproduces that on the same frame: the stored calibration is off, not the rectifier. u_stereo_vrefine measures the residual dy = a + b (y - cy) on rectified frames (Shi-Tomasi patches, half-res NCC search, full-res NCC refinement, gated on NCC, d > 0, unclamped peaks and vertical curvature), fits it robustly (median + Tukey IRLS) and runs the controller: duty cycle, >= 50 matches over >= 3 frames, 0.2 px deadband, |a| <= 6 px, |b| <= 2 px/100 px. u_stereo_rectify folds the correction in as v_offset = a/f, v_slope = b: a symmetric per-eye vertical affine in normalized rectified coordinates applied BEFORE the principal point and the alpha = 0 crop, so P1/P2 describe the corrected frames. Adds u_stereo_rectify_rect_from_raw (the inverse map). tests_stereo_vrefine: the fake with the SR model converges to +0.000 px in one rebuild; degenerate frames never apply; clamps hold; and a REAL SR raw frame (62 KB JPEG + calibration + 63 OpenCV SIFT matches as an independent judge): signed dy median +1.715 px -> +0.075 px. Co-Authored-By: Claude Opus 5.5 --- src/xrt/auxiliary/util/CMakeLists.txt | 2 + src/xrt/auxiliary/util/u_stereo_rectify.c | 63 +- src/xrt/auxiliary/util/u_stereo_rectify.h | 29 + src/xrt/auxiliary/util/u_stereo_vrefine.c | 592 ++++++++++++++++++ src/xrt/auxiliary/util/u_stereo_vrefine.h | 203 ++++++ tests/CMakeLists.txt | 8 + .../fixtures/gen_stereo_vrefine_sr_fixture.py | 104 +++ tests/fixtures/stereo_vrefine_sr_fixture.h | 89 +++ tests/fixtures/stereo_vrefine_sr_raw.jpg | Bin 0 -> 62426 bytes tests/tests_stereo_vrefine.cpp | 547 ++++++++++++++++ 10 files changed, 1627 insertions(+), 10 deletions(-) create mode 100644 src/xrt/auxiliary/util/u_stereo_vrefine.c create mode 100644 src/xrt/auxiliary/util/u_stereo_vrefine.h create mode 100644 tests/fixtures/gen_stereo_vrefine_sr_fixture.py create mode 100644 tests/fixtures/stereo_vrefine_sr_fixture.h create mode 100644 tests/fixtures/stereo_vrefine_sr_raw.jpg create mode 100644 tests/tests_stereo_vrefine.cpp diff --git a/src/xrt/auxiliary/util/CMakeLists.txt b/src/xrt/auxiliary/util/CMakeLists.txt index be73fa7270..12e3914326 100644 --- a/src/xrt/auxiliary/util/CMakeLists.txt +++ b/src/xrt/auxiliary/util/CMakeLists.txt @@ -37,6 +37,8 @@ add_library( u_stereo_camera.h u_stereo_rectify.c u_stereo_rectify.h + u_stereo_vrefine.c + u_stereo_vrefine.h u_color_encoding.h u_debug.c u_debug.h diff --git a/src/xrt/auxiliary/util/u_stereo_rectify.c b/src/xrt/auxiliary/util/u_stereo_rectify.c index 418e95ce35..25fcdd62b0 100644 --- a/src/xrt/auxiliary/util/u_stereo_rectify.c +++ b/src/xrt/auxiliary/util/u_stereo_rectify.c @@ -283,9 +283,27 @@ u_stereo_rectify_undistort_pixel( * */ -//! RAW pixel of eye @p e -> rectified pixel under (f, cx, cy). false = behind. +/* + * The vertical correction (u_stereo_rectify_input::v_offset / v_slope): output + * normalized row y -> uncorrected rectified row, and back. s = -1 left, +1 right. + */ +static inline double +vcorr_src(const struct u_stereo_rectify_result *r, uint32_t e, double y) +{ + const double s = e == 0 ? -0.5 : 0.5; + return y * (1.0 + s * r->v_slope) + s * r->v_offset; +} + +static inline double +vcorr_out(const struct u_stereo_rectify_result *r, uint32_t e, double y_src) +{ + const double s = e == 0 ? -0.5 : 0.5; + return (y_src - s * r->v_offset) / (1.0 + s * r->v_slope); +} + +//! RAW pixel of eye @p e -> normalized OUTPUT coordinates (correction applied). static bool -raw_to_rect(const struct u_stereo_rectify_result *r, uint32_t e, double u, double v, double *ou, double *ov) +raw_to_rect_norm(const struct u_stereo_rectify_result *r, uint32_t e, double u, double v, double *ox, double *oy) { double x, y; u_stereo_rectify_undistort_pixel(&r->raw[e], u, v, &x, &y); @@ -294,16 +312,36 @@ raw_to_rect(const struct u_stereo_rectify_result *r, uint32_t e, double u, doubl if (q[2] <= 1e-12) { return false; } - *ou = r->f * q[0] / q[2] + r->cx; - *ov = r->f * q[1] / q[2] + r->cy; + *ox = q[0] / q[2]; + *oy = vcorr_out(r, e, q[1] / q[2]); + return true; +} + +//! RAW pixel of eye @p e -> rectified pixel under (f, cx, cy). false = behind. +static bool +raw_to_rect(const struct u_stereo_rectify_result *r, uint32_t e, double u, double v, double *ou, double *ov) +{ + double x, y; + if (!raw_to_rect_norm(r, e, u, v, &x, &y)) { + return false; + } + *ou = r->f * x + r->cx; + *ov = r->f * y + r->cy; return true; } +bool +u_stereo_rectify_rect_from_raw( + const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_u, double *out_v) +{ + return eye < 2 && raw_to_rect(r, eye, u, v, out_u, out_v); +} + bool u_stereo_rectify_map_point( const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_x, double *out_y) { - double p[3] = {(u - r->cx) / r->f, (v - r->cy) / r->f, 1.0}, q[3]; + double p[3] = {(u - r->cx) / r->f, vcorr_src(r, eye, (v - r->cy) / r->f), 1.0}, q[3]; mat3t_vec(M3(r->rot[eye]), p, q); // raw ray = R_e^T * rectified ray if (q[2] <= 1e-12) { return false; @@ -352,6 +390,11 @@ u_stereo_rectify_compute(const struct u_stereo_rectify_input *in, struct u_stere const double nx = in->width, ny = in->height; out->width = in->width; out->height = in->height; + out->v_offset = in->v_offset; + out->v_slope = in->v_slope; + if (!(fabs(in->v_offset) < 0.5) || !(fabs(in->v_slope) < 0.5)) { + return false; // not a small correction (or NaN) + } // 0. Intrinsics at the frame size (pixel centres: u' = (u + 0.5) * s - 0.5). double sx = in->calib_width ? nx / (double)in->calib_width : 1.0; @@ -428,11 +471,11 @@ u_stereo_rectify_compute(const struct u_stereo_rectify_input *in, struct u_stere double ax = 0, ay = 0; for (int k = 0; k < 4; k++) { double x, y; - u_stereo_rectify_undistort_pixel(&out->raw[e], corners[k][0], corners[k][1], &x, &y); - double p[3] = {x, y, 1.0}, q[3]; - mat3_vec(M3(out->rot[e]), p, q); - ax += q[0] / q[2]; - ay += q[1] / q[2]; + if (!raw_to_rect_norm(out, e, corners[k][0], corners[k][1], &x, &y)) { + return false; + } + ax += x; + ay += y; } ccx[e] = (nx - 1) * 0.5 - fc * ax * 0.25; ccy[e] = (ny - 1) * 0.5 - fc * ay * 0.25; diff --git a/src/xrt/auxiliary/util/u_stereo_rectify.h b/src/xrt/auxiliary/util/u_stereo_rectify.h index c9b49a3133..dbcf09f10a 100644 --- a/src/xrt/auxiliary/util/u_stereo_rectify.h +++ b/src/xrt/auxiliary/util/u_stereo_rectify.h @@ -72,6 +72,24 @@ struct u_stereo_rectify_input //! OpenCV extrinsics: x_right = R * x_left + T (T in any unit, e.g. mm). double R[3][3]; double T[3]; + /*! + * Online vertical-alignment correction (the service's refinement, see + * u_stereo_vrefine.h); both 0 = none, the plain Bouguet result. The + * calibration leaves the RIGHT eye's rows displaced by + * D(y) = v_offset + v_slope * y (normalized rectified units, + * y = (v - cy) / f) + * relative to the left eye's. The correction is split symmetrically: the + * output row of eye e samples the uncorrected rectified row + * y_src = y * (1 + s_e * v_slope / 2) + s_e * v_offset / 2, + * s_left = -1, s_right = +1 — a per-eye vertical affine in normalized + * rectified coordinates (equivalently a per-eye rectification homography + * diag(1, 1 + s_e B/2, 1) + a row shift, folded into the maps). Rows then + * agree to second order (B^2, A*B ~ 1e-5). Because it is folded in BEFORE + * the principal point and the alpha = 0 crop are computed, the result's f / + * cx / cy / P1 / P2 describe the corrected frames: one pinhole for both + * eyes and a pure +x baseline, exactly the contract of a rectified pair. + */ + double v_offset, v_slope; }; //! The rectified geometry. @@ -95,6 +113,8 @@ struct u_stereo_rectify_result //! The valid-region zoom applied to f (alpha = 0): >= 1 when the lens and //! rotations would otherwise leave black borders. double crop_scale; + //! The vertical correction folded in (copied from the input). + double v_offset, v_slope; }; /*! @@ -115,6 +135,15 @@ bool u_stereo_rectify_map_point( const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_x, double *out_y); +/*! + * The inverse of @ref u_stereo_rectify_map_point: the rectified (output) pixel + * of eye @p eye where the RAW pixel (@p u, @p v) lands, correction included. + * @return false when the ray points behind the rectified camera. + */ +bool +u_stereo_rectify_rect_from_raw( + const struct u_stereo_rectify_result *r, uint32_t eye, double u, double v, double *out_u, double *out_v); + /*! * Float map of eye @p eye on a grid subsampled by @p sub (1 = full * resolution; 2 = NV12 chroma: output sample (x, y) sits at full-res diff --git a/src/xrt/auxiliary/util/u_stereo_vrefine.c b/src/xrt/auxiliary/util/u_stereo_vrefine.c new file mode 100644 index 0000000000..6240c7ca8c --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_vrefine.c @@ -0,0 +1,592 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: online vertical-alignment + * refinement. See u_stereo_vrefine.h. + * @ingroup aux_util + */ + +#include "util/u_stereo_vrefine.h" + +#include +#include +#include + + +/* + * + * Measurement. + * + */ + +#define PR_HALF 5 //!< half-res patch radius (11x11) +#define PR_FULL 10 //!< full-res patch radius (21x21) +#define FINE_R 2 //!< full-res search radius around the doubled coarse peak +#define MIN_EIG 12.0 +#define MIN_CURV 0.004 + +void +u_stereo_vrefine_measure_defaults(struct u_stereo_vrefine_measure_params *p, uint32_t max_disparity) +{ + p->max_disparity = max_disparity; + p->max_dy = 10; + p->min_ncc = 0.90f; + p->max_samples = 160; +} + +//! A left patch, copied contiguous, with its sums (integer NCC). +struct lpatch +{ + uint8_t px[(2 * PR_FULL + 1) * (2 * PR_FULL + 1)]; + int32_t side; + double n, sum, var_n; //!< var_n = n * sum(l^2) - sum(l)^2 +}; + +static bool +lpatch_init(struct lpatch *lp, const uint8_t *img, uint32_t pitch, int32_t x, int32_t y, int32_t rad) +{ + const int32_t side = 2 * rad + 1; + uint32_t s = 0, s2 = 0; + for (int32_t j = 0; j < side; j++) { + const uint8_t *row = img + (size_t)(y - rad + j) * pitch + (x - rad); + uint8_t *o = lp->px + j * side; + for (int32_t i = 0; i < side; i++) { + o[i] = row[i]; + s += row[i]; + s2 += (uint32_t)row[i] * row[i]; + } + } + lp->side = side; + lp->n = (double)(side * side); + lp->sum = s; + lp->var_n = lp->n * (double)s2 - (double)s * (double)s; + return lp->var_n > 1e-6; +} + +//! sum(l * r) over the right patch centred at (x, y). Integer, vectorizable. +static uint32_t +dot_at(const struct lpatch *lp, const uint8_t *img, uint32_t pitch, int32_t x, int32_t y) +{ + const int32_t side = lp->side, rad = side / 2; + uint32_t acc = 0; + for (int32_t j = 0; j < side; j++) { + const uint8_t *row = img + (size_t)(y - rad + j) * pitch + (x - rad); + const uint8_t *l = lp->px + j * side; + uint32_t a = 0; + for (int32_t i = 0; i < side; i++) { + a += (uint32_t)l[i] * row[i]; + } + acc += a; + } + return acc; +} + +static double +ncc_from(const struct lpatch *lp, double dot, double sr, double sr2) +{ + double var_r = lp->n * sr2 - sr * sr; + if (!(var_r > 1e-6)) { + return -1.0; + } + return (lp->n * dot - lp->sum * sr) / sqrt(lp->var_n * var_r); +} + +//! NCC against the right patch at (x, y), right sums computed directly. +static double +ncc_direct(const struct lpatch *lp, const uint8_t *img, uint32_t pitch, int32_t x, int32_t y) +{ + const int32_t side = lp->side, rad = side / 2; + uint32_t s = 0, s2 = 0; + for (int32_t j = 0; j < side; j++) { + const uint8_t *row = img + (size_t)(y - rad + j) * pitch + (x - rad); + for (int32_t i = 0; i < side; i++) { + s += row[i]; + s2 += (uint32_t)row[i] * row[i]; + } + } + return ncc_from(lp, (double)dot_at(lp, img, pitch, x, y), (double)s, (double)s2); +} + +static float +parabola(double m, double c, double p) +{ + double den = m - 2.0 * c + p; + if (!(fabs(den) > 1e-12)) { + return 0.0f; + } + double o = 0.5 * (m - p) / den; + return (float)(o > 0.5 ? 0.5 : o < -0.5 ? -0.5 : o); +} + +uint32_t +u_stereo_vrefine_measure(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + const struct u_stereo_vrefine_measure_params *p, + struct u_stereo_vrefine_sample *out, + uint32_t cap) +{ + if (gray == NULL || p == NULL || out == NULL || cap == 0 || eye_width < 64 || height < 64) { + return 0; + } + const int32_t hw = (int32_t)(eye_width / 2), hh = (int32_t)(height / 2); + const int32_t max_d = (int32_t)p->max_disparity, max_dy = (int32_t)p->max_dy; + const int32_t max_d_h = (max_d + 1) / 2, max_dy_h = (max_dy + 1) / 2; + + // 2x-downscaled eyes (box filter) + the structure-tensor integral images. + uint8_t *half = malloc((size_t)2 * hw * hh); + double *integ = malloc(sizeof(double) * 5 * (size_t)(hw + 1) * (hh + 1)); + if (half == NULL || integ == NULL) { + free(half); + free(integ); + return 0; + } + for (int32_t e = 0; e < 2; e++) { + uint8_t *dst = half + (size_t)e * hw * hh; + const uint8_t *src = gray + (size_t)e * eye_width; + for (int32_t y = 0; y < hh; y++) { + const uint8_t *r0 = src + (size_t)(2 * y) * pitch, *r1 = r0 + pitch; + for (int32_t x = 0; x < hw; x++) { + dst[(size_t)y * hw + x] = + (uint8_t)((r0[2 * x] + r0[2 * x + 1] + r1[2 * x] + r1[2 * x + 1] + 2) >> 2); + } + } + } + const uint8_t *L = half, *R = half + (size_t)hw * hh; + const size_t iw = (size_t)hw + 1; + double *Ixx = integ, *Iyy = integ + iw * (hh + 1), *Ixy = integ + 2 * iw * (hh + 1); + double *Rs = integ + 3 * iw * (hh + 1), *Rs2 = integ + 4 * iw * (hh + 1); + memset(integ, 0, sizeof(double) * 5 * iw * (hh + 1)); + for (int32_t y = 0; y < hh; y++) { + double rxx = 0, ryy = 0, rxy = 0; + for (int32_t x = 0; x < hw; x++) { + double gx = 0, gy = 0; + if (x > 0 && x < hw - 1 && y > 0 && y < hh - 1) { + gx = 0.5 * ((double)L[(size_t)y * hw + x + 1] - L[(size_t)y * hw + x - 1]); + gy = 0.5 * ((double)L[(size_t)(y + 1) * hw + x] - L[(size_t)(y - 1) * hw + x]); + } + rxx += gx * gx; + ryy += gy * gy; + rxy += gx * gy; + size_t o = (size_t)(y + 1) * iw + (x + 1); + Ixx[o] = Ixx[o - iw] + rxx; + Iyy[o] = Iyy[o - iw] + ryy; + Ixy[o] = Ixy[o - iw] + rxy; + } + } + for (int32_t y = 0; y < hh; y++) { + double rs = 0, rs2 = 0; + for (int32_t x = 0; x < hw; x++) { + double v = R[(size_t)y * hw + x]; + rs += v; + rs2 += v * v; + size_t o = (size_t)(y + 1) * iw + (x + 1); + Rs[o] = Rs[o - iw] + rs; + Rs2[o] = Rs2[o - iw] + rs2; + } + } +#define BOX(I, x0, y0, x1, y1) \ + ((I)[(size_t)(y1)*iw + (x1)] - (I)[(size_t)(y0)*iw + (x1)] - (I)[(size_t)(y1)*iw + (x0)] + \ + (I)[(size_t)(y0)*iw + (x0)]) + + // Grid of cells, one best Shi-Tomasi patch per cell. The patch must leave + // room for the full-resolution refinement too. + const int32_t margin = PR_HALF + 2; + const int32_t ux0 = margin, ux1 = hw - margin, uy0 = margin, uy1 = hh - margin; + uint32_t want = p->max_samples > 0 ? p->max_samples : 160; + double area = (double)(ux1 - ux0) * (double)(uy1 - uy0); + int32_t cs = (int32_t)sqrt(area / (double)want); + if (cs < 2 * PR_HALF + 1) { + cs = 2 * PR_HALF + 1; + } + const int32_t side_h = 2 * PR_HALF + 1; + struct lpatch lp_h, lp_f; + double fine[2 * FINE_R + 3][2 * FINE_R + 3]; + uint32_t n_out = 0; + + for (int32_t cy0 = uy0; cy0 + cs <= uy1 && n_out < cap; cy0 += cs) { + for (int32_t cx0 = ux0; cx0 + cs <= ux1 && n_out < cap; cx0 += cs) { + // Best corner in the cell (stride 2). + double best = MIN_EIG; + int32_t bx = -1, by = -1; + for (int32_t y = cy0; y < cy0 + cs; y += 2) { + for (int32_t x = cx0; x < cx0 + cs; x += 2) { + int32_t x0 = x - PR_HALF, y0 = y - PR_HALF, x1 = x + PR_HALF + 1, + y1 = y + PR_HALF + 1; + double a = BOX(Ixx, x0, y0, x1, y1) / (side_h * side_h); + double c = BOX(Iyy, x0, y0, x1, y1) / (side_h * side_h); + double b = BOX(Ixy, x0, y0, x1, y1) / (side_h * side_h); + double mn = 0.5 * (a + c) - sqrt(0.25 * (a - c) * (a - c) + b * b); + if (mn > best) { + best = mn; + bx = x; + by = y; + } + } + } + if (bx < 0) { + continue; + } + + // Coarse: half resolution, every integer (d, dy) in the window. + if (!lpatch_init(&lp_h, L, (uint32_t)hw, bx, by, PR_HALF)) { + continue; + } + int32_t d_lo = -1, d_hi = max_d_h; + if (d_hi > bx - PR_HALF) { + d_hi = bx - PR_HALF; // right patch x = bx - d must stay inside + } + if (d_lo < bx + PR_HALF - (hw - 1)) { + d_lo = bx + PR_HALF - (hw - 1); + } + int32_t y_lo = -max_dy_h, y_hi = max_dy_h; + if (by + y_lo - PR_HALF < 0) { + y_lo = PR_HALF - by; + } + if (by + y_hi + PR_HALF > hh - 1) { + y_hi = hh - 1 - PR_HALF - by; + } + if (d_hi - d_lo < 2 || y_hi - y_lo < 2) { + continue; + } + double cbest = -2.0; + int32_t cd = 0, cdy = 0; + for (int32_t dy = y_lo; dy <= y_hi; dy++) { + for (int32_t d = d_lo; d <= d_hi; d++) { + const int32_t rx = bx - d, ry = by + dy; + const int32_t x0 = rx - PR_HALF, y0 = ry - PR_HALF, x1 = rx + PR_HALF + 1, + y1 = ry + PR_HALF + 1; + double c = ncc_from(&lp_h, (double)dot_at(&lp_h, R, (uint32_t)hw, rx, ry), + BOX(Rs, x0, y0, x1, y1), BOX(Rs2, x0, y0, x1, y1)); + if (c > cbest) { + cbest = c; + cd = d; + cdy = dy; + } + } + } + if (cbest < p->min_ncc - 0.15 || cd == d_lo || cd == d_hi || cdy == y_lo || cdy == y_hi) { + continue; // weak, or a clamp on the window: not a measurement + } + + // Fine: full resolution around the doubled peak. + const int32_t fx = 2 * bx, fy = 2 * by; // full-res left patch centre + if (fx - PR_FULL < 0 || fx + PR_FULL >= (int32_t)eye_width || fy - PR_FULL < 0 || + fy + PR_FULL >= (int32_t)height) { + continue; + } + const uint8_t *Rf = gray + eye_width; + if (!lpatch_init(&lp_f, gray, pitch, fx, fy, PR_FULL)) { + continue; + } + int32_t pd = 2 * cd, pdy = 2 * cdy; + for (int iter = 0; iter < 4; iter++) { + double fb = -2.0; + int32_t md = 0, mdy = 0; + for (int32_t j = -FINE_R - 1; j <= FINE_R + 1; j++) { + for (int32_t i = -FINE_R - 1; i <= FINE_R + 1; i++) { + int32_t d = pd + i, dy = pdy + j; + int32_t rx = fx - d, ry = fy + dy; + double c = -2.0; + if (d >= -2 && d <= max_d && dy >= -max_dy && dy <= max_dy && + rx - PR_FULL >= 0 && rx + PR_FULL < (int32_t)eye_width && + ry - PR_FULL >= 0 && ry + PR_FULL < (int32_t)height) { + c = ncc_direct(&lp_f, Rf, pitch, rx, ry); + } + fine[j + FINE_R + 1][i + FINE_R + 1] = c; + if (abs(i) <= FINE_R && abs(j) <= FINE_R && c > fb) { + fb = c; + md = i; + mdy = j; + } + } + } + if (abs(md) < FINE_R && abs(mdy) < FINE_R) { + pd += md; + pdy += mdy; + // Re-centre the 3x3 on the peak (still inside the table). + const int32_t cj = mdy + FINE_R + 1, ci = md + FINE_R + 1; + double c = fine[cj][ci], up = fine[cj - 1][ci], dn = fine[cj + 1][ci]; + double lf = fine[cj][ci - 1], rt = fine[cj][ci + 1]; + bool nbr_ok = up > -1.5 && dn > -1.5 && lf > -1.5 && rt > -1.5; + if (!nbr_ok || c < p->min_ncc || (2.0 * c - up - dn) < MIN_CURV || + (2.0 * c - lf - rt) < MIN_CURV) { + break; + } + float sx = parabola(lf, c, rt), sy = parabola(up, c, dn); + // d = left x - right x: the right patch at x - (pd + i), + // a higher score at i = +1 means a larger d. + float d = (float)pd + sx; + float dyv = (float)pdy + sy; + if (d > 0.5f) { + struct u_stereo_vrefine_sample *o = &out[n_out++]; + o->x = (float)fx; + o->y = (float)fy; + o->d = d; + o->dy = dyv; + o->ncc = (float)c; + } + break; + } + pd += md; + pdy += mdy; // peak on the fine window's edge: move and retry + } + } + } +#undef BOX + free(half); + free(integ); + return n_out; +} + + +/* + * + * Robust fit. + * + */ + +static int +cmp_double(const void *a, const void *b) +{ + double x = *(const double *)a, y = *(const double *)b; + return x < y ? -1 : x > y ? 1 : 0; +} + +static double +median_of(double *v, uint32_t n) +{ + qsort(v, n, sizeof(double), cmp_double); + return n % 2 ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +bool +u_stereo_vrefine_fit_samples(const struct u_stereo_vrefine_sample *s, + uint32_t n, + double cy, + struct u_stereo_vrefine_fit *out) +{ + memset(out, 0, sizeof(*out)); + if (s == NULL || n < 8) { + return false; + } + double *tmp = malloc(sizeof(double) * n); + double *w = malloc(sizeof(double) * n); + if (tmp == NULL || w == NULL) { + free(tmp); + free(w); + return false; + } + for (uint32_t i = 0; i < n; i++) { + tmp[i] = s[i].dy; + } + out->median_dy = median_of(tmp, n); + + double ym = 0.0, yv = 0.0; + for (uint32_t i = 0; i < n; i++) { + ym += s[i].y; + } + ym /= n; + for (uint32_t i = 0; i < n; i++) { + yv += (s[i].y - ym) * (s[i].y - ym); + } + out->slope_determined = sqrt(yv / n) >= 40.0; + + double a = out->median_dy, b = 0.0, sigma = 0.1; + for (int it = 0; it < 10; it++) { + for (uint32_t i = 0; i < n; i++) { + tmp[i] = fabs(s[i].dy - (a + b * (s[i].y - cy))); + } + sigma = 1.4826 * median_of(tmp, n); + if (sigma < 0.1) { + sigma = 0.1; + } + const double c = 4.685 * sigma; + double sw = 0, swy = 0, swyy = 0, swd = 0, swyd = 0; + for (uint32_t i = 0; i < n; i++) { + double y = s[i].y - cy; + double r = s[i].dy - (a + b * y); + double u = r / c; + w[i] = fabs(u) < 1.0 ? (1.0 - u * u) * (1.0 - u * u) : 0.0; + sw += w[i]; + swy += w[i] * y; + swyy += w[i] * y * y; + swd += w[i] * s[i].dy; + swyd += w[i] * y * s[i].dy; + } + if (!(sw > 0.0)) { + break; + } + double na, nb; + double det = sw * swyy - swy * swy; + if (out->slope_determined && det > 1e-9 * sw * sw) { + nb = (sw * swyd - swy * swd) / det; + na = (swd - nb * swy) / sw; + } else { + nb = 0.0; + na = swd / sw; + } + bool conv = fabs(na - a) < 1e-4 && fabs(nb - b) < 1e-7; + a = na; + b = nb; + if (conv) { + break; + } + } + uint32_t inl = 0; + const double c = 4.685 * sigma; + for (uint32_t i = 0; i < n; i++) { + if (fabs(s[i].dy - (a + b * (s[i].y - cy))) < c) { + inl++; + } + } + out->a = a; + out->b = b; + out->sigma = sigma; + out->inliers = inl; + free(tmp); + free(w); + return true; +} + + +/* + * + * Controller. + * + */ + +void +u_stereo_vrefine_config_defaults(struct u_stereo_vrefine_config *c, uint32_t height, double cy) +{ + memset(c, 0, sizeof(*c)); + c->height = height; + c->cy = cy; + c->min_matches = 50; + c->min_frames = 3; + c->max_frames = 12; + c->deadband_px = 0.2; + c->max_offset_px = 6.0; + c->max_slope = 0.02; + c->fast_period_ns = 250ll * 1000 * 1000; + c->fast_phase_ns = 5000ll * 1000 * 1000; + c->slow_period_ns = 30000ll * 1000 * 1000; +} + +void +u_stereo_vrefine_init(struct u_stereo_vrefine *r, const struct u_stereo_vrefine_config *cfg) +{ + memset(r, 0, sizeof(*r)); + r->cfg = *cfg; +} + +void +u_stereo_vrefine_reset(struct u_stereo_vrefine *r) +{ + struct u_stereo_vrefine_config cfg = r->cfg; + u_stereo_vrefine_init(r, &cfg); +} + +void +u_stereo_vrefine_restart(struct u_stereo_vrefine *r, int64_t now_ns) +{ + r->win_n = 0; + r->win_frames = 0; + r->started = true; + r->phase_start_ns = now_ns; + r->next_due_ns = now_ns; +} + +bool +u_stereo_vrefine_due(struct u_stereo_vrefine *r, int64_t now_ns) +{ + if (!r->started) { + u_stereo_vrefine_restart(r, now_ns); + } + return now_ns >= r->next_due_ns; +} + +static double +clampd(double v, double lim) +{ + return v > lim ? lim : v < -lim ? -lim : v; +} + +static void +schedule_after_window(struct u_stereo_vrefine *r, int64_t now_ns) +{ + bool fast = now_ns - r->phase_start_ns < r->cfg.fast_phase_ns; + r->next_due_ns = now_ns + (fast ? r->cfg.fast_period_ns : r->cfg.slow_period_ns); + r->win_n = 0; + r->win_frames = 0; +} + +enum u_stereo_vrefine_result +u_stereo_vrefine_push(struct u_stereo_vrefine *r, int64_t now_ns, const struct u_stereo_vrefine_sample *s, uint32_t n) +{ + if (!r->started) { + u_stereo_vrefine_restart(r, now_ns); + } + for (uint32_t i = 0; i < n && r->win_n < U_STEREO_VREFINE_WINDOW_CAP; i++) { + r->win[r->win_n++] = s[i]; + } + r->win_frames++; + + if (r->win_n < r->cfg.min_matches || r->win_frames < r->cfg.min_frames) { + if (r->win_frames >= r->cfg.max_frames) { + schedule_after_window(r, now_ns); + return U_STEREO_VREFINE_DROPPED; + } + r->next_due_ns = now_ns + r->cfg.fast_period_ns; + return U_STEREO_VREFINE_ACCUMULATING; + } + + struct u_stereo_vrefine_fit fit; + if (!u_stereo_vrefine_fit_samples(r->win, r->win_n, r->cfg.cy, &fit) || fit.inliers < r->cfg.min_matches || + !isfinite(fit.a) || !isfinite(fit.b)) { + schedule_after_window(r, now_ns); + return U_STEREO_VREFINE_DROPPED; + } + r->windows++; + r->last_residual_dy = fit.median_dy; + r->last_matches = fit.inliers; + if (!r->have_initial) { + r->have_initial = true; + // The first window after a reset is measured without any correction. + r->initial_dy = fit.median_dy; + r->initial_a = fit.a; + r->initial_b = fit.b; + } + + double na = clampd(r->a + fit.a, r->cfg.max_offset_px); + double nb = clampd(r->b + fit.b, r->cfg.max_slope); + double extent = + r->cfg.cy > (double)r->cfg.height - 1.0 - r->cfg.cy ? r->cfg.cy : (double)r->cfg.height - 1.0 - r->cfg.cy; + double effect = fabs(na - r->a) + fabs(nb - r->b) * extent; + enum u_stereo_vrefine_result res = U_STEREO_VREFINE_STABLE; + if (effect > r->cfg.deadband_px) { + r->prev_a = r->a; + r->prev_b = r->b; + r->a = na; + r->b = nb; + r->applied = true; + r->updates++; + r->phase_start_ns = now_ns; // unstable: back to the settling cadence + res = U_STEREO_VREFINE_UPDATED; + } + schedule_after_window(r, now_ns); + return res; +} + +void +u_stereo_vrefine_revert(struct u_stereo_vrefine *r) +{ + r->a = r->prev_a; + r->b = r->prev_b; + if (r->updates > 0) { + r->updates--; + } + r->applied = r->updates > 0 || r->a != 0.0 || r->b != 0.0; +} diff --git a/src/xrt/auxiliary/util/u_stereo_vrefine.h b/src/xrt/auxiliary/util/u_stereo_vrefine.h new file mode 100644 index 0000000000..38fd8494cb --- /dev/null +++ b/src/xrt/auxiliary/util/u_stereo_vrefine.h @@ -0,0 +1,203 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: online VERTICAL-ALIGNMENT + * refinement of a service-rectified pair. + * + * Why: a device's stored calibration can be slightly off for the frames it + * actually delivers. On a Leia SR laptop the rectifier (and OpenCV's own + * stereoRectify, on the same frame and the same calibration) left a signed + * vertical residual of +1.71 px, fitted as dy = +2.6 px at the centre with a + * -0.47 px / 100 px slope in y (a small pitch / vertical-scale mismatch). The + * x term was +0.04 px / 100 px (<= 0.15 px at the image edge, under the match + * noise), so the model is + * + * dy(y) = a + b * (y - cy) (right row - left row, output pixels) + * + * with no roll term. It is measured on the RECTIFIED frames the service + * delivers, so every measurement is a RESIDUAL of the correction already + * applied, and the controller integrates it (a closed loop: new total = + * applied + residual). The correction is folded into the rectification maps + * as u_stereo_rectify_input::v_offset = a / f, v_slope = b (see there for the + * math); frames and the rectified calibration therefore stay consistent. + * + * Two layers, both pure (no threads, no OS) so they are unit-tested: + * + * 1. **Measurement** — @ref u_stereo_vrefine_measure: Shi-Tomasi-selected + * patches of the left eye (one per grid cell, on a 2x-downscaled copy), + * zero-mean NCC search over disparity x vertical offset at half + * resolution, then a full-resolution NCC refinement +-2 px around it with + * parabolic sub-pixel peaks. A match is kept only with NCC >= min_ncc, a + * positive disparity, a peak strictly inside both windows, and a real + * vertical curvature (no aperture-problem edges). + * 2. **Controller** — @ref u_stereo_vrefine: duty cycle, accumulation over + * frames, robust fit (median start, Tukey IRLS), clamps, the >= N matches + * over >= F frames gate and the deadband that decides when maps rebuild. + * + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! One matched feature (full-resolution output pixels). +struct u_stereo_vrefine_sample +{ + float x, y; //!< left-eye position (patch centre) + float d; //!< horizontal disparity, left x - right x + float dy; //!< right y - left y, SIGNED + float ncc; +}; + +struct u_stereo_vrefine_measure_params +{ + uint32_t max_disparity; //!< full-res px; the search is -2..max + uint32_t max_dy; //!< full-res px; the search is +-max_dy + float min_ncc; //!< 0.90 is a good default + uint32_t max_samples; //!< patches tried per frame (grid cells) +}; + +//! Defaults for a pair of @p eye_width px per eye whose nearest subject gives +//! at most @p max_disparity px. +void +u_stereo_vrefine_measure_defaults(struct u_stereo_vrefine_measure_params *p, uint32_t max_disparity); + +/*! + * Match features between the halves of a rectified GRAY8 side-by-side image + * (left eye x < eye_width). Allocates its own scratch (called off the frame + * path). @return the number of samples written (<= @p cap). + */ +uint32_t +u_stereo_vrefine_measure(const uint8_t *gray, + uint32_t pitch, + uint32_t eye_width, + uint32_t height, + const struct u_stereo_vrefine_measure_params *p, + struct u_stereo_vrefine_sample *out, + uint32_t cap); + +//! A robust fit of dy = a + b * (y - cy). +struct u_stereo_vrefine_fit +{ + double a, b; //!< px, px per px + double median_dy; //!< signed median of the samples + double sigma; //!< robust residual scale, px + uint32_t inliers; //!< samples with a non-zero final weight + bool slope_determined; //!< false = rows too clustered, b fixed at 0 +}; + +/*! + * Median start, then Tukey-biweight IRLS. @p cy is the pivot row. The slope is + * only fitted when the samples span enough rows (std >= 40 px); else b = 0. + * @return false with fewer than 8 samples. + */ +bool +u_stereo_vrefine_fit_samples(const struct u_stereo_vrefine_sample *s, + uint32_t n, + double cy, + struct u_stereo_vrefine_fit *out); + + +/* + * + * Controller. + * + */ + +#define U_STEREO_VREFINE_WINDOW_CAP 1024 //!< samples one window holds + +struct u_stereo_vrefine_config +{ + uint32_t height; //!< per-eye rows (the deadband's extent) + double cy; //!< the rectified principal point row + uint32_t min_matches; //!< per window, default 50 + uint32_t min_frames; //!< per window, default 3 + uint32_t max_frames; //!< a window that has not filled by then is dropped, default 12 + double deadband_px; //!< rebuild when the correction moves by more, default 0.2 + double max_offset_px; //!< |a| clamp, default 6 + double max_slope; //!< |b| clamp, px per px, default 0.02 (2 px / 100 px) + int64_t fast_period_ns; //!< sampling period while settling, default 250 ms + int64_t fast_phase_ns; //!< settling phase after a (re)start or an update, default 5 s + int64_t slow_period_ns; //!< then one window every, default 30 s +}; + +void +u_stereo_vrefine_config_defaults(struct u_stereo_vrefine_config *c, uint32_t height, double cy); + +enum u_stereo_vrefine_result +{ + U_STEREO_VREFINE_ACCUMULATING = 0, //!< window not full yet + U_STEREO_VREFINE_STABLE = 1, //!< window fitted, residual within the deadband + U_STEREO_VREFINE_UPDATED = 2, //!< window fitted, new correction: rebuild the maps + U_STEREO_VREFINE_DROPPED = 3, //!< window expired without enough matches +}; + +struct u_stereo_vrefine +{ + struct u_stereo_vrefine_config cfg; + + //! The correction (total, output px) the maps currently carry. + double a, b; + bool applied; //!< a correction was ever applied since the last reset + uint32_t updates; //!< UPDATED results since the last reset + //! The previous correction, for u_stereo_vrefine_revert(). + double prev_a, prev_b; + + // The window. + struct u_stereo_vrefine_sample win[U_STEREO_VREFINE_WINDOW_CAP]; + uint32_t win_n; + uint32_t win_frames; + + // Schedule. + int64_t phase_start_ns; + int64_t next_due_ns; + bool started; + + // Stats for logs / the probe. + bool have_initial; + double initial_dy; //!< signed median dy of the first fitted window (pre-correction) + double initial_a, initial_b; + double last_residual_dy; //!< signed median dy of the latest fitted window + uint32_t last_matches; //!< inliers of the latest fit + uint32_t windows; //!< fitted windows since the last reset +}; + +//! Zero correction, settling phase from the first due() call. +void +u_stereo_vrefine_init(struct u_stereo_vrefine *r, const struct u_stereo_vrefine_config *cfg); + +//! Drop the correction and the window (calibration change): back to zero. +void +u_stereo_vrefine_reset(struct u_stereo_vrefine *r); + +//! Keep the correction, drop the window, re-enter the settling phase (camera re-open). +void +u_stereo_vrefine_restart(struct u_stereo_vrefine *r, int64_t now_ns); + +//! Should the frame at @p now_ns be measured? +bool +u_stereo_vrefine_due(struct u_stereo_vrefine *r, int64_t now_ns); + +/*! + * Feed one measured frame. On UPDATED, @ref a / @ref b hold the new total + * (clamped) correction for the maps. + */ +enum u_stereo_vrefine_result +u_stereo_vrefine_push(struct u_stereo_vrefine *r, int64_t now_ns, const struct u_stereo_vrefine_sample *s, uint32_t n); + +//! The maps could not be rebuilt with the last UPDATED correction: undo it. +void +u_stereo_vrefine_revert(struct u_stereo_vrefine *r); + +#ifdef __cplusplus +} +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 66f2e4ea92..4cf4dd3456 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -69,6 +69,7 @@ set(tests tests_m_matrix_projection tests_stereo_camera tests_stereo_rectify + tests_stereo_vrefine ) if(XRT_BUILD_DRIVER_QWERTY) list(APPEND tests tests_qwerty_camera_profile tests_qwerty_velocity) @@ -716,3 +717,10 @@ target_include_directories(tests_stereo_camera PRIVATE ${PROJECT_SOURCE_DIR}/src # synthetic lens live in the pattern file). target_sources(tests_stereo_rectify PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c) target_include_directories(tests_stereo_rectify PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display ${CMAKE_CURRENT_SOURCE_DIR}/fixtures) + +# ADR-043 R2: the online vertical-alignment refinement — on the distorted fake +# with an injected, uncalibrated offset + slope, and on one REAL raw frame from +# a Leia SR laptop (fixtures/stereo_vrefine_sr_raw.jpg, decoded with stb_image). +target_sources(tests_stereo_vrefine PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display/sim_display_stereo_camera_pattern.c) +target_include_directories(tests_stereo_vrefine PRIVATE ${PROJECT_SOURCE_DIR}/src/xrt/drivers/sim_display ${CMAKE_CURRENT_SOURCE_DIR}/fixtures ${PROJECT_SOURCE_DIR}/src/external/stb) +target_compile_definitions(tests_stereo_vrefine PRIVATE STEREO_FIXTURE_DIR="${CMAKE_CURRENT_SOURCE_DIR}/fixtures") diff --git a/tests/fixtures/gen_stereo_vrefine_sr_fixture.py b/tests/fixtures/gen_stereo_vrefine_sr_fixture.py new file mode 100644 index 0000000000..68176eb425 --- /dev/null +++ b/tests/fixtures/gen_stereo_vrefine_sr_fixture.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# Copyright 2026, The DisplayXR Project +# SPDX-License-Identifier: BSL-1.0 +"""Generate the REAL-hardware fixture for the vertical-alignment refinement. + +ADR-043 R2. One raw side-by-side GRAY8 frame (1280x480) from a Leia SR laptop's +tracking camera (a ceiling scene, camera 150), recompressed as a JPEG (the PNG +is 297 KB of sensor noise; JPEG q90 keeps the statistics: the signed residual +reads the same +1.7 px), plus the device's calibration as the plug-in hands it +to the runtime (left eye = camera 2 = M2/D2, right = camera 1 = M1/D1, +R' = R^T, T' = -R^T T) and an INDEPENDENT reference: SIFT matches found by +OpenCV on OpenCV's own rectification of the same frame, mapped back to RAW +pixels through OpenCV's maps. tests_stereo_rectify maps them forward through +the runtime's (corrected) rectification and asserts the row residual. + +The runtime never depends on OpenCV; run this offline and commit both outputs: + + python3 tests/fixtures/gen_stereo_vrefine_sr_fixture.py tests/fixtures +""" +import sys + +import cv2 +import numpy as np + +M1 = np.array([[4.7623373282393203e+02, 0, 3.1361746988405270e+02], [0, 4.7573318336977724e+02, 2.4590743710177546e+02], [0, 0, 1]]) +D1 = np.array([1.2332390163153338e-01, -1.8065702171496908e-01, -1.7852161465149287e-03, 7.0567885578275071e-04, 8.7485984937824945e-02]) +M2 = np.array([[4.7753198103487864e+02, 0, 3.2061581703236254e+02], [0, 4.7719412297375959e+02, 2.2625301107241927e+02], [0, 0, 1]]) +D2 = np.array([1.0906040224963771e-01, -1.2479660062568124e-01, -1.1093907046283672e-03, 1.0008674090348705e-03, 2.5815446523384768e-02]) +R = np.array([9.9998661420584478e-01, -2.5373925908838452e-03, 4.5092181108025075e-03, 2.5193814304075575e-03, 9.9998884428889723e-01, + 3.9954993384893533e-03, -4.5193059576863282e-03, -3.9840854151836471e-03, 9.9998185130384509e-01]).reshape(3, 3) +T = np.array([1.1996226177065833e+02, -3.9419132551517166e-01, -8.7010595776014110e-01]).reshape(3, 1) + + +def bilinear(m, x, y): + x0, y0 = int(np.floor(x)), int(np.floor(y)) + fx, fy = x - x0, y - y0 + return (m[y0, x0] * (1 - fx) * (1 - fy) + m[y0, x0 + 1] * fx * (1 - fy) + m[y0 + 1, x0] * (1 - fx) * fy + + m[y0 + 1, x0 + 1] * fx * fy) + + +def main(): + src, outdir = sys.argv[1], sys.argv[2] + raw = cv2.imread(src, 0) + cv2.imwrite(outdir + "/stereo_vrefine_sr_raw.jpg", raw, [cv2.IMWRITE_JPEG_QUALITY, 90]) + raw = cv2.imread(outdir + "/stereo_vrefine_sr_raw.jpg", 0) # what the test sees + L, Rr = raw[:, :640], raw[:, 640:] + KL, DL, KR, DR = M2, D2, M1, D1 + Rl2r, Tl2r = R.T, -R.T @ T + R1, R2, P1, P2, _, _, _ = cv2.stereoRectify(KL, DL, KR, DR, (640, 480), Rl2r, Tl2r, alpha=0, + flags=cv2.CALIB_ZERO_DISPARITY) + mL = cv2.initUndistortRectifyMap(KL, DL, R1, P1, (640, 480), cv2.CV_32FC1) + mR = cv2.initUndistortRectifyMap(KR, DR, R2, P2, (640, 480), cv2.CV_32FC1) + a, b = cv2.remap(L, *mL, cv2.INTER_LINEAR), cv2.remap(Rr, *mR, cv2.INTER_LINEAR) + sift = cv2.SIFT_create(8000) + ka, da = sift.detectAndCompute(a, None) + kb, db = sift.detectAndCompute(b, None) + good = [x for x, y in cv2.BFMatcher().knnMatch(da, db, k=2) if x.distance < 0.7 * y.distance] + pa = np.float32([ka[g.queryIdx].pt for g in good]) + pb = np.float32([kb[g.trainIdx].pt for g in good]) + d, dy = pa[:, 0] - pb[:, 0], pb[:, 1] - pa[:, 1] + ok = (d > 0) & (d < 200) & (np.abs(dy - np.median(dy)) < 5) + pa, pb, dy = pa[ok], pb[ok], dy[ok] + rows = [] + for (xa, ya), (xb, yb) in zip(pa, pb): + if not (1 <= xa < 638 and 1 <= ya < 478 and 1 <= xb < 638 and 1 <= yb < 478): + continue + rows.append((bilinear(mL[0], xa, ya), bilinear(mL[1], xa, ya), bilinear(mR[0], xb, yb), bilinear(mR[1], xb, yb))) + fit = np.linalg.lstsq(np.c_[np.ones(len(dy)), pa[:, 1] - P1[1, 2]], dy, rcond=None)[0] + with open(outdir + "/stereo_vrefine_sr_fixture.h", "w") as f: + def p(*x): + print(*x, file=f) + p("// Copyright 2026, The DisplayXR Project") + p("// SPDX-License-Identifier: BSL-1.0") + p("// GENERATED by gen_stereo_vrefine_sr_fixture.py (OpenCV %s) — do not edit." % cv2.__version__) + p("// A Leia SR laptop tracking camera: calibration as the plug-in hands it over, and") + p("// SIFT matches (OpenCV, on OpenCV's rectification) mapped back to RAW pixels.") + p("#pragma once") + p("// clang-format off") + p("") + p("struct stereo_vrefine_sr_match { double lx, ly, rx, ry; }; // RAW pixels, left / right eye") + p("") + p("static const double k_sr_K[2][4] = { // fx fy cx cy; [0] = left = camera 2") + p(" {%.17g, %.17g, %.17g, %.17g}," % (KL[0, 0], KL[1, 1], KL[0, 2], KL[1, 2])) + p(" {%.17g, %.17g, %.17g, %.17g}," % (KR[0, 0], KR[1, 1], KR[0, 2], KR[1, 2])) + p("};") + p("static const double k_sr_D[2][5] = {") + p(" {%s}," % ", ".join("%.17g" % v for v in DL)) + p(" {%s}," % ", ".join("%.17g" % v for v in DR)) + p("};") + p("static const double k_sr_R[3][3] = {%s};" % ", ".join("{%s}" % ", ".join("%.17g" % v for v in r) for r in Rl2r)) + p("static const double k_sr_T[3] = {%s};" % ", ".join("%.17g" % v for v in Tl2r.ravel())) + p("// OpenCV's rectification of the same frame: signed dy median %+.3f px over %d matches," % (np.median(dy), len(dy))) + p("// fit dy = %+.3f %+.4f * (y - cy) px." % (fit[0], fit[1])) + p("static const double k_sr_ocv_median_dy = %.6f;" % np.median(dy)) + p("static const struct stereo_vrefine_sr_match k_sr_matches[] = {") + for r in rows: + p(" {%.4f, %.4f, %.4f, %.4f}," % r) + p("};") + p("// clang-format on") + print("matches", len(rows), "median dy", np.median(dy), "fit", fit) + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/stereo_vrefine_sr_fixture.h b/tests/fixtures/stereo_vrefine_sr_fixture.h new file mode 100644 index 0000000000..f34b96935f --- /dev/null +++ b/tests/fixtures/stereo_vrefine_sr_fixture.h @@ -0,0 +1,89 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +// GENERATED by gen_stereo_vrefine_sr_fixture.py (OpenCV 4.10.0) — do not edit. +// A Leia SR laptop tracking camera: calibration as the plug-in hands it over, and +// SIFT matches (OpenCV, on OpenCV's rectification) mapped back to RAW pixels. +#pragma once +// clang-format off + +struct stereo_vrefine_sr_match { double lx, ly, rx, ry; }; // RAW pixels, left / right eye + +static const double k_sr_K[2][4] = { // fx fy cx cy; [0] = left = camera 2 + {477.53198103487864, 477.19412297375959, 320.61581703236254, 226.25301107241927}, + {476.23373282393203, 475.73318336977724, 313.6174698840527, 245.90743710177546}, +}; +static const double k_sr_D[2][5] = { + {0.10906040224963771, -0.12479660062568124, -0.0011093907046283672, 0.0010008674090348705, 0.025815446523384768}, + {0.12332390163153338, -0.18065702171496908, -0.0017852161465149287, 0.00070567885578275071, 0.087485984937824945}, +}; +static const double k_sr_R[3][3] = {{0.99998661420584478, 0.0025193814304075575, -0.0045193059576863282}, {-0.0025373925908838452, 0.99998884428889723, -0.0039840854151836471}, {0.0045092181108025075, 0.0039954993384893533, 0.99998185130384509}}; +static const double k_sr_T[3] = {-119.96359513724906, 0.69511170577718517, 0.33072915426274135}; +// OpenCV's rectification of the same frame: signed dy median +1.713 px over 63 matches, +// fit dy = +1.196 +0.0029 * (y - cy) px. +static const double k_sr_ocv_median_dy = 1.713043; +static const struct stereo_vrefine_sr_match k_sr_matches[] = { + {42.6564, 426.7630, 27.4999, 444.9824}, + {42.6564, 426.7630, 27.4999, 444.9824}, + {45.4189, 430.8741, 28.6479, 450.2414}, + {46.1918, 398.6357, 29.8170, 418.2359}, + {49.3301, 403.5512, 32.8850, 423.4252}, + {52.4935, 393.9206, 36.4898, 413.5976}, + {72.5452, 440.0057, 60.4815, 459.6938}, + {75.2405, 393.6760, 58.5862, 413.1019}, + {75.5995, 402.3505, 59.3827, 421.6918}, + {76.2475, 440.4224, 60.4815, 459.6938}, + {79.4465, 399.2680, 63.2164, 418.6798}, + {82.6062, 402.6128, 66.1633, 421.5816}, + {84.8178, 441.8156, 69.7202, 460.9757}, + {86.8865, 393.6164, 69.8893, 412.8817}, + {91.7771, 407.4662, 75.3370, 426.7444}, + {91.9771, 431.8771, 75.2770, 451.4111}, + {94.5774, 436.6982, 80.3050, 455.8034}, + {108.1426, 393.5991, 92.4104, 412.8610}, + {116.8991, 303.6753, 94.2693, 323.5830}, + {123.2176, 358.5301, 106.7471, 378.3642}, + {123.5129, 359.1039, 106.7471, 378.3642}, + {125.8485, 371.8545, 109.2153, 391.4623}, + {128.5398, 367.5976, 111.9416, 387.3667}, + {128.5398, 367.5976, 111.9416, 387.3667}, + {129.3160, 353.3312, 112.7393, 372.7762}, + {136.2990, 357.6942, 119.9553, 377.4528}, + {138.7735, 348.3766, 122.0013, 367.9796}, + {138.7846, 358.1171, 121.9061, 378.6802}, + {139.6527, 342.8509, 123.1754, 362.7422}, + {143.6656, 371.9719, 126.6894, 391.4378}, + {148.4452, 364.5681, 131.8349, 384.1829}, + {163.9910, 357.7573, 146.8525, 377.0127}, + {417.9524, 378.5617, 399.9272, 396.7706}, + {447.3165, 371.4063, 429.8571, 389.2461}, + {457.6725, 365.3000, 436.5449, 383.7210}, + {463.4951, 378.0696, 445.4244, 395.7342}, + {482.1422, 364.9758, 463.9877, 383.3140}, + {482.7975, 392.2909, 464.6279, 409.6623}, + {490.8043, 387.2012, 471.6787, 405.1959}, + {507.0309, 398.0456, 493.1158, 415.8319}, + {512.5277, 397.9995, 493.1158, 415.8319}, + {524.7873, 406.4124, 505.5220, 424.1204}, + {532.0674, 446.3396, 512.3051, 463.5432}, + {532.5904, 432.4044, 512.7141, 449.9389}, + {533.2227, 439.8077, 513.0821, 456.9149}, + {551.1292, 409.8321, 531.1552, 427.1079}, + {551.1292, 409.8321, 531.1552, 427.1079}, + {586.0905, 437.3112, 566.1793, 455.1066}, + {587.0881, 446.6747, 566.9643, 463.7476}, + {588.3311, 442.3581, 568.1820, 460.1863}, + {589.0005, 449.9309, 568.6943, 466.9713}, + {590.7910, 426.7519, 570.5395, 443.9697}, + {590.7910, 426.7519, 570.5395, 443.9697}, + {598.5165, 441.1796, 578.2290, 458.3316}, + {599.2280, 435.7159, 578.2069, 452.9794}, + {602.1029, 440.5636, 581.8928, 457.8357}, + {603.0875, 449.2186, 582.8894, 466.3867}, + {605.0008, 446.1069, 585.2899, 464.0975}, + {606.7680, 452.2210, 586.3251, 468.9636}, + {607.5725, 442.2518, 587.0052, 460.8015}, + {611.3437, 453.8625, 590.4322, 472.3528}, + {613.6957, 446.9045, 592.9471, 463.1025}, + {618.1396, 410.9941, 597.7014, 427.6205}, +}; +// clang-format on diff --git a/tests/fixtures/stereo_vrefine_sr_raw.jpg b/tests/fixtures/stereo_vrefine_sr_raw.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d1ed289244b9212d27ae1217fee00e77d0a92d86 GIT binary patch literal 62426 zcmXWCe_WF1{y%;rD(VdTK?@Vn!ll?wiLwF(1$&1`35vBku9byN@sx>}DNY5=+VK!S z0#_L!9zn7*`?1-wQ!cD2+LkzQYHCJ;5jiszHO=gEpYP*)!++e5>xPT#x?ay;ua5&C zt-wkKEs+L5AOHXXU%*MSDiO(mWtbngx zxdOfl{>djRKV7+M4Px!uwZ4Aqke?!us}XAv9}U1KUJ$DXIP5CmyZBIKid= zjHBOVW^uFm`6mlb6&499PFJ2e``tgQ&MTDF-(R?RspjAHT3v(wYU8yZe`>z{^PQHv z|9$xA*Y*x$XV+u1rT5Ri{(+~1FGg+lv6qfl&ev1ZGp^aW`F9J89|r-i|4myQuyk8? z9@mitr@I#aIMZQUNZor1exLp0+i1HP(p(b9NNQ45)Ufk#!|Xug0K3uvDb?v;0O2P~IS=`L+Y5B7C|BX0S>9CV?R^4RZ&lQ9r1VDox?ppZd@ z%JL(_MF?!HqZ6G~rMa-wq|XDcnk=_B1Z^cdi?FKE8hC(&yUSKFY@X^6RC)c)czCe# zp&cR>q^etA4JQC6wHl1oRagrDeUVfD(S;BP*NUC7ZK)eA60AAuxwOLh@w`&`!+Xv z{m#RtBQX}y>Mc=K{|NggMD40{D^Ez3^j1|ysYle!YmM=7SH37-ecw_g@8<6g&r7tg z|JoqVjnm&5hZbJ7$?!DE%Z|dsC0s#C$S~9VlPbVB~O$z^Mbuna|+I8&YOGOBJIHgzZ zT4l`Me#*pPvWWEx6CP5|AG{E&^mWy1Oss55?7_1&drkhfL19Gna=uWAQy8;z$BHJB z#w{U&(F-o4F4!M)eMRUFW}C53hUQ5hD#M^PK8tzabJx5(a-o#$vE@xPXFO32TX+&0 z&(de_Z9|#W(Nphw;4m^w=Tm=c3EM6i8Mr!4H6ccy#VF{ASqGjI_}(&?kxAwXMdQ2V z^E<;=pH}5%{q3ImOicugII4v6odc;AveITV#Xn9*%tffuEfm7+Vrq3&**W*s)N#iC z?_K)Ce=izeYSgZ1{x$FEWKE%1mF}8g1Ac^Y>P}7#H>et4y`$VqH)1IHqbBJoX5*C> zl!WW=Ib0!SbCEXab*}D|@5%Np0kT4^yYGypPq@Un^@eLWV=MO{NS1gh%FiS*Cv^e8 zOs287iN={<27K3!9g`w9X9(W2k0Gs7)V-lqlyx}~Q4AWd({ww+C>20#z&FaOoLpR= z>g>;6bK0Q}zmLE*Sy(8>ozoPCiL3NCzVnHX`Uv!E-U?hFu%~!pUv_2#4L71xbO7zZ zbYTZU7w%=6@;L!T1x?}qnn3xoeZ`6=0srj!fTfb)A6})1de={t?cm>ITb5#<c+?6TBXMfHJCUO`r%(CoMr5m0 zZrA~B=u4PxQ=TASH*}s05ML@iI-+W`vn71XpmNTHu+8t>?jIx-;7pcQz?xx}Tzpwc zI%!+z;WeA{ZFudNz-KK@kn(&);qO|)vOL(nZxeDY#~nh3uj za-Cx%Fn3I57=v$44s~8PvncASaGM^lq>JsbP+F%sN&Ru z7p>fc;9TzZOpWyvzVAyyJ2n*Aj^FY^Wg|EK)*nc&B+T?KZ@jn%Vj|6H_!8Kydetfk zIm=X9ww^tt_`z|PQRK)w9n#m(zZ=>W8utbTOW96sEd0L4(G@f z#T|T4TLNi4FMS`e!Mc*Llr@r|fmV4h{4_kT3$7fMt-7=DQdTfA$=qBk$a37$b#~(z z6e7xpbx%!L?1@O&QJY44v}R&Fe(8B-xD}!hv*q%Mq#PJR<=Civ3v?u550=C>YkeWL z%dNp21GH+dJ1>c73*is5mA;W>jDv2jc9=V)hL9q!p(LI-krsbS%#o%ryX>3_3HP!7 z0hJaJ%^8^21|vpmcAzQ9*yfCCTMZP!DfpA-43`d2 z0Thj!_ut+noB3ki_^vj#AKJC{vmcqbgwJCSVs*6QDd(=~TJWszd|49>e{L3f5j>R} zOz)B*$tCYonmzo%!b?ks0Uv=%s9aogAj5XE!9`Bm6qETD|5y4+0=YfdehRP2ys0}p z6v)-^4H}i)pn>z3oOD~GQKQ9RYP&Bq_%WE;mtxM(TvpIIckGhKCV1398|$U@sk zRV-W5G59m?&_9DYQo)W6vh$IAc7pyi*~?@WuZ}X{!#~B}0|J@R7P^fL9(K10v-WRf z=v)}uWsTs`03^u_&*eXXe8-{h{|L-JK~DBq&IY4h!{>2^SmhYuVAA#gX;9OaZb(R3 zB|)qnwm@vL%D)s<{(jl<5iPpi9CBkP5Q=ot$8Jr$3Oh){Rg5zVM3TbB{(wYdY;%WI z4Xj-g1@MMiCgbgV3)y)f0)?uAF>>Q$Ue>&d1OkbgIon2=uQeZk7RXe6;G*mh=6elf zA(b9`U%=LrcY08A_@bWF?`-&s<0iZ?y}VFA;hq_aI4z!{z9MQG+g_J6LTrI7i!SU) zAx7snEixd)c@O$zV$sBiGT)fCO#++N1usJ%GzA+*Wk8Vwt!xP?USxYX+|B0^qoNtV z-DYs&M^#kJkoi*2D(6Tg`8%XW2KVF|&W`slrhat?7OOngH5b@OS)3V^xxxL$7);o8 zkL}l7_S6g9xmabz`J(Zpwl*VlMC}>3*eklTjmcwSg3H1Wci^a~Q;Vq`Ys=JO?;oj- z$WWp_>3?HJYpUp`rnnP9=9;?)rd)bY(=qfV?Ez6IY@lZ!&xLq+@2hdT?jm$2OgC;rWuOyIr0uq%;jE^OqC1VIZ`rbexatiDV;|^Za@GJA zB&ybtbl{{mFfu^x`ho7KNJ@?pLs^^elCcPpIFHvSTP;XW{ioqoVFR@C{e3~oy;ICs zC3c~;q@92}@uD3@u$SU1nbCetKJX&^3-PMjhe9E`Ir6+`G&|j6+;kf_H{jAN{R%PN z-q2|yET)tj)jZlT`av~RHEYssf){B6TB9Ztp2abUp}m~eU;?^91z=EY$S}GDOJl;u zDF>i^65Iq8qbGoiAE|@7UB072eJH`QHp{p>GJu3MntiIqlj3~W0}1U(X2L?JH0g@T z#hya^r8ZRRk6uKi76^KDOZW8M9~Cw66ldRsm}1cc#eGU;&ctiZbkH|dGd*$gN!p8^ zbRVk0HLneY^(j|b7E&Ioi_I%xx7|-^JWVhuBe{i4oYfvFA!jf_^IH^f` z)y&|<-&E5)CtWW&i|{?(Hd@V$o~xf+nQ)>{L8?8JeIjJ0ul$z^Lfu)E)HtH7keg+` zvznD>uNT<{%YQu|lQM(m)2TLev#td^^im(D`(`50Ug+AUFGM7Gc{x3`D%j(~ z@7yx1uvG1sRu9A~uWi4vo1b_Sezs=Yi?ies?C(5t&AwRGoJI_9d4gH(S@vO%_S^a( z9eyjRs?3MdjT`Q1G6ptovuCQatQBVI3y0WJ?)R%1@GNQkA%g$BPo+r@I@r5y^2R&1 zugI+YR{K)g+A}Kug&i3)t9AC*@W{d??6uN^g)b}pLl-S~QOSj&f$!NB>cVz+v>z<6 z(bLP$8R}Eq_IXP92y9Y$ePY{IZFMn_^Vy24GIOD=If3VxrAyBFHD=`CM8 zMTwDYo7Ei_4HPG@3*At*=*q|p@quBgkQwRrHMm=9R5zoI_^jDkO8 z5Zho45LjevqE^)8g&i`s2G9r&m~J~VoeEpR8b1dd2s4ZcATDBre8y1qe;otK?LXoc zl3_u=vul+r30k|aV+m`R>?jLQnU-d6Umb~~Gm&qjoU}(`1?3>c7*lW_OJ1_W%Bgh#RWe3dod;2_zW2-DHL32L%YY+lk8jmkLchDyUM{yxa1 z{Ggs10Nh>c*Qnj zJBpkPN?aPnllz#um`sc6PSnN~C{S?af2JjZ9vG^J&^9qaId%g9cv5wn4R8FMbWfrV zdgbXj{ap^RK8F_5gZO^J$k=xP{t-Bv&Ac)*t64oj#`0P+`>S$|!4*rd!jV2ZWVk+2 zIT^RBJMLl4N=Ki1WhMdfs9*BT`H0|b50e5Qa{lQ$c8o^E_m*fO3icN7kQpj4heTYo z+YqPWXI%|4$Q)jO`t{B)5)QDx?;tF3*G>&>Qa!13Cc(rfmzB@%N%)f9%SHD5N$~6p ze4pC6P_*=F7-vkKNc^iTx?l#RoyQidlJknlT)M${D4Vj)CTaec;yeMZZctiF5t(u4 zw!=imca{tOP6iNM<7GvqTfFCpIv5hBpr97bZ;ZWy)E!6f+K~G2Y5tSoHapaX_r;C3 zqmEniKXCU!Z^XSw!Z-+Z)D`Lq#0%4dzRQw}6O6mh6yPS)X96cM=m$F zI0y(0uZq@P_D}_zLYG%uu~mJ7GUpkWumv9ebrNqA6X((n{*R?wi6FW>?f5ya327?5 zViW)ySpJJWUOtg=;q1F(#TK|VDJH}(JA3t4AcTJrnMmSjw4d+r%@2I_xgYEC>0w(? z6Q(24Iz?f_^CGy3Fp^nF07^G0r|%*`;!EushHWe_0&=s$?|)A;9ZuF}pzp8$GKJD@ zt7SH?#F-9d$5W6?jfjlt#5r9I&v=NzO@GvgxYp`LY_NsGh+q|urx2_HbEU!V8^rK> z)-y#v6=7_O<9_m2Oe!B-)wgc*`tg?O>qNvA0bIG_SB~X&nET1m%jFTQoZj?R@*ay4 zh|Z9>dh8B~vi@aY2bb;}{TA!hW$*`auv&SktsZiA2cwuXUGM(>h>#WAd-=zrCtH>z z!elsvp!2scD_l2dOHev_%_=(P&TT5fk_QN_#I;HGI>QZ9o*qr&l+lXp?^I$R3X)E#sYqQ zau(6(RvzQmEhIV~2)qXG>OJN)LQlVJC|{z4Un%p+GF@MRMRrawyr2l7-68Yv_;03Y zurfWb1nU>OP&;WJ}ico|P4994!LQI>gu}Wp$qpUd>}X-Y-UDftVI2W8aAs1tnEJ zDD%yQ;O(jE;sTeJ(_C%TvqXHEoTq_(Yu}l>SxYKo{6zrJg-rxVTcUz$wuWV*L;JT(up#!OzeUN zVt*}<_R}{-9~K@x zDudSE5b$6^I9FpM)v$5pQHl{2LZZAFJVPCSQ_PmDPk$q%5=h^a-!%fsYxlgYKH#}% znLlP+FlbRTyM98#^l5Gg%=6o!rGi=p&plH;&`=(YZq}@9y&1--ZpR;{b`lVOh|NFQ zNTqml94WtUUtL=rJ+y|s4Gps;_D*w?rz}kvlp0s#?e6Db?hSs=CK3Y;PTKZWCcO3D z8FJRq!J=1j4DNXQ7G^YCq)?sIhM(e90rsG=1m9vAHd*3mau-jTZU@_~aHW_jV7s8V zeSjcG)7F@T9ZA8no&NgFn@i5M6qH2-MNTGlK?zG(%Cp{lEdl*+rLVQLc~yBCO8@~R z#1COxygE@;Nyfmd(9NBA=nMyn(5{a56|mDYk&YIf{iJ_;u+O;B#(Z@bX)VygPGDex z&ey*>R6d-o!80{`?Ni-l3_%?R|EDyRXw4W^3ZtdpR&9stT@&={xy-oeJk6jNs^+kX z`Vn}oFFGd|qmqdsDqjoAmvi^8u#}dS(Y@t3=SqJj%`E@s2jKmMzYyT;a`_}^d_=YpMih)$5hyN>XQ92B*1xM*A z3oigd$uNNz(zMY2#aFN;F8$4l4Fvou1r0J`uHGl<<-qzMn5z#FT)NLAyijl#+>@DG zoKoG{wiNcaNjf6a>7;)Z!h5%vqEd1EClca`x|{?8%EF>>4K_*d-S$l*Op!-D)d zjclJDA)*!OjzNtAYF8+!xaao-X7j~O%6#CYZd_y;rzRO{$7lvm781b`OA?y4yITu% zDW;q_1`l$URD#vz5s2+!7+6*8+m{Lqek*G9irh4<4R!@)89mE!5)hY+brj(N#D^^C zOHA#<%H)L7u%Yv~a&j06oj~F-61xl&eIORzBOu;FINN!^MLtbVLb|KcLJ*xx}%cb|7vx8;I@4i z_#-!9)oQ6Pz?xW^Sjz9go&CU2Uj_G(Z=Hk)PG!+o+`V4P8U zxL?lPEo!qS(;)a~_xR5jlRX1!*!4u{4ME=dieMgHpX+ZIPZzVLD#wl1jpwcvXSTvU zBO(iWQa|am&bjm%9t$)-26fe7L{Y>2qeuY^?U85%6If zR*A%D9yN}o3rkPfZ9bgf4&nz!ya5GgIg?_9C?6h7s>AzTR=gAZp}1$kx0o35+La#k zz90K&?+z13HRc&3%Rxu-BMMi$|5d@aB=znXNF% zU7g{u`VvC#QC#EQ;Wc{BfSYfS__c1{l3O&YjBCJ6(kN|DJkI2LgdLE;(*BXL%4w;J zbdz!U#sgp}`cRz8q|R=-(tp)zzS#jw8K)P8R)`W?IEwZ!y!8_y4uoY0s&9%xwn3%T z5fuszysUAI=}dt?W(^wL5)dr}XFsUs?q8+G#;+NaZcXiMtsT3pCbfT?h zchq*IK^y8RL%{_Cwp#=paeR{bWiUsLg^if9x5L~|X^%b|{t)Ivb;`jiZ5h1qu4u{W zM+j`?e&Q>z^q#@o(n}z=X$=5t3fTVM$ID<<2~xc~SWYHNWT+F8(#_!iYrX~@2Cyuk zic7R18$@?Vs?@H@niHg4P($zq!qRg+leYXJR+uk0zj;mnBDjMv_wv#m|4!S8u-@PJ z5jc1@BtPgwcB+wVn$ZTvnGnFx-Hlfyetc(A7jP8(-X}a&UV#w&aiR-RmzyxFkrbf> z>>GMS`iu}aktnvKe`$vH2-q@sR4*GHJ*r$Waz)TnU*l8h2vzs`9*-$K#(pzMKux`4 zhYgeCe0T?Zg}B}B>a!K&G~$aNS8Un7N(^uvCw6oNmN+RXB6;l?6;^xk_|m+# z?Qg2Ew%GJ&r zeCi*(jMT%`%gqH2q}D24^Uiz*CaT#s)9tP8@N$jQANhLWu{j@-+*M-u&P}J5@tSH_ z1D97ZKEd)q@}*5*$WDSa=ds4`kbt6N1!5y%rPu;{Zc!kQzD~?H;_D({1bl@%kAkYz z1r(urh@@O@fM`~mzD|`YL^>BNd5PLCSMYkl1A>vULo_J4a)vNLF9OyV9`w>=y5~|F ztXtW6#mOtNAwZV1zjmtb)}X3|5UUjOyHLr$eLs3xl}UQ zBL&}7y1SeoxYr}25)Dw4`QO!xmLK`1Z0->__Wi>-k|po&ExM(!1FM-4+;-!_WYVxV z233RL_3K|yO<%Wi-OiJ*?lh97Si^}>SA9r{U$)}64Fc&yd1BDkVD?#Lcu}nD_tQsk zT{{0~J)Dv1{kN>XAXVXPfkzjw&z_lN1IRp(XZ&Gws}PraAx^x))d~o+4XbYd=*jhV zh@mudvAk;Aj7x)pz7iy(&;)evs1Ty!wQnqs&Yy4Ob-)g##q);{t^ZA?1&Df6qaB#0 zEpWmF4IU*nS1#=gzlwo2;o3lqYhKlbK=F-QGxaW%>TiaJ`cc{|1#FNwE} zT;OcA^vSU8!LPl^*k5rdpGZ$gJw{TtHO;GQw9;zZsPg*K;nLbd7c7)6!pjSw^@`Rb z>n`Zi3Zi*W04&F3-HA=oi)*qvNshS3qVL@$*lnv!2L$GuR{kU<6jR=U7c;3!vmkjRmT0s_G8W`i$JZ`6N=a65JHZSg84&~B`^sy5eSRD?e8D^fZ1F3ondme{(FDYM<9IP?t+Gu z0yhuj(DR`hS~>-PDP&If7hy6f@09O)Pe|7|!!yPrxV5q7Dbn4g!vq2Z9W6wx<_^~$ zX52?*PB%st$YroPZAUI~DGXSdh`(eAMG4Gj7GiGv!u9``Tx>m6k3{!+e8#7=!BFx^ zx-oQVhS{;Kd>G;VWaDbes}kq0GP1Ld$!9$+_e0i`4nGLrJ!Qi5Njfgn&CTK~X_2tB z?CUUQtMf@);4ezg;RJ+^7M1vpEqQpRgY_ywfD>5=vzl{{Dj53$X@n(bTYS2jrEkTn z<|h+)DqElWqVZsf2ey-{;wKFP{voxjL)k#G0D8xKMw~7v3uiNCg!>Tx>R0*B2R6E0 z#mM;mLnErS`}8`(T2fBzQ*Bouh*PAHuWn%auJpWtRxk}6ytcrr&O3|du=Ko`DFuVF zkVuYd!wZk6^p-LB*rQNkoX-8sMD_AsWC!m~AaSn#6QBBdv@qfJ zMhTp^W1Bz}&jIOjDB9I$cihS55d*PWvAo|o2*wTIyf%%fIUljo=cU4<pRv%**194Dnr;%TLz3axUA%L8XM+1jK$;yz4e_0l>U8(Y8SuZ-{7J zU*r_+9uto5Dz6VLxf)dLRvu9jMZZSrh^om&c0v&mhQ(6lpPevuGf*(IXi>f>GbFd) z1JCuee^m4%G?}PXAHtAbo0J}*ji%Nu!&`{jqa(KF+Ns>dycPz3g&)o0Id zD~YhRg}-`-7lG!{-om-_u%H*zudl5;l_s|3Y_rHC6Q*6-++FEDM;aod#BB8?`v>ke zyW}}zhq1+lu$+e-W(I{pQI@Z2vv*22&(^QUWZtVN3*ygUYyL}R*x`ZvaXK9~Hv5|U{82}2ZQsR@02Vr{ z&kXkKzH}$Bcpf3hysFOt(b66FDFwaW-|(`U;7|I%rG75pO;CVfg0)~)ceI;HXz&;u zFnt8BUs2qXGL{PC}!<&{OEmUWb2je~qE@uxuV zI7e$ZiZ|3hqK?05WWxmCY`^6STuhElH5mJp-SxEdO94#_XgZ-=@FV6kLSKmF@_ z{c((F_+)(+sAUN?@I>Ooj&BDHq*CO+3BUQeTu+E+Jn2tx1K(SNcBhKc>qRnw&$$b^ zp$&zjvexS-wa)B6C_l)C!U_{Cb?hI7BLAr9Ii#-UpLZnOYYpZydTKMdHWILA43YBK(@T4cQ{eF+!=~JIY z*r?jw1D?X59N&&p})nVg}7M(ApLr^eI@8-*j5Oz+R!KlOC%vhY$2VQGw_j&JSBsgow(!vK| z3hAM=jPHCZdKdr9Om7LvsFK!A-Wky%p(MFl*+=55pJAbP8}v0d$Oq_7ia6P|R7t>w zg38Yi5yMb9V~=VEL?POtq1lETsLZCV;YFm>h-k;m#IClsB7rlOjt7ZG5k57B|iKP)aJIGFYLhX;GoJDuMRRJJ+=oLmKx(5gHLI{^|kw| zUBmE-n!W4<40J}LhRkY1D%v-Y`L((*AZ(IOaAyQ8*paQ){H3&ZGP)P|n z3%~{2InxbdbSrE|H&kA1=3vz7UX&2dqrq_%(zLJd{a0ZWr1-y~xG6n9F?3<7lIl-< zyI7Ma#=5@&VJ#pZs#5%x^1KOm)*q$=mKHFYNv22qjPd3qBbO5DYKT4$JeyQ7-E*Mg zsZM?M9@Nhe(0lPLf}(7LmwUG`sV}P$HnOydaD9&uX4`#8$y0m?-@Voy1o_7nQORfH zA~3It+3AXzRezOW8LQ6wk%_wQ9>9NguHk%vEvQ@8-tnSsUN;B;1+#kJw~-#fQdPnK zYAfH^KPXS>BL7kiI3Bpk93lBBueesVQ8MrbD;dCO|SJ_Zy(eaVEn&g7FU&+TuWH(XC35@LkDBAHB*x_%G&UdqolwdxMmT)lc ziocyxUKZ|V%_PRUPOnTHz2IJ9@l<4iK8SY+=wj)RMy6Uhwji3ENWAHrA8M9mpgjVZ zL~?P8#1G?4ROP?sQimg`qcxFmn1lJ6ZfM8IU?>pQQ@jp*1QuVRTa&hUqgL`ql)gb_ zj2Q7B=b0-4@*Y_%eOGR#&%L zwj)u~SUsWL*RMaY(JRi;6g~|@&FR7@_GNNo#TiUrt;FwGGVKjFI^PZ=LV;wdmdZ{L zXV@evw)7Hf_dGt_tHpJ;1pBTSRRz!PwizF!?YJu)X@{f~+Z8?R$g7>U zK>lQ+sNAc|DR0LDcSsN>yTxAHah3$_0e9@o0DsY(E*dDSBt{zh$9LU9Spn=1bZAZ( zRIFMv!B|8`u9IX$;XAN*TxSu~AZQqp)8%sT`z3Mpjj ztnO6=4U9HMp@r|wGyMhRR_?tB-uQTX*a0#6d6UP_Q({nuv_@c~SJL?7EW1PcShl7G zKGJ*-*XbCkxeu+$6FmsF`la#n^vJ6ZtK)1e$BATa({E4)&66i2CvC(M4InC(T*}!A zT_su2e>(h2xez{_&Rc4IHFVw&W$8dn6Y+l@Wd4Ji3z!I#yRI0{N1JmNa$&5rGgQkbL~C15Ajh@9&?~h^@1W z$pEKQ^R2$O*u3VvpN_y&h`+he*d5Kz@O4bx|gz)obU*lY`rToHbqPXXh{;<;IE z&=zv>8s^>*{vumK+rvROKl%--zJKIg%fijDSKB#N5wKoMy*=F)T8DpwyE1IyDP~=D zlGqeetGgZIWIdIuOV!Q)x=US(Hvc<48I+0w)31-|8V#$A`_b{|K3UYB$ph z`GyNIV4#~6`jC@>_beA;T3+n@qOEOm*PXFYNesI8+C>zqmde^|wOY*tgK>9h^Plb? z$j(z+a;ZyUaoeJ2de}192?OYB^Mw4R#yvTqaaTxNh@Hk6&9`ld6O7++12ZGSLQwwG zbq1?z8M}WjL>15Kj8CpSKdU47zBeED$m9Ch2qEv+3r@CUyJ=BUU9fw_YwT~b$ijMl z`gy{9J*iw^!L`llIw+2}^&yr6viOfcZ1bk*T-@LV?lbA{H8l5|k5cd75<992UWkP_ z=g76Ser*K8M6XnkwcExWUrhb^Itw{>wHA^P;^lE0lNCz~&~&-*+Eo*Ot=Q`Bn?HB;5rQ1`0sD7JUV?){G`#W?BqJQ^ zl2Tf{?evxtpnLh|Dt*H$unMWPUnOTRS~=5p1Y*>C%d`zgtRL0tTt*A3G_ z{z-yJSzt)yZLXg_J1eHQ;(Q5rAJ{-Na_#{Zn3^1C4)4i}_StbSlq97xBAZk;49N`q z{(CM(Z=9BShkkLq@K6(B+Z%$l=YDPW*FW4f61Cy}MMiVu;~%d3xm|ob%n|oMo~yB5 zyXTg;4N(fyA#;tv)0j<`I_bjmd=0u`pm|G}~6YW67ELB$!~UsAWVD zIt_uZUew5>7f(SKEO$`8v-O}bWRVGDIjXtF0;>zD8!Taxvj+3>o(r8x>`0>ul`>{u zO?X+oM^sUj1@$L?07KqugkJMHLN2_pq0t{XKJJ{P8er}pO0t4i;;J#Tx}fbh-X+sb z{voA!_1|F?3+r=w{KzaPgZ@h3$4Wy$ho=TF6!gUDf{Jo^PXY>T3I)R(69Pt@oss(m zbM@&%oy4G~09eYx`U2|m#uYK8kT+P+&83^BHNLZ?vgox$o7Blz_m4b&&2GC~a;8?y z5k1-b?XyCp?xCw2)`au{1K@es$3WmhT>@yhJS@M`^@I?_D43{6@yrVMQ>GRT*s=xh zT}kt@gB~Eb)U{b{`p?tB63C3g<=&Ad&`=rr>G>ji`|kNU?PquW+jI*;lwx-- z$h;X4Fm;?_2?VrIe{?gxY)jetZRtX$H&`cB$(}ZOHyImWNm3_8Sc8Mi>eGJ4yE-UF zNGD(Wy6!8{Py-hD3@0~^vj!sX|8K6ukpm9s%{o~a@)f8jkL}^Ltf!6L^V#Ve?N~@o z6;HHp{X`{cZ48)oY2Ob7UDXX)U^5L@0X25O&=I$O4RRKGdjn4>@JZ&YU8mQG%ql_` zN%Bx8QQ4dcP_YA%ncE!~%{xkjO5ZkAcXsnAND}`|){qJwQEV8vqD%kJ{s>6H%dC+UV4*oJmr11sg(Y<#=A!ltLum$LghCpW}#o750 zAX&QMxqc0R2HKpSI$elSNn9MVV760dY$RDyyoSLOv+lppr9<7S=cif_8*RjnlARku z39fo&x+mGmBJnM^BMC!GGJPk!VUQ*cMuf6-5lIme+2gBRAD2 z?*h|=`#R{K^Az-JUY*t+y}yZ3c!W1B=vni(4qTEPD!s@Vx%RuynqDa@?Vo2cWUs5g za|ms=yLzkjxC09IJA4HGq%tGF645Db#7;8V33Qo|ak>C_UTJc14+cp9VP&P8j6eG% z^k70+8qM80w`k@+mw(6t?7VlWw^XxeS4D+*P6zp}BcaE9wJXHt{1TR9Yx2$W%2D_B zc;?-AG(pEY5y!1pIxdYXOt-@bCfGPLp5Xxft68YjEIV4byr(iT*vx}{nAML6}&T)D`3 zetTnRs8}?p+}fM^U2wU_rZ#mHGk7g%)p+@L5ZZmzHiVYnu`~Fa(41(2d>lG?D zwi4DJMLEb+E#M~L+^qi(ihcai8sJN3Z=C`IbUxSWAk4qR-UqtqA(kHNo6JALgFlAma<4k-4$ErxWMh zYL=!z*)jplY0(04vaG?qKQebhfp1&6IFTup0yu7%WjX0!mtGc3U8?NaOZ7LlqAI%u z1$vLAs70{VsS9r{5Ld|seyPKPV~bXZr;@&$MO+%XBW6!@{so3>vXB^b zvqc@%v&h!_Bm|`urQTEJg_EkC6;*j+&Q+}s;$&B#lL=U*jXHN|2Q$p7N|!cmfCcrP z*`a18X!IVB)zK-%jfldpb$PJQ5x#qB_4#UZHBJcD^~|0gPtHya~tCsH(h{1 z{lN?&U;3)p{6k5Xb42~0a#quZju(W3-k%vlD@9aP zX+obWb$Cz42_j5KRK}rRUO;gq*wR-M@7Oc~X;!ZgN$y^~h!XRpX}JOawYgQFMFw(l z!bmWk_`|)kzqQ5b4yN4Gg^SgKDw;pR&V~%+gB^*q7?&Qk9V5e@f7F^6E%}p4Nx3zU znim-vRl~UjgNa^aeHue%r24eooDdfo?AY*O9Z0UCn25SyS)zm7c*AIxCtzwHpNUDh zr6q|4^Pu65@C*T)#~uM3&&$Fa&uhN)PwBGV23EqF_V0s@vxX#(#*;EStu|-UUQm}e ziav1baFWOpt^hjW91#No4n~LX;DIsv@)E4}I3`G_o$hjw-?OV9men$L9w|Ktd!P#& z@7L=vhi{G3yt_=)jxe0Cl{a3+W)iy4&7(r;4L*He4m=N*eIPTC3j}`z&SATt`nwI^ z2Mf$%MJIrEy}9xebMWsinhkAMG254K8eKy93A~mR+ zSJaQFG{=J?JjHo@u!o05w6kwKCb%h2ft^9&VrVz(c0hb>n$ML7X*&=26j=2?vk;MI z2%xFq_S9WL{cbkYzy2GV_fpsZ0FzgV*VJARu?PhB(mhYWXbvwm5xKel3szPT1+j@P`pd#y}~7R2HPHRm{2j81c#TL#b5r8WP{zNUr!Jer5YBO z@o*L3H&b_E^D7>ubI1tQebMm|cm#?rVl-@8pDR2W+NTIT@BIXG)G*!-yx>)N2Tv3K z3hORu-2d&bGs_69)wh)F)Fq&5*tEh9oLT4%hIe!Rl8a~BgAc4Ec#1s_-rZDwxv;zR zN@F|a;i9DZ7?@)#aW+--a1hrhivsp7n;v#|k`i*Vb51)LFkrdW{}L}6SP4q6*d41B zqV{l^QQfS|1=9YH4QEp7y1xQWBGxW=A+CMpPe8gfkZ1pFC!lk^+JW<79p3IQm3lpu zaRBZIcB)=LO)-QKwg*ifUaveh#kh3AHvJgA#klc`N<3QMQS>UyTNnct0KgNst9Nd( z=4-9Ewp)H`Wy0Z}KgzvKwjE)iWakgMBKOs0jC*Lh{u=Pe%vfAPpyO#z~ zsY5T3o*K1VLd&(5Hha1`BGs5J#y}))6^Xs}AcI-hk%SZ`ljV;XW%A^%25T%bk_Q3TS?YaGH8;L~v#Ic#&K3-Ln>W+62# zsyi%YX+H|ye2suAMU<^0IU2bB-LT)lE+{&N?ANr_J+~}8E0-rXHU*#C0|tzqHr|?+ zD<*2HvUUVxZlqmHJHSk9k==h+|j>xt= zFVp$l&wJ!dTK;^Bu?`xf!?Nqi35fEj-h9{9KLg}TjgQY-rgWy@>c)p$=SQW8@MQDx|E*9>G z%xtQ4XEwXU74IYUfYX4$OrhQqPGqN@ANc<%Zu(r4f{AiB2TlCkeS90We>hzljz>>A z&4NQ88`D-SnAeE`mj^-h+)M>-qQi{H2^bHg&N%u6EHCEAb=nww?{rG=^&W_jjVUU& zc$pkuPt>K52u6v|xg^_&dgUnsSBZ~VLhpL)HSH)5{n3}JsS*vw z3`t#UUNQOllj@Mu+~KEN=*YY>7z)#{)_q1#G0-ZbV4Ubj>C(h>CWO&n>O-;MD6K60 z$1wY=TWc$58`y8+051H`y0wm_YUPvq!HVjRzcT_WYTEVFI5E_=O_k;dfknmegjoN{ zM~x(83dvZ?y}@*0#-V3e#XVC z(^p}HIzS(HFFBtQNZitZXjhPHHclb+W$+qW5S&f`UQtWvw9Yl0dJn>Dm48QlPfv^6Kr-C>1NWo=wlbyXg&+IaPnZ@-&XX zm3H$2BDwU4&{y@EES(P})e12>w~Sv!>{u*H#j6F%KB&y$Ocf8;dMMC z=vA{XIvSf5I$4ph9sx7eNXZPOBWd~hX+TE=8e+k(v%dabJA+OTwODiXp|fH~d(3z? zpvz9s35!hAfob>dHgG3~kzBtmT2L*r$Vuf+X2d2JCs6rcE=6{kA)$Bf`j-pf-yqchPJu zXXT-iC-2V5kEI7=V4@?tv{$B&_XJ56pTxZ~?eU-|Li*AZBP`asPe8gNEp{eA^JTye zHNt#8lwSHB%=69~70`tsynMxPWHTfGayU%8N1WrsAWh>H{w8~AzTfc7(q%;|hectR z3&M$7xKODP#y4(CMj(UvfZ0b?jk&LH_Z?!{#mc;_tb!>jp+sOEeCsB!U z2A8T<+ykMEP&*nGMBeDL=zN_vmer-JYBCxkL26cJ|?c^w>zCRJ zzy+cJl4RbUEM>b`a9I;)pn(g;bP$P-{?^(R>h)Uax8v}hf>etOkCM=E*r%axqAe@M z6@4UN8{Kt>VVtP+)_+l2p@TLE@C=TcaWt|QP+8-Wrv38MO;~&A0IOKD$n#T>d`|7Pc4IgTE|rkm?x;6l;>s|8Z%yx4<+KUMVC8=J8+^% z@}fj+qp!7D(dYbefxcOhsM_peALuAMnpU|p@f1CHI76%Pxm89ZeT^wPaP%b}!bEqu zD?jQ>&HZfqlVtaxkjBT^JD$t`6Rp*Rp~o^lFaw+AL0)PA4o2}lm;f+vu0XsdVkc!y zUBK1ujQGr=#~(Djk=1GKm~j}o=ANNPaz<*$a7(tr-iKXO1Ver@x)HLt5NOIm6GD1< zE!l65$`VKsz^qark_t|oi-a2nTRJF#Rb~4@`Vxoc=DxkUfhKB2m=c5x^S^@aBEHc1 z*kBhJ+=yP%?g`hjc5-LuMur?n#1y?D2CjHK!dWC3UXQN}jU!NermzU1AcqxYDKS%B9HT?)i`0hfd4(w?+}sS|{zL2Ef>+lOsIHz^qFW-`2m;d+ z_C3;_HIXm;&Z0=n3fM5S#5{`e!q(U>Kstfd^NgWv%cxikum^d!$Onb2#m%c*U7|5M zWx8iNQpXQPN-=ugLPdFCKG!(eZMwD4GC2VH_Z!FZz;dy>@rMytK#93Z@knfF)e14> z{#bLPGN#O=OzrYK`l~q7XKzVMk=Yqta5CS~i6zvs_B}ngs&k-*8x#mjQ+7PYw_;~J z=?MbB39_PTUULSWnH>te#DED~4|dNO*!pk$(rzgD!FgWbt5f+*I)3u~rO@ zH|R(Mr3odDjF9>Ib&Kq%NNWy+cqtAoYum8C5&BmaGUJH<@tm`3wa%E3zg<{_*0y9Q z@1eFr5lxu=2zU}(&12`jdF)c*u>ox15M_zWwJIt&zxujfZKRMk-L!XqbP6r`;(I6uVYDkb2hl7aZkptbRU1z;PVr56CddNo53T!TU8s(Tcy4vT8h@993FW_>j~ zaO;X?BHGKL0f(M|Afu+#nh}=Zj0AQCIn`fh2R9+Y)?Be*V#k#hy12|$T|N@X=xM%3 zkoJlhX(otx~Vwyq0EtNIn79mo`5O@g458jWX6(eERS`+q-egWcFP6U-YqH0 z#H@krf>$d-0i*U=24l$_&O9|pJ@RU4B-;W`N_PJS1ad2_lW?#=yH_b?&JR8Y%!wTB z5hCX;nXQ(|3R5geA)*CG5`ZV2R;Y=+A6Wj&>q$C7cN%@ifCKF*D3r4ww<0Oj`U`PX z2B`fn#kK>c-BwA77Nky!w=ExJ_yeIuq4Imv=>oX@>KJ`wEZeqoxM4}c(SYKdH6zV~ z&1x}A)9t{m<#t0ze^n(W6|%he#3KR6ixMOMzlnh6wI#kK*^OVFd=yTZN`~3~OvuIg zz_d1;4?H}*oyO-}X#kcxSf)6^J!EazLIF<3)UmN?(otP43W z_Txg3V5Iec9(ASr1>#1JxEjQcsRR*wh%NbEVA)*{1S5N{EQrN3$1C9E`a;Z~1jA4b zB>3#$CfX<14p4RMLI!I{>BnD2V2bu-VUS)oHO8-+t>5av7GBT`03kitJ2{?LRRA&- z{qBt3R7%)CH%(?EF5<=%Tw=_A*wQ@+e9H|qMlbzG<>5pj*yjZ@f5t*VP+(G!uGJUzM|!&N0>MhFn!7U|z>ADUzHQMrkI2#kv>3q|pX4@H@C}oq|k{ zeWu;r!`%5-7?f0Ma|@(}W32RcvCsgfMGSM=4(R+3I4xtGoOw>TuVUinm4^Jr$*93a z4#6&8AGhGfycJ;THLzRU9NKJJ#qQlrW3mFZaOgX*B4f7X1LBRWoK{J}er$&Y*~Je+ zPdet94^|=F@GXnh4=hz+Q0j123*2_5ucRH0g-z&ijyBeS6F*0~j!RFzidCk@zK;oV zU}{Mp#wHKBsuRJ9Kt_(EWQrZ?rx=J?k`!u;FR3M7`BmM%)vy?3Wbkc#Ei5;SAuxd; zK$KU-Ncl6io_h;}ypWNgI=JHbiO!hJC|5SS+>F5#XlU__AVQX_Ep$)no)PQJJzb#X zrDT$|Lti^rBrCs}QPU9@XE4YL)=*P{G=oDTV9@cKP>LlFh3WW4tqYUDI3v223pRS+ zx&+U^x8g!eYHN_51({qZ-`3$lKFDK0pp}+S^a3%GLc*ZF7RYi3A?ahR)|+EFbh-o# zwbBDXc$QOVevKxeMAIZE7^bN zIvBR(k`{}^QSIh!Sz3gBbuWw0<09Z#;kEA*k#asT_7hbfGH&JVdd$+BJa@R=Db}V#D{l z&S~A!IVB>f!6ntw8)$`jWkJRWSjp2Sbg$))L+Tp}0Lq5pZa8IJS&;gGt5L+PmlROD z)17sb!~amO7R}Xi_hh@snyXM4l48PCii`8XAl>;083KZlmo?*o3a+wR+`X-P@tNEm z?Zw^A_z3jy36EAWftyImBW~6aia%~+5W~_hXd>rjFuT*)F9%s1qjs^xgkR3HuEUeq^@2)_z-IXnaDB$)cGQ{wR zlr@Nj+F$Cn(crtsp-G);cWzrHrH7(K_#w8e5*Us#A%jDA+O#tkpbxg1h$wb}nsR1scE;H|-IXwe z$QFPkd&y9QYMcm+&}^=IZKo^VoIGO-)dfFwv)n2tj{-Z&oTKXj=zGc$#K@A_@8>qw zJME!Ze+kvt9cWB5F}6LuSujobv@vkuOA=^7n|({!h!y;<=O;J@IOk6#+qjt63c*T` z0p_+19e!epTMUPL{Wt8t>FHhLkEZ)C@&@^rukGFY=6I07OcNQzpKk8zw|#WwQH0N~ zNB!zkJ192>E=_7?eM4+ZicKy{gqv@KmgY7YL22pWS3tA}fM#b|&AGI0>9ER}b>I{4@UoFsPQ>jqq4 zKBht~Q+%Bkx+T+&r`|g)aZA_f!fBs?PkE1I%j`9^jmYaIXH^vdFl2Jn??xk2TlhH0n{}x^JOm^WZ+POjx(tyd(_B^tWgrKX*0FT%sIpe}G1 z>Bk#%iN)V2&cC#kMb6Ami^)`w3HS9Dr(w-<$eoBHmVxkjomr+*wjHf*%M5MC83lV{ z^&x`}YP{^^bc_)k>8r9CMje8fMQ%P!TEeq|p>%HTW9g;j*WYO`hfyL9r zd1?nxNZ1UIIk?MvAl5@_`#KKY#zo*{Oj|w$@6IUiP8Tp8l)BBtwg1hcOY;L19(184 z;JiOtNCR=;u<^y>gMd}rsz|hwyrwxUz!wW8vk>u%wE5}+w7w*Wva&`Xw_cr!`ny~7c584DDp>EzI}C~ zEP1G}D8o%}b!c~<|Kq06ocj?!_9Zv=c=QUhbpGwF|6|&~xu!9`?T_*_p+p|WTjHsk zGjeBR-Jj08{P^hk)GAVzh|GS7dD)6JTcp3gagL>e{zD$lyxfBvALFWC_P;atBmm@1 z|9X5piKM$t0B~&6!Gf=MU~^CKU9zTxwt|ko z8^B=NkIm|9tAgN^3pM#iRKiYhKLxU#9V*_KU&xyfi|7>hr_4^ZoL3*`HbPckoGQcm zFdWvpe%@3ay4LRg&_vnaa37i4j)Hcd5ET*-`CW6t#28p+cB0$lie*0iB?PNE_5{wz5;+X612JFfYcnWEf}f$Q*aI zfnh|Tb5#VUH`_?`1!qtF=i!nGr1nZd$QG7|;}o4mxyI3zvHJp}GFq_FV`- z@$X;vKz%JVk(lMK`V-tZmKIpDz9f>8tZrx9qOxuBsnbs1(}$TkXOWAAO6<3j886&6 z)t*(cHzl)pKxeb$;UVe8^{~?g+V2*15n%ik5Vf?VO!I|yvm)>WGps)m#E?PX2hQby592%9^`KYWO`Qw96|CKlY7GB>|L~HC0^>7TX#@7$N zgq>;5@}%9GvZ^a_=HN%F^^w|=3BM}TbCLVqBsze%_KmiKgM0c|>el|s1EXg;7o=nJe5Da$y2pc>^CQJp zi;7<|G;^e5z2w7Deo1!1{yGy4!MUiB-rM(Hzy;(rH$BZ)I71kH_ej6-#B78S?r--Y zae#~QS~8`^9ow?V>wOhT@#w$ql&~oBikwQ0fnhdtL#6$uxZQoX{GOv0iY#sUm=<-% z6H#xB-PA22W6ZitjAE~ILRvMDJ?9;lLCj~r&bwm>fJFG}7eJo2OCz?^7omcbx8V3x z#$g1sR1pS@;7Z0hp|IkE`kMIAdZ0e}wGP%Y`^=d~07W20ZfOp+@1J(ojYxPzv8&B_ zXYNK1fG4r!TeW!Hbb8>AK00}58(5CAknSo(51M#5wOq&#JF_p&*$ni_FwiF=;)rUU z2GoA6*m-d930Umv^9trBvxgWgM#hJa*QRiF=)y#{}EqXKcr~^fVEaW6h&E^XCDkP7osaw^wu_ZnOPGq~5 z*3^2&#m|-}XaAfE2>-37jy<5F{_$XK$y9i=h79qN=T1+! z3)Aiy1CcJFUq6h1#Dx3xtLq^{*y;9z!u>`RFZq1q^ltT2;E-qQQAnp>@0P3^uUQdc zihmW148@%OU2@Q_-ul`)V@KKO`Zd9LF>XTy4BO6#Y@uc<_Sp90pk}O>3FxV|?=4d1 z*iUTh5=*3*mxEwdfjY(Nhlb#Iu!oS1DYPVsZ2WO4fi4|t$~~^0i#oBKJ!92w%^?HE zoz{ZvPYqDB?{8E3l4?faFiJp*^jyOqboCHs15JMntzHtlnFteW3JnN9WsLU9087uj znYSY-!oD|RY7`XdU3%yj00a*87K^E{m(>IqXEheDc}=k*>SOmu1ov&Rk; znh;`OI=qr+KO)xSLPt$o#Yf!|fA~+9?!FzK_rRi6n+1+iKB`d}aw3;44_9pfN_XEb z)-2h3cg23Pw|jrE6M6bLGFVX#_H~)dDgKp!az9wJw{>c;&gkUG`1qPa^lRO52Kshy zM097}!HShhJ30Zz(+W!u@t!jcoLUZ-%yH~)MH%=%{vfaf{s3&l*LDsX2#~D3cqx`P z?H3EFf=P-W8{x5?VF7k;71uVpU zM-gUZ7HzW&Z!o@%l?^nErP_ZbA8(LW6|jwzSI0OHmd3ZOcz)dWLn`~DBDC{KyDq8* z%UJ+0&2bwB4Q4kFMS~qnSW=vQe;vj^tGF2!j>el%z~G*o;#bKZ1M34Wpm9aRL3=YJ zK!Q~xb*FF+-uWbA^kjVp%LccqHk+R~UjD;1->obzfoc8T=i%AEoj=JX1D*~Lg`0v|#p;y5Frxmb>ju6%DR}vg^g4hQuUdX6V4!4+M+9j?? z)MA2ha9l`z%MANw6d3p#Ps7Q8nE>jhRJJD^jct~jCcM0TK_99nfI-thT0UP04yX4R zkr-&Fm}Xh$qvwQwgu?|FZ!pX$$!8N;5eYl%&R%4Evs$bQzH~&&_Tcj?s^K(U@7%hV zh7e@-_ijm7lLHd&aIkYF$L;WS8THl&R$frtX$-o|oay%Ejn< zzU0$|EY=u@fl9L1Se$m})Ea9gWD*{JTrb?VD0V-lVXxr2Fw|Q;x(@=?<|n^+xEdbA z+y0H5nxEiHeDzSDVfZ>TtMyH~)a&>@{h}Uj@3Oe%0Hz+43d#J6DcYgcNt__;@e7I)AD_AfO=)9>tFdD? zGv@li*}7NtRGA`xm=S~HnSPP|6-T)SFe<+{V0nxcbcvJ)a=Fm=eJp1B^h(c{a&vlY zcC;65$)O5>br~w)Dl+@QD;ZM_J$DwEZ))hUSo5&Uur?ZG{;1Q>@Pn^P zB)qNrn72B^bALte-&3VD@sj9o7*tpN?}Pao36Ll9^)_)JYwYJjYZc?wQTq=Ap>CQ8 zoid!ymDvTpFB1h#Z;stbDH^6(3kvU{sE8x-JP8mI*#Q2Ccds~#EJ4l{#-#5-lik0ud(HF^Xk0*v~wNJQXo?H&p00P@+C%GXV|%| zAJYYouH!%~#I72tsB^~FIY>7cnXN8|1@&Qs2B=j9{U(zIag4O#Q;ykTaE3uQWkkpK z+2G+2{Yu|gZ*+sTVkzFeoOfTkp*`N2g}~uYdChP3J%La6Td_0AzC8>kch&9Hy`nbS zkm9v+_(|m~Pfv7~lJu!<^&pvKAl=k^66Q zs?AuL{e_8W)UU9%lcqo^SYn3d?vs)-T~Kzp{ZNyf=ke3<{2LP^*TQW>YhKl6Lc~9J z^s70iWR=pvTZ>R%56OduE5j2iCNZ*EYZw1}(<>6>Gn#TrM%J<)SB!ZEQ>VSAUeDKq zX4}5UO4Ptasg2NoUKsq&UoWUum+p(U8MM_HT6)O5~zZ*8?1Rnx8+^u8=jie>e~qqKgaAY{pTgWrsyp$Q97?h!8T}P*1l4< zi`OBr6U1n~ivNjA6nO=|WV;@f@p}LEd)+TOd*^=^Z0BQNsa^moY*_aZxu!n*ZR($EU@9 zAuF5Ps$(g=dmo@TFDbG4b!312+0#{b#XGkYKZ+WH{lQI7&1l@P>bap4@qyrb@zA-z z$CN-pjn7SW#CWV%RLYg`xL^CsE*X0i!If`#d{_o8a`K?!Eno|?uh4tglR~*(94>y_ zrR)urde!zO{*QULModHdVdO%$Aa#lkZeY8JT3EB2fj!$CIkk)Vko1P4OP}0}$UoF| z5#y3)DyQcD#Mf)`E2zstC@ zD!15o?|j=o#+Y4x2hk^r_b5m5bwK4}Z3r%bqX{7c{Szu4Q-;iE^Q)I9@rde(ZFeD+PqM(CJ{X;)7%@XN6M zElQ*5#EJPjqXG_5O_zPlFInj;GRSK%HXJ@m^A4!eOU#dRK6%OxJDA5g`36k>9+>b+ zcZS4!wH~yj1e(zITMvuf{|B5u1fZBA=QvYcDOn=r2K> zUdK5fyjn0Z@kqP5ApJi?(o7Vn-HL`fza#1&DjIhwbmlp@mX=lL>oiWE zzE}C9jY@Mn&Y{-W%Tba?-kzO(wdKpd!|Wfy)w;R4S!?|n!=e3$4m)2aYb0O$B{w}Y z<5m7|B8r`R%=Vvm_FBen^xHq>)z4vl5-Hiak;Eg(P65oXn-%#1Nob$FeAI9TT}b2m zKDU%%*t-oBzcSE^#HEB$pESpmJla0#0)q|n4!(Xat=N6zwuJjZ8x2t}eO?DaMqu)3 zOh!c)7iA`7D2tp2KF&3vJ(9EcNf{46-+jyf8&0Gc`OHmJ}f(jom8rRIqCb(m{&a?99kEZStCT6AL%z+ch{2?H2h2RBSbw!f4Zq z%kJ+5&fIDyd09GNOZvAfiOI?zY^AUvFxx5{rom3bC(GbtLeG0B4H;2hq^4|+`mQH`1J@wV|QohM3 z4AG!|s`Wf`zEFv;v7F!7d64K%yZme<7_)iqy+>u4W}Gm$K`!G!-k!w6Mgb*pK#B5`%zaU+e~x*){i5MYO`_a(#gvN7c&hhWE?`D_ zv68OAd$5S#7Y~7Mg$FG7xVP-{(YuQVJJCN>xNG4?B}AtMEt{Mr`LF_ zF^IH1YqzBC-5T$$(bgOD#1jKdJ`-EK?j9(_3)wxh+~<}(_x0OpJ*4EU061;7P(l8? zSGoB9)&>W@>t2&%P|Nsh+lQU^bQNmX6u|W80j_wG*dZl8w0buSqg@dWLg{~`ICS9q zN*O}V&^%uKY=N}04O&xreAZypg#hbPxR_*rrcscUuur+~K`rR4N=)yc&xc1OC>T@u z2rF6()y6O40W$otgyR>O zYP=D4?0z?<-2tgbTQ3P24S*p3cDDtLDE55P7;HB0VSaRIKze2Udk>7Iyn0guV-P8s z|3WNNI=IIp0nG%%7jFs!BfxX~)kC^b)SS=d(jg;ZS=eCc{DEP=jf=KsztuGlr4PX? z<8jDa-x<+U>Se)CHGHIS;vZVi?3Cf^enDs)-rTGDD*mgKewJwm$vsaw4IuPt)REn| zw}WJ=43^e9Mb6h>bZ%JCn4i&ihc3VMjqfrTK^Ujs5&>;mDQKt zK1no*9eeywy7t?6b;XC8sT1U?zM3;?(L0E9PEy#O!6E9+Tnin_oX)mr{NE3c=53=( zXCH11jmEpBfgA7TD z>2CVGMY8uQ3`y?kjm>dG?~^^T=6&_07}Kz<4&6igWgcfJe|ACEC=#7_flp38^HNGq zF&^yaV{Qan#(U`kd1TRW(w!CZO12br`9DG6ztnb&HU^pPi9s}5RffZc0IiaMT9*@o zysT94KAjvFW7at^_2HC#W*-xToIUgc#?NI*mB(%HT4KQdlqYrx_ncetw4VSG9cb0N zSiwtu#*o|4(waxxI2fYHKDR5zthye>$PI0~`CNbQNe24&u`PhU{yZnG*D)=cS1RrR zK-lJuxU3$?7Th)g2291iZ&XanPe!E=RNmj86ZlV~@_(4l3BO~X-j;v*Pk?`eU3qr5;-MbV<)J9vc4!y8?$^idN-Bz<;eS32lyZvo zL30T6-><6myoWt(78u`id=oU69Q|gXre2EkR}~=9K@rG<-^=BINaVVaKKZreNmOH13GrS)%5`LPd|DrUfOnGuhC3w|K^TCb@9bNtErt?;*a}E{sTks~DeHW@~qV=gy zGUa7XTs`R7K-)0fChtAtBCUEG!J#XC)2hv8p>gkbx>wg1g`cG_u*ZsrPYNeB1}rFE zJ5E2KM-{0rt)opM03j$UiD_96J(L%2AqW}=Uonto1>EC=E%^aP>bll7wRcqU*;Npn4)*7ZC}zU zL%XrAtwnL*XM~fr3u)D(;C_0CGa)rqSxsUW zJ+*EaJ&MrHz7Yyi;k<&IX<$#&(trcU6GKz5xo0mwc;St`&)yBx+9(ME+a`^6Uq&kf z(7I@+gslqTUHMS)RG_Jw2@_QAlRW(_Yjx#{<3N?*aH8CnJm0>`uEGHI3>9UB_*7Ip zg1IukvU#mGc(ev}IzRBM!k5a~SnMaBULdruLO6SM)UFW4rg2Ua};X-;JT3(~1(-z$kRywRhEoi&rBTVg0f41V^>0NQrqUB-8kMgHZ^TtLc4tHiJ=N9Jpuvd*r}p zYjfhh!^C;R)49O3?R~wuMYB2ihI-LJ+L4R~o!I2svyfGr^=DCA@Ln2-wcf!Hf+ab* z_oZPwJrU=io-=lt;Ma*%ePNuxdUvsOkH$E1Q>`}zD@-5eoJiE*JF+n?2%+|(*`aav z9mZnlW#;V=#^5r$H-2w~%~&btMZKuW_x9)Kmo|=_NFfgW4=`#D3z5wD6D{G3mc9XA z`YZQD-cJNA(rh*DO2g2QXY)yo&kvf4l7bJcV~`58zq^Fz+>Y-WzmT1Mymo7%!LQ?< zks;&9A4b;ve;IsX^Y;kM|FKqyYT@1&=kdDm+mH8TT#gxt55G_Eu_qX>e;ekrc0MB) zNpA=hcS#mYFMFSgJv#hZ6{q=p)Gvx#UiS4o@%vHDB#ZvT(W#ZCe;}c7qi@U4zt{O3 zKY2y3O>vyOqVQzT{mRx4tj)~R6z5XP>tTBvV!l0g%*5>|uWLa8~)YXUUc>Lhk4qfJn zWpfX$ZwN2hx9MZVh8*JdeN}IGOR=-E8hEJBXY-jmGnN*-8YGpMcvb4t0qewhQcheRmgS?5t6?yA?$Gz(>zM23#-j-B&30;fp8 zAGk^hkzOt077`;feDCQYW~X}SzPi_9@23^#B4k2NiT(!|SU4<$dm8TkiTYODUT-B` zS!wEu_jmT~6Xhx+b1U9`IinAmcmAxNS?K#u%eql9`AE5XiYxYDP4H+yTj7UV`^ddd zyVrNrcZu>Uh}q44eXZ0N#gIz%rpd^oc%cvS9XW+IZDEx7+-lr*pS zH@QmrwiHV)S=e)p2fOW+TV>B$+RaJ%5Fao&F>Ce5!c(wv77sP>Qw%!ySpT+LJ*LB2 zp=47NU>VZ?dJ;!fcPr-mk@*zDx&dMb!86rD;l$hhyQEibjD>)wX^3L=Y80~zB6y3H zq#?ip^oP1*oYrF+iB0w_zsj+Z_bU%8eJJ0&gC14#T6A&W`8R-lnU}fWMNQO)@=~M0 zJ8u4m=XQ()UC&umQr^B$e%!{sZB?@TT&ea&o=Z*0{-?$rZ&m6U;P2p7v$n?7dY#k_ zASKlYXQMhSMnKV12Y`CX^C7vw{3R>A2v)-w z)@!|y-;WC?oBTXwA_}(+rw4d9>a;OyTA(_PbXn=^YMko;>`a&xfCDx{+>!$UylluZ zl!j=Ah%?`LMJhKX1Ia+!@ckwEvRY_Il{#fL=?1MxoT1mvk)W=ZNH%4eYZi#G&UMh9 zu-s~uF}MBH9($`nhT+FkLMM5|fxkTyGJwUP^~}ypl>jPhXCx!S;`%eyMpn=@UeilG zQg-gYQDS|NrbE2256k?Q?=o3-VF9L!>44=c7yuIu7RUt2mQ1gsZQocMUtLyz5L+beT(YttpN#8l|tB35fZVskH9wPzd%9$<~e5FPnY`s$~Jvge6y}*cG=N zI$XfWD+bEPswG)xP0cPXakkv<06)6AP7I9M6MG;#6`mWN&x2Juy#!4qyA-3>OFoj$ z+BRr^eJ1tKV4C4)HwYcB*T>o{FXD<_d|-yY7DT)9-zw^N#^fj-kMPKApqk!9XlJ@y z*&;Vj)VbqZ-)Ht$>(IV7YA(^{Vq@?ZRn0^jt6s-5@MAUa{3*0#p4H8}k;MS?{lN!T zw-293xjTplz?tKiiH*O-S}$61dHREJxcGDkgunKgnFMGV`zHuHX-ocQmCU!C+hWhm z#{lf$!kPOwWO&ggqH}fk=+3#X11C`H7g=da1-etXChusVUX)y}_Hq42eb29YHs!BJ zq69X_vG5{y38IkTj6pPd99qxkmbukdD76(s2a3&aA)wEldX8bO-eq(ZZu{zrg(jbE zh3Uq6yry;qxCf*ek-#gK8599{%^nE#0^qgsCo4h5(}#JO>!cHwTK=U<`^OQ&F0EI_ z!&zTut=}4`roKmO=x}uL`l)Efj+)k`d=*(hA2i;uv`yKWhQ%m-O9gio6}eBb9>>G= z09XxYuxGR43Aw7fC@yr&v}&*Qd_jFp~NKM|Y>UG+FW8=7V#KtUJ)xN$opck~N#&q;ftZ^&%+ zR&cq|Rc5@teNWqn+VYH7di~Ajj~^A#BeJ~rqgzzBx?8IIyO_57)Xd&U2f7XFHAW)C zSer_QKd0dn9r2fTe(bl@f7cf2=6R^QGFI!ud;FDPol6Ogs~*zHEA5y}g*Zd~#=wDs zt3F~c&YhIAo|B2Hf?~gV^8n?rj_Nxo402gg`YFd150pO330IbSRsI{AFJcbA3jdvH zD_khmY(I2pXs1!jklNYQV=*(yR$EsBPghl+Dl=5eq<(y6QUTc(l@08BIM)(F3U`6+S~Yf-zX1vv(`oz)-^(ogZ&HhR$qQMmp*YqPMg zZ2(s8GT>MPsfLdRarmBdzk0|5JIsp$e{B$FOksY#;@h5uy7q6>$ysn`(Q>D-d z%=mA)-eNEyWT(RBG<5*>2!BYAbZCh2h&mtg)wA zQRftf9#_UV5x*)kk1}+qZop%34#Er;C~WHoyFdgZT!mrZuyd5e|{Yc69_>GDh0; z%5zfy^S%*bTL*i?4}kRc)v4TdB@6jnI8p91blrN*KYN(q%&8ehnG;hM&)Fljt<*dK z{oeYJpaT&1Suozfr>R4yDV zAD-#5I$)D+1-I*Pi;1gm5oEWHN!)bH2ab%8Wdu6qTZn#do#=OQgJy)mrN1nd$mL2Z zRd#_nO5$A4*>$@}Qaw5>D#LWIaB?_mX(5OCz}T;&m``> zPMn%8(*FB;Ex1GYSG1RDT`N7+oJK?PK#vx)Q0(y=`XTc2%r(WPnUB8FexGgLK+m%@mLlqgvv@N zTTIg2fkY^ZN+;l~M7Z$;|EKFT*%j6I!n z&oi_qofQ#T?b9tv&_Ar}Jl;Q-a2%^j)KaCZCt;W2&hjclP*n&OVAR*ZBxK0pFICu6 zSlw`w_i>6vCRvsuhy7-B5{43Zy6!{+Rh{nQz(nC?KW*DK5DAsZ6d9Z z8Hpl5SVIl--ZR@)>zBR>7k4 zy*}7x{>f7#Bx7M^w|yoAm}!UKY>}L_^dc!+4ATKg56hc?9D(G@;6+L)F;z=R32Oz6 zg9YwMcpFnNA1}ZllIX*e>$aQ6;TeU6CGP_L2z|??rZ%SD(=tnCwZJd4QV-sRx07Ms zNCv&X5OXXnYb^MJ)>xEynHz~dUCzji#MNQkgnvlnQaA~0$TH*gR+i!NA01H>P^&5F zXmq8mg$?LdU<*Dbxd-|cM+X3lpZ8HfS28|R$PqsrXkI^k`7Zn4yQk2^^;^v9H#|2RGqRtBv(I|$5 zXOePP47%LOgDeMmuYH*tm~S27lADAlXN6((jowVSIA)+L7MjUhu;W8lShBYbc$D!` z6a^^T{(apC;Uk=A_I_Z;7>+2HBYzbuvTZ^Q36HON_P!LUs&P}YW-?&pTdHrCN>A%L z%W2!!{U&xiv8l$<{545cDv?5__(1W!(Nd&?=nn~^Siz6I#z*>1 zMB6L%kE(F8T#R?EJQbpGLr{YWgfsVTrDP|M7@9^;Ryh8pwYjf+>_^b@e(vHKl?My+ zq>aKo(DJ>X=*=24T@pl?p{q>%r0|7g@*t@GpBi!2e?a#VKfTxm-rB-wILlDKF{iXcg7+laG4PiH*Y^c*S^BO8d-{yhy&G<|FU2o8IGaO$&* z_=%-^(taYtGejDlS1?_~9c1v`PR+jfD4_T>hY50ev)xrLinB@ln<@Ks^k$$fcCe|^ zha>Es{&|#xwo@X?*v1#dg8f3lfCfvQ+`?++`6Lu2LZy4)2RAo0EUc@M8IId+^fWOW zQoA^%fk6d2k-17d20*qquJsnF|kA&L!-lD?ip6RaChQ1xZ8o3oqcpfE9Q2 zzctDaPTbHP?F&&xkT0?f)x4|1CqMZ`2wWF6I?+7FItW-1ChO z-u^JjD(|s}nVzsRsG}vTjMDF2HC~*5&~^hWI}1)IthE?_&k~V-C8rR+vFs>OPsHQXd&By*NQP^+0J7bN%%ZZD*3fxi~ z$zb*df_%to0TGaUEZS8KjJ@~IQE~+6H}`a_pJBfFE`A*j&DT`b)pXr~Jh;-+t2&YY zjx3T>6J2QFLKu1-qebE+_jP9?(Injxk`M$748+9_2LGzd=M6&H9v1FMjnv3Gu!o@VUoosBn%eIj zzF?UQ!*+EAv8JU_1kL$MleS1Zn;@QH;xuBT(M6*D&Z7OyQ}Kq#N80bhMC!8~r`)Q8 zh{*8twtryF*`EhF363O))oje58!45llGKiq*;L0@-cE#HY1M^(J)`N*g+SwfH_UMI zFzLy@X70r07=k&bEnrY=%PT=OefHniAA4pg!oXPNzaNzb#3}2u;r2rFES}u*k80+^ zJ||C%zdQK-s@|Bs`|Z{lm1FJi;Wk0=I+M14`o|h6fzKTN(a(G|D^}BJM=`E5TbH2(0#;1c%Mf zr{7>|NQ3 zO;yY_Q9d@0WnJzs{@ARd=J5ep`;-u~NgJB#Dh~;gH0YsPJQl*dDa5kiS4MF%v# zqG9VH)ztZyay+Wt@++wFhUJZv{$nzCUKI}nb4sf_lNxdAJ9WrTo_qxhmLO zVO_$-m*KeC3KjRL2?vaoDYOb`4K+DZ*~-antKR=WjvP>1H?NftB4%mEZ`@_r!pcRh z>n((rAFDLg^3xqBFOgd%{>Aar6+aeimFoF>2`z#L?;7h}{&66<=%xJ^!YtlmwxaNj zrF#^bjpMUZ-}2pU*&%@J8OJ{aL6D!NA@j9;>UEeJ%ucWM{+n`=C zH;hmGuMq0k^!NpZs^odeV=7+MGHG9P7#WpFMo)9?Y9a0CP-l)8(ut#2(BvTf3;vv& z7lK+LapWsEtwF5B6@(Wt2f79aBJ^crTDL;rt5M+rx$PmYpIjSyx}(PQyB>WIh6+}m z9U{S?&V&~Ahm66&j_PeA81a|$_xifuEzG7ojaeum%cGcxVLe1$jS))$*clo@4v(c4 zPVsQumYgHrDmHFL)-YH0xL+*)oje7Fdw>b@aw;;yG2dJsz{e&n2;DjYwuPiXo3HWe z;Mn(8>OI_KKx17`P#9R*C=ym1Lp}5WE9A>5v7*r%sVS?mo@9ZEhP-K@E2D$nw&Jd! z^HLHDI%+trr({ae!U(pko5V&=u2hxLU6D!Kaptvv@WK`8{Eu-{5@5e@c}m$fuap~K zZf27!yFB;RsBvCgY^??zE7Yan!dZ$g=KY?op`~y7^6^?JFKFJD9R>BzzIfTQy4K%o zL2)|kxcD&6H<8(+owG4hrt^N-3+BnO>5JF(2DT$Xfly`JzbXkV+`U$T9+u=WQ3JcY zc?9#MbF~Lsy>V*~_RaKhwTH_eHVPlGg@0g4l>~ty27mth3@Cfv7{tbUIs+XEg`19zy`*077J9J?FeR@ zsYklaFuNIpkb&5+xG;B*jIvgFAe8#HR;i)U)2U3C%IS~vS5_m!Qxn-k^E<{vs|v3& zCD#ttc>f*^{FXVy?pL>V^PFrkEZ2zN3NT+P7E_DaDax z=2R*XM5!5`c+Tv#$J?~WaU2_!7;(@Qi+L6WU)%UuK{Bh@S7K0ZgcrA1(4203k>kM< z>$=0A7Y?{~8B-uw!Hc?Q>Duqven*|{8#CABxUKBI*xL~M?VR6n)Z%3puoS&3t4mla zYx{fCIV^bGN@eBw{gK=SQG@aeiWmLy6f!Iw4H~<~I<)RWXq1W1A{z6MmaeS~`q@D? z^0X*(m>A4iGW^&0xJQXLVup}HvfxMJeg+~Qu&GeNfpt7}hRe33n#ZQ)XBCZ{HLt^!I@s~*5aW!OD&>SC`Ox`Mtf)UGb3r(X`YbWIl@ z6-R?@oN}Yivw?Bt|5X@${UKq+s5R>n>0}FVY_s0}#%J{-@L+v>&6*W=n#SvX5g~R0 z0x@H;d9J0)7HxSsyIrC$LTAMaHOhy5r$aK=8b+Q zz!(cP-UH_%F0#-KM;w~|thDi1{82vbeOmT88$TkP?mvk)md!mS{^>9tdpJHlDOJ|NqFFW=FG+W2 z6hm6SjlRfRsJHVjLJP5>j--Q~IQQoIeR`GAu%A!Z>sIws`?jYlY9xc2*Sua4`01qN zm3xdKc@`2!N98SIGoO-nVw}x&n82a1l%y`N-h86oahmy?&Ur&h=C!lDMKz8P8fr!i z2x5I_TO;!%5Cw2>;+m~95HMS$%RFooNezT zt%ZjrHYS^d-lBZow-*^EE^zt?1xH}d+ ze5CGpLPukqNN4GK@JL5vrh*~~Ro|=}pBcY2UA%9E+N((I9w?K+yJMi03 z-pd2H$J_C-o>>h6H=%(pNwAok_J!q&`k{BJ8zWvaBwoVy#>r4ErmNU<(Kqe*QpFiA!HUD0cW4(p>4;Z^~4NbN44)cTjQx z`_}Bx`j~}tsMI$QwR(Sfw98}dA0!^+Ye@^$S;FstJ|KHsniM03DnzB!YuIV8YV3qGXuvG7qu zSM8TESnd>MhqsTIb^A=ayOSIK;_7!6fN#I{q{g6m>N$Lh`ua~r^FLZi8~Dlr-b6g3 z_!F`UkbB_7pQJY1I*diEqFTP@4_gp~LHW5P`AGiruML3jrYn4Z3?Z%rtYvdh0|b9k zeI$i8Z~@oGP7Vz;@D0>h7NM5uHVh!yaL zS9BoS9=4xhv|ciXzeVzyjb4T^)t_OunX-%i^|p97T3<#aA6;EdK?|kgroiiG8%<1{ zxK(bW>+sno47=g&GRH1wQzoM!8PtO{L37XB#neQR;GGWNuQ4CNIXo$xWoxoK6HVlJ zjgRj`umv+^zQzBvaQ@rHx!x73SlawjW_y7TOs2LuuR9)X&@+fXjW}^VT~VOIXve0q zaB40GK&FstD|*Y+_qe8NEI)S<3|{6f09(+L_Wa(o9j!`xfrhJ7lne7^4gYwROT&ak z&>gwb6_Q+8uO?i|1bowGVc=hcrwV2fTBE9d{Kjb~b|PFbrJ8~q zR~vVoskfcMLhF^*=ay%_RwMUp!cH6wUI8jZn9RTeKK21cZW{Z=6%UW zC^&#%F`6-^MYTxdv^T)QQwHy9jL_PvTfn2ehls#&7iE}oj09iAB5MOej5*%E-jo#M zrni8E#%VvIp>*rPC|4g$pAA_cK)Dd+NzW(sWZ@T+eSJy@k8LPxa3mx+n9r<7iF2%% zuG_oOi;o$!+TwFXUCM@LM2FP;lNKv7O%d8f2WFf+CpRTyuRDZH6}&$qeQKhQbQiMv zl;_w9`>S!@?VDkel+8vYCz8C_c>7$Hkb5q?S8U9DM$NSz#PIX~%89-T?I(=6-Q)X7 z!F*wdgj|H)fWl5}BZTd%&GUzq{acHvt5)1t+sBj)sn(dO0ztmvQ*D!3?3R?Bb5e-K z`N|JjslSYFx@0AUAD63ENSkA|8+0l)>~O zC5iBn#ESa{vvYAxsp>dvHS0&05oMDkGAc!mtBFN@rOXcrnvS2cF4Pq?KfY-HG^^Xn z{eCc%gqAREr&C7TC~3Z4(l8<45IvpB@m#aJUT+Ld(o5TYqy6h{_`u}*S_(DgGusAP zX!woH*j+d0cK~TjMRUGUeDl5`P?y}4u1_OBt7KV%;3hQHdy$l%E5>=vbBL`ysq4t= zYN}{#v1xjUMf;esiE5wq!ZkF0+gGsDPr6MlxjwG`x?XH} z=|y5uo7A1p5UiLkC=6J@JRMo;uYCm0btkuLrI3OcGA5s#6wR&60<4RH3oBIMg!A+o zvm8$wfEgi2iXgm)Uez0-lH_=O5?{o_Pnd9bHYl%?g6gn%X#jNGMs}hwoFe%aHBqFq zC_T2j8sYUKjj4x7(rW~xP0xyV8(=uPeicmq3X|3w@KsnXAcK{X6Z6tt&CBF_N`-MB z<{k$b9)tk8>i#&MPX}w@i|{0Jn$5Ow;J^-jE3ITyXK(TL6(F$+y*eGoP0>wL2ucq5 zh(G3_Z9ge5>DV4LwgEQJw@+FuI6hUi)AgU7m-lKs7s|8KD zeTOfcoAw5x^fh1=8Im2AP2ZZIlN@iF` z@vxV<@71xSOv>ScuGybRbE3+JZiNbC-iBt5AkL1ivKPXA( zqEc6f2n1l+HMe`Mcv*WBuf|^$GA~Ai^N`|=XzFzxf!UA%{m~#8&YAz|3ze$kIp(wS zbyYNdDbXh;$B0$OSx?I1yleGgxzJ3p0{dkYr5Urcb^Kk*$4xicETzSW-^B16p6z^Amcgt)|zq0^STF|T`9#}l61>cj(M=z}JSG6V>{jXB`dlu237jF$|(0n%k4pZSl)^ z9$P~XmWZdc1qzqLr&9?K<0b)9Sb47Q)N_ve-wJG`BXYCG8u|EU?C&?ETpgN;{{M$< z6Sa?G_g&!$G zdS^}*mqtWY-~1l`Sg}UFmDJPIPo}|YfU>?}`wMzOfd2VbGxRcRP`swYvfl%V?1$ zw(4c<@`sAA9yHdz_T5f1@Lug#ZY?oe91v36&W%%k)tie9GiQ)HOQPk|1cdj`dmg1T zVWA)T&5^J{59jkoKh?bG+Lj&?#Qs{Y`1mt__ZI!V?PJ?}1j-uv9oH&fZA&Ly;&Abq zbn9e^aaPF0{6h+}yx~}r%fs%pWrwBi|pRd+4qRbaRo86*A_@|jR(8F^DxDzSRA zjXCHE=XI)GpHM|cbk|Gaw@}Yc^d}V($fxGIIaMS5f$HFBji<}Sg)NAqLs)NO^FZ13 zkOVWbmtneKY!0_lI}j2%q`i#hZT?j4y*9}>${teI2MP@UZe&*^jPxZ`U+7_4@-^Klof}Pi87(zKuOKJ!d56)=7Ay zD~0ul(&M$CIRAYVs>0T23AB7lt8-gtE{sBUd7p_PmfcocR1gkw#dLU65Wk#O+X0kK z28i%_EWrvru|C2iRs6S|y#sLT<639dU{bZOmmZv<5E+{}GCwPs|mi3@e z#wj-UxFCL|f8>OrPCg=Xu&W8VK>xnJr(HCLtg(hkCyA2&^W5^g*lCg~U3 zTon7Gii|SjTt&stLr0h;GkgRRBba_jGX1pjc$=eUHzh&685C10Aaob()cYThBMU`& z>EG94vI*5@eQX@h_0-E}SN0A!Z8vso5w=;!-KNoj^;ZM06XVr1Ywj=fXdFX`?|k5l zeG`oXTU}7)&*sUIg3|r?YCyPHOHTAykBof_S%&LwcV8Xqi<{^ zz7E7sZ(oez1u-%1x<=B?l56(h>{PUhzf}Iy7S}Q_>g(EJyxqU6QDR$E{cGl^`5#bA zw~`O+=sDTo9`A=*!Uy4yqTH~^Yv`GCS{sSxzKjl~*~T3Q-2U9C zYTV^yJ5)^HPtvNVx#zkq{h;c&tT#jChaP<}4VIf$c=iDv&g=2K3DQ zn*tVHg`fbh8hoq_?Lex6nQ&-TRZ6;oYHN=yn99@p)+;4eg2H|!ZPdE98mBkK{{-QX z!#>Q#(z?aN%xq7YNv#`MxNB-=l|OLzO7e{8=q<(%KDg)T*0@~w^gTr40r*CzA@_C) zihH!f7B;4yKvFLtL!S&d2%9vV6tl-tDIHB@=^K^__Vg&FzKuk7*|eZLcIVV;5;a`@ zBz(z&t;m{+o-@Su>={2Srwk#7R@c_OX=eJgM>l!xVgISRqy5+7_SNKtZC&cU`K??K zu`RY`>C9{*GV1}(FlmD|ski)U8@0l?>enYasTbw+ogeYc3 zk#2cc5SbO=WhjDG62dcp92l7j*;nd5qyK>50L>USv-`r|tFKAu=1qQ}TI7wkC^vFm z-OfPf&H@@pb~w-aep>LX`2GKW6)Kzc1ho=5{a8x&g){1%i`zNaIvK!V@7F$|}v5Xd0lccq7D+KCdz*lxU z$erbu60s^=ev(nu50L6sfDftEG%KE|(Get=mGvnO`jlM{s~gmFwdf_{Z%UPmtc2v> zrY_6D>^$QSF5g3oIi`lQD0iPW)S&-<%`vvl=|Ds-7G8v}+#jLf|FG}FKOronhycIx z!C7*}EI-L<)-(QKe&^cocC3Y3%%>(w9TWwz#XKv}e+ZhM7gRi{P&iBox1LaJ`oVTT z^%5<Yf&Ycq~>4>}sWr=xT26{hi??w}XN00{tN;W$D`h#Ii0_GtMJR zPcC3Anm`_19Ya6DE}Ge(z^`72A_L~Ove5)}u-ezQTwvyu(bwqgP|alb-d7nmS?P(I z@fv+S+$OS{vR79|U)vydfn%kRuo7Xp0He>D4U!AHIxpJn&|r*u4B~JEXF|1^v&zK# za#-#b_S@DCn;%!diS#c$QJhu;-ITDm%AT+tyK^$Q`>~ACk#1Fv$Y<6SgA}jb zu}uio-1W{3QdxJEA_uxI54#eJ8Bw<7e9eDujA5GoNl}s`*v7f%)VSkvqOl~oKof)n z_BhP;E91XOoByezz|ZFM6_tk6r-M|a5a>wAlRsg|pXA%yw|pbbYK2m@@9IFjNnNW+ zeEqGgIE4e`g{@u{SK7V7ULUFAoUl!#R z`CGn{aTnDrTQ@w(Cm`D`U9q9LD6H%|f0^fdJGn;_;i!y%ovzzSJTC-9+L=$rit9c4 zaEb`2%foHt5eeh6~0*fus5_PcNi|q zvZ+2mY4+LCM08ogGax|E$j(p|Ch zl3goB5Q#Ilf*>!G!+~y!Ssh-3tQYRJFlsPN$e@JyGC_Xg7xxo*@q9sky)QsKvT~3zA6l0p3kzGG_)X}Nj8rN`>fAHgdQ2F5>Nc}vYEBue z4SNIIdeki#h~(R{?&10SPF96mk~vf8YRcR};3+F}bR#1Pi1qaRq&#A5+mU&4+%f&m z5Fj3JTcQ(7pASgbj!tF2XMXy+^%w*vWR7k!m-$!0PKFSf4=~R7pMTYM@ocI#?w}$6 z%~+pe0cF~!9zAj@{t6!jv<8%^UuOY%e$gcw9Dj#}b@FHq&OmD=PAuQXp~D@I)E`4G zem7^Rdi@M-^PD4-7pAkV1lOi1cXd>LH$pJjD+E>qe!BbY1=IzqyN}^Pw&~@ z(iBPTP~)MuuEb!06_uQkG2bs*B^qi?tOR#S{+CJ!p4FQ}Xpi{S<_G88Y7Iiqc}N=F zFkjSxGj}7|hvLO4GIhCz@`tM~7s2LQRTkf0e49ppNVfWXaD_W9hYn(8zjp*kI-tO9bQRbT9PNgxM>Gr;A-vPo{htCu05yrM z?O2{ET}3`Ovi2UUZB=1}DlqN*wn01mn!(%C8Xum2d~gr!S0iUfAqk1^tkHy&Ab<_Z z(aOuFWP`Lt(9Q8I3g|5m+Sa_nR9Yu=$9;wx0qS(}G3oLC2d<_aG zk!tH@^|0jp7TVV?qa)I*8#JZl7RYDar9*S1CKd#*sh-a8wXsz7MWdjBf1lIOI+>~D zk0zRE;LYb&foW$P8#~0uATIQG0+SQ(uhv8c6her>6`H#zTFD~Tn)let1E&0Pa8SQT zTgpXB^BG(G&TEm31<&NbnhpQgZiHsuD1O^D)S#w%QSl~30R!)8d9=x`(-xZVe^(xD zt@u=(te*LFHDJ7X1O5ULS(#^#aKa#5Fuk+OOxX5VtVf>$e;_`0)3AG4%Gy zbj9Cqe`4hgJJ!znlTRvCE))&}+~NZM)oSy2|HJ%(Z~A(w!OlNl_hB#&^+^20a>a83 z0QYY3=7ZR0H{SL9hFHeZI^mBqVKyQr6|JlnZfcpB{U9Y<7wLqrPgC$kj1#(l5%W?9 z85V>uId5s)!m58UG?RxFuVl&!?Tu0JVn%u?iFzMl5BU>0Zko(WmTul2WFmPyR`_1d z=cKr8@!^NY%aa*U-f)D`xXn8+)yUy9L4Rn&2>AvO~Pp-IAvPmb6 zbl1+VcUaA7H%^oBuuGLnnAN3H`ABgr6D)~kiI=^Kai?P|(kgx~Wi5Rw=foMfmZUe( zb%mIucl!^>KeOvL5J*K?y>j^PU-%!;Cjs$l$0n)Ek0<5wTQ7!nymoj~D$u&j8%0e# z9H&FpmI7R!=Eb|SrK{`}%qem4ndCw(s+7*-G)T&jKm()_0Lgj+v6YQ|Nm#jU00~d9 zn)uRjWw773i8#xJlM;*Qr7xq5tRpn=ODrJjQnaSWO$j;^4%vWdJfUSotj={kM+wN> z$b83#oR|HHvVr$vWGexd{?`#Ek!kEm^ctTZ8?~U}FO2hWUQa-BJPIo3F7o6buzwbu zeMaLtOtBX@QOdb46EKx`p}8tgUq?T`)g@+dXzo%+M4@Zhiat#y;2d2rPv|Ikxc`nk zf)ZfUeDM$0VDuWP@d?47jETA9xUf3+Ws_hwXR%Sdvv0*FDcYqJpURr-rv4eAnx}K^kb&*vTCe}3UF00 z#?MUc6GIS5W!n{0g{D-P*bwP>0`#Mdge^D>)gD4bzs}01s#!D~xeQQ{&=2`TKf1M( zAA@w%Hm%;3TU3`1V2%Ug=ch(|XES5F%8;r9UVnJIq*HSnW5V^-f+K(ZHM4@mikokL zRD4h!4h6+e`Wa0=t4^)a>RvpjtYM#}SmAj(CHik&5vyLPa5M~T7tC#+d1}6Nrv`^U zBQ;#nGW{_&C)o=rQW(p2qwZmWnbjMPgUIeu&E7oRL#|8_Ojc#B8@tLX>T=Fg>)-(( z=wi+~-Xt<>ggd#t21X~spvrYb3C#yKoh)psL@ydmKYjC@Z(7Vhv}>1*{L#|Oaclk~ z9^ZMVm`KA3QFBKV+PKK1?Pk9cA%(dwi(84EqWkB{0`78r%?IQ^XD$K+2Oe-_=97Zj z=s}^pxWYy-`!BOxj{r5IN^Q?p$%!;MikF1@0yHl2HJgw9UKWFvZRuz~`MV2Jh6c2? z$n=X1Z}AdtMxyNg1B%jjZTJVqs6y#BF``l3irwedhAaW+8%PrIWMpv3O95g(?>(Nj zRxfl5&z#-JBG+sL3m6s5lO<*0D#di`$oRY0%F#A6VmZk@ly@Y<^pV3mU!un42}jvd zv`aZtChP3z2&e-%uU=sU04e^R;=ETWtJ>-(QM|R_j8Rg*2Q3Be<)UMR+T8=-C5pS z=&UD^`)naX3_J14_k6?!5=5xSJ6RWdqfHnX-MAd;ZGy5AKNN)vZI_o$gw30Dg;L9! z=g-MjI_WHfeoM+i$3qC3m_-gAB@`|EXg+_yB33nVd^Is^^|@%KU#z8UsUbo+63{`5 zjZMG5+HtL5`u**PNA$=g?VO|d?7RkpMv&@`S9rJYEMeo&PvXaR$vB)<`*Ic?4Ey$x zKfz{^>q<+YI@V^=+Qj172Rn@8|Mh0Yh9|j3q3Q4+*370tgT5?QHESX4-vqkcNqTf) z#|yBv?hV@hx1)hm=}tV1+flwnYI;L;O`f5dAwjcn1+$E?=X|=UDTruUf{8J$(^K7@%xtZ{BNSTh_(xHqVc#mi8sT}&=0v^9>yOg=qLPp7V&?+(YWxTx#QsJW}$cC<@+ z8A~Jc0C!sU@Gp}&!6VS>pQz_I>blzX^*3cdB)bbU_y9Wb~U#6Y$2N|Z*7%;iI>N92(D^TpxRB~+w*!Tx>w(=2pr)*6)fzXEFVWSRLm|rf!H6@xNWCX)wFhuV`SG0{ME zw| z)3$HP37HE$iwKm;{SV00iFe>vv_eN~boFgcF8l%K_|owEI~zFnvt?DJQ(@sexf zd#^GnbPKS(V5UAdFtkaZ1IM9yb z6skj`_gvW!O8%4??S6gDq-TSc8jMaVPq9nFdb7xUqSvQ<|dVd=oB zocGct%}S>Ay#Y&LcF}5J!a8(;UB+S0pxa|4Q-0Tl6-Pxu`oWOeO3k>`w_@Y!Ep(e_o!YTx0ISFe736n^<{! zn2*vp+((y1iScIiwbBMvfGZ5EWo+NCr+ZXvTWdT#asOmr=lYpLQF~sX4_{nv{6*15 zP27a5UPz6Bma!fj1o7^+UnGh$<4V&iq*RdGqtGe&mQ0`xLEOSB6z`g$AUq*2Y1l zt-sszocYn%i?`YQR9p{|d&i2c{sr)<(Y%?dOx5{tTGFvW$4){W6(_${fi2YzqGV6N zI9OUe;wTrfNJo?>%vitJBG71v4leyvpXJCQ?lo$nvNlf@h?V<2m%y}5dYOO@co=T9 zpl*F{d-jVVQ}5WgXrznOsXZe`)rmf$;AW`>z#*??ms$m8|LC8>q}(t8>ZhBV=BK^+g1ll~79wdJ}z z+zOb0-GikjH<<~B<6Xs%H6+BIU3b8?1uy=2p?R*zk6K#w9w)O!6<(kLgvchCZAIT^ zi21Z$*tvXmO&Z3n_GU}6wfJdzoDD|N4YDngAEGxZ+`C=x6^f}{}TXb&ig_@hbNS|`HdZl+W zl+9&b>#VtQEDMzkt?(OpikT;7nH2m7)S*=WiQLz4;r7 z)Zu#+`T!*heXnz@n7P_q`LY-J%)yy%b7r5u#QR=GXS z;-TpDaFDOrul4-0qMuy%ep}ZGNf%N*W6wo*{XHj1aczPk#4viLG>sH2J0Y-^a8T|C zwNF~}6fa3Mb=Ispk=sfgCscL)vPt_@Y;-z!`^g$nrOX((t#@1dnb0&~8H$h&k+1m% zzt=xZWcWQy^JE3ZVJ-7#qmoG@pIVGN^@}^;hx$4(=8r*){_ggd4Ip)RWf*`?S>IAf zb)HRnf}jrY+t+Y6$`M0#wStN|!gL(}2!gM$@o}6(FNAdoZ8LBj$?LLY0RbjNM{ye^ z5sKite6e}r)#81_In7`-+9^2x!RNW@en#;K;lPXqY*_sO3W9ORd96LxUh?@kBvyVv zDO*FeWiZ?KLshe~^zh=xjR;jAF<5`BeC}dgpQ6l^F@B}IbJcvs|EQlS78_NOAV!1~ zdVRPgmILn6Ruv9;$$`!+b$E4=#VyQn7(w|0to%kbh~>g_QDM}j#{Bak4Vj<+d*OM8 ztE^Xz*Zi5yLPBq;3CTLg0bQ=8KvvLwd~+~Y%6LDMo z)Bk{46>0n7(kn7EVlLXptt)Q)5~1!;^*ka%f+5k&1wdqoCm)mW8;AlZ)?2P;)f42a zh^-WHM7T?WOyoAoViLYBMNW-Xo|3CyKI?vMVh$rR^c~YE57D+PoJ6RDUXODaK{F~5 zsr<1$jS~k@Czc?r&thp634xc%tGchRBJfHuMZ|P{;KXDj<_m7OFCk3Z$isJI8h+G% z#{9|q{{d~4oBaJG&~@0uC*U_<2+)vIk4GUCcQr90LeF$P116r$RR#yk@ib{{o)(DM z)++Sm0y45AN5%Hj+R>+uj9x6}(uO@H>R-O)?a+?eJW96jvTzE6_~L4>s>T6L8lnW=C} zy>ABrw=01i8h&&y*^fB?Aq^{VD}FNdqVpnux(a_UpRP}WPdj)A9pbET*F0MtJk1&f zt*RUv69E)oD{^*U7Zj3^$88N}nQc{XcXS;HO4~Hvzu#QJfiK5rGrN;UC&w2uqUTRC@zkIaM_=74$FH zG=hFrvZm>8J8!QT-Y)u3tFA_n3OZZEl8^Hi&s<&`wLYp!HaH|l@HRYcm+xqe(|-E_ zmKb@RKi#k$-t944TQK#|g4jliC-J5<;i9V4pP9b$hvmReani^MTZQ4~cvHa^Nv5rz zH`AMiUfS*Um;MA}wI}@{{n;?6wM*hxn-_r_RfevkLXo=wMK~bHS(R9Ucl!=9Bi2(r z_T{|H+TCf!#G*(Kq#rL=qQT^gpa^!1lc2*HKXuPL#^wnB)y;n~vQ$}g)6s?RfjZu` zSYONTX9Y_-dl@l(Tst`_fwsT0L;W|zGCw$BhiBCM8!dfxH6}E#xQbJ9766MPW`L0c zMz5)-bNi+QiPKUQR*?kh2=!tlRl@H?n}j8Mc$(B%^-AO4v(xA6N1Yc5h(*Y_4 z09xW~CeFXQS!bq&qG9`~_0}=_otI9JD!9gCJmu+btJKr5jYhvVhbt^ac8;CYTA(p! zP&t?xpieIG^!VTM`rOWkr2@w)^E-+LV=S}yUo!=4=X^VE~$o_hR*+^g(jBWmNb(!p+$V_6WSwe*l>0ZQCkw%g^P6 zu5Vey@FuOAx?ic@Gxo37qT=-&2*l1)Tl%R>mLbcBm)Lbwd>fgUC1=`GWL)INO4A!k zTtMaus8WdIF!uBp_`P4+gGWuwR%#?@r&3@NRFZ!7V98vY9z;)7te|hjo)3Qr{fYJA zfA)eN@A$#>gV*S%2XUw2NQ)Qk>EtNrx2=Fcj)bF1&}PO>kVZs>1od(JtC)9{y|rul}_=;}eJ28`TVn==}mRvN5&wqH|g3*Vh+|{{iV@AHJ$p?`}zL ztd4Ord27YK$}&gUW9Ifr?7p_pP7;=$491Zl$sW7Ga@Ko+IC6P>rYzUR9*ek}Qh3mr z_WuGeEzr{b(z9cZf+58=+xDmUjj8Gu_U);gWoXLL78<3!2))cN;NF>z>v%5Tznx#7st-%-^3c-<&Q~4TZeqJ$* z&yG)PX%`^p z@Tg>yDsj|P6tG-zROM2S&VIFAbIGS}^zTa%#EZ{yT@ltf;ya_2@=Y#YW<@(jj$t;CV z6#CZ_s@w(-xsD0P;apy|a-)93gkCovB+2F zSFxy-4FeX5v_r+Pu@^UaZ&p z>~Ns{+$u*M25XOUF`U!^Ixrar98-doR>!C0YhP7XwD9$npcr{in85m0U`?@ukHVo5 zW$JW`<^T>W(Eb_NyF~W@6%OTh9`)#Pl?#A#j-CfJ9~k80*FZ;_;RW6<`h z#_}+E&MBt+1He6MSgt(9^gfj#VYK3?U6ZtDy;kW(Dr0v(l;Xf(XRSyIp4j)GNTi%} zrQi+^G~6yT*mH_Oz{l%IMik zuL}dWa3h~zO6nfuup>N@dRH&2f|pKvo=>@~yp=g8pGv=K%Opntk`6wVCd?h9C)3)b zM8O5B%xW+Xa(K;Kh%P#e`&Gn|5aqiJ^HNI4ZT!Vmr6y8BcLeqM7BdnZIb?jBf9bN~Npl@M|{B6v{J<2K286 z@y~)^NV${b0S|v#{7N1n@NVeQ{5y0x#}a$j*q;U3+rs_Qq-TM6uXew4mC z@nlz*HdY0|QvQ34^ij=lHhwC%(lk#HYga5v%*}$ws9b<6=F9IE_+siwCYwN#S!80( zAd{&ioc^^!UlMpx5Bz{6Ip_qO(4HmmssdBPbFpr_q~rP4*M&9j4@<2_=P?9pWX-i-1Vww#kxA8)=dHt{?7iNTF}t7 zyF2|>)&Zfyk~4gzSnzAzEbmRFq%jUcuif;dn(WE}B#ibIN=t&xoy1fxAuH2xH3@P+ z;}qAKM+E%GwN{*xiWGh|$?6k>bvps~nDy*^D>_$J>UsW^O6L*ANIB-RuBXgx$rtBQ?(wHtzbI3H3NLc~=Y6dtU4@z!ueX2

?r25u*I2`ajX#-?~=~6E}`Sz}h!f%ATL&iuym1y1pf=DEK z*E=h6PfBj{o=-{<@Z2A*SrToG;Ck1kcw+F}cy7${03N>ek>bWIvB~@qwhd#O@F5B^ zGv75)?#aO1aauZ8iA98B7mIR^hd#Bd2aF(=Mpw3AqZlBLYUB9Q-ax7LM%cY@MQB?1 z=Ee{~u$9Lc&1hTr_Q@0nvno9S=NYYQPl!5Pn^WwpjE($N)p&2j_WJGIR<>}fTu9r#6f<}I`!&n+HCJ~Re|YU4x6g-GHbWdb-lZC_&ttm z(EK^BDP=}(AA%39b)e>!iYTU*3NlVUwdMEs{{Yy35HFx+DIMYhFnjf`TgCT>b1(#x zo@>nYTfZ?$2N*s3*BPr^0_A~iy(^W~)z>rpLXnHhV!-m{>89=fu$V++B-eL{JfIO(AKz8+^HP(_o3h+NzQXo ztY6+Z#YV~U^V*?h8@kg(WFEX#JxTnVc5--Lzn&|bXUM=momP{8eNAL)k{z2)2lcKl zMgiC!%z5>xRL9VcF-pxJKq!hp7y-ptGBFtG?kTb&G+;5o^)*?o&;#@56xN6Vklg#y zY*k$D=A)(Mz41x(rz;XiKaD9Y4JPA|4LQ#rT419dn8hLdD?-Jy2+8PbX}~ILEaN0% zkw!Dpl|jcmd(`l;7(MgH=Uqzt!FKlpuhO!$MI*mXwU{^@5NVsZJY(EajP3eWy9NSy znDA8ob<*z`Z2fB^S0SY=Ka2FNgO(oO(ywW!cJv%@59?Nm0G@d?hE~Vj$)_dG9aN5b zjB{F6a|UC;#wzRVXvtxWVvBPekus|1oEp>8q$bIs9kMI*Gxk>2*IW2ErF{G3wvnJ? z>4RTMct1>&QMg1bnAPIux{2d=J|(`-A#3|$g-ehV9I(CPpIaVZQ^Mh z;YU;HPX{%T;$0%!Skuf`0CgK!_BH(4=$7Bw8s?p7s!5ty_-CeweaJw=;m2saFJU}%600RF2XU2KCV4xnP9!akYEwMtDB=)NE zxgebM?kTG=z~_us=7V(hT4t+y>5wB3sUEHGSTT~3B97gu-e)+$%G1(LI3pF;d^3pM zc&ai~5gO;7*{^PUfL*^XDkC=B0#7u@lMl{0sHBvJJx8@d(h|dvP7OsQjkGo~?tLl; zmjmw}Ip(=bpm#VOGJ5yxRG$C?)7%Q7sY({#<>xKbeJc)matXoWsIz1ooCDgIuTj_5 zkPZRG>^ya7#Kt1UddUnl9zblMZ$A(;sn}p{Z z2lK6a#wiiUJan#hcFeyqz^JBj0KusuU_nv{?Z1aq3R48JydlloSKEMFWC z#;iw$Z0Da}O5X6EruuHF7NKiDlXq(%btk5C-oE$v74b&n;zxs;?)hR%%UL{`3}^3g z*1cxZ>oKSwoo-l4`$0MFgIz|OJM9HQgRcdFHKS zEG0vs0QaspTZi{g!2>c<2loN`E`E%u`2OyuPO5#2vc#7{% zirUr*OiukuoE%pazA4p4)fdtgUc~1$p`mN{($3LIbjplD;Nvyunq~crTLsP1qL4GW zxvtAez3{WbME5Y0*ttC`&^$dKfJ9L-p2}Gqav6!QPVm=<{vX|i#5QCeuG!+fSHqtU zeje(G3_5O^bm}=Oz?$VgJA5hdewE-IO8VDDwzj&ADUG7QIj@SwB$j93DPkAcR(`Xs z$D&xY(qw_@-nm%5DZ#XdwUjVCZKqs(REVfyX8AsctyOP}NsK*?1kgmg5z#ajQO|r^d|?K_Z62)^kQnWK>K_gOObBxgl7Xlfu_2kWP5| z8hWyvWarwZJ3!!56R16Xs(sr>UTU?$`Gy-O6%Z%X(TaRX7##OB?-gm;*@9+nKuiDV`G7u{V4FV+v%`E(2zr@VE)7k^bI#%3KfHLJq>kMI_QxX zaL6^U9O*KGtOzv*t8{F%>zjq^8W#M8MHR~Gx-Xp(3@3n<$GX?UgW|3H7uQQN7=#nN zOv5B+nr@Th&l&i$Qu`i+o9124tiz~LUsrqv@uT?D;q{!?L=8&Vv=>M?D&U^g==Dch zh|i>6qMR{_vGv7J@P4-*x8d7ckl?bYNj;cx{(iL{xvGT(E;g^~Yx&#wR~Mb)$ftF{ zwhYQo_pyrhDRm;MU-*|jeR|g~JkLH6#!DT?>T7`byR93YPQuX_1{jHBp685Wyz)V~ zjz~2SlL`PEsEmdgBc6W>*STgMxVX{c5ptr?4YD(b@pSYWM(x^!$GlDVcS$d3xQ>O;7XO*3HudP;gWC!}vEaQW_H7dvm z;Qmz{(-5EiYNL4Jqrn~Xnx{xl6(no94?l%SqmH;0AW~QQQWeSPk6}PTgYDjvqjdZ! zr9d9Tx1|>cD7<480(bSN5H^qJNwYbv*ot{b7oY?A3axpso#tAm7Liapj(swQ}H(QHtm8!uh3FILCU6rp`_C(mPAW5n z9(k)#$L25@z$XHor$)(jJU9Tk9Ok~G{f#wMPZjHWBRB5lwiBxRj12y@_AZqeDikhr zj2~+4G+jAmbzc3m-o0PJ9uSK1KwdD)xa9N|>UNeeSj4fz8lVHPt29$tf`UoF^)xF6 zP*mg|;+;i7cWD*$4GhR6jk>9(O{ZJUZXtM;qXTFd>0V9a{{R8~n0c{Bx&xkr*R^;@ ziS&Oa-N1~73^^UwHRn40qQtw9V4j}j*C=6WH6$glr#yZJzb@>bHt0K#@vlbsb*AsI zd7A<5B@2(I`Neu;KuYagouCdoty{F#qKZOZ9o!F;HELy+K?M(#oVi;aE2xwC$FmG?N_$T`|2Zrz85*gX2B#jjX4$hUZWgi_0a% z#ks6-GMLZT>s)ul`I_5Ff;^Ajw*Y@S^B*_o2RZhpOK^HrVY0c;=Sp0a%}SDIY-8^; z0rbs!e8p1U1RQORc>D!c-b;U;DhY=j^IX4(B-!R6Aok|Du%*8Wk|>B^NaCfLf~H3Y z6%;3)qO9bE=f8TXAqI?=BomyzG-2TY?<`$SKz*yAo~az#{_X!hPQN} z{`VO@G3izpT%RF}FCCZp)a$NXI;a~}N2Oi1)+P#mU`{%kw-&!T;G(8`=Cv&RM{^hw zFi(`^V05pxJ_P>HJ|FRigfxE?>RN@=lH7jqMIQhrCr{zEhMsS1op*tS63jy-zWpF zPpxO#NxtaF+#-J}@h=u$wdaZT3yA~mG7!Hn7|mk&>dlZ}nvDiHYE6NW5b*l0sHdi(s%URET#VEWQY0a89{v2QU{!R_DLtTSYNYPG~BLC4b-RNBZp98*oa zi;tH%pvLizL8fHqu4%?r?Mjj!=QS?>0GAlg(wZNfkbacl!G|A}HdB-p1}RS@<@S*O04e0{;8m!vVRD5@Ae>_rwQd6^YHK1zDlwFy zX4|y%J*v`!72IGDTRVQa2ED8HTGtMR@s~p`cBR()SqU9O4nJD`5z^o(I0{vOW1Ls0 zcpJm=>emsSn&dzeC$#_GdSpJk1?WenIG$UJwiUTya(cE zf!j!yMv05tXCR){;T|W__e{Q0l0n`1X1x8a{LMvhI0c4%a1DM`**OjQ!>q~Kzqk@ANppeH{{;`O;q@&ZREl7G+AveCkU#Ynd$M@npy znR<+S3P|udJXGZ559iveH{Ln=QAH>RZhLp8j+BSNCNut8WqH47L=17qwt~Kk-uN zzY?`;9pAdVkIj#s;Mb7;*j^#iw0o^))_WC_ZGPQ+yMUuA$_0EYYiJ>sNJ|ymak;yD z*VJAX{h>8Uej3<#QsT-cmc+*CcE_A8ev0@<#%ZtkOGni36YW;XBL(LKfyGIv>%ij= zv<|iN?z^j<3*!T9VzJz=ay#|nw{<;Pqz%1}52tZok-BmVJvkzcRaMx(0{~~!{P(CX zTO_L=m($v^JVWBEFAQ5mCRUP3jsi#5r&{M%;--j@h$PXZl$`EnQT`PzkHl>*>QzYe zh}51i$U1&i(b!%`s>#loZ476CSwS_@>DnfvKm2+}Z5ZjCV~@(ZeIwz1wW~8SwTmh0 zAJ)4a2jE7teD4sGONDsO=<}b_x@!;FJ6zQ6JhjxccsK_l9R78k@e}rF*M1)OX6IU; zRnu;*U?a>(VcON;d%%)yt&mvYkol6!r%@7@9S0WE?FDps72&~aY)xL*ykAUPjK=t*R?QPw^joJq&C~vuS#!` zgMfauI$P}n2Aa`v%WyqA(?m)FoZz2Ij#dkRIrgYW9P>v$rj#Z}>00nM=#!JhLv}xQ z$F4e7c1iT0ag5`ddOtkp6?aaUE$l@`4srFag`Iih1a{3~>P)6qz~ZVF+&BZ8X<|DA zk4kWT+jGV%S4DF7A$jB;^{&zqF`C1^C?{+W!l;JY`RF|>O4#p(QCMf7&%IuUM!Q!~ z?GtCXCyJElmunoG<&2g+2WofxBezk+0?8X5ryl0FEbJO6kTa<`?M}VWE_Dry4ACyq zALc%lW-kxlNgxd@8-dP48tr@^4x{0p5$RgEF*vfFZZ@kQn;hc5OTGwr%SiaEr8cLh zvxkI%JPa^Z>*)Ug4c?`dxKubEg1e0~#0w%Qe*NoeDbi31usNw}fz%Z>f3Y_V@p*~w zgM(g6`$Ks8Ujs>_IC)YVo$B6M<0IC*X5-_B*vPmUkV()+4&rOQ_#N@0_g?W$wwsvPy3wT-1IIlnPMeH*{aTKfOf)50D zHS_n5^e-hPp4%HRenlLL@#Tm^ew&*lmLrU0;Me5$hb&)EyR(5C8|3HNM_T$DO0!6< zrHxwzU@kj=F;3Lf$8Bz^Sy|Zb2e`#^o;AIFGs1VJSBU^p!`{3`Xt%PENcQbfI)(X3 z&%HyDpF!T5_}!7tDbYUn0B4@{pxRu!LlwgDf=~3S&ZS=fjMRb1Oy?{(>(rX{{{RTs ziysf#ry#yrf$nR193tdokxxJSngb8 z*Hd{Qd$}BOo-y>Qj^8$O#aFpR(Do`R`>Ta0Fnr|I>hVb4#;G^!4H zCJ&*dDo1KcD58oeqKYV@iYTP-j8dwT&>U3g3a~wTaro8q2~q(g9-fswwWucu z!#?$YP`2{pjfoip1cTPG?pc_kfN*oqy=rN9C16;jf0+*i_sw)L<6_`sA6oi{_Al3k zv|84h(hMJ&LFx!StLlh#Y(xsSN1&}6ZDk}~%>G_Kz22`yt*Zft=N)s_wJf#G`33UA zufgIoji=@3schs|E3EjF`wzz+AJv47wWj+#gbvk>`(wiXA<=w2b#EV-$vei(djh-) z_Y3L!4>Gg3}R0Z+21ww$+BB?+_*R=gPQxd_E-3ZHQ$7ATeQzC z`ecm*7CeFcHR=~S>63TOfsUl+yjSCm$+fxB^@y7cvn*}+bD!r|=GH~ctWrfT_?!j+ zA6op*MB1vq!Uh1W>#$^(RI|-9a9Df;t-MG~bRsEwH&vU&3M> z;Zz#>Tj2iy#Qh6a(wggD_?>N_UU9Y?m=^@sO&{$KdExCvzq7nY;r{>=+X6t7a-m_K zqZQ8lV*R79e1G8C^oG-I>|%CSmP>iWn}DyJB7L!dNp5qV^~HQty~dAeZyPd6B$;3s z@}t_k!zv^vY2E9e{=GOj+Esx7`kI7nE_f^lsiWivECBjdy*@D4Fsk8Mflv+z_NB=kF-e{=`qb95{hD-> zk`(mytuwC1#O5aF(2lzhi7^ukNwZFHm z4geCU$MFizy1z?_K1_gnkOY9c!imi`8!K>8@g7kJ|QUi7U76I65YKk z(0&)$U3^OMRgZ^s;}xyd&mLsNkhS^=`#gLnpWv;grjzR{sY!ZMdPfmOTxPx2&sQS_ z8yz}VL20j)U`w1H>s=<-nIqk1|Hluse9^ zkHWo2{uhUZd}X8fdf!LXe!+jCjq%80U{4~x%kU41pz)uC^!-v9qnOJW5*W@vanycw z(N~##QPiKqz9P`IL?u^$mgIK?*5HXeStugNckI&itv!s=AfQmr_W`#Bs;+mFJWH0N1ZS_`P!{*sSc4vF7~e2d)iyjl=mkQm3a{&ymX< z*n{(P zj-=ID{$|4Qo|R7eMo3R^Ju8d9ZPzdOaZm;*ayrtPGCB>Z0FpXW$siN_>LJpW zig_c+T936`vr?$d$p`eQQG%{>(wQSOE_&5A7;mi$!4*12TaIy4q-&AE#VaC*1Eo;u zMHEp*6j4PKQAHIj2pc?S6(hyYFhHh|7oOd5S0p8ZvYw+qol9v9^RXy6PDux)T)UU$ zU`Z{-K&$3PxKK_ynn_`3*d7NRt3yG(U$z*_jjX%|J^icK{t|pkhvA2b^y`UXGTP5@ zt8(AnQgL5!%c|MD>ot_3ds$RSg%41AREJdnjjBh~)YiJt^LEL#bI^*t4!nxs0y9^R zyg>k904tUFiL5lbDoBs>TgXFxU#W1K0d3;yJ*MH+CO!bAeQ1 zSc1tT9hg$vk4~AdpS7E|)O9P%w_?pCje-whz^vz(#!d(Tf!3l~43G04%B&>&6nxpF zTaxNQIn7NR`K{dXxS&G!rTwI9}UMF0T*_pdGZrFtaSBDWtfiavw2aYLyc>CWr`8O}difM<6jpGs1{Fys;MTV4>i`%j4M zmCjWd$&YX=xxJLjbs-?&vE)@^yb=!{m0oS65uPw}?OaZ+F6)4}!0V4ntpQOP86ckd zsZ_$b+{EUqJ0rGNG^w4Zr{hyD005uE(y7QnbIyLW(Sk8ZKl;@7QWO)OX}vu;r5?Y^ zg(E#b8V@J1_n-`qr9D&SIi@m|Jolvc?TQBimWn8%iYTIrD58o~IAKyS%7aoX6Bz@i zrCYa<$EGus84_bY)WLF2JBqnvvRuf(bX6=t?lD#&U{@gZ=xWpoWN(+K>IG@U%(xiO z^{+_%pS(!^75K&B%l1+u`$9&<`V)`jYxN2}PZPSS%I@8tV_KHl&eb@4e?wZ9y8CQU z?f8!Mvt_QZar?km*w*c@i5~zoo?bigSf3HKzwI9kOn05u>?T9*D^thXjgODM2eV&5 zE}~cxW>)9!k)P7PH9j$1X_9!p>Rky$8g#0qB3=L^gWkID+1ug`pTl1fM|-N+CZl6= zncpR8(rE7PqYHg@fC_Pc9tF^c*UeQq^~l>61>{{S7XTK@pSzI0&k zwv;&|u=T8M55&V)@W!8TWZdR3tZ%XJpIVOi(RI4udc+L!!X;o7dJkcW$rzeuehGH({Hw0E{SmXH==ayazGR3L5isF-a$QiaItfr^z70g_MBn%uV;9@Qjlp+F#K z(zpB{Znxefu|Nsf-ew1S_4yk&E^&@&9kRHp(x8!gk%QaQ(ws`P0SM@Goev(u@(00S40fY3bR}2h%fyX(ldF0EHjyh(moxSr}n#ISHZ6HJQ z5O_7k$mQQHjGszw53LyVpp4U${M7Q}80k^bMLe$P#yQ0xjfnL6)ysd~#6dlK3WGfT z>}S17SrIU;xQ_U!88`gd0-CXq*x*&98wPStG18~XuP2OCWEnhn@BJ$@@J~OD6j4PK zQAHF{MHEz)&$PEp5&G4l;lV#nY9>PG1JbnZ^!cZT21Z^~0vPtKeI_^4H2a#cr;xg#&}0 z`8?B?P%OE~1KZZ6iv9Q?^{WZ0Is(p1HaHcZu3sCyKI$9=5=3#y>x|XEgZ?gCpA1Q7 zCgbF07aqCqULo<*;bV9*89Ys=NN)7$UlPE_4HkM4UTN^&)?XHQ*2_sZ0?I4CxLo6p z@UOLe1>w6NhCUX*f(xTOuB{}pu;7q56=%VICDS}hqfX{u^ytB5a87b_f!o@)Exb^5 zpASkVUGF4M8y3$51O9VdCxJC-?IpXic2-#2vhL@K`D*wB5JhA2<0IOzyk&B9?+9N^ zpcVikpZ18r{{R~BFb9?w1x5!n$rK)P4tT{yv?~Kh+Xr3_J9MWj4WrW?Y0I~02atVg zNRM39s7I9i9{oF4vG{ToweaSb4%Sd(-_9#)B{DEy?;Z_!w~VeD^Tk?&oCQ}3KLLuw z@`-LZBi4^AB>I6@bgvKG>W*;UTwsOX-T9={Eh2*9p^3zTGC^-rD{H_}Y}X`}mne{* z^utw|Ze@z1p+ z%o_(7#YfcUfzpo&wLgCrA@rk>Wr3xxxk+t6%o zeEreenHO#tf;g<*tq$e;r3q8@Zb!-h?bfw?H+A+cYRXa@ zbaNwQ4uZc-^c``ow2KS3c1D)j8Hnz11#R2w3j0|YvL5wrEoqsEL9`5V)unH$vXH|E z>+4!@Yf@k^f1kcaX?1;M-D&g0;1w}2I3CpBggztw*!XdCr_b{QGTR?gF<%4t=;@kv zuc$PVOBzEok*Euvjf%n2d_iv@hB%$2iJ4J%0x)x5U;fXHsr)SXx1_;&a)upB$&{j< z*ap7GlUy=x!wcV literal 0 HcmV?d00001 diff --git a/tests/tests_stereo_vrefine.cpp b/tests/tests_stereo_vrefine.cpp new file mode 100644 index 0000000000..5c96322688 --- /dev/null +++ b/tests/tests_stereo_vrefine.cpp @@ -0,0 +1,547 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera (ADR-043) R2: the online vertical-alignment + * refinement (u_stereo_vrefine + the correction folded into + * u_stereo_rectify). + * + * - the correction math: map <-> inverse round trip, and a pair displaced by + * a known (A, B) lands on the same row once the correction is folded in; + * - the measurement recovers a known UNCALIBRATED offset + slope injected into + * the sim_display distorted fake; + * - CLOSED LOOP on the fake (render -> rectify -> measure -> fit -> rebuild): + * converges to <= 0.2 px residual, judged by the probe's own NCC matcher, + * not by the refiner's; + * - degenerate input (flat, uncorrelated noise, vertical stripes) never + * applies anything, and the clamps hold for any input; + * - REAL SR hardware: one raw frame from a Leia SR laptop + its calibration + * (fixtures/stereo_vrefine_sr_*): the refinement, run on the runtime's own + * rectification of that frame, brings the signed dy median of OpenCV SIFT + * matches (an independent reference) from ~+1.7 px to <= 0.5 px; + * - [.perf] (hidden): cost of one measurement. + */ + +#include "catch_amalgamated.hpp" + +#include "util/u_stereo_camera.h" +#include "util/u_stereo_rectify.h" +#include "util/u_stereo_vrefine.h" + +#include "sim_display_stereo_camera_pattern.h" +#include "stereo_vrefine_sr_fixture.h" + +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#endif +#define STB_IMAGE_STATIC +#define STB_IMAGE_IMPLEMENTATION +#define STBI_ONLY_JPEG +#include "stb_image.h" +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr double k_pi = 3.14159265358979323846; +const double k_ideal_fx_640 = 320.0 / std::tan(34.0 * k_pi / 180.0); +const uint32_t W = 640, H = 480; + +double +median(std::vector v) +{ + if (v.empty()) { + return 0.0; + } + std::sort(v.begin(), v.end()); + size_t n = v.size(); + return n % 2 ? v[n / 2] : 0.5 * (v[n / 2 - 1] + v[n / 2]); +} + +u_stereo_rectify_input +input_from_truth(const sim_stereo_camera_truth &t) +{ + u_stereo_rectify_input in; + std::memset(&in, 0, sizeof(in)); + in.width = t.eye_width; + in.height = t.eye_height; + for (int e = 0; e < 2; e++) { + in.eye[e].fx = t.fx[e]; + in.eye[e].fy = t.fy[e]; + in.eye[e].cx = t.cx[e]; + in.eye[e].cy = t.cy[e]; + in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + std::memcpy(in.eye[e].d, t.dist[e], sizeof(t.dist[e])); + } + std::memcpy(in.R, t.R, sizeof(in.R)); + std::memcpy(in.T, t.T_mm, sizeof(in.T)); + return in; +} + +//! Rectify @p raw with the correction (a px, b) — the way the service rebuilds. +struct rectifier +{ + u_stereo_rectify_input in; + u_stereo_rectify_result geo; + u_stereo_rectify_lut lut; + bool have_lut = false; + + bool + build(double a_px, double b) + { + if (have_lut) { + u_stereo_rectify_lut_fini(&lut); + have_lut = false; + } + // a is in pixels of the current output; the normalized offset uses its f. + double f = geo.f > 0.0 ? geo.f : 1.0; + in.v_offset = geo.f > 0.0 ? a_px / f : 0.0; + in.v_slope = b; + if (!u_stereo_rectify_compute(&in, &geo)) { + return false; + } + have_lut = u_stereo_rectify_lut_init(&lut, &geo, 1); + return have_lut; + } + void + apply(const uint8_t *raw, uint8_t *out) const + { + u_stereo_rectify_lut_apply(&lut, 1, raw, 2 * geo.width, out, 2 * geo.width); + } + ~rectifier() + { + if (have_lut) { + u_stereo_rectify_lut_fini(&lut); + } + } +}; + +//! The probe's matcher (u_stereo_camera_match_block) on a grid: signed dy of +//! every confident, unclamped block — an independent judge of the rows. +std::vector +probe_dy(const uint8_t *img, uint32_t eye_w, uint32_t h, uint32_t max_d) +{ + std::vector out; + const uint32_t B = 32; + for (uint32_t y = 48; y + B + 16 <= h; y += 24) { + for (uint32_t x = 96; x + B <= eye_w; x += 24) { + u_stereo_camera_block_match m; + if (!u_stereo_camera_match_block(img, 2 * eye_w, eye_w, h, x, y, B, B, max_d, 12, &m) || + m.dx_at_edge || m.dy_at_edge || m.ncc < 0.93f || m.dx < 0.5f) { + continue; + } + out.push_back(m.dy); + } + } + return out; +} + +struct fake +{ + sim_stereo_camera_distorted d; + fake(double dy_px, double slope) + { + REQUIRE(sim_stereo_camera_distorted_init(&d, W, H, k_ideal_fx_640, 50.0, 2.0, 0.6)); + sim_stereo_camera_distorted_set_vmisalign(&d, dy_px, slope); + } + ~fake() + { + sim_stereo_camera_distorted_fini(&d); + } +}; + +} // namespace + + +TEST_CASE("stereo vrefine: the correction folded into the rectification", "[stereo_vrefine]") +{ + fake fk(0.0, 0.0); + u_stereo_rectify_input in = input_from_truth(fk.d.truth); + u_stereo_rectify_result r0, r1; + REQUIRE(u_stereo_rectify_compute(&in, &r0)); + const double A = 3.1 / r0.f, B = -0.006; + in.v_offset = A; + in.v_slope = B; + REQUIRE(u_stereo_rectify_compute(&in, &r1)); + + // Still the rectified-pair contract: one pinhole, a pure +x baseline. + CHECK(r1.P[0][1][1] == r1.f); + CHECK(r1.P[1][1][2] == r1.P[0][1][2]); + CHECK(r1.P[1][0][3] == Catch::Approx(r1.f * r1.t_rect[0])); + CHECK(r1.v_offset == A); + CHECK(r1.v_slope == B); + + // map_point and rect_from_raw are inverses, correction included. + for (uint32_t e = 0; e < 2; e++) { + for (double v = 20; v < H - 20; v += 55) { + for (double u = 20; u < W - 20; u += 70) { + double x, y, u2, v2; + REQUIRE(u_stereo_rectify_map_point(&r1, e, u, v, &x, &y)); + REQUIRE(u_stereo_rectify_rect_from_raw(&r1, e, x, y, &u2, &v2)); + CHECK(u2 == Catch::Approx(u).margin(1e-6)); + CHECK(v2 == Catch::Approx(v).margin(1e-6)); + } + } + } + + // A pair whose UNCORRECTED rows differ by D = f (A + B y_n) lands on one row. + double worst = 0.0; + for (double v = 40; v < H - 40; v += 40) { + for (double u = 60; u < W - 60; u += 80) { + double yn = (v - r0.cy) / r0.f; + double vr = v + r0.f * (A + B * yn); + double lx, ly, rx, ry, ul, vl, ur, vr1; + REQUIRE(u_stereo_rectify_map_point(&r0, 0, u, v, &lx, &ly)); + REQUIRE(u_stereo_rectify_map_point(&r0, 1, u - 20.0, vr, &rx, &ry)); + REQUIRE(u_stereo_rectify_rect_from_raw(&r1, 0, lx, ly, &ul, &vl)); + REQUIRE(u_stereo_rectify_rect_from_raw(&r1, 1, rx, ry, &ur, &vr1)); + worst = std::max(worst, std::fabs(vr1 - vl)); + } + } + std::printf("[math] D = %.2f px + %.4f/px: corrected rows differ by at most %.4f px\n", A * r0.f, B, worst); + CHECK(worst < 0.02); +} + +TEST_CASE("stereo vrefine: measurement recovers an injected offset + slope", "[stereo_vrefine]") +{ + // The SR laptop's measured model, in the fake's virtual frame. + const double a_true = 2.6, b_true = -0.0047; + fake fk(a_true, b_true); + rectifier rc; + rc.in = input_from_truth(fk.d.truth); + std::memset(&rc.geo, 0, sizeof(rc.geo)); + REQUIRE(rc.build(0.0, 0.0)); + std::vector raw(2 * W * H), rect(2 * W * H); + sim_stereo_camera_render_gray_distorted(&fk.d, 7, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + + u_stereo_vrefine_measure_params mp; + u_stereo_vrefine_measure_defaults(&mp, 96); + std::vector s(512); + uint32_t n = u_stereo_vrefine_measure(rect.data(), 2 * W, W, H, &mp, s.data(), (uint32_t)s.size()); + u_stereo_vrefine_fit fit; + REQUIRE(u_stereo_vrefine_fit_samples(s.data(), n, rc.geo.cy, &fit)); + // The misalignment is defined in the virtual frame (ideal_fx); the + // rectified frame is that frame zoomed by f / ideal_fx about its centre. + const double k = rc.geo.f / fk.d.truth.ideal_fx; + const double a_exp = k * a_true, b_exp = b_true; + std::printf( + "[measure] %u matches (%u inliers): a %+.3f px (expect %+.3f), b %+.5f (expect %+.5f), sigma %.3f\n", n, + fit.inliers, fit.a, a_exp, fit.b, b_exp, fit.sigma); + CHECK(n >= 50); + CHECK(fit.slope_determined); + CHECK(fit.a == Catch::Approx(a_exp).margin(0.15)); + CHECK(fit.b == Catch::Approx(b_exp).margin(0.0008)); + for (uint32_t i = 0; i < n; i++) { + CHECK(s[i].d > 0.5f); + CHECK(s[i].ncc >= 0.9f); + } +} + +TEST_CASE("stereo vrefine: CLOSED LOOP on the fake converges to <= 0.2 px", "[stereo_vrefine]") +{ + const double a_true = 2.6, b_true = -0.0047; + fake fk(a_true, b_true); + rectifier rc; + rc.in = input_from_truth(fk.d.truth); + std::memset(&rc.geo, 0, sizeof(rc.geo)); + REQUIRE(rc.build(0.0, 0.0)); + std::vector raw(2 * W * H), rect(2 * W * H); + + u_stereo_vrefine_config cfg; + u_stereo_vrefine_config_defaults(&cfg, H, rc.geo.cy); + u_stereo_vrefine ref; + u_stereo_vrefine_init(&ref, &cfg); + u_stereo_vrefine_measure_params mp; + u_stereo_vrefine_measure_defaults(&mp, 96); + std::vector s(512); + + // Before: the probe sees the injected offset. + sim_stereo_camera_render_gray_distorted(&fk.d, 1, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + double before = median(probe_dy(rect.data(), W, H, 96)); + + // 60 frames at 30 Hz = 2 s of the settling phase. + int64_t now = 1000000000; + uint32_t updates = 0, measured = 0; + for (uint64_t frame = 0; frame < 60; frame++, now += 33333333) { + if (!u_stereo_vrefine_due(&ref, now)) { + continue; + } + sim_stereo_camera_render_gray_distorted(&fk.d, frame, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + uint32_t n = u_stereo_vrefine_measure(rect.data(), 2 * W, W, H, &mp, s.data(), (uint32_t)s.size()); + measured++; + if (u_stereo_vrefine_push(&ref, now, s.data(), n) == U_STEREO_VREFINE_UPDATED) { + updates++; + REQUIRE(rc.build(ref.a, ref.b)); + CHECK(std::fabs(ref.a) <= cfg.max_offset_px); + CHECK(std::fabs(ref.b) <= cfg.max_slope); + } + } + sim_stereo_camera_render_gray_distorted(&fk.d, 999, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + std::vector after = probe_dy(rect.data(), W, H, 96); + std::vector absd; + for (double v : after) { + absd.push_back(std::fabs(v)); + } + double med_after = median(after), abs_after = median(absd); + std::printf( + "[closed loop] %u frames measured, %u map rebuilds: applied a %+.3f px, b %+.5f; probe signed dy median " + "%+.3f -> %+.3f px (|dy| median %.3f, %zu blocks); initial window %+.3f px\n", + measured, updates, ref.a, ref.b, before, med_after, abs_after, after.size(), ref.initial_dy); + CHECK(before > 2.0); + CHECK(updates >= 1); + CHECK(after.size() > 40); + CHECK(std::fabs(med_after) <= 0.2); + CHECK(abs_after <= 0.25); + // Rows everywhere, not just at the centre: the slope was fitted. + CHECK(ref.b == Catch::Approx(b_true).margin(0.001)); + // Converged: further windows are STABLE (the deadband), no more rebuilds. + uint32_t late = 0; + for (uint64_t frame = 60; frame < 120; frame++, now += 33333333) { + if (!u_stereo_vrefine_due(&ref, now)) { + continue; + } + sim_stereo_camera_render_gray_distorted(&fk.d, frame, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + uint32_t n = u_stereo_vrefine_measure(rect.data(), 2 * W, W, H, &mp, s.data(), (uint32_t)s.size()); + if (u_stereo_vrefine_push(&ref, now, s.data(), n) == U_STEREO_VREFINE_UPDATED) { + late++; + REQUIRE(rc.build(ref.a, ref.b)); + } + } + CHECK(late == 0); + CHECK(std::fabs(ref.last_residual_dy) <= 0.2); +} + +TEST_CASE("stereo vrefine: degenerate frames never apply; clamps always hold", "[stereo_vrefine]") +{ + u_stereo_vrefine_config cfg; + u_stereo_vrefine_config_defaults(&cfg, H, 239.5); + u_stereo_vrefine_measure_params mp; + u_stereo_vrefine_measure_defaults(&mp, 96); + std::vector s(512); + std::vector img(2 * W * H); + + uint32_t lcg = 12345; + auto rnd = [&]() { + lcg = lcg * 1664525u + 1013904223u; + return (uint8_t)(lcg >> 24); + }; + for (int kind = 0; kind < 3; kind++) { + for (uint32_t y = 0; y < H; y++) { + for (uint32_t x = 0; x < 2 * W; x++) { + uint8_t v = 128; + if (kind == 1) { + v = rnd(); // uncorrelated noise between the eyes + } else if (kind == 2) { + v = ((x % W) / 7) % 2 ? 200 : 40; // vertical stripes: dy unobservable + } + img[y * 2 * W + x] = v; + } + } + u_stereo_vrefine ref; + u_stereo_vrefine_init(&ref, &cfg); + uint32_t total = 0; + int64_t now = 0; + for (int f = 0; f < 60; f++, now += 250000000) { + uint32_t n = + u_stereo_vrefine_measure(img.data(), 2 * W, W, H, &mp, s.data(), (uint32_t)s.size()); + total += n; + CHECK(u_stereo_vrefine_push(&ref, now, s.data(), n) != U_STEREO_VREFINE_UPDATED); + } + std::printf("[degenerate] kind %d: %u matches over 60 frames, applied %d\n", kind, total, ref.applied); + CHECK_FALSE(ref.applied); + CHECK(ref.a == 0.0); + CHECK(ref.b == 0.0); + } + + // Clamps: an absurd consistent residual saturates at the limits, never beyond. + u_stereo_vrefine ref; + u_stereo_vrefine_init(&ref, &cfg); + std::vector big(60); + for (size_t i = 0; i < big.size(); i++) { + big[i].x = 320; + big[i].y = (float)(20 + 7 * i); + big[i].d = 30; + big[i].ncc = 0.99f; + big[i].dy = (float)(15.0 + 0.05 * (big[i].y - 239.5)); + } + int64_t now = 0; + for (int w = 0; w < 10; w++) { + for (uint32_t f = 0; f < cfg.min_frames; f++, now += 250000000) { + u_stereo_vrefine_push(&ref, now, big.data(), 20); + } + CHECK(std::fabs(ref.a) <= cfg.max_offset_px); + CHECK(std::fabs(ref.b) <= cfg.max_slope); + } + CHECK(ref.a == cfg.max_offset_px); + CHECK(ref.b == cfg.max_slope); + + // Too few frames or matches: nothing applies. + u_stereo_vrefine_init(&ref, &cfg); + CHECK(u_stereo_vrefine_push(&ref, 0, big.data(), 60) == U_STEREO_VREFINE_ACCUMULATING); // 1 frame + CHECK(u_stereo_vrefine_push(&ref, 1, big.data(), 0) == U_STEREO_VREFINE_ACCUMULATING); // 2 frames + CHECK(u_stereo_vrefine_push(&ref, 2, big.data(), 0) == U_STEREO_VREFINE_UPDATED); // 3 frames, 60 matches + u_stereo_vrefine_init(&ref, &cfg); + for (int f = 0; f < 11; f++) { + CHECK(u_stereo_vrefine_push(&ref, f, big.data(), 4) == U_STEREO_VREFINE_ACCUMULATING); // 44 < 50 + } + // 12th frame: 48 < 50 matches when the window expires -> dropped, not applied. + CHECK(u_stereo_vrefine_push(&ref, 12, big.data(), 4) == U_STEREO_VREFINE_DROPPED); + CHECK_FALSE(ref.applied); +} + +TEST_CASE("stereo vrefine: duty cycle — settle fast, then every 30 s", "[stereo_vrefine]") +{ + u_stereo_vrefine_config cfg; + u_stereo_vrefine_config_defaults(&cfg, H, 239.5); + u_stereo_vrefine ref; + u_stereo_vrefine_init(&ref, &cfg); + std::vector ok(30); + for (size_t i = 0; i < ok.size(); i++) { + ok[i].x = 320; + ok[i].y = (float)(20 + 15 * i); + ok[i].d = 30; + ok[i].ncc = 0.99f; + ok[i].dy = (float)(0.01 * ((int)(i % 3) - 1)); // aligned rows: STABLE windows + } + const int64_t step = 16666667; // 60 Hz + uint32_t measured_fast = 0, measured_slow = 0; + for (int64_t t = 0; t < 65ll * 1000000000; t += step) { + if (u_stereo_vrefine_due(&ref, t)) { + u_stereo_vrefine_push(&ref, t, ok.data(), (uint32_t)ok.size()); + (t < cfg.fast_phase_ns ? measured_fast : measured_slow)++; + } + } + std::printf("[duty] measured %u frames in the first 5 s, %u in the next 60 s\n", measured_fast, measured_slow); + CHECK(measured_fast >= 15); + CHECK(measured_fast <= 21); + CHECK(measured_slow <= 3 * 3); // two or three windows of min_frames + CHECK_FALSE(ref.applied); +} + +TEST_CASE("stereo vrefine: REAL Leia SR frame — residual +1.7 px -> <= 0.5 px", "[stereo_vrefine]") +{ + int w = 0, h = 0, ch = 0; + uint8_t *raw = stbi_load(STEREO_FIXTURE_DIR "/stereo_vrefine_sr_raw.jpg", &w, &h, &ch, 1); + REQUIRE(raw != nullptr); + REQUIRE(w == 1280); + REQUIRE(h == 480); + + rectifier rc; + std::memset(&rc.in, 0, sizeof(rc.in)); + std::memset(&rc.geo, 0, sizeof(rc.geo)); + rc.in.width = 640; + rc.in.height = 480; + for (int e = 0; e < 2; e++) { + rc.in.eye[e].fx = k_sr_K[e][0]; + rc.in.eye[e].fy = k_sr_K[e][1]; + rc.in.eye[e].cx = k_sr_K[e][2]; + rc.in.eye[e].cy = k_sr_K[e][3]; + rc.in.eye[e].model = U_STEREO_RECTIFY_MODEL_RADTAN5; + for (int k = 0; k < 5; k++) { + rc.in.eye[e].d[k] = k_sr_D[e][k]; + } + } + std::memcpy(rc.in.R, k_sr_R, sizeof(rc.in.R)); + std::memcpy(rc.in.T, k_sr_T, sizeof(rc.in.T)); + REQUIRE(rc.build(0.0, 0.0)); + + auto sift_dy = [&](std::vector *ys) { + std::vector dys; + for (const stereo_vrefine_sr_match &m : k_sr_matches) { + double ul, vl, ur, vr; + if (u_stereo_rectify_rect_from_raw(&rc.geo, 0, m.lx, m.ly, &ul, &vl) && + u_stereo_rectify_rect_from_raw(&rc.geo, 1, m.rx, m.ry, &ur, &vr)) { + dys.push_back(vr - vl); + if (ys != nullptr) { + ys->push_back(vl); + } + } + } + return dys; + }; + std::vector dy0 = sift_dy(nullptr); + double before = median(dy0); + + std::vector rect(1280 * 480); + u_stereo_vrefine_config cfg; + u_stereo_vrefine_config_defaults(&cfg, 480, rc.geo.cy); + u_stereo_vrefine ref; + u_stereo_vrefine_init(&ref, &cfg); + u_stereo_vrefine_measure_params mp; + // A subject at 0.4 m on this 120 mm, f ~ 513 px pair: ~154 px. + u_stereo_vrefine_measure_defaults(&mp, (uint32_t)(rc.geo.f * rc.geo.baseline / 400.0)); + std::vector s(512); + uint32_t first_n = 0, updates = 0; + int64_t now = 0; + // One recorded frame stands in for the live stream (the service measures + // consecutive frames; the matcher is deterministic, so a repeated frame + // only repeats its matches). 30 frames = 7.5 s at the settling cadence. + for (int f = 0; f < 30; f++, now += cfg.fast_period_ns) { + rc.apply(raw, rect.data()); + uint32_t n = u_stereo_vrefine_measure(rect.data(), 1280, 640, 480, &mp, s.data(), (uint32_t)s.size()); + if (f == 0) { + first_n = n; + } + if (u_stereo_vrefine_push(&ref, now, s.data(), n) == U_STEREO_VREFINE_UPDATED) { + updates++; + REQUIRE(rc.build(ref.a, ref.b)); + } + } + std::vector dy1 = sift_dy(nullptr); + double after = median(dy1); + std::printf( + "[SR] %u refiner matches per frame, %u rebuilds: applied a %+.3f px, b %+.3f px/100 px; initial window " + "%+.3f px, last residual %+.3f px (%u inliers)\n" + "[SR] OpenCV SIFT reference (%zu matches): signed dy median %+.3f px (OpenCV's own rectification %+.3f) " + "-> %+.3f px\n", + first_n, updates, ref.a, ref.b * 100.0, ref.initial_dy, ref.last_residual_dy, ref.last_matches, dy1.size(), + before, k_sr_ocv_median_dy, after); + CHECK(before == Catch::Approx(k_sr_ocv_median_dy).margin(0.3)); // the runtime reproduces OpenCV + CHECK(before > 1.2); + CHECK(updates >= 1); + CHECK(std::fabs(after) <= 0.5); + CHECK(std::fabs(ref.a) <= cfg.max_offset_px); + CHECK(std::fabs(ref.b) <= cfg.max_slope); + stbi_image_free(raw); +} + +TEST_CASE("stereo vrefine: cost of one measurement", "[stereo_vrefine][.perf]") +{ + fake fk(2.6, -0.0047); + rectifier rc; + rc.in = input_from_truth(fk.d.truth); + std::memset(&rc.geo, 0, sizeof(rc.geo)); + REQUIRE(rc.build(0.0, 0.0)); + std::vector raw(2 * W * H), rect(2 * W * H); + sim_stereo_camera_render_gray_distorted(&fk.d, 3, raw.data(), 2 * W); + rc.apply(raw.data(), rect.data()); + u_stereo_vrefine_measure_params mp; + u_stereo_vrefine_measure_defaults(&mp, 154); + std::vector s(512); + const int N = 20; + uint32_t n = 0; + auto t0 = std::chrono::steady_clock::now(); + for (int i = 0; i < N; i++) { + n = u_stereo_vrefine_measure(rect.data(), 2 * W, W, H, &mp, s.data(), (uint32_t)s.size()); + } + double ms = std::chrono::duration(std::chrono::steady_clock::now() - t0).count() / N; + auto t1 = std::chrono::steady_clock::now(); + REQUIRE(rc.build(1.0, 0.001)); + double build_ms = std::chrono::duration(std::chrono::steady_clock::now() - t1).count(); + std::printf("[perf] measure %.2f ms per 1280x480 frame (%u matches, max disparity 154); map rebuild %.1f ms\n", + ms, n, build_ms); +} From 19552f11272e2db760dc6a2f2b042ebe0d618a8a Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 12:14:19 -0700 Subject: [PATCH 27/37] feat(stereo-camera): the service refines row alignment online; probe reports it A per-camera worker measures rectified frames the camera thread parks for it (every 250 ms for 5 s after an open or an update, then every 30 s; 2.7 ms per measurement at -O2, off the frame path), rebuilds geometry + LUTs when the correction moves, and the camera thread swaps them in between frames and bumps calibration_generation. WARN on first application, INFO after, re-verified on re-open, DXR_STEREO_CAMERA_REFINE=0 kill switch. Stream stats carry the state, applied a/b, match count and dy before/after; camera probe prints them. Co-Authored-By: Claude Opus 5.5 --- src/xrt/include/xrt/xrt_stereo_camera.h | 22 ++ src/xrt/ipc/server/ipc_server_stereo_camera.c | 300 ++++++++++++++++++ src/xrt/targets/cli/cli_cmd_camera.c | 30 ++ 3 files changed, 352 insertions(+) diff --git a/src/xrt/include/xrt/xrt_stereo_camera.h b/src/xrt/include/xrt/xrt_stereo_camera.h index 73fdcd7b57..6fa17a3ea0 100644 --- a/src/xrt/include/xrt/xrt_stereo_camera.h +++ b/src/xrt/include/xrt/xrt_stereo_camera.h @@ -157,6 +157,16 @@ struct xrt_stereo_camera_frame_info uint32_t reserved; }; +//! State of the service's online vertical-alignment refinement (R2). +enum xrt_stereo_camera_refine_state +{ + XRT_STEREO_CAMERA_REFINE_NONE = 0, //!< not a service-rectified RECTIFIED stream + XRT_STEREO_CAMERA_REFINE_OFF = 1, //!< DXR_STEREO_CAMERA_REFINE=0 + XRT_STEREO_CAMERA_REFINE_MEASURING = 2, //!< no window fitted yet + XRT_STEREO_CAMERA_REFINE_ALIGNED = 3, //!< fitted, the calibration was already within the deadband + XRT_STEREO_CAMERA_REFINE_APPLIED = 4, //!< a correction is folded into the maps +}; + struct xrt_stereo_camera_stream_stats { float source_frame_rate; @@ -165,6 +175,18 @@ struct xrt_stereo_camera_stream_stats uint64_t frames_skipped; uint64_t frames_acquired; uint64_t mean_latency_ns; + + // R2 refinement (diagnostics; the service's view of the rows). + uint32_t refine_state; //!< enum xrt_stereo_camera_refine_state + uint32_t refine_updates; + uint32_t refine_windows; + uint32_t refine_matches; //!< inliers of the latest fitted window + float refine_offset_px; //!< applied a + float refine_slope_per_100px; //!< applied b * 100 + float refine_initial_dy_px; //!< signed median dy of the first window (uncorrected) + float refine_residual_dy_px; //!< signed median dy of the latest window + float refine_measure_ms; //!< mean cost of one measurement (worker thread) + uint32_t reserved; }; /*! diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index b2d39bfcac..d3ca57ff22 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -40,6 +40,22 @@ * calibration (common f / principal point, rightFromLeft = pure +x baseline) * comes from the same geometry, so frames and numbers cannot disagree. * + * Online vertical-alignment refinement (R2) — a device's stored calibration + * can be slightly off for its frames (a Leia SR laptop: +1.7 px of row + * residual that OpenCV's own stereoRectify reproduces on the same frame). A + * per-camera WORKER thread measures the residual dy(y) = a + b (y - cy) on + * rectified frames the camera thread hands it (a copy of the luma, only when + * u_stereo_vrefine_due(): every 250 ms for 5 s after an open or an update, + * then one 3-frame window every 30 s; ~2.7 ms per measurement at -O2, i.e. + * < 0.4 ms per frame amortised at 30 Hz). When the fitted correction moves by + * more than 0.2 px it rebuilds the geometry + LUTs with the correction folded + * in (u_stereo_rectify_input::v_offset / v_slope; ~8 ms, on the worker) and + * parks them; the camera thread swaps them in between two frames under the + * lock and bumps calibration_generation, so the rectified calibration a + * consumer re-reads describes exactly the frames it gets. Clamped (|a| <= 6 + * px, |b| <= 2 px / 100 px), gated (>= 50 matches over >= 3 frames), + * re-verified on every re-open, and DXR_STEREO_CAMERA_REFINE=0 turns it off. + * * Transports — R1 implements SHARED_MEMORY only. The GPU transports are * designed to reuse this ring's state machine unchanged, only the slot storage * differs: @@ -77,6 +93,7 @@ #include "util/u_misc.h" #include "util/u_stereo_camera.h" #include "util/u_stereo_rectify.h" +#include "util/u_stereo_vrefine.h" #include #include @@ -93,6 +110,7 @@ DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_enabled, "DXR_STEREO_CAMERA", true) DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_dev_allow, "DXR_STEREO_CAMERA_DEV_ALLOW", false) +DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_refine, "DXR_STEREO_CAMERA_REFINE", true) #define MAX_STREAMS (XRT_STEREO_CAMERA_MAX_CAMERAS * XRT_STEREO_CAMERA_MAX_STREAMS_PER_CAMERA) #define LINGER_NS (2000ll * 1000 * 1000) @@ -100,6 +118,7 @@ DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_dev_allow, "DXR_STEREO_CAMERA_DEV_ALLOW #define SUSPEND_AFTER_NS (1000ll * 1000 * 1000) #define REOPEN_BACKOFF_NS (1000ll * 1000 * 1000) #define STATS_LOG_NS (5000ll * 1000 * 1000) +#define REFINE_SAMPLES 512 //!< matches one measurement may return /* @@ -239,6 +258,34 @@ struct scam_rectifier // stats uint64_t ns_sum; uint64_t count; + + /* + * Online vertical-alignment refinement. Everything below is guarded by the + * manager lock except where noted; the worker never touches geo / lut. + */ + bool refine; //!< enabled (DXR_STEREO_CAMERA_REFINE) + struct u_stereo_rectify_input in; //!< the RAW calibration, rebuilt with a correction + struct u_stereo_vrefine ref; //!< the controller + struct u_stereo_vrefine_measure_params mp; + struct ipc_server_stereo_camera *mgr; + uint64_t camera_id; + //! Luma of one rectified SBS frame. Written by the camera thread while + //! !job_pending, read by the worker while job_pending. + uint8_t *job_gray; + bool job_pending; + int64_t job_ns; + struct u_stereo_vrefine_sample *samples; //!< worker-only + //! A rebuilt geometry + LUTs the camera thread swaps in before its next frame. + bool pending_ready; + struct u_stereo_rectify_result pending_geo; + struct u_stereo_rectify_lut pending_lut[2]; + bool quit; + bool worker_started; + struct os_thread worker; + struct os_cond worker_cond; + uint64_t measure_ns_sum; + uint32_t measure_count; + bool stable_logged; }; struct scam_camera @@ -550,11 +597,36 @@ rectifier_destroy(struct scam_rectifier **rp) } u_stereo_rectify_lut_fini(&r->lut[0]); u_stereo_rectify_lut_fini(&r->lut[1]); + if (r->pending_ready) { + u_stereo_rectify_lut_fini(&r->pending_lut[0]); + u_stereo_rectify_lut_fini(&r->pending_lut[1]); + } + if (r->refine) { + os_cond_destroy(&r->worker_cond); + } + free(r->job_gray); + free(r->samples); free(r->buf); free(r); *rp = NULL; } +//! Stop and join the refinement worker (manager lock NOT held). +static void +rectifier_stop_worker(struct ipc_server_stereo_camera *m, struct scam_rectifier *r) +{ + if (r == NULL || !r->worker_started) { + return; + } + os_mutex_lock(&m->lock); + r->quit = true; + os_cond_signal(&r->worker_cond); + os_mutex_unlock(&m->lock); + os_thread_join(&r->worker); + os_thread_destroy(&r->worker); + r->worker_started = false; +} + /*! * Build the rectifier from the plug-in's RAW calibration: per-eye K + * distortion at image_width x image_height (rescaled to the frame's eye size @@ -600,6 +672,34 @@ rectifier_create(const struct scam_camera *cam) return NULL; } r->apply = rectify_apply_cpu; + r->in = in; + r->mgr = cam->mgr; + r->camera_id = cam->camera_id; + r->refine = debug_get_bool_option_stereo_camera_refine(); + if (r->refine) { + r->job_gray = malloc((size_t)2 * in.width * in.height); + r->samples = U_TYPED_ARRAY_CALLOC(struct u_stereo_vrefine_sample, REFINE_SAMPLES); + if (r->job_gray == NULL || r->samples == NULL) { + free(r->job_gray); + free(r->samples); + r->job_gray = NULL; + r->samples = NULL; + r->refine = false; + } + } + if (r->refine) { + os_cond_init(&r->worker_cond); + struct u_stereo_vrefine_config cfg; + u_stereo_vrefine_config_defaults(&cfg, in.height, r->geo.cy); + u_stereo_vrefine_init(&r->ref, &cfg); + // The disparity window: a subject at 0.3 m (the probe uses 0.4 m). + double dmax = r->geo.f * r->geo.baseline / 300.0; + dmax = dmax < 24.0 ? 24.0 : dmax > in.width / 2.0 ? in.width / 2.0 : dmax; + u_stereo_vrefine_measure_defaults(&r->mp, (uint32_t)dmax); + } else { + U_LOG_W("stereo camera %llu: vertical-alignment refinement OFF (DXR_STEREO_CAMERA_REFINE=0)", + (unsigned long long)cam->camera_id); + } if (r->geo.t_rect[0] > 0.0) { // Contract: SBS left half = the camera's LEFT lens, T.x < 0 (xrt_plugin.h). U_LOG_W( @@ -615,6 +715,176 @@ rectifier_create(const struct scam_camera *cam) return r; } +/*! + * The refinement worker: measures the frame the camera thread parked, feeds + * the controller and, when the correction moved, rebuilds geometry + LUTs off + * the frame path and parks them for the camera thread to swap in. + */ +static void * +refine_worker(void *ptr) +{ + struct scam_rectifier *r = (struct scam_rectifier *)ptr; + struct ipc_server_stereo_camera *m = r->mgr; + const uint32_t w = r->in.width, h = r->in.height; + + os_mutex_lock(&m->lock); + while (!r->quit && !m->shutting_down) { + if (!r->job_pending) { + os_cond_wait(&r->worker_cond, &m->lock); + continue; + } + const int64_t t_job = r->job_ns; + const struct u_stereo_vrefine_measure_params mp = r->mp; + os_mutex_unlock(&m->lock); + + int64_t t0 = os_monotonic_get_ns(); + uint32_t n = u_stereo_vrefine_measure(r->job_gray, 2 * w, w, h, &mp, r->samples, REFINE_SAMPLES); + int64_t t1 = os_monotonic_get_ns(); + + os_mutex_lock(&m->lock); + r->measure_ns_sum += (uint64_t)(t1 - t0); + r->measure_count++; + enum u_stereo_vrefine_result res = u_stereo_vrefine_push(&r->ref, t_job, r->samples, n); + if (res != U_STEREO_VREFINE_UPDATED) { + // One INFO when a correction is confirmed, then one per slow-cadence + // window (every 30 s) — never one per settling window. + bool settling = t_job - r->ref.phase_start_ns < r->ref.cfg.fast_phase_ns; + if (res == U_STEREO_VREFINE_STABLE && (!r->stable_logged || !settling)) { + r->stable_logged = true; + U_LOG_I( + "stereo camera %llu: row alignment check — residual %+.2f px over %u matches, " + "correction a %+.2f px b %+.3f px/100 px unchanged", + (unsigned long long)r->camera_id, r->ref.last_residual_dy, r->ref.last_matches, + r->ref.a, r->ref.b * 100.0); + } + r->job_pending = false; + continue; + } + r->stable_logged = false; + const double a = r->ref.a, b = r->ref.b, residual = r->ref.last_residual_dy; + const uint32_t matches = r->ref.last_matches, updates = r->ref.updates; + struct u_stereo_rectify_input in = r->in; + in.v_offset = a / r->geo.f; // a is in pixels of the frames currently delivered + in.v_slope = b; + os_mutex_unlock(&m->lock); + + int64_t b0 = os_monotonic_get_ns(); + struct u_stereo_rectify_result g; + struct u_stereo_rectify_lut lut[2]; + memset(lut, 0, sizeof(lut)); + bool ok = u_stereo_rectify_compute(&in, &g) && u_stereo_rectify_lut_init(&lut[0], &g, 1) && + u_stereo_rectify_lut_init(&lut[1], &g, 2); + double build_ms = (double)(os_monotonic_get_ns() - b0) * 1e-6; + + os_mutex_lock(&m->lock); + if (!ok) { + u_stereo_rectify_lut_fini(&lut[0]); + u_stereo_rectify_lut_fini(&lut[1]); + u_stereo_vrefine_revert(&r->ref); + U_LOG_W( + "stereo camera %llu: vertical correction a %+.2f px b %+.3f px/100 px rejected by the " + "rectifier — kept the previous maps", + (unsigned long long)r->camera_id, a, b * 100.0); + } else { + if (r->pending_ready) { + u_stereo_rectify_lut_fini(&r->pending_lut[0]); + u_stereo_rectify_lut_fini(&r->pending_lut[1]); + } + r->pending_geo = g; + r->pending_lut[0] = lut[0]; + r->pending_lut[1] = lut[1]; + r->pending_ready = true; + if (updates == 1) { + U_LOG_W( + "stereo camera %llu: vertical-alignment refinement APPLIED — the calibration left " + "the rows %+.2f px off (fit %+.2f px %+.3f px/100 px about the centre row, %u " + "matches); correction a %+.2f px, b %+.3f px/100 px, maps rebuilt in %.1f ms", + (unsigned long long)r->camera_id, r->ref.initial_dy, r->ref.initial_a, + r->ref.initial_b * 100.0, matches, a, b * 100.0, build_ms); + } else { + U_LOG_I( + "stereo camera %llu: vertical correction updated — residual %+.2f px over %u " + "matches; now a %+.2f px, b %+.3f px/100 px (update %u, %.1f ms)", + (unsigned long long)r->camera_id, residual, matches, a, b * 100.0, updates, + build_ms); + } + } + r->job_pending = false; + } + os_mutex_unlock(&m->lock); + return NULL; +} + +/*! + * Camera thread, lock held, before rectifying a frame: swap in maps the worker + * rebuilt. The old LUTs are only ever read by this thread, so they can go now. + */ +static void +rectifier_swap_pending_locked(struct scam_camera *cam) +{ + struct scam_rectifier *r = cam->rect; + if (r == NULL || !r->pending_ready) { + return; + } + u_stereo_rectify_lut_fini(&r->lut[0]); + u_stereo_rectify_lut_fini(&r->lut[1]); + r->geo = r->pending_geo; + r->lut[0] = r->pending_lut[0]; + r->lut[1] = r->pending_lut[1]; + r->pending_ready = false; + // Consumers re-read the rectified calibration (f / principal point move + // by the alpha = 0 re-crop) on the next frame's generation change. + cam->calibration_generation++; +} + +/*! + * Camera thread, lock held: should this rectified frame be measured? Starts + * the worker on first use. + */ +static bool +rectifier_want_sample_locked(struct scam_rectifier *r, int64_t now) +{ + if (r == NULL || !r->refine || r->job_pending || r->pending_ready || r->quit) { + return false; + } + if (!u_stereo_vrefine_due(&r->ref, now)) { + return false; + } + if (!r->worker_started) { + if (os_thread_init(&r->worker) != 0 || os_thread_start(&r->worker, refine_worker, r) != 0) { + U_LOG_W("stereo camera %llu: refinement worker failed to start — refinement OFF", + (unsigned long long)r->camera_id); + r->refine = false; + return false; + } + r->worker_started = true; + } + return true; +} + +//! Camera thread, lock NOT held: copy the luma of rectified frame @p rf. +static bool +rectifier_copy_luma(struct scam_rectifier *r, const struct xrt_plugin_stereo_camera_frame *rf) +{ + const uint32_t W = 2 * r->in.width, H = r->in.height; + if (rf->width != W || rf->height != H || rf->planes[0] == NULL) { + return false; + } + for (uint32_t y = 0; y < H; y++) { + const uint8_t *src = rf->planes[0] + (size_t)y * rf->pitches[0]; + uint8_t *dst = r->job_gray + (size_t)y * W; + if (rf->format == XRT_PLUGIN_STEREO_CAMERA_FORMAT_BGRA8) { + for (uint32_t x = 0; x < W; x++) { + const uint8_t *p = src + 4 * (size_t)x; + dst[x] = (uint8_t)((p[0] + 2 * p[1] + p[2] + 2) >> 2); // B + 2G + R: luma enough + } + } else { + memcpy(dst, src, W); // GRAY8, or NV12's Y plane + } + } + return true; +} + //! Does any started stream on @p cam want the service's rectified image? static bool camera_wants_rectified_locked(struct ipc_server_stereo_camera *m, const struct scam_camera *cam) @@ -886,6 +1156,10 @@ camera_thread(void *ptr) cam->src = src; cam->last_frame_ns = 0; u_stereo_camera_rate_meter_restart(&cam->rate_meter); + if (cam->rect != NULL && cam->rect->refine) { + // Keep the correction, re-verify it at the settling cadence. + u_stereo_vrefine_restart(&cam->rect->ref, os_monotonic_get_ns()); + } U_LOG_W("stereo camera %llu: source opened", (unsigned long long)cam->camera_id); set_state_locked(cam, XRT_STEREO_CAMERA_STATE_WAITING); } @@ -906,9 +1180,17 @@ camera_thread(void *ptr) const struct xrt_plugin_stereo_camera_frame *rfp = NULL; if (camera_wants_rectified_locked(m, cam)) { struct scam_rectifier *rect = cam->rect; + rectifier_swap_pending_locked(cam); + bool sample = rectifier_want_sample_locked(rect, now); os_mutex_unlock(&m->lock); bool ok = rect->apply(rect, &f, &rf); + bool copied = ok && sample && rectifier_copy_luma(rect, &rf); os_mutex_lock(&m->lock); + if (copied) { + rect->job_ns = f.time_ns > 0 ? f.time_ns : now; + rect->job_pending = true; + os_cond_signal(&rect->worker_cond); + } now = os_monotonic_get_ns(); rfp = ok ? &rf : NULL; } @@ -1062,6 +1344,7 @@ ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr) os_thread_destroy(&m->cams[i].thread); } os_cond_destroy(&m->cams[i].cond); + rectifier_stop_worker(m, m->cams[i].rect); rectifier_destroy(&m->cams[i].rect); } for (uint32_t i = 0; i < MAX_STREAMS; i++) { @@ -1612,6 +1895,23 @@ ipc_handle_stereo_camera_stream_stats(volatile struct ipc_client_state *ics, out_stats->frames_skipped = s->ring.skipped; out_stats->frames_acquired = s->ring.acquired; out_stats->mean_latency_ns = s->latency_count > 0 ? s->latency_sum_ns / s->latency_count : 0; + const struct scam_rectifier *r = m->cams[s->camera].rect; + if (r != NULL && s->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { + const struct u_stereo_vrefine *ref = &r->ref; + out_stats->refine_state = !r->refine ? XRT_STEREO_CAMERA_REFINE_OFF + : ref->applied ? XRT_STEREO_CAMERA_REFINE_APPLIED + : ref->windows > 0 ? XRT_STEREO_CAMERA_REFINE_ALIGNED + : XRT_STEREO_CAMERA_REFINE_MEASURING; + out_stats->refine_updates = ref->updates; + out_stats->refine_windows = ref->windows; + out_stats->refine_matches = ref->last_matches; + out_stats->refine_offset_px = (float)ref->a; + out_stats->refine_slope_per_100px = (float)(ref->b * 100.0); + out_stats->refine_initial_dy_px = (float)ref->initial_dy; + out_stats->refine_residual_dy_px = (float)ref->last_residual_dy; + out_stats->refine_measure_ms = + r->measure_count > 0 ? (float)((double)r->measure_ns_sum / r->measure_count * 1e-6) : 0.0f; + } os_mutex_unlock(&m->lock); return XRT_SUCCESS; } diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index d10ac11c78..2755a72bbd 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -34,6 +34,14 @@ * (SIM_DISPLAY_FAKE_STEREO_CAMERA_DISTORT=1) the depths read back 2.00 m * (background) and 0.60 m (bar); `--raw` shows the misalignment it fixes. * + * The service stats line is followed by the ONLINE VERTICAL REFINEMENT state + * (R2): applied correction a / b, match count, and the signed dy of its first + * (uncorrected) window next to its latest one and this probe's own last-frame + * measurement. It settles in ~1 s after a stream starts, so give the probe a + * few seconds (the default 90 frames is ~3 s at 30 Hz). On the fake, inject a + * calibration error with SIM_DISPLAY_FAKE_STEREO_CAMERA_DY=2.6 and + * _DY_SLOPE=-0.47 (px per 100 px) next to _DISTORT=1. + * * Connects as a DIAG client, like `clients`: on Windows run it NON-elevated. */ @@ -570,6 +578,8 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) bool require_rectified = false; uint32_t max_d_arg = 0; // 0 = from the camera's geometry struct row_params rp = {.max_d = 0, .max_dy = 24, .min_ncc = 0.90f}; + char rows_txt[64] = ""; + bool have_rows = false; for (int i = 3; i < argc; i++) { if (strcmp(argv[i], "--raw") == 0) { output = XRT_STEREO_CAMERA_OUTPUT_RAW; @@ -780,6 +790,9 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) ra.mode_dx[k], ra.mode_count[k]); } } + snprintf(rows_txt, sizeof(rows_txt), "%+.2f px (%d blocks)", ra.dy_signed_median, + ra.blocks); + have_rows = true; // A constant vertical offset fails even when |dy| scatter is small. float sm = ra.dy_signed_median < 0 ? -ra.dy_signed_median : ra.dy_signed_median; rows_ok = ra.dy_median <= 0.5f && sm <= 0.5f; @@ -824,6 +837,23 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) st.source_frame_rate, st.delivered_frame_rate, (unsigned long long)st.frames_published, (unsigned long long)st.frames_skipped, (unsigned long long)st.frames_acquired, (double)st.mean_latency_ns * 1e-6); + // R2: the service's online vertical-alignment refinement. + static const char *const rs[] = {"n/a", "OFF (DXR_STEREO_CAMERA_REFINE=0)", "MEASURING", + "ALIGNED (calibration within 0.2 px)", "APPLIED"}; + if (st.refine_state != XRT_STEREO_CAMERA_REFINE_NONE && + st.refine_state <= XRT_STEREO_CAMERA_REFINE_APPLIED) { + printf( + "vertical refinement: %s — correction a %+.2f px, b %+.3f px/100 px; %u update(s), " + "%u window(s), %u matches in the latest, %.2f ms per measurement\n", + rs[st.refine_state], st.refine_offset_px, st.refine_slope_per_100px, st.refine_updates, + st.refine_windows, st.refine_matches, st.refine_measure_ms); + if (st.refine_windows > 0) { + printf( + "vertical refinement: signed dy BEFORE (first window, uncorrected) %+.2f px -> " + "latest window %+.2f px; this probe's last frame %s\n", + st.refine_initial_dy_px, st.refine_residual_dy_px, have_rows ? rows_txt : "n/a"); + } + } } unmap_readonly(&map); From f3322ba9a4660a1ba0d6b12b6e334d9e4e327c68 Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 12:14:19 -0700 Subject: [PATCH 28/37] docs(stereo-camera): R2 online vertical-alignment refinement as built Co-Authored-By: Claude Opus 5.5 --- docs/roadmap/stereo-camera-source.md | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/roadmap/stereo-camera-source.md b/docs/roadmap/stereo-camera-source.md index 2b38111571..859e3fbe4d 100644 --- a/docs/roadmap/stereo-camera-source.md +++ b/docs/roadmap/stereo-camera-source.md @@ -262,8 +262,32 @@ against the sim fake). source; **budget: ≤ 2 ms per frame on the camera thread**, above which the rows split across threads (`u_stereo_rectify_lut_apply_rows`) or the GPU backend takes over. Setup (geometry + both LUTs) 9–18 ms, once per calibration; LUT memory 6 bytes per output sample (3.7 MB + 0.9 MB). -- **Not in R2:** the GPU backend (seam in place), per-frame recalibration (`calibrationGeneration` - stays 0), and a rectifier for a camera whose calibration changes at runtime. +- **Online vertical-alignment refinement** (`auxiliary/util/u_stereo_vrefine.{h,c}` + the service + worker). Why: on a Leia SR laptop the rectified rows were still +1.71 px apart, and OpenCV's own + `stereoRectify` on the same frame and calibration gave the same +1.71 px — the stored calibration + is slightly off for the frames, not the rectifier. Model `dy = a + b·(y − cy)` (right − left, + output px); the x (roll) term measured 0.04 px / 100 px on that data and is left out. A worker + measures rectified frames the camera thread hands it (Shi-Tomasi patches on a 2× downscale, NCC + over d × dy, full-res NCC refinement ±2 px with parabolic sub-pixel; gated on NCC ≥ 0.9, d > 0, + unclamped peaks, real vertical curvature), fits robustly (median start, Tukey IRLS; the slope + only when rows spread ≥ 40 px), and when the correction moves by > 0.2 px rebuilds the maps with + it folded into the geometry: `v_offset = a/f`, `v_slope = b`, split symmetrically as a per-eye + vertical affine in normalized rectified coordinates **before** the principal point and the + alpha = 0 crop are computed, so P1/P2 describe the corrected frames (still one pinhole, pure +x + baseline) and `calibrationGeneration` bumps on the swap. Duty cycle: every 250 ms for 5 s after + an open or an update, then one ≥ 3-frame window every 30 s; 2.7 ms per measurement at -O2 on the + worker (< 0.4 ms per frame amortised at 30 Hz), 8 ms per rebuild. Safety: |a| ≤ 6 px, + |b| ≤ 2 px / 100 px, ≥ 50 matches over ≥ 3 frames, re-verified on every re-open, WARN on first + application and INFO after, `DXR_STEREO_CAMERA_REFINE=0` kill switch. Fake: + `SIM_DISPLAY_FAKE_STEREO_CAMERA_DY` / `_DY_SLOPE` (px, px per 100 px) inject an uncalibrated + misalignment. `probe` prints the state, the applied a/b and dy before → after. + Tests (`tests_stereo_vrefine`): the fake with the SR model (+2.6 px, −0.47 px / 100 px) converges + to +0.000 px (probe matcher, 338 blocks) in one rebuild; flat / uncorrelated / vertical-stripe + frames never apply; clamps hold. **Real SR frame** (`tests/fixtures/stereo_vrefine_sr_raw.jpg`, + 62 KB, + the device calibration + 63 OpenCV SIFT matches as an independent reference): SIFT + signed dy median +1.715 px → **+0.075 px**. +- **Not in R2:** the GPU backend (seam in place), and a rectifier for a camera whose calibration + changes at runtime (the refinement's reset hook is there for it). ## F. Risks From 54e8402e0f4968667a5f38787bdbaf803538465c Mon Sep 17 00:00:00 2001 From: David Date: Sun, 27 Sep 2026 13:20:23 -0700 Subject: [PATCH 29/37] fix(cli): probe row gate = |signed median dy| + MAD, not the unsigned |dy| median On the SR laptop with a face at 0.6 m the refined rows were aligned (signed median -0.10/-0.18 px) but --rectified FAILED: block-match noise alone put the unsigned |dy| median at 1.03/1.10 px. Gate on |signed median| <= 0.5 px, MAD about it <= 1.5 px and >= 20 blocks; print the gate numbers; keep |dy| stats informational. Co-Authored-By: Claude Opus 5.5 --- src/xrt/targets/cli/cli_cmd_camera.c | 30 +++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index 2755a72bbd..cf5b71986c 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -23,9 +23,10 @@ * rectified pair must bring to ~0 (median / p90 / max |dy|) and the dominant * horizontal disparities, each converted to a depth Z = f * B / d with the * RECTIFIED calibration. `--rectified` insists on RECTIFIED output and fails - * (exit 5) if the frames came back RAW-flagged or the rows do not align - * (median |dy| > 0.5 px, or a SIGNED median dy beyond +-0.5 px: a constant - * vertical offset). Matching is zero-mean NCC with sub-pixel refinement; the + * (exit 5) if the frames came back RAW-flagged or the rows do not align: + * |SIGNED median dy| > 0.5 px (a constant vertical offset), a robust spread + * (MAD about that median) > 1.5 px, or fewer than 20 accepted blocks. The + * unsigned |dy| statistics are printed but not gated (block-match noise). Matching is zero-mean NCC with sub-pixel refinement; the * disparity window comes from the camera's own f * B (a subject at 0.4 m), * the vertical one is +-24 px, blocks under NCC 0.90 or peaking ON either * bound are counted and excluded (a bound hit is a clamp, not a measurement — @@ -452,7 +453,8 @@ struct row_alignment int dx_edge; //!< rejected: peak ON the disparity bound (a clamp, not a measurement) int dy_edge; //!< rejected: peak ON the vertical bound float dy_signed_median; //!< right y - left y, px (a constant vertical offset shows here) - float dy_median, dy_p90; //!< |dy|, px + float dy_median, dy_p90; //!< |dy|, px (informational: block-match noise on a face scene inflates these) + float dy_mad; //!< median |dy - signed median|, px: the robust spread the gate uses int outliers; //!< accepted blocks with |dy| > 1 px int modes; float mode_dx[2]; @@ -537,6 +539,13 @@ measure_rows(const uint8_t *gray, qsort(ady, (size_t)n, sizeof(float), cmp_float); out->dy_median = ady[n / 2]; out->dy_p90 = ady[(n * 9) / 10]; + // Robust spread about the signed median (MAD). ady[] is free again once its stats are read. + for (int i = 0; i < n; i++) { + float r = sdy[i] - out->dy_signed_median; + ady[i] = r < 0 ? -r : r; + } + qsort(ady, (size_t)n, sizeof(float), cmp_float); + out->dy_mad = ady[n / 2]; for (int i = 0; i < n; i++) { out->outliers += ady[i] > 1.0f; } @@ -794,8 +803,15 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) ra.blocks); have_rows = true; // A constant vertical offset fails even when |dy| scatter is small. + // The gate is ROBUST: |signed median| <= 0.5 px (the rows' actual offset) with a + // bounded MAD spread and enough blocks. The unsigned |dy| median is NOT gated: on a + // face scene block-match noise alone put it at ~1.0 px while the signed median was + // -0.1 px (SR laptop, 2026-09-27). float sm = ra.dy_signed_median < 0 ? -ra.dy_signed_median : ra.dy_signed_median; - rows_ok = ra.dy_median <= 0.5f && sm <= 0.5f; + rows_ok = sm <= 0.5f && ra.dy_mad <= 1.5f && ra.blocks >= 20; + printf(" row gate: |signed median| %.3f px (<= 0.5), MAD %.3f px (<= 1.5), %d blocks " + "(>= 20)\n", + sm, ra.dy_mad, ra.blocks); } } if (out_dir != NULL) { @@ -870,8 +886,8 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) } if (!rows_ok) { printf( - "--rectified: FAIL — rows do not align (median |dy| or |signed median dy| > 0.5 px, or too " - "few matched blocks)\n"); + "--rectified: FAIL — rows do not align (|signed median dy| > 0.5 px, MAD > 1.5 px, or fewer " + "than 20 matched blocks)\n"); return 5; } printf("--rectified: PASS — rows aligned\n"); From 6b9df2d002bfdc6a161101e5dfa58dc4ca28f995 Mon Sep 17 00:00:00 2001 From: dfattal Date: Sun, 27 Sep 2026 15:18:47 -0700 Subject: [PATCH 30/37] fix(ipc): PRESENT_OWNER quota counts owners (executables), not connections On the SR laptop the B1 browser's "3D Camera (DisplayXR)" never appeared: [CAP] refusing client pid=... ('ChromiumStereoCamera'): class PRESENT_OWNER quota 2/2 exhausted (#960). One browser already takes both slots (GPU process weaves, browser process binds the window), so its video-capture utility - the XR_DXR_stereo_camera client, which declares PRESENT_OWNER by enabling XR_DXR_weave - was the third connection and was refused (xrCreateInstance -10 in the capture service). A connection from the same executable as an admitted PRESENT_OWNER is now a sibling and takes no new slot; the quota is compared against DISTINCT owner executables, so two different browsers still fill it. A peer whose image path is unreadable is never a sibling (fails closed). Verified on the SR laptop: all three Chromium processes admitted, the page lists and opens "3D Camera (DisplayXR)" (sbs 1280x480), stream RECTIFIED. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/xrt/ipc/server/ipc_server_handler.c | 45 ++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/src/xrt/ipc/server/ipc_server_handler.c b/src/xrt/ipc/server/ipc_server_handler.c index 704523a83c..f622631a4b 100644 --- a/src/xrt/ipc/server/ipc_server_handler.c +++ b/src/xrt/ipc/server/ipc_server_handler.c @@ -232,6 +232,48 @@ verify_client_class(volatile struct ipc_client_state *ics, uint32_t declared, co return XRT_CLIENT_CLASS_APP; } +// PRESENT_OWNER quota counts OWNERS, not connections: one browser is several +// processes of one executable (its GPU process weaves, the browser process binds +// the window, and its video-capture utility reads the stereo camera through +// XR_DXR_stereo_camera), and all of them are the one owner competing for the +// panel. A connection from the same executable as an already-admitted +// PRESENT_OWNER is a SIBLING and takes no new slot; otherwise the quota is +// compared against the number of DISTINCT owner executables. A peer whose image +// path the OS will not reveal ("") is never a sibling and counts as its own +// owner (fails closed). Caller holds global_state.lock; the exe lookups run at +// admission only. +static uint32_t +count_present_owners_locked(struct ipc_server *s, volatile struct ipc_client_state *ics, const char *ics_exe) +{ + static char seen[IPC_MAX_CLIENTS][512]; // under global_state.lock + uint32_t n = 0; + for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { + volatile struct ipc_client_state *o = &s->threads[i].ics; + if (o == ics || o->server_thread_index < 0 || !o->class_verified || + o->client_state.client_class != XRT_CLIENT_CLASS_PRESENT_OWNER) { + continue; + } + char exe[512]; + exe[0] = '\0'; + ipc_server_peer_exe_path(o->peer_pid, exe, sizeof(exe)); + if (exe[0] != '\0' && ics_exe[0] != '\0' && strcmp(exe, ics_exe) == 0) { + return 0; // a sibling of an admitted owner: no new slot + } + bool dup = false; + for (uint32_t k = 0; k < n && exe[0] != '\0'; k++) { + if (strcmp(seen[k], exe) == 0) { + dup = true; + break; + } + } + if (!dup) { + snprintf(seen[n], sizeof(seen[n]), "%s", exe); + n++; + } + } + return n; +} + // Count connected clients of a class, EXCLUDING ics. Caller holds global_state.lock. static uint32_t count_class_locked(struct ipc_server *s, volatile struct ipc_client_state *ics, uint32_t client_class) @@ -2132,7 +2174,8 @@ ipc_handle_instance_describe_client(volatile struct ipc_client_state *ics, struct ipc_server *s = ics->server; os_mutex_lock(&s->global_state.lock); - uint32_t in_use = count_class_locked(s, ics, verified); + uint32_t in_use = verified == XRT_CLIENT_CLASS_PRESENT_OWNER ? count_present_owners_locked(s, ics, exe) + : count_class_locked(s, ics, verified); uint32_t quota = ipc_server_client_class_quota(s, verified); bool refused = quota != 0 && in_use >= quota; if (!refused) { From 298b6a510ff01dd53263d3c7ad13cd05295049d7 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:31 -0700 Subject: [PATCH 31/37] feat(util): stereo camera consent policy, platform store, keyed persistentId (ADR-043 R3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit u_camera_consent: the spec §7.1 decision tree (sharing off > dev override > no identity > delegating client > OS camera switch > stored decision > Allow-once > prompt) over two injected vtables so it is testable with fakes. u_camera_consent_store: the real store — HKCU\Software\DisplayXR\CameraConsent on Windows (plus HKLM / HKCU Delegating lists and the CapabilityAccessManager webcam switch), camera_consent.json (0600) in the user config dir on POSIX plus the installer's system delegating list. persistentId becomes HMAC-SHA-256 under a 32-byte per-user secret the store generates once (u_sha256: compact FIPS 180-4 / RFC 2104, pinned by test vectors). u_client_class: the PRESENT_OWNER owner-counting rule as a pure function. tests_camera_consent covers every rule, Allow-once bound to the pid, prompt timeout / unavailable retried, the keyed id's stability and unlinkability, and the owner count with a CAMERA_CONSUMER sibling. Co-Authored-By: Claude Fable 5.1 --- src/xrt/auxiliary/util/CMakeLists.txt | 10 +- src/xrt/auxiliary/util/u_camera_consent.c | 278 +++++++ src/xrt/auxiliary/util/u_camera_consent.h | 256 ++++++ .../auxiliary/util/u_camera_consent_store.c | 750 ++++++++++++++++++ src/xrt/auxiliary/util/u_client_class.c | 57 ++ src/xrt/auxiliary/util/u_client_class.h | 58 ++ src/xrt/auxiliary/util/u_sha256.c | 156 ++++ src/xrt/auxiliary/util/u_sha256.h | 47 ++ tests/CMakeLists.txt | 1 + tests/tests_camera_consent.cpp | 420 ++++++++++ 10 files changed, 2032 insertions(+), 1 deletion(-) create mode 100644 src/xrt/auxiliary/util/u_camera_consent.c create mode 100644 src/xrt/auxiliary/util/u_camera_consent.h create mode 100644 src/xrt/auxiliary/util/u_camera_consent_store.c create mode 100644 src/xrt/auxiliary/util/u_client_class.c create mode 100644 src/xrt/auxiliary/util/u_client_class.h create mode 100644 src/xrt/auxiliary/util/u_sha256.c create mode 100644 src/xrt/auxiliary/util/u_sha256.h create mode 100644 tests/tests_camera_consent.cpp diff --git a/src/xrt/auxiliary/util/CMakeLists.txt b/src/xrt/auxiliary/util/CMakeLists.txt index 12e3914326..6a7837b56b 100644 --- a/src/xrt/auxiliary/util/CMakeLists.txt +++ b/src/xrt/auxiliary/util/CMakeLists.txt @@ -33,6 +33,13 @@ add_library( u_builders.h u_capture_dims.c u_capture_dims.h + u_camera_consent.c + u_camera_consent.h + u_camera_consent_store.c + u_client_class.c + u_client_class.h + u_sha256.c + u_sha256.h u_stereo_camera.c u_stereo_camera.h u_stereo_rectify.c @@ -170,7 +177,8 @@ if(WIN32) target_sources(aux_util PRIVATE u_windows.c u_windows.h) # shell32 needed for SHGetFolderPathA in u_file_logging.c # advapi32 needed for ACL/security APIs (AppContainer write access) in u_file_logging.c - target_link_libraries(aux_util PRIVATE shell32 advapi32) + # bcrypt: BCryptGenRandom seeds the stereo camera persistentId secret (u_camera_consent_store.c) + target_link_libraries(aux_util PRIVATE shell32 advapi32 bcrypt) endif() # WIL is lacking official support on MINGW. diff --git a/src/xrt/auxiliary/util/u_camera_consent.c b/src/xrt/auxiliary/util/u_camera_consent.c new file mode 100644 index 0000000000..8a246d0cea --- /dev/null +++ b/src/xrt/auxiliary/util/u_camera_consent.c @@ -0,0 +1,278 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Stereo camera consent policy (see u_camera_consent.h). Pure: every + * fact comes through the injected vtables. + * @ingroup aux_util + */ + +#include "util/u_camera_consent.h" +#include "util/u_logging.h" +#include "util/u_sha256.h" + +#include "os/os_time.h" + +#include +#include +#include + +#ifdef XRT_OS_WINDOWS +#include +#endif + +static bool +exe_known(const char *exe) +{ + return exe != NULL && exe[0] != '\0'; +} + +static void +random_bytes(uint8_t *out, size_t n) +{ + // A per-process fallback key only needs to be unpredictable, not + // cryptographic-grade: mix the clock, the address space and libc rand. + uint64_t seed = (uint64_t)os_monotonic_get_ns() ^ (uint64_t)(uintptr_t)out; +#ifdef XRT_OS_WINDOWS + seed ^= (uint64_t)GetCurrentProcessId() << 32; +#endif + srand((unsigned)(seed ^ (seed >> 32))); + for (size_t i = 0; i < n; i++) { + seed = seed * 6364136223846793005ull + 1442695040888963407ull; + out[i] = (uint8_t)((seed >> 56) ^ (uint8_t)rand()); + } +} + +void +u_camera_consent_init(struct u_camera_consent *c, + const struct u_camera_consent_store_ops *store, + void *store_ctx, + const struct u_camera_consent_env_ops *env, + void *env_ctx) +{ + memset(c, 0, sizeof(*c)); + c->store = store; + c->store_ctx = store_ctx; + c->env = env; + c->env_ctx = env_ctx; + c->prompt_enabled = true; + c->prompt_timeout_ms = 60000; +} + +const char * +u_camera_consent_why_str(enum u_camera_consent_why why) +{ + switch (why) { + case U_CAMERA_CONSENT_WHY_KILL_SWITCH: return "kill switch (DXR_STEREO_CAMERA=0)"; + case U_CAMERA_CONSENT_WHY_SHARING_OFF: return "camera sharing is off (user toggle)"; + case U_CAMERA_CONSENT_WHY_DEV_OVERRIDE: return "DXR_STEREO_CAMERA_DEV_ALLOW=1 (dev override)"; + case U_CAMERA_CONSENT_WHY_NO_IDENTITY: return "the peer executable could not be verified"; + case U_CAMERA_CONSENT_WHY_DELEGATING: return "registered consent-delegating client"; + case U_CAMERA_CONSENT_WHY_OS_DENIED: return "the OS camera privacy setting denies this app"; + case U_CAMERA_CONSENT_WHY_STORED_ALLOW: return "stored decision: Allow"; + case U_CAMERA_CONSENT_WHY_STORED_DENY: return "stored decision: Deny"; + case U_CAMERA_CONSENT_WHY_ALLOW_ONCE: return "Allow once (this process)"; + case U_CAMERA_CONSENT_WHY_PROMPT_ALLOW: return "prompt: Allow (stored)"; + case U_CAMERA_CONSENT_WHY_PROMPT_ALLOW_ONCE: return "prompt: Allow once"; + case U_CAMERA_CONSENT_WHY_PROMPT_DENY: return "prompt: Deny (stored)"; + case U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE: return "no consent prompt available (headless / prompt off)"; + case U_CAMERA_CONSENT_WHY_PROMPT_TIMEOUT: return "the consent prompt was not answered"; + default: return "?"; + } +} + +bool +u_camera_consent_sharing_enabled(struct u_camera_consent *c) +{ + if (c->kill_switch) { + return false; + } + if (c->store != NULL && c->store->sharing_enabled != NULL && !c->store->sharing_enabled(c->store_ctx)) { + return false; + } + return true; +} + +static bool +once_find(const struct u_camera_consent *c, const char *exe, long pid) +{ + for (uint32_t i = 0; i < c->once_count; i++) { + if (c->once[i].pid == pid && strcmp(c->once[i].exe, exe) == 0) { + return true; + } + } + return false; +} + +static void +once_add(struct u_camera_consent *c, const char *exe, long pid) +{ + if (once_find(c, exe, pid)) { + return; + } + uint32_t i = c->once_count < U_CAMERA_CONSENT_ONCE_MAX ? c->once_count++ : (U_CAMERA_CONSENT_ONCE_MAX - 1); + snprintf(c->once[i].exe, sizeof(c->once[i].exe), "%s", exe); + c->once[i].pid = pid; +} + +void +u_camera_consent_forget_once(struct u_camera_consent *c) +{ + c->once_count = 0; +} + +static void +decide(struct u_camera_consent_decision *out, enum u_camera_consent_verdict v, enum u_camera_consent_why why) +{ + out->verdict = v; + out->why = why; +} + +void +u_camera_consent_evaluate(struct u_camera_consent *c, + const char *exe, + const char *app_name, + long pid, + struct u_camera_consent_decision *out) +{ + memset(out, 0, sizeof(*out)); + const struct u_camera_consent_store_ops *st = c->store; + void *sc = c->store_ctx; + + // 1. sharing off beats everything, the dev override included: a user who + // switched the camera off must not find a dev box ignoring it. + if (c->kill_switch) { + decide(out, U_CAMERA_CONSENT_DISABLED, U_CAMERA_CONSENT_WHY_KILL_SWITCH); + return; + } + if (st != NULL && st->sharing_enabled != NULL && !st->sharing_enabled(sc)) { + decide(out, U_CAMERA_CONSENT_DISABLED, U_CAMERA_CONSENT_WHY_SHARING_OFF); + return; + } + // 2. dev override — one WARN per service lifetime, never silent. + if (c->dev_override) { + if (!c->dev_override_logged) { + c->dev_override_logged = true; + U_LOG_W("stereo camera consent: DXR_STEREO_CAMERA_DEV_ALLOW=1 — every consumer is ALLOWED without " + "consent (development override; never set this on a user's machine)"); + } + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_DEV_OVERRIDE); + return; + } + // 3. no identity, no consent: nothing to store a decision against. + if (!exe_known(exe)) { + decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_NO_IDENTITY); + return; + } + // 4. a registered delegating client prompts per origin itself. + if (st != NULL && st->is_delegating != NULL && st->is_delegating(sc, exe)) { + out->delegating = true; + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_DELEGATING); + return; + } + // 5. the OS switch: a camera that ignores it would surprise the user. + if (c->env != NULL && c->env->os_camera_allowed != NULL && !c->env->os_camera_allowed(c->env_ctx, exe)) { + decide(out, U_CAMERA_CONSENT_OS_DENIED, U_CAMERA_CONSENT_WHY_OS_DENIED); + return; + } + // 6. the stored decision. + enum u_camera_consent_stored stored = U_CAMERA_CONSENT_STORED_NONE; + if (st != NULL && st->get != NULL && st->get(sc, exe, &stored)) { + if (stored == U_CAMERA_CONSENT_STORED_ALLOW) { + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_STORED_ALLOW); + return; + } + if (stored == U_CAMERA_CONSENT_STORED_DENY) { + decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_STORED_DENY); + return; + } + } + // 7. "Allow once" already given to this process. + if (once_find(c, exe, pid)) { + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_ALLOW_ONCE); + return; + } + // 8. ask. + if (!c->prompt_enabled || c->env == NULL || c->env->prompt == NULL) { + decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE); + return; + } + enum u_camera_consent_prompt_answer a = c->env->prompt(c->env_ctx, exe, app_name, pid, c->prompt_timeout_ms); + switch (a) { + case U_CAMERA_CONSENT_PROMPT_ALLOW: + if (st != NULL && st->set != NULL && !st->set(sc, exe, U_CAMERA_CONSENT_STORED_ALLOW)) { + U_LOG_W("stereo camera consent: could not persist Allow for %s — it will be asked again", exe); + once_add(c, exe, pid); + } + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_PROMPT_ALLOW); + return; + case U_CAMERA_CONSENT_PROMPT_ALLOW_ONCE: + once_add(c, exe, pid); + decide(out, U_CAMERA_CONSENT_ALLOWED, U_CAMERA_CONSENT_WHY_PROMPT_ALLOW_ONCE); + return; + case U_CAMERA_CONSENT_PROMPT_DENY: + if (st != NULL && st->set != NULL && !st->set(sc, exe, U_CAMERA_CONSENT_STORED_DENY)) { + U_LOG_W("stereo camera consent: could not persist Deny for %s", exe); + } + decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_PROMPT_DENY); + return; + case U_CAMERA_CONSENT_PROMPT_TIMEOUT: + decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_PROMPT_TIMEOUT); + return; + case U_CAMERA_CONSENT_PROMPT_UNAVAILABLE: + default: decide(out, U_CAMERA_CONSENT_REFUSED, U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE); return; + } +} + + +/* + * + * Keyed persistentId. + * + */ + +void +u_camera_consent_persistent_id_keyed( + const uint8_t *key, size_t key_len, const char *device_identity, const char *consumer_exe, char out[64]) +{ + // Length-prefixed fields: "ab" + "c" never collides with "a" + "bc". + uint8_t msg[2 * 1024 + 64]; + size_t n = 0; + const char *fields[2] = {device_identity != NULL ? device_identity : "", + consumer_exe != NULL ? consumer_exe : ""}; + for (int i = 0; i < 2; i++) { + size_t len = strlen(fields[i]); + if (len > 1024) { + len = 1024; + } + msg[n++] = (uint8_t)(len >> 8); + msg[n++] = (uint8_t)len; + memcpy(msg + n, fields[i], len); + n += len; + } + uint8_t mac[32]; + u_hmac_sha256(key, key_len, msg, n, mac); + // 128 bits is plenty for an opaque id and keeps it within the 64-char field. + int w = snprintf(out, 64, "dxrcam-"); + for (int i = 0; i < 16; i++) { + w += snprintf(out + w, 64 - (size_t)w, "%02x", mac[i]); + } +} + +void +u_camera_consent_persistent_id(struct u_camera_consent *c, + const char *device_identity, + const char *consumer_exe, + char out[64]) +{ + if (!c->secret_ready) { + bool ok = c->store != NULL && c->store->get_secret != NULL && c->store->get_secret(c->store_ctx, c->secret); + if (!ok) { + random_bytes(c->secret, sizeof(c->secret)); + U_LOG_W("stereo camera consent: no persistent secret in the store — persistentId is keyed per " + "service run (stable until restart)"); + } + c->secret_ready = true; + } + u_camera_consent_persistent_id_keyed(c->secret, sizeof(c->secret), device_identity, consumer_exe, out); +} diff --git a/src/xrt/auxiliary/util/u_camera_consent.h b/src/xrt/auxiliary/util/u_camera_consent.h new file mode 100644 index 0000000000..4a75c4258d --- /dev/null +++ b/src/xrt/auxiliary/util/u_camera_consent.h @@ -0,0 +1,256 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Stereo camera consent policy + store (ADR-043 R3, spec §7). + * + * The POLICY is platform-neutral and takes every fact it needs through two + * injected vtables, so the decision tree can be unit-tested with fakes: + * - @ref u_camera_consent_store_ops — the persistent per-user store (Windows: + * HKCU\Software\DisplayXR\CameraConsent; POSIX: camera_consent.json in the + * user config dir) plus the registered consent-DELEGATING client list + * (installer-written under HKLM / a system JSON, or the user list written by + * `displayxr-cli camera trust`). u_camera_consent_store_default() is the + * real one; the CLI uses it directly. + * - @ref u_camera_consent_env_ops — what only the service process knows: the + * OS camera privacy switch for a given executable and the tray prompt. + * + * Decision order at every start / calibration read (first hit wins): + * 1. sharing off (DXR_STEREO_CAMERA=0 kill switch, or the user's "Share the + * 3D camera with apps" toggle) -> DISABLED + * 2. DXR_STEREO_CAMERA_DEV_ALLOW=1 (dev override, logged once) -> ALLOWED + * 3. no verifiable peer executable -> REFUSED + * 4. registered consent-delegating client (the browser) -> ALLOWED + * (its own per-origin prompt + indicator is the consent) + * 5. OS camera privacy denies this executable -> OS_DENIED + * 6. stored per-app decision (Allow / Deny) -> as stored + * 7. "Allow once" granted earlier to this same process -> ALLOWED + * 8. tray prompt (" wants to use the 3D camera — + * Allow / Allow once / Deny"): Allow + Deny are written to the store, + * Allow once is remembered for the process; no prompt available + * (headless, DXR_STEREO_CAMERA_PROMPT=0) or unanswered in time -> REFUSED + * + * The keyed persistentId (§7.5) also lives here: HMAC-SHA-256 under a 32-byte + * secret the store generates once per user, so a page cannot derive the + * device from the id and two installs never share one. + * + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define U_CAMERA_CONSENT_EXE_MAX 512 +#define U_CAMERA_CONSENT_SECRET_SIZE 32 +//! "Allow once" grants remembered per (executable, pid) for the service's lifetime. +#define U_CAMERA_CONSENT_ONCE_MAX 16 + +//! A stored per-executable decision. +enum u_camera_consent_stored +{ + U_CAMERA_CONSENT_STORED_NONE = 0, + U_CAMERA_CONSENT_STORED_ALLOW = 1, + U_CAMERA_CONSENT_STORED_DENY = 2, +}; + +//! What the prompt came back with. +enum u_camera_consent_prompt_answer +{ + U_CAMERA_CONSENT_PROMPT_UNAVAILABLE = 0, //!< no UI (headless service, prompt disabled) + U_CAMERA_CONSENT_PROMPT_ALLOW = 1, + U_CAMERA_CONSENT_PROMPT_ALLOW_ONCE = 2, + U_CAMERA_CONSENT_PROMPT_DENY = 3, + U_CAMERA_CONSENT_PROMPT_TIMEOUT = 4, //!< shown, not answered in time +}; + +//! The outcome a caller acts on (maps 1:1 to the XR results, spec §8). +enum u_camera_consent_verdict +{ + U_CAMERA_CONSENT_ALLOWED = 0, + U_CAMERA_CONSENT_REFUSED = 1, //!< XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR + U_CAMERA_CONSENT_DISABLED = 2, //!< XR_ERROR_STEREO_CAMERA_DISABLED_DXR + U_CAMERA_CONSENT_OS_DENIED = 3, //!< XR_ERROR_PERMISSION_INSUFFICIENT +}; + +//! Which rule decided (for the log line and the tests). +enum u_camera_consent_why +{ + U_CAMERA_CONSENT_WHY_KILL_SWITCH = 0, + U_CAMERA_CONSENT_WHY_SHARING_OFF, + U_CAMERA_CONSENT_WHY_DEV_OVERRIDE, + U_CAMERA_CONSENT_WHY_NO_IDENTITY, + U_CAMERA_CONSENT_WHY_DELEGATING, + U_CAMERA_CONSENT_WHY_OS_DENIED, + U_CAMERA_CONSENT_WHY_STORED_ALLOW, + U_CAMERA_CONSENT_WHY_STORED_DENY, + U_CAMERA_CONSENT_WHY_ALLOW_ONCE, + U_CAMERA_CONSENT_WHY_PROMPT_ALLOW, + U_CAMERA_CONSENT_WHY_PROMPT_ALLOW_ONCE, + U_CAMERA_CONSENT_WHY_PROMPT_DENY, + U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE, + U_CAMERA_CONSENT_WHY_PROMPT_TIMEOUT, +}; + +struct u_camera_consent_decision +{ + enum u_camera_consent_verdict verdict; + enum u_camera_consent_why why; + //! The executable is a registered consent-delegating client: it shows its + //! own prompt + indicator, follows its own visibility rule, and is never + //! given RAW frames (it exposes cameras to third-party content). + bool delegating; +}; + +/*! + * The persistent store. Every call takes the full, OS-derived executable path + * of the peer. All functions may be NULL (treated as: no entry / not + * delegating / sharing on / no secret). + */ +struct u_camera_consent_store_ops +{ + bool (*get)(void *ctx, const char *exe, enum u_camera_consent_stored *out); + //! NONE removes the entry. Returns false if the store could not be written. + bool (*set)(void *ctx, const char *exe, enum u_camera_consent_stored value); + bool (*is_delegating)(void *ctx, const char *exe); + //! The user's "Share the 3D camera with apps" toggle (default on). + bool (*sharing_enabled)(void *ctx); + bool (*set_sharing_enabled)(void *ctx, bool enabled); + //! The per-user secret, generated on first use. false = no store (the id + //! is then keyed with a per-process random key: stable for the service's + //! lifetime, never across restarts, which is the safe failure). + bool (*get_secret)(void *ctx, uint8_t out[U_CAMERA_CONSENT_SECRET_SIZE]); +}; + +/*! + * What the hosting process supplies. NULL = OS allows / no prompt available. + */ +struct u_camera_consent_env_ops +{ + //! The OS camera privacy switch for @p exe (Windows: CapabilityAccessManager + //! ConsentStore\webcam, global + desktop-app + per-app). true elsewhere. + bool (*os_camera_allowed)(void *ctx, const char *exe); + //! Show the consent prompt and BLOCK until answered or @p timeout_ms. The + //! caller holds no locks of the policy while this runs. + enum u_camera_consent_prompt_answer (*prompt)( + void *ctx, const char *exe, const char *app_name, long pid, uint32_t timeout_ms); +}; + +struct u_camera_consent +{ + const struct u_camera_consent_store_ops *store; + void *store_ctx; + const struct u_camera_consent_env_ops *env; + void *env_ctx; + + bool kill_switch; //!< DXR_STEREO_CAMERA=0 + bool dev_override; //!< DXR_STEREO_CAMERA_DEV_ALLOW=1 + bool prompt_enabled; //!< DXR_STEREO_CAMERA_PROMPT (default on) + uint32_t prompt_timeout_ms; + bool dev_override_logged; + + struct + { + char exe[U_CAMERA_CONSENT_EXE_MAX]; + long pid; + } once[U_CAMERA_CONSENT_ONCE_MAX]; + uint32_t once_count; + + //! Secret for persistentId: the store's, else a per-process random one. + uint8_t secret[U_CAMERA_CONSENT_SECRET_SIZE]; + bool secret_ready; +}; + +void +u_camera_consent_init(struct u_camera_consent *c, + const struct u_camera_consent_store_ops *store, + void *store_ctx, + const struct u_camera_consent_env_ops *env, + void *env_ctx); + +/*! + * Run the decision tree for one peer. Not thread-safe by itself: the caller + * serialises evaluations (the service does so per stream start) and holds no + * lock the prompt might need. @p exe may be "" (no verifiable identity). + */ +void +u_camera_consent_evaluate(struct u_camera_consent *c, + const char *exe, + const char *app_name, + long pid, + struct u_camera_consent_decision *out); + +//! Human-readable rule name for @ref u_camera_consent_why. +const char * +u_camera_consent_why_str(enum u_camera_consent_why why); + +//! Forget every "Allow once" grant (e.g. when the user stops sharing). +void +u_camera_consent_forget_once(struct u_camera_consent *c); + +//! Is sharing on right now (kill switch + user toggle)? +bool +u_camera_consent_sharing_enabled(struct u_camera_consent *c); + +/*! + * The keyed persistentId (spec §7.5): HMAC-SHA-256(secret, device | consumer), + * rendered as "dxrcam-" + 32 hex chars into @p out[64]. The same (secret, + * device, consumer) always gives the same id; any other secret gives an + * unrelated one; the id reveals nothing about the device. + */ +void +u_camera_consent_persistent_id(struct u_camera_consent *c, + const char *device_identity, + const char *consumer_exe, + char out[64]); + +//! Same, with an explicit key (tests, and the pure form of the above). +void +u_camera_consent_persistent_id_keyed( + const uint8_t *key, size_t key_len, const char *device_identity, const char *consumer_exe, char out[64]); + + +/* + * + * The real store (u_camera_consent_store.c). + * + */ + +//! The platform store: Windows registry / POSIX JSON. @p ctx is unused (NULL). +const struct u_camera_consent_store_ops * +u_camera_consent_store_default(void); + +/*! + * Add / remove an executable from the USER-level delegating list (the + * machine-level list is the installer's). `displayxr-cli camera trust`. + */ +bool +u_camera_consent_store_set_delegating(const char *exe, bool delegating); + +/*! + * List the store for diagnostics: calls @p cb for every stored app decision + * (kind 'a' with "allow"/"deny"), every delegating entry (kind 'd', value + * "user"/"system") and the sharing toggle (kind 's', "on"/"off"). + */ +void +u_camera_consent_store_list(void (*cb)(void *ctx, char kind, const char *exe, const char *value), void *ctx); + +//! Where the user store lives (for messages); false if unknown. +bool +u_camera_consent_store_path(char *out, size_t cap); + +//! The OS camera privacy switch for @p exe (Windows ConsentStore; true elsewhere). +bool +u_camera_consent_os_camera_allowed(const char *exe); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_camera_consent_store.c b/src/xrt/auxiliary/util/u_camera_consent_store.c new file mode 100644 index 0000000000..44ce477d3a --- /dev/null +++ b/src/xrt/auxiliary/util/u_camera_consent_store.c @@ -0,0 +1,750 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief The real stereo camera consent store (ADR-043 R3, spec §7.1). + * + * Windows — registry, per user: + * HKCU\Software\DisplayXR\CameraConsent + * Sharing REG_DWORD 1 (default) / 0 — the user's sharing toggle + * Secret REG_BINARY 32 bytes, generated once — keys persistentId + * Apps\ REG_DWORD value NAME = full executable path, data 1 = Allow, 2 = Deny + * Delegating\ REG_SZ value name = any id, data = executable path (user-level list, + * `displayxr-cli camera trust`) + * HKLM\Software\DisplayXR\CameraConsent\Delegating same shape, written by installers + * The OS camera privacy switch is read from CapabilityAccessManager\ConsentStore\webcam + * (HKLM global policy, HKCU global, NonPackaged = desktop apps, and the per-app entry + * whose key name is the path with '\' replaced by '#'). + * + * POSIX — JSON, mode 0600, in the user config dir ($XDG_CONFIG_HOME/displayxr or + * ~/.config/monado, like every other runtime file): camera_consent.json + * {"sharing": true, "secret": "<64 hex>", "apps": {"": "allow"|"deny"}, + * "delegating": ["", ...]} + * plus the system delegating list an installer may write: + * Linux /etc/displayxr/camera-delegating.json + * macOS /Library/Application Support/DisplayXR/camera-delegating.json + * {"delegating": ["", ...]} + * + * Paths compare the way the OS does: Windows case-insensitively with either + * separator, POSIX byte-exact. + * + * @ingroup aux_util + */ + +#include "util/u_camera_consent.h" +#include "util/u_logging.h" + +#include "xrt/xrt_config_os.h" + +#include +#include +#include + +#ifdef XRT_OS_WINDOWS +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#else +#include "util/u_file.h" +#include "util/u_json.h" +#include +#include +#include +#include +#include +#endif + + +/* + * + * Shared helpers. + * + */ + +static char +norm_ch(char c) +{ +#ifdef XRT_OS_WINDOWS + if (c == '/') { + return '\\'; + } + if (c >= 'A' && c <= 'Z') { + return (char)(c - 'A' + 'a'); + } +#endif + return c; +} + +static bool +path_equal(const char *a, const char *b) +{ + if (a == NULL || b == NULL || a[0] == '\0' || b[0] == '\0') { + return false; + } + while (*a != '\0' && *b != '\0') { + if (norm_ch(*a) != norm_ch(*b)) { + return false; + } + a++; + b++; + } + return *a == '\0' && *b == '\0'; +} + +static void +fill_random(uint8_t *out, size_t n) +{ +#ifdef XRT_OS_WINDOWS + if (BCryptGenRandom(NULL, out, (ULONG)n, BCRYPT_USE_SYSTEM_PREFERRED_RNG) == 0) { + return; + } +#else + int fd = open("/dev/urandom", O_RDONLY | O_CLOEXEC); + if (fd >= 0) { + size_t got = 0; + while (got < n) { + ssize_t r = read(fd, out + got, n - got); + if (r <= 0) { + break; + } + got += (size_t)r; + } + close(fd); + if (got == n) { + return; + } + } +#endif + // Last resort: still unpredictable enough to key an id for one install. + uint64_t x = (uint64_t)(uintptr_t)out ^ 0x9e3779b97f4a7c15ull; + for (size_t i = 0; i < n; i++) { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + out[i] = (uint8_t)(x ^ (uint64_t)rand()); + } +} + + +#ifdef XRT_OS_WINDOWS + +/* + * + * Windows: registry. + * + */ + +#define CONSENT_KEY "Software\\DisplayXR\\CameraConsent" +#define CAM_CONSENT_STORE_WEBCAM "Software\\Microsoft\\Windows\\CurrentVersion\\CapabilityAccessManager\\ConsentStore\\webcam" + +static bool +reg_read_dword(HKEY root, const char *sub, const char *name, DWORD *out) +{ + HKEY k; + if (RegOpenKeyExA(root, sub, 0, KEY_READ, &k) != ERROR_SUCCESS) { + return false; + } + DWORD type = 0, data = 0, size = sizeof(data); + LSTATUS rc = RegQueryValueExA(k, name, NULL, &type, (LPBYTE)&data, &size); + RegCloseKey(k); + if (rc != ERROR_SUCCESS || type != REG_DWORD) { + return false; + } + *out = data; + return true; +} + +static bool +reg_read_string(HKEY root, const char *sub, const char *name, char *out, size_t cap) +{ + HKEY k; + if (RegOpenKeyExA(root, sub, 0, KEY_READ, &k) != ERROR_SUCCESS) { + return false; + } + DWORD type = 0, size = (DWORD)cap; + LSTATUS rc = RegQueryValueExA(k, name, NULL, &type, (LPBYTE)out, &size); + RegCloseKey(k); + if (rc != ERROR_SUCCESS || (type != REG_SZ && type != REG_EXPAND_SZ)) { + return false; + } + out[cap - 1] = '\0'; + return true; +} + +static bool +reg_write_dword(HKEY root, const char *sub, const char *name, DWORD v) +{ + HKEY k; + if (RegCreateKeyExA(root, sub, 0, NULL, 0, KEY_WRITE, NULL, &k, NULL) != ERROR_SUCCESS) { + return false; + } + LSTATUS rc = RegSetValueExA(k, name, 0, REG_DWORD, (const BYTE *)&v, sizeof(v)); + RegCloseKey(k); + return rc == ERROR_SUCCESS; +} + +static bool +reg_delete_value(HKEY root, const char *sub, const char *name) +{ + HKEY k; + if (RegOpenKeyExA(root, sub, 0, KEY_WRITE, &k) != ERROR_SUCCESS) { + return true; // nothing to delete + } + LSTATUS rc = RegDeleteValueA(k, name); + RegCloseKey(k); + return rc == ERROR_SUCCESS || rc == ERROR_FILE_NOT_FOUND; +} + +static bool +win_get(void *ctx, const char *exe, enum u_camera_consent_stored *out) +{ + (void)ctx; + *out = U_CAMERA_CONSENT_STORED_NONE; + DWORD v = 0; + if (!reg_read_dword(HKEY_CURRENT_USER, CONSENT_KEY "\\Apps", exe, &v)) { + return false; + } + *out = v == 1 ? U_CAMERA_CONSENT_STORED_ALLOW : v == 2 ? U_CAMERA_CONSENT_STORED_DENY : U_CAMERA_CONSENT_STORED_NONE; + return true; +} + +static bool +win_set(void *ctx, const char *exe, enum u_camera_consent_stored value) +{ + (void)ctx; + if (value == U_CAMERA_CONSENT_STORED_NONE) { + return reg_delete_value(HKEY_CURRENT_USER, CONSENT_KEY "\\Apps", exe); + } + return reg_write_dword(HKEY_CURRENT_USER, CONSENT_KEY "\\Apps", exe, + value == U_CAMERA_CONSENT_STORED_ALLOW ? 1u : 2u); +} + +//! Walk the values of a Delegating key: each value's DATA is an executable path. +static bool +win_delegating_has(HKEY root, const char *exe, char *match_name, size_t match_cap) +{ + HKEY k; + if (RegOpenKeyExA(root, CONSENT_KEY "\\Delegating", 0, KEY_READ, &k) != ERROR_SUCCESS) { + return false; + } + bool found = false; + for (DWORD i = 0; !found; i++) { + char name[256]; + char data[1024]; + DWORD name_len = sizeof(name), type = 0, data_len = sizeof(data); + LSTATUS rc = RegEnumValueA(k, i, name, &name_len, NULL, &type, (LPBYTE)data, &data_len); + if (rc != ERROR_SUCCESS) { + break; + } + if (type != REG_SZ && type != REG_EXPAND_SZ) { + continue; + } + data[sizeof(data) - 1] = '\0'; + if (path_equal(data, exe)) { + found = true; + if (match_name != NULL) { + snprintf(match_name, match_cap, "%s", name); + } + } + } + RegCloseKey(k); + return found; +} + +static bool +win_is_delegating(void *ctx, const char *exe) +{ + (void)ctx; + return win_delegating_has(HKEY_LOCAL_MACHINE, exe, NULL, 0) || win_delegating_has(HKEY_CURRENT_USER, exe, NULL, 0); +} + +static bool +win_sharing_enabled(void *ctx) +{ + (void)ctx; + DWORD v = 1; + if (reg_read_dword(HKEY_CURRENT_USER, CONSENT_KEY, "Sharing", &v)) { + return v != 0; + } + return true; +} + +static bool +win_set_sharing_enabled(void *ctx, bool enabled) +{ + (void)ctx; + return reg_write_dword(HKEY_CURRENT_USER, CONSENT_KEY, "Sharing", enabled ? 1u : 0u); +} + +static bool +win_get_secret(void *ctx, uint8_t out[U_CAMERA_CONSENT_SECRET_SIZE]) +{ + (void)ctx; + HKEY k; + if (RegCreateKeyExA(HKEY_CURRENT_USER, CONSENT_KEY, 0, NULL, 0, KEY_READ | KEY_WRITE, NULL, &k, NULL) != + ERROR_SUCCESS) { + return false; + } + DWORD type = 0, size = U_CAMERA_CONSENT_SECRET_SIZE; + LSTATUS rc = RegQueryValueExA(k, "Secret", NULL, &type, out, &size); + if (rc == ERROR_SUCCESS && type == REG_BINARY && size == U_CAMERA_CONSENT_SECRET_SIZE) { + RegCloseKey(k); + return true; + } + fill_random(out, U_CAMERA_CONSENT_SECRET_SIZE); + rc = RegSetValueExA(k, "Secret", 0, REG_BINARY, out, U_CAMERA_CONSENT_SECRET_SIZE); + RegCloseKey(k); + return rc == ERROR_SUCCESS; +} + +static const struct u_camera_consent_store_ops win_ops = { + .get = win_get, + .set = win_set, + .is_delegating = win_is_delegating, + .sharing_enabled = win_sharing_enabled, + .set_sharing_enabled = win_set_sharing_enabled, + .get_secret = win_get_secret, +}; + +const struct u_camera_consent_store_ops * +u_camera_consent_store_default(void) +{ + return &win_ops; +} + +bool +u_camera_consent_store_set_delegating(const char *exe, bool delegating) +{ + char name[256]; + bool present = win_delegating_has(HKEY_CURRENT_USER, exe, name, sizeof(name)); + if (!delegating) { + return !present || reg_delete_value(HKEY_CURRENT_USER, CONSENT_KEY "\\Delegating", name); + } + if (present) { + return true; + } + // Value name: the executable's base name (unique enough; data is what matters). + const char *base = exe; + for (const char *p = exe; *p != '\0'; p++) { + if (*p == '\\' || *p == '/') { + base = p + 1; + } + } + HKEY k; + if (RegCreateKeyExA(HKEY_CURRENT_USER, CONSENT_KEY "\\Delegating", 0, NULL, 0, KEY_WRITE, NULL, &k, NULL) != + ERROR_SUCCESS) { + return false; + } + LSTATUS rc = RegSetValueExA(k, base, 0, REG_SZ, (const BYTE *)exe, (DWORD)strlen(exe) + 1); + RegCloseKey(k); + return rc == ERROR_SUCCESS; +} + +static void +win_list_delegating(HKEY root, const char *origin, void (*cb)(void *, char, const char *, const char *), void *ctx) +{ + HKEY k; + if (RegOpenKeyExA(root, CONSENT_KEY "\\Delegating", 0, KEY_READ, &k) != ERROR_SUCCESS) { + return; + } + for (DWORD i = 0;; i++) { + char name[256]; + char data[1024]; + DWORD name_len = sizeof(name), type = 0, data_len = sizeof(data); + if (RegEnumValueA(k, i, name, &name_len, NULL, &type, (LPBYTE)data, &data_len) != ERROR_SUCCESS) { + break; + } + if (type == REG_SZ || type == REG_EXPAND_SZ) { + data[sizeof(data) - 1] = '\0'; + cb(ctx, 'd', data, origin); + } + } + RegCloseKey(k); +} + +void +u_camera_consent_store_list(void (*cb)(void *ctx, char kind, const char *exe, const char *value), void *ctx) +{ + cb(ctx, 's', "", win_sharing_enabled(NULL) ? "on" : "off"); + HKEY k; + if (RegOpenKeyExA(HKEY_CURRENT_USER, CONSENT_KEY "\\Apps", 0, KEY_READ, &k) == ERROR_SUCCESS) { + for (DWORD i = 0;; i++) { + char name[1024]; + DWORD name_len = sizeof(name), type = 0, v = 0, v_len = sizeof(v); + if (RegEnumValueA(k, i, name, &name_len, NULL, &type, (LPBYTE)&v, &v_len) != ERROR_SUCCESS) { + break; + } + if (type == REG_DWORD) { + cb(ctx, 'a', name, v == 1 ? "allow" : v == 2 ? "deny" : "?"); + } + } + RegCloseKey(k); + } + win_list_delegating(HKEY_LOCAL_MACHINE, "system", cb, ctx); + win_list_delegating(HKEY_CURRENT_USER, "user", cb, ctx); +} + +bool +u_camera_consent_store_path(char *out, size_t cap) +{ + snprintf(out, cap, "HKCU\\%s", CONSENT_KEY); + return true; +} + +//! "Deny" in a ConsentStore Value means off; anything else (Allow, absent) is on. +static bool +consent_store_value_denies(HKEY root, const char *sub) +{ + char v[32]; + return reg_read_string(root, sub, "Value", v, sizeof(v)) && _stricmp(v, "Deny") == 0; +} + +bool +u_camera_consent_os_camera_allowed(const char *exe) +{ + // Global (policy, then user), then the desktop-app class, then this app. + if (consent_store_value_denies(HKEY_LOCAL_MACHINE, CAM_CONSENT_STORE_WEBCAM) || + consent_store_value_denies(HKEY_CURRENT_USER, CAM_CONSENT_STORE_WEBCAM) || + consent_store_value_denies(HKEY_CURRENT_USER, CAM_CONSENT_STORE_WEBCAM "\\NonPackaged")) { + return false; + } + if (exe == NULL || exe[0] == '\0') { + return true; + } + char sub[1600]; + int n = snprintf(sub, sizeof(sub), "%s\\NonPackaged\\", CAM_CONSENT_STORE_WEBCAM); + for (const char *p = exe; *p != '\0' && n < (int)sizeof(sub) - 1; p++) { + sub[n++] = (*p == '\\' || *p == '/') ? '#' : *p; + } + sub[n] = '\0'; + return !consent_store_value_denies(HKEY_CURRENT_USER, sub); +} + +#else // POSIX + +/* + * + * POSIX: JSON file. + * + */ + +#define CONSENT_FILE "camera_consent.json" +#ifdef XRT_OS_MACOS +#define SYSTEM_DELEGATING_FILE "/Library/Application Support/DisplayXR/camera-delegating.json" +#else +#define SYSTEM_DELEGATING_FILE "/etc/displayxr/camera-delegating.json" +#endif + +static bool +user_path(char *out, size_t cap) +{ + int n = u_file_get_path_in_config_dir(CONSENT_FILE, out, cap); + return n > 0 && (size_t)n < cap; +} + +//! Parse the user file; NULL = absent / unreadable. An empty object otherwise. +static cJSON * +load_user(bool create) +{ + char path[1024]; + if (!user_path(path, sizeof(path))) { + return NULL; + } + size_t size = 0; + char *content = u_file_read_content_from_path(path, &size); + cJSON *root = NULL; + if (content != NULL) { + root = cJSON_Parse(content); + free(content); + } + if (root == NULL && create) { + root = cJSON_CreateObject(); + } + return root; +} + +static bool +save_user(cJSON *root) +{ + // u_file_open_file_in_config_dir creates the directory; we then clamp the + // mode — the file holds the id-keying secret. + FILE *f = u_file_open_file_in_config_dir(CONSENT_FILE, "w"); + if (f == NULL) { + return false; + } + char *text = cJSON_Print(root); + bool ok = text != NULL && fputs(text, f) >= 0; + free(text); + fclose(f); + char path[1024]; + if (user_path(path, sizeof(path))) { + (void)chmod(path, S_IRUSR | S_IWUSR); + } + return ok; +} + +static bool +posix_get(void *ctx, const char *exe, enum u_camera_consent_stored *out) +{ + (void)ctx; + *out = U_CAMERA_CONSENT_STORED_NONE; + cJSON *root = load_user(false); + if (root == NULL) { + return false; + } + bool found = false; + cJSON *apps = cJSON_GetObjectItem(root, "apps"); + cJSON *it = NULL; + cJSON_ArrayForEach(it, apps) + { + if (it->string != NULL && cJSON_IsString(it) && path_equal(it->string, exe)) { + found = true; + *out = strcmp(it->valuestring, "allow") == 0 ? U_CAMERA_CONSENT_STORED_ALLOW + : strcmp(it->valuestring, "deny") == 0 ? U_CAMERA_CONSENT_STORED_DENY + : U_CAMERA_CONSENT_STORED_NONE; + break; + } + } + cJSON_Delete(root); + return found; +} + +static bool +posix_set(void *ctx, const char *exe, enum u_camera_consent_stored value) +{ + (void)ctx; + cJSON *root = load_user(true); + if (root == NULL) { + return false; + } + cJSON *apps = cJSON_GetObjectItem(root, "apps"); + if (apps == NULL || !cJSON_IsObject(apps)) { + cJSON_DeleteItemFromObject(root, "apps"); + apps = cJSON_AddObjectToObject(root, "apps"); + } + cJSON_DeleteItemFromObject(apps, exe); + if (value != U_CAMERA_CONSENT_STORED_NONE) { + cJSON_AddStringToObject(apps, exe, value == U_CAMERA_CONSENT_STORED_ALLOW ? "allow" : "deny"); + } + bool ok = save_user(root); + cJSON_Delete(root); + return ok; +} + +static bool +delegating_array_has(const cJSON *arr, const char *exe) +{ + const cJSON *it = NULL; + cJSON_ArrayForEach(it, arr) + { + if (cJSON_IsString(it) && path_equal(it->valuestring, exe)) { + return true; + } + } + return false; +} + +static cJSON * +load_system(void) +{ + size_t size = 0; + char *content = u_file_read_content_from_path(SYSTEM_DELEGATING_FILE, &size); + if (content == NULL) { + return NULL; + } + cJSON *root = cJSON_Parse(content); + free(content); + return root; +} + +static bool +posix_is_delegating(void *ctx, const char *exe) +{ + (void)ctx; + bool yes = false; + cJSON *sys = load_system(); + if (sys != NULL) { + yes = delegating_array_has(cJSON_GetObjectItem(sys, "delegating"), exe); + cJSON_Delete(sys); + } + if (!yes) { + cJSON *user = load_user(false); + if (user != NULL) { + yes = delegating_array_has(cJSON_GetObjectItem(user, "delegating"), exe); + cJSON_Delete(user); + } + } + return yes; +} + +static bool +posix_sharing_enabled(void *ctx) +{ + (void)ctx; + bool on = true; + cJSON *root = load_user(false); + if (root != NULL) { + cJSON *s = cJSON_GetObjectItem(root, "sharing"); + if (cJSON_IsBool(s)) { + on = cJSON_IsTrue(s); + } + cJSON_Delete(root); + } + return on; +} + +static bool +posix_set_sharing_enabled(void *ctx, bool enabled) +{ + (void)ctx; + cJSON *root = load_user(true); + if (root == NULL) { + return false; + } + cJSON_DeleteItemFromObject(root, "sharing"); + cJSON_AddBoolToObject(root, "sharing", enabled); + bool ok = save_user(root); + cJSON_Delete(root); + return ok; +} + +static int +hexval(char c) +{ + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; +} + +static bool +posix_get_secret(void *ctx, uint8_t out[U_CAMERA_CONSENT_SECRET_SIZE]) +{ + (void)ctx; + cJSON *root = load_user(true); + if (root == NULL) { + return false; + } + cJSON *s = cJSON_GetObjectItem(root, "secret"); + if (cJSON_IsString(s) && strlen(s->valuestring) == 2 * U_CAMERA_CONSENT_SECRET_SIZE) { + bool ok = true; + for (int i = 0; i < U_CAMERA_CONSENT_SECRET_SIZE && ok; i++) { + int hi = hexval(s->valuestring[2 * i]), lo = hexval(s->valuestring[2 * i + 1]); + ok = hi >= 0 && lo >= 0; + out[i] = (uint8_t)((hi << 4) | lo); + } + if (ok) { + cJSON_Delete(root); + return true; + } + } + fill_random(out, U_CAMERA_CONSENT_SECRET_SIZE); + char hex[2 * U_CAMERA_CONSENT_SECRET_SIZE + 1]; + for (int i = 0; i < U_CAMERA_CONSENT_SECRET_SIZE; i++) { + snprintf(hex + 2 * i, 3, "%02x", out[i]); + } + cJSON_DeleteItemFromObject(root, "secret"); + cJSON_AddStringToObject(root, "secret", hex); + bool ok = save_user(root); + cJSON_Delete(root); + return ok; +} + +static const struct u_camera_consent_store_ops posix_ops = { + .get = posix_get, + .set = posix_set, + .is_delegating = posix_is_delegating, + .sharing_enabled = posix_sharing_enabled, + .set_sharing_enabled = posix_set_sharing_enabled, + .get_secret = posix_get_secret, +}; + +const struct u_camera_consent_store_ops * +u_camera_consent_store_default(void) +{ + return &posix_ops; +} + +bool +u_camera_consent_store_set_delegating(const char *exe, bool delegating) +{ + cJSON *root = load_user(true); + if (root == NULL) { + return false; + } + cJSON *arr = cJSON_GetObjectItem(root, "delegating"); + if (arr == NULL || !cJSON_IsArray(arr)) { + cJSON_DeleteItemFromObject(root, "delegating"); + arr = cJSON_AddArrayToObject(root, "delegating"); + } + // Remove any existing entry, then re-add when requested. + for (int i = cJSON_GetArraySize(arr) - 1; i >= 0; i--) { + cJSON *it = cJSON_GetArrayItem(arr, i); + if (cJSON_IsString(it) && path_equal(it->valuestring, exe)) { + cJSON_DeleteItemFromArray(arr, i); + } + } + if (delegating) { + cJSON_AddItemToArray(arr, cJSON_CreateString(exe)); + } + bool ok = save_user(root); + cJSON_Delete(root); + return ok; +} + +void +u_camera_consent_store_list(void (*cb)(void *ctx, char kind, const char *exe, const char *value), void *ctx) +{ + cb(ctx, 's', "", posix_sharing_enabled(NULL) ? "on" : "off"); + cJSON *user = load_user(false); + if (user != NULL) { + cJSON *it = NULL; + cJSON_ArrayForEach(it, cJSON_GetObjectItem(user, "apps")) + { + if (it->string != NULL && cJSON_IsString(it)) { + cb(ctx, 'a', it->string, it->valuestring); + } + } + cJSON_ArrayForEach(it, cJSON_GetObjectItem(user, "delegating")) + { + if (cJSON_IsString(it)) { + cb(ctx, 'd', it->valuestring, "user"); + } + } + cJSON_Delete(user); + } + cJSON *sys = load_system(); + if (sys != NULL) { + cJSON *it = NULL; + cJSON_ArrayForEach(it, cJSON_GetObjectItem(sys, "delegating")) + { + if (cJSON_IsString(it)) { + cb(ctx, 'd', it->valuestring, "system"); + } + } + cJSON_Delete(sys); + } +} + +bool +u_camera_consent_store_path(char *out, size_t cap) +{ + return user_path(out, cap); +} + +bool +u_camera_consent_os_camera_allowed(const char *exe) +{ + (void)exe; + return true; // the OS switch (macOS TCC) does not see this consumer: nothing to read +} + +#endif diff --git a/src/xrt/auxiliary/util/u_client_class.c b/src/xrt/auxiliary/util/u_client_class.c new file mode 100644 index 0000000000..51dfa4079c --- /dev/null +++ b/src/xrt/auxiliary/util/u_client_class.c @@ -0,0 +1,57 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Pure client-class rules (see u_client_class.h). + * @ingroup aux_util + */ + +#include "util/u_client_class.h" + +#include "xrt/xrt_instance.h" + +#include + +bool +u_client_class_counts_toward_present_owner(uint32_t client_class) +{ + return client_class == XRT_CLIENT_CLASS_PRESENT_OWNER; +} + +bool +u_client_class_may_create_session(uint32_t client_class) +{ + return client_class != XRT_CLIENT_CLASS_CAMERA_CONSUMER; +} + +uint32_t +u_client_class_present_owner_count(const struct u_client_class_peer *peers, uint32_t n, const char *candidate_exe) +{ + const char *seen[64]; + uint32_t distinct = 0; + const bool candidate_known = candidate_exe != NULL && candidate_exe[0] != '\0'; + for (uint32_t i = 0; i < n; i++) { + const struct u_client_class_peer *p = &peers[i]; + if (!u_client_class_counts_toward_present_owner(p->client_class)) { + continue; + } + const char *exe = p->exe != NULL ? p->exe : ""; + if (exe[0] != '\0' && candidate_known && strcmp(exe, candidate_exe) == 0) { + return 0; // a sibling of an admitted owner: no new slot + } + bool dup = false; + for (uint32_t k = 0; k < distinct && exe[0] != '\0'; k++) { + if (strcmp(seen[k], exe) == 0) { + dup = true; + break; + } + } + if (!dup) { + if (distinct < sizeof(seen) / sizeof(seen[0])) { + seen[distinct] = exe; + } + distinct++; + } + } + return distinct; +} diff --git a/src/xrt/auxiliary/util/u_client_class.h b/src/xrt/auxiliary/util/u_client_class.h new file mode 100644 index 0000000000..1ccc868da4 --- /dev/null +++ b/src/xrt/auxiliary/util/u_client_class.h @@ -0,0 +1,58 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Pure rules about IPC client classes (#960, ADR-035 D1/D6) that the + * service applies at admission, kept free of server state so they can + * be tested host-side. + * @ingroup aux_util + */ + +#pragma once + +#include "xrt/xrt_compiler.h" + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +//! One already-admitted connection, as the PRESENT_OWNER count sees it. +struct u_client_class_peer +{ + uint32_t client_class; //!< enum xrt_client_class (the VERIFIED class) + const char *exe; //!< OS-derived image path; "" when the OS would not reveal it +}; + +/*! + * Does a connection of this class hold the panel as an owner? Only + * PRESENT_OWNER does: a CAMERA_CONSUMER from the browser's executable (its + * video-capture utility, ADR-043 R3) reads a camera, it never presents. + */ +bool +u_client_class_counts_toward_present_owner(uint32_t client_class); + +/*! + * The PRESENT_OWNER quota counts OWNERS (distinct executables), not + * connections. Returns the number of distinct owner executables among + * @p peers (only PRESENT_OWNER peers count), or 0 when @p candidate_exe is a + * SIBLING of one of them — a second process of an already-admitted owner takes + * no new slot. A peer whose path is unknown ("") is its own owner and is never + * anyone's sibling (fails closed); an unknown candidate is never a sibling. + */ +uint32_t +u_client_class_present_owner_count(const struct u_client_class_peer *peers, uint32_t n, const char *candidate_exe); + +/*! + * May a connection of this class create a session / compositor? RELAY may + * not create a native compositor (its own rule in the handler); a + * CAMERA_CONSUMER may not create a session at all. + */ +bool +u_client_class_may_create_session(uint32_t client_class); + +#ifdef __cplusplus +} +#endif diff --git a/src/xrt/auxiliary/util/u_sha256.c b/src/xrt/auxiliary/util/u_sha256.c new file mode 100644 index 0000000000..dea5f69cb6 --- /dev/null +++ b/src/xrt/auxiliary/util/u_sha256.c @@ -0,0 +1,156 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Compact SHA-256 + HMAC (see u_sha256.h). Straight FIPS 180-4; the + * test vectors in tests_camera_consent.cpp pin it. + * @ingroup aux_util + */ + +#include "util/u_sha256.h" + +#include + +static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, +}; + +#define ROTR(x, n) (((x) >> (n)) | ((x) << (32 - (n)))) + +static void +block(struct u_sha256 *s, const uint8_t p[64]) +{ + uint32_t w[64]; + for (int i = 0; i < 16; i++) { + w[i] = ((uint32_t)p[4 * i] << 24) | ((uint32_t)p[4 * i + 1] << 16) | ((uint32_t)p[4 * i + 2] << 8) | + (uint32_t)p[4 * i + 3]; + } + for (int i = 16; i < 64; i++) { + uint32_t s0 = ROTR(w[i - 15], 7) ^ ROTR(w[i - 15], 18) ^ (w[i - 15] >> 3); + uint32_t s1 = ROTR(w[i - 2], 17) ^ ROTR(w[i - 2], 19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + s0 + w[i - 7] + s1; + } + uint32_t a = s->h[0], b = s->h[1], c = s->h[2], d = s->h[3]; + uint32_t e = s->h[4], f = s->h[5], g = s->h[6], h = s->h[7]; + for (int i = 0; i < 64; i++) { + uint32_t S1 = ROTR(e, 6) ^ ROTR(e, 11) ^ ROTR(e, 25); + uint32_t ch = (e & f) ^ (~e & g); + uint32_t t1 = h + S1 + ch + K[i] + w[i]; + uint32_t S0 = ROTR(a, 2) ^ ROTR(a, 13) ^ ROTR(a, 22); + uint32_t maj = (a & b) ^ (a & c) ^ (b & c); + uint32_t t2 = S0 + maj; + h = g; + g = f; + f = e; + e = d + t1; + d = c; + c = b; + b = a; + a = t1 + t2; + } + s->h[0] += a; + s->h[1] += b; + s->h[2] += c; + s->h[3] += d; + s->h[4] += e; + s->h[5] += f; + s->h[6] += g; + s->h[7] += h; +} + +void +u_sha256_init(struct u_sha256 *s) +{ + static const uint32_t iv[8] = {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + memcpy(s->h, iv, sizeof(iv)); + s->len = 0; + s->buf_len = 0; +} + +void +u_sha256_update(struct u_sha256 *s, const void *data, size_t len) +{ + const uint8_t *p = (const uint8_t *)data; + s->len += len; + while (len > 0) { + size_t take = 64 - s->buf_len; + if (take > len) { + take = len; + } + memcpy(s->buf + s->buf_len, p, take); + s->buf_len += take; + p += take; + len -= take; + if (s->buf_len == 64) { + block(s, s->buf); + s->buf_len = 0; + } + } +} + +void +u_sha256_final(struct u_sha256 *s, uint8_t out[32]) +{ + uint64_t bits = s->len * 8; + uint8_t pad = 0x80; + u_sha256_update(s, &pad, 1); + uint8_t zero = 0; + while (s->buf_len != 56) { + u_sha256_update(s, &zero, 1); + } + uint8_t lenb[8]; + for (int i = 0; i < 8; i++) { + lenb[i] = (uint8_t)(bits >> (56 - 8 * i)); + } + u_sha256_update(s, lenb, 8); + for (int i = 0; i < 8; i++) { + out[4 * i] = (uint8_t)(s->h[i] >> 24); + out[4 * i + 1] = (uint8_t)(s->h[i] >> 16); + out[4 * i + 2] = (uint8_t)(s->h[i] >> 8); + out[4 * i + 3] = (uint8_t)(s->h[i]); + } +} + +void +u_sha256(const void *data, size_t len, uint8_t out[32]) +{ + struct u_sha256 s; + u_sha256_init(&s); + u_sha256_update(&s, data, len); + u_sha256_final(&s, out); +} + +void +u_hmac_sha256(const void *key, size_t key_len, const void *data, size_t len, uint8_t out[32]) +{ + uint8_t k[64]; + memset(k, 0, sizeof(k)); + if (key_len > 64) { + u_sha256(key, key_len, k); + } else { + memcpy(k, key, key_len); + } + uint8_t ipad[64], opad[64]; + for (int i = 0; i < 64; i++) { + ipad[i] = k[i] ^ 0x36; + opad[i] = k[i] ^ 0x5c; + } + uint8_t inner[32]; + struct u_sha256 s; + u_sha256_init(&s); + u_sha256_update(&s, ipad, 64); + u_sha256_update(&s, data, len); + u_sha256_final(&s, inner); + u_sha256_init(&s); + u_sha256_update(&s, opad, 64); + u_sha256_update(&s, inner, 32); + u_sha256_final(&s, out); +} diff --git a/src/xrt/auxiliary/util/u_sha256.h b/src/xrt/auxiliary/util/u_sha256.h new file mode 100644 index 0000000000..bf76a54ef9 --- /dev/null +++ b/src/xrt/auxiliary/util/u_sha256.h @@ -0,0 +1,47 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief Compact SHA-256 + HMAC-SHA-256 (FIPS 180-4 / RFC 2104). Used to key + * identifiers that must not be derivable without a secret (the stereo + * camera persistentId, ADR-043 §7.5). Not a general crypto library. + * @ingroup aux_util + */ + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct u_sha256 +{ + uint32_t h[8]; + uint64_t len; + uint8_t buf[64]; + size_t buf_len; +}; + +void +u_sha256_init(struct u_sha256 *s); + +void +u_sha256_update(struct u_sha256 *s, const void *data, size_t len); + +void +u_sha256_final(struct u_sha256 *s, uint8_t out[32]); + +//! One-shot digest. +void +u_sha256(const void *data, size_t len, uint8_t out[32]); + +//! HMAC-SHA-256 of @p data under @p key (any key length). +void +u_hmac_sha256(const void *key, size_t key_len, const void *data, size_t len, uint8_t out[32]); + +#ifdef __cplusplus +} +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4cf4dd3456..b0f6e10c66 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -68,6 +68,7 @@ set(tests tests_comp_color_policy tests_m_matrix_projection tests_stereo_camera + tests_camera_consent tests_stereo_rectify tests_stereo_vrefine ) diff --git a/tests/tests_camera_consent.cpp b/tests/tests_camera_consent.cpp new file mode 100644 index 0000000000..4d22ffa372 --- /dev/null +++ b/tests/tests_camera_consent.cpp @@ -0,0 +1,420 @@ +// Copyright 2026, The DisplayXR Project +// SPDX-License-Identifier: BSL-1.0 +/*! + * @file + * @brief XR_DXR_stereo_camera R3 (ADR-043): the consent policy against fake + * store / environment vtables, the keyed persistentId, the SHA-256 / + * HMAC it rests on, and the client-class rules the service applies at + * admission (CAMERA_CONSUMER outside the PRESENT_OWNER quota). + */ + +#include "catch_amalgamated.hpp" + +#include "util/u_camera_consent.h" +#include "util/u_client_class.h" +#include "util/u_sha256.h" +#include "xrt/xrt_instance.h" + +#include +#include +#include +#include +#include + +namespace { + +struct fake_store +{ + std::map apps; + std::set delegating; + bool sharing = true; + bool have_secret = true; + uint8_t secret[32] = {1, 2, 3}; + int sets = 0; + bool set_fails = false; +}; + +bool +fs_get(void *ctx, const char *exe, u_camera_consent_stored *out) +{ + auto *s = (fake_store *)ctx; + auto it = s->apps.find(exe); + if (it == s->apps.end()) { + return false; + } + *out = it->second; + return true; +} +bool +fs_set(void *ctx, const char *exe, u_camera_consent_stored v) +{ + auto *s = (fake_store *)ctx; + s->sets++; + if (s->set_fails) { + return false; + } + if (v == U_CAMERA_CONSENT_STORED_NONE) { + s->apps.erase(exe); + } else { + s->apps[exe] = v; + } + return true; +} +bool +fs_is_delegating(void *ctx, const char *exe) +{ + return ((fake_store *)ctx)->delegating.count(exe) > 0; +} +bool +fs_sharing(void *ctx) +{ + return ((fake_store *)ctx)->sharing; +} +bool +fs_set_sharing(void *ctx, bool on) +{ + ((fake_store *)ctx)->sharing = on; + return true; +} +bool +fs_secret(void *ctx, uint8_t out[32]) +{ + auto *s = (fake_store *)ctx; + if (!s->have_secret) { + return false; + } + memcpy(out, s->secret, 32); + return true; +} + +const u_camera_consent_store_ops fake_store_ops = {fs_get, fs_set, fs_is_delegating, fs_sharing, fs_set_sharing, + fs_secret}; + +struct fake_env +{ + bool os_allowed = true; + u_camera_consent_prompt_answer answer = U_CAMERA_CONSENT_PROMPT_UNAVAILABLE; + int prompts = 0; + std::string last_exe; + long last_pid = 0; +}; + +bool +fe_os(void *ctx, const char *exe) +{ + (void)exe; + return ((fake_env *)ctx)->os_allowed; +} +u_camera_consent_prompt_answer +fe_prompt(void *ctx, const char *exe, const char *app, long pid, uint32_t timeout_ms) +{ + (void)app; + (void)timeout_ms; + auto *e = (fake_env *)ctx; + e->prompts++; + e->last_exe = exe; + e->last_pid = pid; + return e->answer; +} + +const u_camera_consent_env_ops fake_env_ops = {fe_os, fe_prompt}; + +struct rig +{ + fake_store store; + fake_env env; + u_camera_consent c; + rig() + { + u_camera_consent_init(&c, &fake_store_ops, &store, &fake_env_ops, &env); + } + u_camera_consent_decision eval(const char *exe = "/apps/caller", long pid = 100) + { + u_camera_consent_decision d; + u_camera_consent_evaluate(&c, exe, "Caller", pid, &d); + return d; + } +}; + +} // namespace + +TEST_CASE("camera consent: sharing off wins over everything", "[camera_consent]") +{ + rig r; + r.c.dev_override = true; + r.store.delegating.insert("/apps/caller"); + SECTION("kill switch") + { + r.c.kill_switch = true; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_DISABLED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_KILL_SWITCH); + } + SECTION("user toggle") + { + r.store.sharing = false; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_DISABLED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_SHARING_OFF); + CHECK(!u_camera_consent_sharing_enabled(&r.c)); + } + CHECK(r.env.prompts == 0); +} + +TEST_CASE("camera consent: dev override allows without a prompt, once logged", "[camera_consent]") +{ + rig r; + r.c.dev_override = true; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_DEV_OVERRIDE); + CHECK(!d.delegating); + CHECK(r.c.dev_override_logged); + CHECK(r.env.prompts == 0); + CHECK(r.store.sets == 0); +} + +TEST_CASE("camera consent: no verifiable executable is refused, never prompted", "[camera_consent]") +{ + rig r; + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW; + auto d = r.eval(""); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_NO_IDENTITY); + CHECK(r.env.prompts == 0); +} + +TEST_CASE("camera consent: a delegating client passes without a prompt or a store entry", "[camera_consent]") +{ + rig r; + r.store.delegating.insert("/opt/browser/browser"); + r.store.apps["/opt/browser/browser"] = U_CAMERA_CONSENT_STORED_DENY; // ignored: it is delegating + r.env.os_allowed = false; // checked after delegation + auto d = r.eval("/opt/browser/browser"); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_DELEGATING); + CHECK(d.delegating); + CHECK(r.env.prompts == 0); +} + +TEST_CASE("camera consent: the OS camera switch refuses before the store", "[camera_consent]") +{ + rig r; + r.env.os_allowed = false; + r.store.apps["/apps/caller"] = U_CAMERA_CONSENT_STORED_ALLOW; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_OS_DENIED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_OS_DENIED); + CHECK(r.env.prompts == 0); +} + +TEST_CASE("camera consent: stored decisions are final, no prompt", "[camera_consent]") +{ + rig r; + r.env.answer = U_CAMERA_CONSENT_PROMPT_DENY; + r.store.apps["/apps/caller"] = U_CAMERA_CONSENT_STORED_ALLOW; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_STORED_ALLOW); + + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW; + r.store.apps["/apps/caller"] = U_CAMERA_CONSENT_STORED_DENY; + d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_STORED_DENY); + CHECK(r.env.prompts == 0); +} + +TEST_CASE("camera consent: the prompt's answers", "[camera_consent]") +{ + rig r; + SECTION("Allow is stored and not asked again") + { + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_ALLOW); + CHECK(r.env.prompts == 1); + CHECK(r.env.last_exe == "/apps/caller"); + CHECK(r.env.last_pid == 100); + CHECK(r.store.apps["/apps/caller"] == U_CAMERA_CONSENT_STORED_ALLOW); + d = r.eval(); + CHECK(d.why == U_CAMERA_CONSENT_WHY_STORED_ALLOW); + CHECK(r.env.prompts == 1); + } + SECTION("Allow once binds to the process, not the executable") + { + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW_ONCE; + auto d = r.eval("/apps/caller", 100); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_ALLOW_ONCE); + CHECK(r.store.sets == 0); + d = r.eval("/apps/caller", 100); + CHECK(d.why == U_CAMERA_CONSENT_WHY_ALLOW_ONCE); + CHECK(r.env.prompts == 1); + d = r.eval("/apps/caller", 101); // a new process of the same app asks again + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_ALLOW_ONCE); + CHECK(r.env.prompts == 2); + u_camera_consent_forget_once(&r.c); // the user stopped sharing + d = r.eval("/apps/caller", 100); + CHECK(r.env.prompts == 3); + } + SECTION("Deny is stored") + { + r.env.answer = U_CAMERA_CONSENT_PROMPT_DENY; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_DENY); + CHECK(r.store.apps["/apps/caller"] == U_CAMERA_CONSENT_STORED_DENY); + d = r.eval(); + CHECK(d.why == U_CAMERA_CONSENT_WHY_STORED_DENY); + CHECK(r.env.prompts == 1); + } + SECTION("unanswered / unavailable refuse and are asked again next time") + { + r.env.answer = U_CAMERA_CONSENT_PROMPT_TIMEOUT; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_TIMEOUT); + r.env.answer = U_CAMERA_CONSENT_PROMPT_UNAVAILABLE; + d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE); + CHECK(r.env.prompts == 2); + CHECK(r.store.sets == 0); + } + SECTION("prompt disabled (headless) refuses without calling it") + { + r.c.prompt_enabled = false; + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE); + CHECK(r.env.prompts == 0); + } + SECTION("an Allow the store cannot persist still allows this process") + { + r.env.answer = U_CAMERA_CONSENT_PROMPT_ALLOW; + r.store.set_fails = true; + auto d = r.eval(); + CHECK(d.verdict == U_CAMERA_CONSENT_ALLOWED); + d = r.eval(); + CHECK(d.why == U_CAMERA_CONSENT_WHY_ALLOW_ONCE); + CHECK(r.env.prompts == 1); + } +} + +TEST_CASE("camera consent: no store and no env still decides safely", "[camera_consent]") +{ + u_camera_consent c; + u_camera_consent_init(&c, nullptr, nullptr, nullptr, nullptr); + u_camera_consent_decision d; + u_camera_consent_evaluate(&c, "/apps/caller", "Caller", 1, &d); + CHECK(d.verdict == U_CAMERA_CONSENT_REFUSED); + CHECK(d.why == U_CAMERA_CONSENT_WHY_PROMPT_UNAVAILABLE); + CHECK(u_camera_consent_sharing_enabled(&c)); + char id[64]; + u_camera_consent_persistent_id(&c, "serial-1", "/apps/caller", id); // per-process key + CHECK(strncmp(id, "dxrcam-", 7) == 0); + CHECK(strlen(id) == 7 + 32); +} + +TEST_CASE("sha256 / hmac vectors", "[camera_consent]") +{ + static const char *hex = "0123456789abcdef"; + auto to_hex = [&](const uint8_t *d, size_t n) { + std::string s; + for (size_t i = 0; i < n; i++) { + s += hex[d[i] >> 4]; + s += hex[d[i] & 15]; + } + return s; + }; + uint8_t out[32]; + u_sha256("", 0, out); + CHECK(to_hex(out, 32) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + u_sha256("abc", 3, out); + CHECK(to_hex(out, 32) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + const char *long_msg = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + u_sha256(long_msg, strlen(long_msg), out); + CHECK(to_hex(out, 32) == "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); + // RFC 4231 test case 2. + u_hmac_sha256("Jefe", 4, "what do ya want for nothing?", 28, out); + CHECK(to_hex(out, 32) == "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"); + // RFC 4231 test case 6: a key longer than one block. + uint8_t key131[131]; + memset(key131, 0xaa, sizeof(key131)); + const char *m6 = "Test Using Larger Than Block-Size Key - Hash Key First"; + u_hmac_sha256(key131, sizeof(key131), m6, strlen(m6), out); + CHECK(to_hex(out, 32) == "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54"); +} + +TEST_CASE("keyed persistentId: stable per key, unlinkable across keys and consumers", "[camera_consent]") +{ + uint8_t k1[32], k2[32]; + memset(k1, 0x11, 32); + memset(k2, 0x22, 32); + char a[64], b[64], c[64], d[64], e[64]; + u_camera_consent_persistent_id_keyed(k1, 32, "serial-0001", "/opt/browser", a); + u_camera_consent_persistent_id_keyed(k1, 32, "serial-0001", "/opt/browser", b); + u_camera_consent_persistent_id_keyed(k2, 32, "serial-0001", "/opt/browser", c); + u_camera_consent_persistent_id_keyed(k1, 32, "serial-0001", "/opt/other", d); + u_camera_consent_persistent_id_keyed(k1, 32, "serial-0002", "/opt/browser", e); + CHECK(std::string(a) == b); + CHECK(std::string(a) != c); + CHECK(std::string(a) != d); + CHECK(std::string(a) != e); + CHECK(strncmp(a, "dxrcam-", 7) == 0); + CHECK(strlen(a) == 39); + // The serial never appears in the id. + CHECK(strstr(a, "0001") == nullptr); + // Length-prefixed fields: ("ab","c") != ("a","bc"). + char f[64], g[64]; + u_camera_consent_persistent_id_keyed(k1, 32, "ab", "c", f); + u_camera_consent_persistent_id_keyed(k1, 32, "a", "bc", g); + CHECK(std::string(f) != g); + // The store's secret is what keys the service's id. + rig r; + memcpy(r.store.secret, k1, 32); + char h[64]; + u_camera_consent_persistent_id(&r.c, "serial-0001", "/opt/browser", h); + CHECK(std::string(h) == a); +} + +TEST_CASE("client class: CAMERA_CONSUMER is outside the PRESENT_OWNER quota", "[camera_consent][client_class]") +{ + CHECK(u_client_class_counts_toward_present_owner(XRT_CLIENT_CLASS_PRESENT_OWNER)); + CHECK(!u_client_class_counts_toward_present_owner(XRT_CLIENT_CLASS_CAMERA_CONSUMER)); + CHECK(!u_client_class_counts_toward_present_owner(XRT_CLIENT_CLASS_APP)); + CHECK(!u_client_class_may_create_session(XRT_CLIENT_CLASS_CAMERA_CONSUMER)); + CHECK(u_client_class_may_create_session(XRT_CLIENT_CLASS_PRESENT_OWNER)); + CHECK(u_client_class_may_create_session(XRT_CLIENT_CLASS_APP)); + + const char *chrome = "C:\\Program Files\\Browser\\browser.exe"; + const char *other = "C:\\Other\\other.exe"; + std::vector peers = { + {XRT_CLIENT_CLASS_PRESENT_OWNER, chrome}, // the GPU process + {XRT_CLIENT_CLASS_PRESENT_OWNER, chrome}, // the browser process + {XRT_CLIENT_CLASS_CAMERA_CONSUMER, chrome}, // the video-capture utility: not an owner + {XRT_CLIENT_CLASS_APP, other}, + }; + SECTION("one owner, siblings take no slot") + { + CHECK(u_client_class_present_owner_count(peers.data(), (uint32_t)peers.size(), other) == 1); + CHECK(u_client_class_present_owner_count(peers.data(), (uint32_t)peers.size(), chrome) == 0); + } + SECTION("the capture utility alone never makes the browser an owner") + { + std::vector only_cam = {{XRT_CLIENT_CLASS_CAMERA_CONSUMER, chrome}}; + CHECK(u_client_class_present_owner_count(only_cam.data(), 1, other) == 0); + CHECK(u_client_class_present_owner_count(only_cam.data(), 1, chrome) == 0); + } + SECTION("two distinct owners; an unknown path is its own owner and nobody's sibling") + { + peers.push_back({XRT_CLIENT_CLASS_PRESENT_OWNER, other}); + CHECK(u_client_class_present_owner_count(peers.data(), (uint32_t)peers.size(), "/x") == 2); + peers.push_back({XRT_CLIENT_CLASS_PRESENT_OWNER, ""}); + CHECK(u_client_class_present_owner_count(peers.data(), (uint32_t)peers.size(), "/x") == 3); + CHECK(u_client_class_present_owner_count(peers.data(), (uint32_t)peers.size(), "") == 3); + } +} From d658a9750473ace40a30ebe90ac5ba666b8c4c3e Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:31 -0700 Subject: [PATCH 32/37] =?UTF-8?q?feat(ipc):=20CAMERA=5FCONSUMER=20client?= =?UTF-8?q?=20class=20=E2=80=94=20camera-only=20consumers=20outside=20the?= =?UTF-8?q?=20panel=20quota?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A browser's video-capture utility (B1) has no window and no weave, yet was classed PRESENT_OWNER by enabling XR_DXR_weave and spent a panel-owner slot. XRT_CLIENT_CLASS_CAMERA_CONSUMER is declared explicitly with the new XrStereoCameraClientInfoDXR (CONSUMER_ONLY bit) on XrInstanceCreateInfo, verified by use (camera handlers only; session_create refuses it), quota 4, and never counted toward the PRESENT_OWNER quota — the handler now uses the pure u_client_class_present_owner_count(). A hybrid runtime always routes a declared consumer to the service. The tester's owner-quota (sibling) rule for the browser's other processes is kept. XR_DXR_stereo_camera spec v2: appends 1004999311–316 (client-info struct, CONSENT_REFUSED / DISABLED / BUSY / STREAM_ENDED results, the stream-ended event); nothing renumbered. Co-Authored-By: Claude Fable 5.1 --- src/external/openxr_includes/openxr/README.md | 2 +- .../openxr/XR_DXR_stereo_camera.h | 78 ++++++++++++++++++- src/xrt/include/xrt/xrt_instance.h | 11 ++- src/xrt/ipc/server/ipc_server_handler.c | 64 ++++++++------- src/xrt/state_trackers/oxr/oxr_instance.c | 13 ++++ src/xrt/targets/openxr/target.c | 8 ++ 6 files changed, 141 insertions(+), 35 deletions(-) diff --git a/src/external/openxr_includes/openxr/README.md b/src/external/openxr_includes/openxr/README.md index 9fc409a17e..f3bafb2a75 100644 --- a/src/external/openxr_includes/openxr/README.md +++ b/src/external/openxr_includes/openxr/README.md @@ -50,7 +50,7 @@ Rules: | 1004999280–289 | `XR_DXR_lift` (continued) | 280 = `XR_TYPE_LIFT_STREAM_STATS_DXR` (spec v1). A second decade because 270–279 is full | | 1004999290–299 | `XR_DXR_weave` (v14+ additions) | 290 = `XR_TYPE_WEAVE_SUBMIT_OVERLAY_UNCHANGED_DXR` (the v4 overlay atlas is unchanged since the previous submit, so a DP may reuse its lens prefilter of the 2D layer, spec v14, ADR-027 Amendment); 291 = `XR_TYPE_WEAVE_SUBMIT_OVERLAY_FILTER_DXR` (the caller's lens-filter strength for the 2D layer, 0..1, omitted = the DP's default, spec v15). The third weave decade (240–249 is full) | | 1004999300–309 | `XR_DXR_stereo_camera` | 300 = `XR_TYPE_STEREO_CAMERA_PROPERTIES_DXR`, 301 = `XR_TYPE_STEREO_CAMERA_CALIBRATION_DXR`, 302 = `XR_TYPE_STEREO_CAMERA_STREAM_CREATE_INFO_DXR`, 303 = `XR_TYPE_STEREO_CAMERA_FRAME_DXR`, 304 = `XR_TYPE_STEREO_CAMERA_STREAM_INFO_DXR`, 305 = `XR_TYPE_STEREO_CAMERA_STREAM_TRANSPORT_DXR`, 306 = `XR_TYPE_STEREO_CAMERA_STREAM_STATS_DXR`, 307 = `XR_TYPE_EVENT_DATA_STEREO_CAMERA_STATE_CHANGED_DXR`, 308 = `XR_TYPE_EVENT_DATA_STEREO_CAMERAS_CHANGED_DXR` (spec v1, ADR-043). 309 = `XR_OBJECT_TYPE_STEREO_CAMERA_STREAM_DXR` — an `XrObjectType`, not an `XrStructureType`. **Relocated from 290–299** on the rebase onto post-lift `main`: 290 had meanwhile been taken by `XR_TYPE_WEAVE_SUBMIT_OVERLAY_UNCHANGED_DXR` (weave spec v14). Safe to move: spec v1, draft PR, no shipped app chains it | -| 1004999310–319 | `XR_DXR_stereo_camera` (continued) | 310 = `XR_STEREO_CAMERA_FRAME_NOT_READY_DXR` — a **success**-class `XrResult` (positive). A second decade because 300–309 is full | +| 1004999310–319 | `XR_DXR_stereo_camera` (continued) | 310 = `XR_STEREO_CAMERA_FRAME_NOT_READY_DXR` — a **success**-class `XrResult` (positive). A second decade because 300–309 is full. Spec v2 (R3, ADR-043 §7): 311 = `XR_TYPE_STEREO_CAMERA_CLIENT_INFO_DXR` (camera-only client declaration on `XrInstanceCreateInfo`); 312 = `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR`, 313 = `XR_ERROR_STEREO_CAMERA_DISABLED_DXR`, 314 = `XR_ERROR_STEREO_CAMERA_BUSY_DXR`, 316 = `XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR` — **error**-class `XrResult`s (negated in the header); 315 = `XR_TYPE_EVENT_DATA_STEREO_CAMERA_STREAM_ENDED_DXR` | | 1004999320+ | **next free** | | `XR_DXR_android_surface_binding` was implemented in #1037 and now has its own diff --git a/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h index 740871487c..ce514d908e 100644 --- a/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h +++ b/src/external/openxr_includes/openxr/XR_DXR_stereo_camera.h @@ -44,13 +44,21 @@ * Privacy: frames bypass the OS camera stack, so the runtime gates them itself * (consent, foreground rule, in-use indicator, kill switch — spec §7). * xrStartStereoCameraStreamDXR and xrGetStereoCameraCalibrationDXR return - * XR_ERROR_PERMISSION_INSUFFICIENT when refused. + * XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR when the user / the stored + * decision refused, XR_ERROR_STEREO_CAMERA_DISABLED_DXR when sharing is off, + * XR_ERROR_PERMISSION_INSUFFICIENT when the OS camera privacy switch is off, + * XR_ERROR_STEREO_CAMERA_BUSY_DXR when the source cannot be opened. * * Spec: docs/specs/extensions/XR_DXR_stereo_camera.md. * * Version history: 1 = initial (enumerate + state, calibration raw/rectified, * streams with start/stop, latest-wins acquire over a pinned 3-slot shared- - * memory ring with per-stream wake handles, stream stats). + * memory ring with per-stream wake handles, stream stats). 2 (R3) = consent + * and privacy as built: XrStereoCameraClientInfoDXR (camera-only client + * class), distinct refusal results (CONSENT_REFUSED / DISABLED / BUSY / + * STREAM_ENDED), the state-changed and cameras-changed events are now + * delivered, XrEventDataStereoCameraStreamEndedDXR, persistentId keyed with + * a per-user secret. Everything appended; nothing renumbered. */ #ifndef XR_DXR_STEREO_CAMERA_H #define XR_DXR_STEREO_CAMERA_H 1 @@ -63,7 +71,7 @@ extern "C" { #endif #define XR_DXR_stereo_camera 1 -#define XR_DXR_stereo_camera_SPEC_VERSION 1 +#define XR_DXR_stereo_camera_SPEC_VERSION 2 #define XR_DXR_STEREO_CAMERA_EXTENSION_NAME "XR_DXR_stereo_camera" // Reserved 1004999300..319 (relocated from 290..309 on the post-lift rebase: 290 was taken by XR_DXR_weave v14). Allocation registry: README.md in this directory. @@ -81,6 +89,25 @@ extern "C" { //! SUCCESS-class result: no frame newer than the last one this stream acquired. //! Not an error — keep showing the previous frame. #define XR_STEREO_CAMERA_FRAME_NOT_READY_DXR ((XrResult)1004999310) +// Spec v2 (R3) — appended, never renumbered. +//! Chain on XrInstanceCreateInfo to declare a camera-only client (spec §2a). +#define XR_TYPE_STEREO_CAMERA_CLIENT_INFO_DXR ((XrStructureType)1004999311) +//! ERROR: the user (or the stored per-app decision) refused this executable the +//! 3D camera (spec §7.1). Retryable after consent; a browser maps it to +//! NotAllowedError. Distinct from XR_ERROR_PERMISSION_INSUFFICIENT, which is +//! the OS camera privacy switch. +#define XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR ((XrResult)-1004999312) +//! ERROR: camera sharing is off — the user's "Share the 3D camera with apps" +//! toggle or the DXR_STEREO_CAMERA=0 kill switch (spec §7.4). +#define XR_ERROR_STEREO_CAMERA_DISABLED_DXR ((XrResult)-1004999313) +//! ERROR: the source cannot be opened right now (the plug-in reported the +//! device unavailable); a browser maps it to NotReadableError. Retry later. +#define XR_ERROR_STEREO_CAMERA_BUSY_DXR ((XrResult)-1004999314) +//! A stream the SERVICE ended (user kill switch, revocation, source removed). +#define XR_TYPE_EVENT_DATA_STEREO_CAMERA_STREAM_ENDED_DXR ((XrStructureType)1004999315) +//! ERROR: this stream was ended by the service (see the STREAM_ENDED event); +//! destroy it and create a new one. +#define XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR ((XrResult)-1004999316) //! Size of XrStereoCameraPropertiesDXR::persistentId, NUL included. #define XR_STEREO_CAMERA_PERSISTENT_ID_MAX_SIZE_DXR 64 @@ -310,6 +337,51 @@ typedef struct XrEventDataStereoCamerasChangedDXR { const void* XR_MAY_ALIAS next; } XrEventDataStereoCamerasChangedDXR; +/* + * Spec v2 (R3): client declaration, service-ended streams. + */ + +typedef XrFlags64 XrStereoCameraClientFlagsDXR; +//! This instance is a CAMERA-ONLY consumer: it will never create a session, and +//! it is not a panel owner even if its executable also runs an XR_DXR_weave +//! present-owner (a browser's video-capture utility process). The service +//! admits it as the CAMERA_CONSUMER client class, outside the present-owner +//! quota. A hybrid runtime routes such an instance to the service (an +//! in-process instance has no cameras). +static const XrStereoCameraClientFlagsDXR XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR = 0x00000001; + +/*! + * Chain on XrInstanceCreateInfo (the extension must be in enabledExtensionNames). + */ +typedef struct XrStereoCameraClientInfoDXR { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrStereoCameraClientFlagsDXR flags; +} XrStereoCameraClientInfoDXR; + +typedef enum XrStereoCameraStreamEndReasonDXR { + //! The user stopped camera sharing from the runtime's tray / menu bar. + XR_STEREO_CAMERA_STREAM_END_REASON_USER_STOPPED_DXR = 1, + //! Camera sharing was switched off (user toggle or kill switch). + XR_STEREO_CAMERA_STREAM_END_REASON_DISABLED_DXR = 2, + //! The camera is gone (plug-in removed it / the service is shutting it down). + XR_STEREO_CAMERA_STREAM_END_REASON_SOURCE_LOST_DXR = 3, + XR_STEREO_CAMERA_STREAM_END_REASON_MAX_ENUM_DXR = 0x7FFFFFFF +} XrStereoCameraStreamEndReasonDXR; + +/*! + * The service ended @c stream (it is now stopped for good: acquire returns + * XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR). A consumer surfaces this as its + * track ending, then destroys the handle; a new stream may be created and + * started again later (subject to §7). + */ +typedef struct XrEventDataStereoCameraStreamEndedDXR { + XrStructureType type; + const void* XR_MAY_ALIAS next; + XrStereoCameraStreamDXR stream; + XrStereoCameraStreamEndReasonDXR reason; +} XrEventDataStereoCameraStreamEndedDXR; + typedef XrResult (XRAPI_PTR *PFN_xrEnumerateStereoCamerasDXR)( XrInstance instance, XrSystemId systemId, uint32_t capacityInput, uint32_t* countOutput, XrStereoCameraPropertiesDXR* cameras); diff --git a/src/xrt/include/xrt/xrt_instance.h b/src/xrt/include/xrt/xrt_instance.h index 9a93cd75ae..3810207955 100644 --- a/src/xrt/include/xrt/xrt_instance.h +++ b/src/xrt/include/xrt/xrt_instance.h @@ -81,7 +81,16 @@ enum xrt_client_class XRT_CLIENT_CLASS_RELAY = 3, //!< headless relay (XR_MND_headless + display_info; the WebXR bridge) XRT_CLIENT_CLASS_PROVIDER_HOST = 4, //!< service-spawned input-provider host (Phase 4; reserved) XRT_CLIENT_CLASS_DIAG = 5, //!< read-only diagnostics (displayxr-cli clients, bridge introspection) - XRT_CLIENT_CLASS_COUNT = 6, + /*! + * XR_DXR_stereo_camera consumer with no compositor session (ADR-043 R3): + * a browser's video-capture utility process, `displayxr-cli camera`. It + * may only use the camera entry points — never a session — and it is + * NOT a panel owner, so it never counts toward the PRESENT_OWNER quota + * even when its executable is the browser's. Declared explicitly + * (XrStereoCameraClientInfoDXR on xrInstanceCreateInfo); verified by use. + */ + XRT_CLIENT_CLASS_CAMERA_CONSUMER = 6, + XRT_CLIENT_CLASS_COUNT = 7, }; /*! diff --git a/src/xrt/ipc/server/ipc_server_handler.c b/src/xrt/ipc/server/ipc_server_handler.c index f622631a4b..3abe61d8f7 100644 --- a/src/xrt/ipc/server/ipc_server_handler.c +++ b/src/xrt/ipc/server/ipc_server_handler.c @@ -20,6 +20,7 @@ #include "server/ipc_server.h" #include "server/ipc_server_peer_creds.h" +#include "util/u_client_class.h" #include "ipc_server_generated.h" #include "xrt/xrt_device.h" #include "xrt/xrt_results.h" @@ -151,6 +152,7 @@ ipc_server_client_class_str(uint32_t client_class) case XRT_CLIENT_CLASS_RELAY: return "RELAY"; case XRT_CLIENT_CLASS_PROVIDER_HOST: return "PROVIDER_HOST"; case XRT_CLIENT_CLASS_DIAG: return "DIAG"; + case XRT_CLIENT_CLASS_CAMERA_CONSUMER: return "CAMERA_CONSUMER"; default: return "?"; } } @@ -164,6 +166,7 @@ ipc_server_client_class_quota(const struct ipc_server *s, uint32_t client_class) case XRT_CLIENT_CLASS_PRESENT_OWNER: return 2; case XRT_CLIENT_CLASS_DIAG: return 4; case XRT_CLIENT_CLASS_PROVIDER_HOST: return 2; + case XRT_CLIENT_CLASS_CAMERA_CONSUMER: return 4; case XRT_CLIENT_CLASS_APP: default: // APP gets the rest of the admitted cap minus the reserved controller slot. @@ -190,8 +193,11 @@ verify_client_class(volatile struct ipc_client_state *ics, uint32_t declared, co case XRT_CLIENT_CLASS_APP: return XRT_CLIENT_CLASS_APP; case XRT_CLIENT_CLASS_RELAY: case XRT_CLIENT_CLASS_PRESENT_OWNER: + case XRT_CLIENT_CLASS_CAMERA_CONSUMER: // Verified by use: a RELAY may never create a compositor (session_create - // refuses), only a PRESENT_OWNER may call the weave_* handlers. + // refuses), only a PRESENT_OWNER may call the weave_* handlers, and a + // CAMERA_CONSUMER may never create a session at all (ADR-043 R3) — the + // class buys camera calls and nothing else, so there is nothing to spoof. return declared; case XRT_CLIENT_CLASS_CONTROLLER: { unsigned long orch = get_orchestrator_workspace_pid(); @@ -234,44 +240,33 @@ verify_client_class(volatile struct ipc_client_state *ics, uint32_t declared, co // PRESENT_OWNER quota counts OWNERS, not connections: one browser is several // processes of one executable (its GPU process weaves, the browser process binds -// the window, and its video-capture utility reads the stereo camera through -// XR_DXR_stereo_camera), and all of them are the one owner competing for the -// panel. A connection from the same executable as an already-admitted -// PRESENT_OWNER is a SIBLING and takes no new slot; otherwise the quota is -// compared against the number of DISTINCT owner executables. A peer whose image -// path the OS will not reveal ("") is never a sibling and counts as its own -// owner (fails closed). Caller holds global_state.lock; the exe lookups run at -// admission only. +// the window), and all of them are the one owner competing for the panel. A +// connection from the same executable as an already-admitted PRESENT_OWNER is a +// SIBLING and takes no new slot; otherwise the quota is compared against the +// number of DISTINCT owner executables. A peer whose image path the OS will not +// reveal ("") is never a sibling and counts as its own owner (fails closed). The +// browser's video-capture utility declares CAMERA_CONSUMER (ADR-043 R3) and is +// not in this count at all — the rule itself is the pure +// u_client_class_present_owner_count() so it can be tested host-side. Caller +// holds global_state.lock; the exe lookups run at admission only. static uint32_t count_present_owners_locked(struct ipc_server *s, volatile struct ipc_client_state *ics, const char *ics_exe) { - static char seen[IPC_MAX_CLIENTS][512]; // under global_state.lock + static char exes[IPC_MAX_CLIENTS][512]; // under global_state.lock + struct u_client_class_peer peers[IPC_MAX_CLIENTS]; uint32_t n = 0; for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { volatile struct ipc_client_state *o = &s->threads[i].ics; - if (o == ics || o->server_thread_index < 0 || !o->class_verified || - o->client_state.client_class != XRT_CLIENT_CLASS_PRESENT_OWNER) { + if (o == ics || o->server_thread_index < 0 || !o->class_verified) { continue; } - char exe[512]; - exe[0] = '\0'; - ipc_server_peer_exe_path(o->peer_pid, exe, sizeof(exe)); - if (exe[0] != '\0' && ics_exe[0] != '\0' && strcmp(exe, ics_exe) == 0) { - return 0; // a sibling of an admitted owner: no new slot - } - bool dup = false; - for (uint32_t k = 0; k < n && exe[0] != '\0'; k++) { - if (strcmp(seen[k], exe) == 0) { - dup = true; - break; - } - } - if (!dup) { - snprintf(seen[n], sizeof(seen[n]), "%s", exe); - n++; - } + exes[n][0] = '\0'; + ipc_server_peer_exe_path(o->peer_pid, exes[n], sizeof(exes[n])); + peers[n].client_class = o->client_state.client_class; + peers[n].exe = exes[n]; + n++; } - return n; + return u_client_class_present_owner_count(peers, n, ics_exe); } // Count connected clients of a class, EXCLUDING ics. Caller holds global_state.lock. @@ -2293,6 +2288,15 @@ ipc_handle_session_create(volatile struct ipc_client_state *ics, ics->peer_pid); return XRT_ERROR_NOT_AUTHORIZED; } + if (cls == XRT_CLIENT_CLASS_CAMERA_CONSUMER) { + // ADR-043 R3: a camera consumer has no session by contract — that is + // what keeps it out of the panel quota. + IPC_WARN(ics->server, + "session_create: denied — CAMERA_CONSUMER client pid %ld may only use the stereo " + "camera entry points (ADR-043 R3).", + ics->peer_pid); + return XRT_ERROR_NOT_AUTHORIZED; + } if ((xsi->is_workspace_controller && cls != XRT_CLIENT_CLASS_CONTROLLER) || (xsi->is_bridge_relay && cls != XRT_CLIENT_CLASS_RELAY)) { IPC_WARN(ics->server, diff --git a/src/xrt/state_trackers/oxr/oxr_instance.c b/src/xrt/state_trackers/oxr/oxr_instance.c index e7e553b01f..1e3e1292dd 100644 --- a/src/xrt/state_trackers/oxr/oxr_instance.c +++ b/src/xrt/state_trackers/oxr/oxr_instance.c @@ -561,6 +561,19 @@ oxr_instance_create(struct oxr_logger *log, else if (i_info.app_info.ext_weave_enabled) { i_info.app_info.declared_client_class = XRT_CLIENT_CLASS_PRESENT_OWNER; } +#ifdef OXR_HAVE_DXR_stereo_camera + // ADR-043 R3: a camera-only consumer says so explicitly (a browser's + // video-capture utility process, `displayxr-cli camera`). It outranks the + // weave-derived PRESENT_OWNER claim: such a process never presents, so it + // must not take a panel-owner slot. The service refuses it any session. + if (extensions->DXR_stereo_camera) { + const XrStereoCameraClientInfoDXR *ci = OXR_GET_INPUT_FROM_CHAIN( + createInfo, XR_TYPE_STEREO_CAMERA_CLIENT_INFO_DXR, XrStereoCameraClientInfoDXR); + if (ci != NULL && (ci->flags & XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR) != 0) { + i_info.app_info.declared_client_class = XRT_CLIENT_CLASS_CAMERA_CONSUMER; + } + } +#endif // #964 Phase A: is this process part of a workspace session? The workspace // controller sets DISPLAYXR_WORKSPACE_SESSION=1 for the apps it launches // (so does the service orchestrator's spawn), and ONLY those clients are diff --git a/src/xrt/targets/openxr/target.c b/src/xrt/targets/openxr/target.c index 540e763ca0..357db8f7fd 100644 --- a/src/xrt/targets/openxr/target.c +++ b/src/xrt/targets/openxr/target.c @@ -200,6 +200,14 @@ xrt_instance_create(struct xrt_instance_info *ii, struct xrt_instance **out_xins return ipc_instance_create(ii, out_xinst); } + // ADR-043 R3: a declared camera-only consumer has no use for an in-process + // instance (it enumerates zero cameras — the service is each camera's + // single owner), so it always goes to the service. + if (ii != NULL && ii->app_info.declared_client_class == XRT_CLIENT_CLASS_CAMERA_CONSUMER) { + U_LOG_W("Hybrid mode: stereo-camera consumer instance — forcing IPC"); + return ipc_instance_create(ii, out_xinst); + } + // Check if we should use IPC mode if (u_sandbox_should_use_ipc()) { // One line per instance create (lifecycle, never per frame): WARN so the From 93bff14125e23a0557d58d1ee90047fe54d50bf1 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:31 -0700 Subject: [PATCH 33/37] feat(stereo-camera): the service enforces consent, foreground and lock; distinct results; events delivered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the R1 DXR_STEREO_CAMERA_DEV_ALLOW gate with the u_camera_consent policy at every stream start and calibration read, evaluated with the manager lock released and one evaluation at a time (the tray prompt blocks the client thread for up to 60 s). Each verdict has its own result — XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED / _DISABLED / _BUSY / _STREAM_ENDED, mapped 1:1 to the new XrResults — so a browser can map consent to NotAllowedError. RAW is refused to PRESENT_OWNER and delegating clients. Foreground rule per published frame (250 ms cache): window-bearing classes need a visible top-level window of the peer pid (EnumWindows / NSRunning- Application); delegating, CAMERA_CONSUMER and DIAG are exempt. Every stream is suspended while the OS session is locked (set_session_locked, fed by the platform hooks); the reported state is SUSPENDED meanwhile. The state-changed and stream-ended events are now delivered: per-connection queues drained by stereo_camera_poll_event from xrPollEvent, mapping a service stream id back to the app's handle. stop_all / set_sharing / get_status serve the UI; the macOS menu-bar item gains the in-use badge, "Stop camera sharing", the sharing toggle, a floating consent panel and the screen-lock / session-switch observers. persistentId is keyed with the store's secret. Consumers receive a READ-ONLY section on every platform (ipc_shmem_create_with_readonly). Co-Authored-By: Claude Fable 5.1 --- src/xrt/auxiliary/util/u_pretty_print.c | 4 + src/xrt/include/xrt/xrt_results.h | 29 + src/xrt/include/xrt/xrt_stereo_camera.h | 48 ++ src/xrt/ipc/client/ipc_client_stereo_camera.c | 37 + src/xrt/ipc/client/ipc_client_stereo_camera.h | 8 + src/xrt/ipc/server/ipc_server_macos_appkit.h | 9 + src/xrt/ipc/server/ipc_server_macos_appkit.m | 300 +++++++ src/xrt/ipc/server/ipc_server_stereo_camera.c | 816 ++++++++++++++++-- src/xrt/ipc/server/ipc_server_stereo_camera.h | 59 ++ src/xrt/ipc/shared/ipc_shmem.c | 105 +++ src/xrt/ipc/shared/ipc_shmem.h | 18 + src/xrt/ipc/shared/proto.json | 17 + src/xrt/state_trackers/oxr/oxr_event.c | 84 ++ src/xrt/state_trackers/oxr/oxr_objects.h | 19 + .../state_trackers/oxr/oxr_stereo_camera.c | 91 +- 15 files changed, 1547 insertions(+), 97 deletions(-) diff --git a/src/xrt/auxiliary/util/u_pretty_print.c b/src/xrt/auxiliary/util/u_pretty_print.c index d9d3b9ac1b..f2dc0f1084 100644 --- a/src/xrt/auxiliary/util/u_pretty_print.c +++ b/src/xrt/auxiliary/util/u_pretty_print.c @@ -195,6 +195,10 @@ u_pp_xrt_result(struct u_pp_delegate dg, xrt_result_t xret) case XRT_ERROR_WEAVE_REFUSED: DG("XRT_ERROR_WEAVE_REFUSED"); return; case XRT_ERROR_IPC_VERSION_SKEW: DG("XRT_ERROR_IPC_VERSION_SKEW"); return; case XRT_ERROR_INPUT_HOST_GEOMETRY_NOT_READY: DG("XRT_ERROR_INPUT_HOST_GEOMETRY_NOT_READY"); return; + case XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED: DG("XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED"); return; + case XRT_ERROR_STEREO_CAMERA_DISABLED: DG("XRT_ERROR_STEREO_CAMERA_DISABLED"); return; + case XRT_ERROR_STEREO_CAMERA_BUSY: DG("XRT_ERROR_STEREO_CAMERA_BUSY"); return; + case XRT_ERROR_STEREO_CAMERA_STREAM_ENDED: DG("XRT_ERROR_STEREO_CAMERA_STREAM_ENDED"); return; case XRT_ERROR_NO_IMAGE_AVAILABLE: DG("XRT_ERROR_NO_IMAGE_AVAILABLE"); return; case XRT_ERROR_VULKAN: DG("XRT_ERROR_VULKAN"); return; case XRT_ERROR_OPENGL: DG("XRT_ERROR_OPENGL"); return; diff --git a/src/xrt/include/xrt/xrt_results.h b/src/xrt/include/xrt/xrt_results.h index e9c3f71ea9..7ef323b31f 100644 --- a/src/xrt/include/xrt/xrt_results.h +++ b/src/xrt/include/xrt/xrt_results.h @@ -326,4 +326,33 @@ typedef enum xrt_result * tell "too early" from "never". */ XRT_ERROR_INPUT_HOST_GEOMETRY_NOT_READY = -44, + + /*! + * XR_DXR_stereo_camera (ADR-043 R3): the user, or the stored per-app + * decision, refused this executable the camera. Retryable after consent. + * Maps to XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR; distinct from + * @ref XRT_ERROR_NOT_AUTHORIZED, which stays the OS-switch / class refusal + * (XR_ERROR_PERMISSION_INSUFFICIENT). + */ + XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED = -45, + + /*! + * XR_DXR_stereo_camera: camera sharing is switched off (user toggle or + * kill switch). Maps to XR_ERROR_STEREO_CAMERA_DISABLED_DXR. + */ + XRT_ERROR_STEREO_CAMERA_DISABLED = -46, + + /*! + * XR_DXR_stereo_camera: the source cannot be opened right now (the + * plug-in reported it unavailable and the retry is pending). Maps to + * XR_ERROR_STEREO_CAMERA_BUSY_DXR. + */ + XRT_ERROR_STEREO_CAMERA_BUSY = -47, + + /*! + * XR_DXR_stereo_camera: the service ended this stream (user stop, + * sharing off, source lost); destroy and recreate it. Maps to + * XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR. + */ + XRT_ERROR_STEREO_CAMERA_STREAM_ENDED = -48, } xrt_result_t; diff --git a/src/xrt/include/xrt/xrt_stereo_camera.h b/src/xrt/include/xrt/xrt_stereo_camera.h index 6fa17a3ea0..ca06e4fcd3 100644 --- a/src/xrt/include/xrt/xrt_stereo_camera.h +++ b/src/xrt/include/xrt/xrt_stereo_camera.h @@ -189,6 +189,48 @@ struct xrt_stereo_camera_stream_stats uint32_t reserved; }; +//! What a delivered event is (R3). +enum xrt_stereo_camera_event_kind +{ + XRT_STEREO_CAMERA_EVENT_CAMERA_STATE = 1, //!< XrEventDataStereoCameraStateChangedDXR + XRT_STEREO_CAMERA_EVENT_STREAM_ENDED = 2, //!< XrEventDataStereoCameraStreamEndedDXR + XRT_STEREO_CAMERA_EVENT_CAMERAS_CHANGED = 3, //!< XrEventDataStereoCamerasChangedDXR +}; + +//! Why the service ended a stream (= XrStereoCameraStreamEndReasonDXR). +enum xrt_stereo_camera_end_reason +{ + XRT_STEREO_CAMERA_END_USER_STOPPED = 1, + XRT_STEREO_CAMERA_END_DISABLED = 2, + XRT_STEREO_CAMERA_END_SOURCE_LOST = 3, +}; + +/*! + * One queued event for a connection (R3). The service queues them per + * connection; the client drains them from xrPollEvent. + */ +struct xrt_stereo_camera_event +{ + uint32_t kind; //!< enum xrt_stereo_camera_event_kind + uint32_t value; //!< CAMERA_STATE: the state; STREAM_ENDED: the reason + uint64_t camera_id; + uint64_t stream_id; //!< STREAM_ENDED only +}; + +//! `stereo_camera_control` ops (DIAG clients and the service's own UI only). +enum xrt_stereo_camera_control_op +{ + //! End every started stream now (the tray's "Stop camera sharing"). + XRT_STEREO_CAMERA_CONTROL_STOP_ALL = 1, + //! arg 0/1: the persistent "Share the 3D camera with apps" toggle. + XRT_STEREO_CAMERA_CONTROL_SET_SHARING = 2, + //! arg 0/1: simulate the OS session lock (diagnostics only; the real + //! lock comes from the platform notification). + XRT_STEREO_CAMERA_CONTROL_SET_LOCKED = 3, + //! out = bit 0 sharing on, bit 1 locked, bits 8..15 started streams. + XRT_STEREO_CAMERA_CONTROL_STATUS = 4, +}; + /*! * Client aspect of an @ref xrt_instance: set by the IPC client instance only * (the camera's single owner is the service); NULL for an in-process instance, @@ -225,6 +267,12 @@ struct xrt_stereo_camera_client xrt_result_t (*stats)(struct xrt_stereo_camera_client *c, uint64_t stream_id, struct xrt_stereo_camera_stream_stats *out_stats); + //! R3: pop one queued event; @p out_has false = none pending. + xrt_result_t (*poll_event)(struct xrt_stereo_camera_client *c, + bool *out_has, + struct xrt_stereo_camera_event *out_event); + //! R3: enum xrt_stereo_camera_control_op (DIAG clients only). + xrt_result_t (*control)(struct xrt_stereo_camera_client *c, uint32_t op, uint32_t arg, uint32_t *out_value); }; #ifdef __cplusplus diff --git a/src/xrt/ipc/client/ipc_client_stereo_camera.c b/src/xrt/ipc/client/ipc_client_stereo_camera.c index a87c9d6db8..3ce861cf1c 100644 --- a/src/xrt/ipc/client/ipc_client_stereo_camera.c +++ b/src/xrt/ipc/client/ipc_client_stereo_camera.c @@ -156,6 +156,29 @@ ipc_client_stereo_camera_stats(struct ipc_connection *ipc_c, return ipc_call_stereo_camera_stream_stats(ipc_c, stream_id, out_stats); } +xrt_result_t +ipc_client_stereo_camera_poll_event(struct ipc_connection *ipc_c, + bool *out_has, + struct xrt_stereo_camera_event *out_event) +{ + *out_has = false; + U_ZERO(out_event); + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_poll_event(ipc_c, out_has, out_event); +} + +xrt_result_t +ipc_client_stereo_camera_control(struct ipc_connection *ipc_c, uint32_t op, uint32_t arg, uint32_t *out_value) +{ + *out_value = 0; + if (ipc_c == NULL) { + return XRT_ERROR_IPC_FAILURE; + } + return ipc_call_stereo_camera_control(ipc_c, op, arg, out_value); +} + /* * @@ -247,6 +270,18 @@ c_stats(struct xrt_stereo_camera_client *c, uint64_t stream_id, struct xrt_stere return ipc_client_stereo_camera_stats(conn(c), stream_id, out); } +static xrt_result_t +c_poll_event(struct xrt_stereo_camera_client *c, bool *out_has, struct xrt_stereo_camera_event *out_event) +{ + return ipc_client_stereo_camera_poll_event(conn(c), out_has, out_event); +} + +static xrt_result_t +c_control(struct xrt_stereo_camera_client *c, uint32_t op, uint32_t arg, uint32_t *out_value) +{ + return ipc_client_stereo_camera_control(conn(c), op, arg, out_value); +} + struct xrt_stereo_camera_client * ipc_client_stereo_camera_client_create(struct ipc_connection *ipc_c) { @@ -262,5 +297,7 @@ ipc_client_stereo_camera_client_create(struct ipc_connection *ipc_c) c->base.stream_get_transport = c_stream_get_transport; c->base.acquire = c_acquire; c->base.stats = c_stats; + c->base.poll_event = c_poll_event; + c->base.control = c_control; return &c->base; } diff --git a/src/xrt/ipc/client/ipc_client_stereo_camera.h b/src/xrt/ipc/client/ipc_client_stereo_camera.h index ab8bf10789..251fa08bfe 100644 --- a/src/xrt/ipc/client/ipc_client_stereo_camera.h +++ b/src/xrt/ipc/client/ipc_client_stereo_camera.h @@ -78,6 +78,14 @@ ipc_client_stereo_camera_stats(struct ipc_connection *ipc_c, * The @ref xrt_stereo_camera_client aspect over @p ipc_c (which must outlive * it). Caller frees with free(). */ +xrt_result_t +ipc_client_stereo_camera_poll_event(struct ipc_connection *ipc_c, + bool *out_has, + struct xrt_stereo_camera_event *out_event); + +xrt_result_t +ipc_client_stereo_camera_control(struct ipc_connection *ipc_c, uint32_t op, uint32_t arg, uint32_t *out_value); + struct xrt_stereo_camera_client * ipc_client_stereo_camera_client_create(struct ipc_connection *ipc_c); diff --git a/src/xrt/ipc/server/ipc_server_macos_appkit.h b/src/xrt/ipc/server/ipc_server_macos_appkit.h index 3192551e96..dd6b29aab9 100644 --- a/src/xrt/ipc/server/ipc_server_macos_appkit.h +++ b/src/xrt/ipc/server/ipc_server_macos_appkit.h @@ -16,6 +16,8 @@ #pragma once +#include + #ifdef __cplusplus extern "C" { #endif @@ -34,6 +36,13 @@ struct ipc_server; void ipc_server_macos_pump_main_thread(struct ipc_server *s); +/*! + * XR_DXR_stereo_camera R3 foreground rule: does @p pid belong to a running + * application that is not hidden (NSRunningApplication)? Any thread. + */ +bool +ipc_server_macos_pid_app_visible(long pid); + #ifdef __cplusplus } #endif diff --git a/src/xrt/ipc/server/ipc_server_macos_appkit.m b/src/xrt/ipc/server/ipc_server_macos_appkit.m index 47413cc4f0..b924ceeeda 100644 --- a/src/xrt/ipc/server/ipc_server_macos_appkit.m +++ b/src/xrt/ipc/server/ipc_server_macos_appkit.m @@ -362,12 +362,123 @@ // service GRACEFULLY ("Quit" → raise SIGTERM → the kqueue EVFILT_SIGNAL path → // service_orchestrator_shutdown, which reaps the controller — no orphan). // Complements the global Ctrl+Space hotkey with a no-TCC, always-visible control. +/* + * + * XR_DXR_stereo_camera R3 (ADR-043 §7): the menu-bar indicator ("3D camera in + * use by ", red dot on the icon), the user kill switch ("Stop camera + * sharing"), the persistent "Share the 3D camera with apps" toggle, the consent + * prompt (a floating panel with Allow / Allow once / Deny — NOT a modal alert, + * which would park this run loop and with it every IPC accept for up to a + * minute), the screen-lock / fast-user-switch suspension, and the foreground + * rule's visibility query. The prompt is requested from an IPC client thread + * (camera_prompt_fn blocks it on a condition); the main-thread pump shows the + * panel and the button actions answer it. + * + */ + +#include "server/ipc_server_stereo_camera.h" +#include +#include +#include + +struct cam_prompt_req +{ + char exe[512]; + char app[128]; + long pid; + enum u_camera_consent_prompt_answer answer; + bool answered; + bool shown; +}; + +static pthread_mutex_t s_cam_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t s_cam_cond = PTHREAD_COND_INITIALIZER; +static struct cam_prompt_req *s_cam_prompt = NULL; //!< the one pending request (consent_lock serialises) +static NSPanel *s_cam_panel = nil; //!< main thread only +static volatile bool s_cam_dirty = true; //!< indicator needs a refresh +static NSImage *s_logo_image = nil; +static NSImage *s_logo_inuse_image = nil; + +bool +ipc_server_macos_pid_app_visible(long pid) +{ + @autoreleasepool { + NSRunningApplication *app = [NSRunningApplication runningApplicationWithProcessIdentifier:(pid_t)pid]; + if (app == nil || app.terminated) { + return false; // not a GUI app the user can see + } + return !app.hidden && app.activationPolicy != NSApplicationActivationPolicyProhibited; + } +} + +//! IPC client thread: ask, block until answered / timeout. +static enum u_camera_consent_prompt_answer +camera_prompt_fn(void *ctx, const char *exe, const char *app_name, long pid, uint32_t timeout_ms) +{ + (void)ctx; + struct cam_prompt_req req; + memset(&req, 0, sizeof(req)); + snprintf(req.exe, sizeof(req.exe), "%s", exe != NULL ? exe : ""); + snprintf(req.app, sizeof(req.app), "%s", app_name != NULL && app_name[0] ? app_name : "An app"); + req.pid = pid; + + struct timespec deadline; + clock_gettime(CLOCK_REALTIME, &deadline); + deadline.tv_sec += timeout_ms / 1000; + deadline.tv_nsec += (long)(timeout_ms % 1000) * 1000000L; + if (deadline.tv_nsec >= 1000000000L) { + deadline.tv_sec += 1; + deadline.tv_nsec -= 1000000000L; + } + pthread_mutex_lock(&s_cam_mutex); + s_cam_prompt = &req; + s_cam_dirty = true; + while (!req.answered) { + if (pthread_cond_timedwait(&s_cam_cond, &s_cam_mutex, &deadline) == ETIMEDOUT) { + break; + } + } + enum u_camera_consent_prompt_answer a = req.answered ? req.answer : U_CAMERA_CONSENT_PROMPT_TIMEOUT; + s_cam_prompt = NULL; // the panel (if up) closes on the next pump; buttons see NULL + s_cam_dirty = true; + pthread_mutex_unlock(&s_cam_mutex); + return a; +} + +static void +camera_indicator_changed(void *ctx) +{ + (void)ctx; + s_cam_dirty = true; +} + +//! Main thread: a button answered (NULL request = already timed out). +static void +camera_prompt_answer(enum u_camera_consent_prompt_answer a) +{ + pthread_mutex_lock(&s_cam_mutex); + if (s_cam_prompt != NULL && !s_cam_prompt->answered) { + s_cam_prompt->answer = a; + s_cam_prompt->answered = true; + pthread_cond_broadcast(&s_cam_cond); + } + pthread_mutex_unlock(&s_cam_mutex); + if (s_cam_panel != nil) { + [s_cam_panel orderOut:nil]; + [s_cam_panel release]; + s_cam_panel = nil; + } +} + @interface DXRStatusTarget : NSObject { @public struct ipc_server *server; NSMenuItem *statusLine; // owned by the menu NSMenuItem *openItem; // owned by the menu + NSMenuItem *cameraLine; // "3D camera: …" + NSMenuItem *cameraStop; // "Stop camera sharing" + NSMenuItem *cameraShare; // "Share the 3D camera with apps" (checkmark) } @end @@ -399,6 +510,46 @@ - (void)menuNeedsUpdate:(NSMenu *)menu statusLine.title = @"Workspace: stopped"; [openItem setEnabled:YES]; } + struct ipc_server_stereo_camera_status st; + if (!ipc_server_stereo_camera_get_status(&st) || st.camera_count == 0) { + cameraLine.title = st.sharing_enabled ? @"3D camera: none" : @"3D camera: sharing off"; + [cameraStop setEnabled:NO]; + } else if (st.started_streams > 0) { + cameraLine.title = [NSString stringWithFormat:@"3D camera in use by %s%s", st.consumers, + st.locked ? " (suspended: screen locked)" : ""]; + [cameraStop setEnabled:YES]; + } else { + cameraLine.title = st.sharing_enabled ? @"3D camera: not in use" : @"3D camera: sharing off"; + [cameraStop setEnabled:NO]; + } + cameraShare.state = st.sharing_enabled ? NSControlStateValueOn : NSControlStateValueOff; +} +- (void)stopCameraSharing:(id)sender +{ + (void)sender; + ipc_server_stereo_camera_stop_all("menu bar: Stop camera sharing"); +} +- (void)toggleCameraSharing:(id)sender +{ + (void)sender; + struct ipc_server_stereo_camera_status st; + bool on = ipc_server_stereo_camera_get_status(&st) ? st.sharing_enabled : true; + ipc_server_stereo_camera_set_sharing(!on, "menu bar: Share the 3D camera with apps"); +} +- (void)consentAllow:(id)sender +{ + (void)sender; + camera_prompt_answer(U_CAMERA_CONSENT_PROMPT_ALLOW); +} +- (void)consentAllowOnce:(id)sender +{ + (void)sender; + camera_prompt_answer(U_CAMERA_CONSENT_PROMPT_ALLOW_ONCE); +} +- (void)consentDeny:(id)sender +{ + (void)sender; + camera_prompt_answer(U_CAMERA_CONSENT_PROMPT_DENY); } @end @@ -407,6 +558,108 @@ - (void)menuNeedsUpdate:(NSMenu *)menu static DXRStatusTarget *s_status_target = nil; static NSStatusItem *s_status_item = nil; +//! Main thread: show the pending consent prompt as a floating panel. +static void +camera_prompt_show(DXRStatusTarget *target, const struct cam_prompt_req *req) +{ + const char *base = req->exe; + for (const char *p = req->exe; *p != '\0'; p++) { + if (*p == '/') { + base = p + 1; + } + } + NSRect frame = NSMakeRect(0, 0, 440, 150); + NSPanel *panel = [[NSPanel alloc] initWithContentRect:frame + styleMask:(NSWindowStyleMaskTitled | NSWindowStyleMaskUtilityWindow) + backing:NSBackingStoreBuffered + defer:NO]; + panel.title = @"DisplayXR"; + panel.level = NSFloatingWindowLevel; + panel.releasedWhenClosed = NO; + [panel setHidesOnDeactivate:NO]; + NSView *v = panel.contentView; + + NSTextField *title = [NSTextField labelWithString:[NSString stringWithFormat:@"\u201c%s\u201d wants to use the 3D camera", req->app]]; + title.font = [NSFont boldSystemFontOfSize:13]; + title.frame = NSMakeRect(20, 108, 400, 22); + [v addSubview:title]; + NSTextField *body = [NSTextField wrappingLabelWithString:[NSString stringWithFormat:@"%s (pid %ld) will receive frames from the display\u2019s stereo camera. You can stop it at any time from the DisplayXR menu.", base[0] ? base : "The app", req->pid]]; + body.font = [NSFont systemFontOfSize:11]; + body.frame = NSMakeRect(20, 58, 400, 44); + [v addSubview:body]; + + NSButton *deny = [NSButton buttonWithTitle:@"Deny" target:target action:@selector(consentDeny:)]; + deny.frame = NSMakeRect(20, 14, 100, 32); + [v addSubview:deny]; + NSButton *once = [NSButton buttonWithTitle:@"Allow once" target:target action:@selector(consentAllowOnce:)]; + once.frame = NSMakeRect(210, 14, 110, 32); + [v addSubview:once]; + NSButton *allow = [NSButton buttonWithTitle:@"Allow" target:target action:@selector(consentAllow:)]; + allow.frame = NSMakeRect(326, 14, 100, 32); + allow.keyEquivalent = @"\r"; + [v addSubview:allow]; + + [panel center]; + [NSApp activateIgnoringOtherApps:YES]; + [panel makeKeyAndOrderFront:nil]; + s_cam_panel = panel; + U_LOG_W("stereo camera: consent prompt shown for %s (pid %ld)", req->exe, req->pid); +} + +//! A copy of the logo with a red "recording" dot bottom-right. +static NSImage * +make_inuse_image(NSImage *logo) +{ + NSSize sz = logo.size; + NSImage *img = [[NSImage alloc] initWithSize:sz]; + [img lockFocus]; + [logo drawInRect:NSMakeRect(0, 0, sz.width, sz.height) + fromRect:NSZeroRect + operation:NSCompositingOperationSourceOver + fraction:1.0]; + CGFloat d = sz.height * 0.45; + NSBezierPath *dot = [NSBezierPath bezierPathWithOvalInRect:NSMakeRect(sz.width - d, 0, d, d)]; + [[NSColor systemRedColor] setFill]; + [dot fill]; + [img unlockFocus]; + img.template = NO; // keep the red; the base stays a template + return img; +} + +//! Main thread, every pump tick: prompt panel + indicator image. +static void +camera_pump(void) +{ + pthread_mutex_lock(&s_cam_mutex); + struct cam_prompt_req *req = s_cam_prompt; + bool show = req != NULL && !req->shown; + if (show) { + req->shown = true; + } + bool close = req == NULL && s_cam_panel != nil; + pthread_mutex_unlock(&s_cam_mutex); + if (show) { + camera_prompt_show(s_status_target, req); + } + if (close) { + [s_cam_panel orderOut:nil]; + [s_cam_panel release]; + s_cam_panel = nil; + } + if (!s_cam_dirty) { + return; + } + s_cam_dirty = false; + struct ipc_server_stereo_camera_status st; + bool in_use = ipc_server_stereo_camera_get_status(&st) && st.started_streams > 0; + if (s_status_item != nil && s_logo_image != nil) { + s_status_item.button.image = in_use && s_logo_inuse_image != nil ? s_logo_inuse_image : s_logo_image; + s_status_item.button.toolTip = + in_use ? [NSString stringWithFormat:@"3D camera in use by %s", st.consumers] : @"DisplayXR Service"; + } +} + + // Build the menu-bar status item + menu. Main thread; called once. static void install_status_item(struct ipc_server *s) @@ -440,6 +693,8 @@ - (void)menuNeedsUpdate:(NSMenu *)menu logo.size = NSMakeSize(w, h); s_status_item.button.image = logo; s_status_item.button.imagePosition = NSImageOnly; + s_logo_image = [logo retain]; + s_logo_inuse_image = make_inuse_image(logo); // camera in-use badge (ADR-043 §7.3) } else { // Fallback if the embedded image fails to decode for any reason. s_status_item.button.title = @"DisplayXR"; @@ -465,6 +720,48 @@ - (void)menuNeedsUpdate:(NSMenu *)menu [menu addItem:[NSMenuItem separatorItem]]; + // XR_DXR_stereo_camera R3: indicator + user kill switch + sharing toggle. + NSMenuItem *camLine = [[[NSMenuItem alloc] initWithTitle:@"3D camera: not in use" action:nil keyEquivalent:@""] autorelease]; + [camLine setEnabled:NO]; + [menu addItem:camLine]; + s_status_target->cameraLine = camLine; + NSMenuItem *camStop = [[[NSMenuItem alloc] initWithTitle:@"Stop camera sharing" + action:@selector(stopCameraSharing:) + keyEquivalent:@""] autorelease]; + camStop.target = s_status_target; + [camStop setEnabled:NO]; + [menu addItem:camStop]; + s_status_target->cameraStop = camStop; + NSMenuItem *camShare = [[[NSMenuItem alloc] initWithTitle:@"Share the 3D camera with apps" + action:@selector(toggleCameraSharing:) + keyEquivalent:@""] autorelease]; + camShare.target = s_status_target; + [menu addItem:camShare]; + s_status_target->cameraShare = camShare; + ipc_server_stereo_camera_set_prompt_provider(camera_prompt_fn, NULL); + ipc_server_stereo_camera_set_indicator_provider(camera_indicator_changed, NULL); + // Screen lock / fast user switch suspend every stream (spec §7.2). + NSDistributedNotificationCenter *dnc = [NSDistributedNotificationCenter defaultCenter]; + [dnc addObserverForName:@"com.apple.screenIsLocked" object:nil queue:nil usingBlock:^(NSNotification *n) { + (void)n; + ipc_server_stereo_camera_set_session_locked(true, "com.apple.screenIsLocked"); + }]; + [dnc addObserverForName:@"com.apple.screenIsUnlocked" object:nil queue:nil usingBlock:^(NSNotification *n) { + (void)n; + ipc_server_stereo_camera_set_session_locked(false, "com.apple.screenIsUnlocked"); + }]; + NSNotificationCenter *wnc = [[NSWorkspace sharedWorkspace] notificationCenter]; + [wnc addObserverForName:NSWorkspaceSessionDidResignActiveNotification object:nil queue:nil usingBlock:^(NSNotification *n) { + (void)n; + ipc_server_stereo_camera_set_session_locked(true, "session resigned active (user switch)"); + }]; + [wnc addObserverForName:NSWorkspaceSessionDidBecomeActiveNotification object:nil queue:nil usingBlock:^(NSNotification *n) { + (void)n; + ipc_server_stereo_camera_set_session_locked(false, "session became active"); + }]; + + [menu addItem:[NSMenuItem separatorItem]]; + NSMenuItem *quit = [[[NSMenuItem alloc] initWithTitle:@"Quit DisplayXR Service" action:@selector(quitService:) keyEquivalent:@"q"] autorelease]; @@ -503,6 +800,9 @@ - (void)menuNeedsUpdate:(NSMenu *)menu inited = true; } + // XR_DXR_stereo_camera R3: consent panel + in-use indicator. + camera_pump(); + // Service the global toggle hotkey (set by the Carbon handler). Done here, // holding the ipc_server, so spawn/SIGTERM can read workspace_controller_pid. if (g_toggle_requested) { diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.c b/src/xrt/ipc/server/ipc_server_stereo_camera.c index d3ca57ff22..f3504cea91 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.c +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.c @@ -11,19 +11,45 @@ * 1280x480 NV12 copy is well under a millisecond, and it keeps an acquire from * ever observing a half-written slot. Only the camera thread closes a source. * - * Privacy (spec §7) — R1 ships the HOOK POINTS, deny-by-default: - * - authorise_locked(): every start and every calibration read. Allowed in R1 - * only with DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment; - * otherwise XRT_ERROR_NOT_AUTHORIZED (XR_ERROR_PERMISSION_INSUFFICIENT). - * R3 replaces the env check with OS consent + the DisplayXR consent store + - * delegating-client registration. - * - client_visible_locked(): the foreground rule, evaluated per published - * frame. R1 always true; R3 wires session visibility / OS lock. - * - output_allowed(): RAW is refused to PRESENT_OWNER clients (the browser, - * which exposes cameras to web pages) — maintainer decision for R1. - * - DXR_STEREO_CAMERA=0: kill switch, zero cameras enumerated. - * - the in-use indicator (tray badge naming consumers) is R3; R1 logs one - * WARN per stream start/stop naming the peer executable. + * Privacy (spec §7) — R3, as built. The plug-in never sees any of it: + * - consent_evaluate(): every stream start and every calibration read runs + * the u_camera_consent policy (aux util, unit-tested with fakes) against + * the OS-derived peer executable: sharing off (DXR_STEREO_CAMERA=0 or the + * user's tray toggle) -> DISABLED; DXR_STEREO_CAMERA_DEV_ALLOW=1 -> allowed + * (one WARN); registered consent-DELEGATING client (the browser, by + * installer / `displayxr-cli camera trust`) -> allowed, its own prompt is + * the consent; OS camera privacy switch -> PERMISSION_INSUFFICIENT; the + * stored per-app Allow/Deny; else the tray prompt (" wants to use the + * 3D camera — Allow / Allow once / Deny"), which the host installs with + * ipc_server_stereo_camera_set_prompt_provider(). The prompt BLOCKS the + * calling client thread (manager lock released, one prompt at a time) for + * up to 60 s; unanswered = refused, retryable. Each verdict has its own + * result: CONSENT_REFUSED / DISABLED / BUSY / STREAM_ENDED, so a browser + * can map consent to NotAllowedError and the rest to NotReadableError. + * - client_visible_locked(): the foreground rule, per published frame, cached + * 250 ms per stream. It applies to window-bearing classes (APP, + * PRESENT_OWNER): the peer pid must own a visible top-level window + * (Windows EnumWindows; macOS NSRunningApplication not hidden; Linux: no + * display connection in the service, always visible). A delegating client + * follows its own rule; CAMERA_CONSUMER / DIAG have no window by contract + * and are exempt (they got explicit consent instead). Every stream is + * suspended while the OS session is locked (ipc_server_stereo_camera_ + * set_session_locked from the platform notification: WTS session change on + * Windows, screenIsLocked / sessionDidResignActive on macOS). + * - output_allowed(): RAW is refused to PRESENT_OWNER and to any delegating + * client (both expose cameras to third-party content). + * - events: XrEventDataStereoCameraStateChangedDXR (effective state: SUSPENDED + * while locked or sharing is off) and XrEventDataStereoCameraStreamEndedDXR + * are queued per connection (16 deep, oldest dropped) and drained by the + * client from xrPollEvent through stereo_camera_poll_event. + * - indicator + user kill switch: ipc_server_stereo_camera_get_status() feeds + * the tray / menu-bar item ("3D camera in use by "); "Stop camera + * sharing" = ipc_server_stereo_camera_stop_all() ends every started stream + * (STREAM_ENDED event, acquire -> STREAM_ENDED); the persistent "Share the + * 3D camera with apps" toggle = ipc_server_stereo_camera_set_sharing() + * (off: zero cameras enumerated, DISABLED on start). + * - persistentId is HMAC-SHA-256(per-user secret, device | consumer) — the + * secret lives in the consent store (§7.5). * * Rectification (R2) — when the plug-in reports CALIBRATED but not * NATIVELY_RECTIFIED, the manager builds a per-camera rectifier at create time @@ -81,6 +107,9 @@ #include "server/ipc_server_peer_creds.h" #include "server/ipc_server_stereo_camera.h" #include "ipc_server_generated.h" +#ifdef XRT_OS_MACOS +#include "server/ipc_server_macos_appkit.h" +#endif #include "xrt/xrt_plugin.h" #include "xrt/xrt_stereo_camera.h" @@ -90,6 +119,7 @@ #include "shared/ipc_shmem.h" #include "util/u_debug.h" #include "util/u_logging.h" +#include "util/u_camera_consent.h" #include "util/u_misc.h" #include "util/u_stereo_camera.h" #include "util/u_stereo_rectify.h" @@ -110,6 +140,7 @@ DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_enabled, "DXR_STEREO_CAMERA", true) DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_dev_allow, "DXR_STEREO_CAMERA_DEV_ALLOW", false) +DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_prompt, "DXR_STEREO_CAMERA_PROMPT", true) DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_refine, "DXR_STEREO_CAMERA_REFINE", true) #define MAX_STREAMS (XRT_STEREO_CAMERA_MAX_CAMERAS * XRT_STEREO_CAMERA_MAX_STREAMS_PER_CAMERA) @@ -119,6 +150,9 @@ DEBUG_GET_ONCE_BOOL_OPTION(stereo_camera_refine, "DXR_STEREO_CAMERA_REFINE", tru #define REOPEN_BACKOFF_NS (1000ll * 1000 * 1000) #define STATS_LOG_NS (5000ll * 1000 * 1000) #define REFINE_SAMPLES 512 //!< matches one measurement may return +#define VISIBILITY_TTL_NS (250ll * 1000 * 1000) //!< foreground-rule cache per stream +#define PROMPT_TIMEOUT_MS 60000u //!< an unanswered consent prompt = refused +#define EVQ_CAP 16 //!< queued events per connection /* @@ -209,6 +243,19 @@ struct scam_stream uint32_t output; //!< what frames actually are (RAW when rectification is unavailable) char consumer[260]; + // R3 privacy facts, settled at create from the OS-derived peer. + char exe[512]; //!< verified peer executable ("" = unknown) + long peer_pid; //!< the pid the foreground rule looks at + uint32_t client_class; //!< verified class of the creating connection + bool delegating; //!< registered consent-delegating client + int64_t vis_checked_ns; + bool visible; + uint32_t consent_why; //!< enum u_camera_consent_why of the start that allowed it + //! The SERVICE ended this stream (user stop / sharing off / source lost): + //! stays until destroy; acquire and start report STREAM_ENDED. + bool ended; + uint32_t end_reason; //!< enum xrt_stereo_camera_end_reason + bool started; bool allocated; uint32_t width, height; @@ -216,6 +263,9 @@ struct scam_stream uint64_t slot_stride; uint64_t section_size; xrt_shmem_handle_t section; + //! READ-ONLY handle to the same memory — what every consumer receives + //! (R3, spec §5.3): a client can map the ring but never write it. + xrt_shmem_handle_t section_ro; uint8_t *map; struct scam_wake wake; struct u_stereo_camera_ring ring; @@ -223,10 +273,9 @@ struct scam_stream struct xrt_stereo_camera_frame_info slot_info[U_STEREO_CAMERA_RING_SLOTS]; #ifdef XRT_OS_WINDOWS - //! Restricted duplicates handed to the last get_section / get_wake reply. - //! The generated dispatch duplicates them into the peer AFTER the handler - //! returns, so they are closed on the next call / stream destroy. - HANDLE sent_section_dup; + //! SYNCHRONIZE-only duplicate handed to the last get_wake reply. The + //! generated dispatch duplicates it into the peer AFTER the handler + //! returns, so it is closed on the next call / stream destroy. HANDLE sent_wake_dup; #endif @@ -320,6 +369,15 @@ struct scam_camera struct ipc_server_stereo_camera *mgr; }; +//! Per-connection event queue (R3), keyed by the connection's owner token. +struct scam_evq +{ + uint64_t owner; + struct xrt_stereo_camera_event ev[EVQ_CAP]; + uint32_t head; + uint32_t count; +}; + struct ipc_server_stereo_camera { struct os_mutex lock; @@ -331,8 +389,26 @@ struct ipc_server_stereo_camera struct scam_stream streams[MAX_STREAMS]; uint64_t next_stream_id; uint64_t next_owner; + + // R3 privacy. + struct u_camera_consent consent; + //! Serialises consent evaluations (one prompt at a time); never held + //! together with @ref lock. + struct os_mutex consent_lock; + bool sharing_enabled; //!< cached user toggle (the store is the truth) + bool locked; //!< OS session locked / switched away + struct scam_evq evq[IPC_MAX_CLIENTS]; }; +//! The one manager of this process, for the service's UI (tray / menu bar). +static struct ipc_server_stereo_camera *g_mgr = NULL; +//! UI providers. Process-wide, not per manager: the tray registers them before +//! ipc_server_main() has created the manager. +static ipc_server_stereo_camera_prompt_fn g_prompt_fn = NULL; +static void *g_prompt_ctx = NULL; +static ipc_server_stereo_camera_indicator_fn g_indicator_fn = NULL; +static void *g_indicator_ctx = NULL; + /* * @@ -398,56 +474,268 @@ static xrt_result_t require_camera_client(volatile struct ipc_client_state *ics, const char *what) { uint32_t cls = ics->client_state.client_class; - if (cls != XRT_CLIENT_CLASS_APP && cls != XRT_CLIENT_CLASS_PRESENT_OWNER && cls != XRT_CLIENT_CLASS_DIAG) { - U_LOG_W("%s: denied — pid %ld is class %s (stereo camera: APP / PRESENT_OWNER / DIAG).", what, - ics->peer_pid, ipc_server_client_class_str(cls)); + if (cls != XRT_CLIENT_CLASS_APP && cls != XRT_CLIENT_CLASS_PRESENT_OWNER && cls != XRT_CLIENT_CLASS_DIAG && + cls != XRT_CLIENT_CLASS_CAMERA_CONSUMER) { + U_LOG_W("%s: denied — pid %ld is class %s (stereo camera: APP / PRESENT_OWNER / CAMERA_CONSUMER / DIAG).", + what, ics->peer_pid, ipc_server_client_class_str(cls)); + return XRT_ERROR_NOT_AUTHORIZED; + } + return XRT_SUCCESS; +} + +//! The control ops are for the service's own UI and the diagnostic CLI. +static xrt_result_t +require_diag_client(volatile struct ipc_client_state *ics, const char *what) +{ + if (ics->client_state.client_class != XRT_CLIENT_CLASS_DIAG) { + U_LOG_W("%s: denied — pid %ld is class %s, not DIAG.", what, ics->peer_pid, + ipc_server_client_class_str(ics->client_state.client_class)); return XRT_ERROR_NOT_AUTHORIZED; } return XRT_SUCCESS; } +/* + * + * R3 privacy: consent, foreground rule, events. + * + */ + +//! The service's view of the OS camera switch (u_camera_consent_store.c). +static bool +env_os_camera_allowed(void *ctx, const char *exe) +{ + (void)ctx; + return u_camera_consent_os_camera_allowed(exe); +} + +//! The tray prompt, if the host installed one. +static enum u_camera_consent_prompt_answer +env_prompt(void *ctx, const char *exe, const char *app_name, long pid, uint32_t timeout_ms) +{ + (void)ctx; + ipc_server_stereo_camera_prompt_fn fn = g_prompt_fn; + if (fn == NULL) { + return U_CAMERA_CONSENT_PROMPT_UNAVAILABLE; + } + return fn(g_prompt_ctx, exe, app_name, pid, timeout_ms); +} + +static const struct u_camera_consent_env_ops env_ops = { + .os_camera_allowed = env_os_camera_allowed, + .prompt = env_prompt, +}; + +static xrt_result_t +verdict_to_xret(enum u_camera_consent_verdict v) +{ + switch (v) { + case U_CAMERA_CONSENT_ALLOWED: return XRT_SUCCESS; + case U_CAMERA_CONSENT_DISABLED: return XRT_ERROR_STEREO_CAMERA_DISABLED; + case U_CAMERA_CONSENT_OS_DENIED: return XRT_ERROR_NOT_AUTHORIZED; + case U_CAMERA_CONSENT_REFUSED: + default: return XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED; + } +} + /*! - * Spec §7.1 authorisation HOOK. R1: deny by default; the dev override is the - * only way in. R3 replaces the body: OS camera consent of the OS-derived peer - * (Windows CapabilityAccessManager ConsentStore\webcam, Android CAMERA of the - * peer uid), then the DisplayXR per-executable consent (HKCU allow list / tray - * prompt; a registered consent-delegating browser passes per executable and - * prompts per origin itself). + * Spec §7.1: the authorisation point for a stream start and a calibration + * read. Manager lock NOT held (the prompt may block for up to a minute); + * evaluations are serialised on consent_lock so two apps never race two + * dialogs. One WARN per decision names the peer and the rule. */ static xrt_result_t -authorise_locked(volatile struct ipc_client_state *ics, const struct scam_camera *cam, const char *what) +consent_evaluate(struct ipc_server_stereo_camera *m, + volatile struct ipc_client_state *ics, + const char *exe, + const char *what, + struct u_camera_consent_decision *out) { - (void)cam; - if (debug_get_bool_option_stereo_camera_dev_allow()) { - return XRT_SUCCESS; + char app_name[XRT_MAX_APPLICATION_NAME_SIZE]; + snprintf(app_name, sizeof(app_name), "%s", (const char *)ics->client_state.info.application_name); + os_mutex_lock(&m->consent_lock); + u_camera_consent_evaluate(&m->consent, exe, app_name, ics->peer_pid, out); + os_mutex_unlock(&m->consent_lock); + xrt_result_t xret = verdict_to_xret(out->verdict); + if (xret == XRT_SUCCESS) { + U_LOG_W("%s: ALLOWED for %s (pid %ld, \"%s\") — %s", what, exe[0] ? exe : "?", ics->peer_pid, app_name, + u_camera_consent_why_str(out->why)); + } else { + U_LOG_W("%s: REFUSED for %s (pid %ld, \"%s\") — %s", what, exe[0] ? exe : "?", ics->peer_pid, app_name, + u_camera_consent_why_str(out->why)); } - U_LOG_W( - "%s: refused for pid %ld — stereo camera consent is not implemented yet (R3); set " - "DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment for development.", - what, ics->peer_pid); - return XRT_ERROR_NOT_AUTHORIZED; + return xret; +} + +//! Is @p exe a registered consent-delegating client (cheap store read)? +static bool +exe_is_delegating(struct ipc_server_stereo_camera *m, const char *exe) +{ + const struct u_camera_consent_store_ops *st = m->consent.store; + return exe[0] != '\0' && st != NULL && st->is_delegating != NULL && st->is_delegating(m->consent.store_ctx, exe); +} + +//! Does @p pid own a visible top-level window right now (platform query)? +#ifdef XRT_OS_WINDOWS +struct vis_probe +{ + DWORD pid; + bool visible; +}; + +static BOOL CALLBACK +vis_enum_cb(HWND hwnd, LPARAM lp) +{ + struct vis_probe *vp = (struct vis_probe *)lp; + DWORD pid = 0; + GetWindowThreadProcessId(hwnd, &pid); + if (pid == vp->pid && IsWindowVisible(hwnd) && !IsIconic(hwnd)) { + vp->visible = true; + return FALSE; + } + return TRUE; +} +#endif + +static bool +pid_has_visible_window(long pid) +{ +#if defined(XRT_OS_WINDOWS) + struct vis_probe vp = {.pid = (DWORD)pid, .visible = false}; + EnumWindows(vis_enum_cb, (LPARAM)&vp); + return vp.visible; +#elif defined(XRT_OS_MACOS) + return ipc_server_macos_pid_app_visible(pid); +#else + (void)pid; + return true; // no display connection in the service: the OS lock is the only gate +#endif } /*! - * Spec §7.2 foreground-rule HOOK, evaluated per published frame. R1: always - * visible. R3: an app's session must be VISIBLE/FOCUSED; a delegating client - * follows its own rule; every stream is suspended while the OS session is - * locked / switched away (Android: android_package_is_visible of the peer). + * Spec §7.2 foreground rule, evaluated per published frame with a 250 ms + * cache. Window-bearing classes must own a visible top-level window; a + * delegating client follows its own rule; CAMERA_CONSUMER / DIAG have no + * window by contract and are exempt. The OS-lock gate is separate (publish + * checks m->locked first). */ static bool -client_visible_locked(volatile struct ipc_client_state *ics) +client_visible_locked(struct scam_stream *s, int64_t now) { - (void)ics; - return true; + if (s->delegating || s->client_class == XRT_CLIENT_CLASS_CAMERA_CONSUMER || + s->client_class == XRT_CLIENT_CLASS_DIAG || s->peer_pid <= 0) { + return true; + } + if (s->vis_checked_ns != 0 && now - s->vis_checked_ns < VISIBILITY_TTL_NS) { + return s->visible; + } + bool v = pid_has_visible_window(s->peer_pid); + if (s->vis_checked_ns != 0 && v != s->visible) { + U_LOG_W("stereo camera: stream %llu (%s) %s — foreground rule", (unsigned long long)s->id, s->consumer, + v ? "visible again, frames resume" : "has no visible window, frames suspended"); + } + s->vis_checked_ns = now; + s->visible = v; + return v; } -//! Maintainer decision (R1): raw frames never reach web pages — the browser -//! (PRESENT_OWNER) gets RECTIFIED only; native clients may ask for RAW. +//! RAW frames never reach third-party content: refused to PRESENT_OWNER and +//! to every consent-delegating client (a browser exposes cameras to pages). static bool -output_allowed(volatile struct ipc_client_state *ics, uint32_t output) +output_allowed(volatile struct ipc_client_state *ics, bool delegating, uint32_t output) { return output != XRT_STEREO_CAMERA_OUTPUT_RAW || - ics->client_state.client_class != XRT_CLIENT_CLASS_PRESENT_OWNER; + (ics->client_state.client_class != XRT_CLIENT_CLASS_PRESENT_OWNER && !delegating); +} + +//! The state a client is told: SUSPENDED while the session is locked or sharing is off. +static uint32_t +effective_state_locked(const struct ipc_server_stereo_camera *m, const struct scam_camera *cam) +{ + if (m->locked || !m->sharing_enabled) { + return XRT_STEREO_CAMERA_STATE_SUSPENDED; + } + return cam->state; +} + +static struct scam_evq * +evq_for_owner_locked(struct ipc_server_stereo_camera *m, uint64_t owner, bool create) +{ + struct scam_evq *free_slot = NULL; + for (uint32_t i = 0; i < IPC_MAX_CLIENTS; i++) { + if (m->evq[i].owner == owner) { + return &m->evq[i]; + } + if (m->evq[i].owner == 0 && free_slot == NULL) { + free_slot = &m->evq[i]; + } + } + if (!create || free_slot == NULL) { + return NULL; + } + memset(free_slot, 0, sizeof(*free_slot)); + free_slot->owner = owner; + return free_slot; +} + +static void +evq_push_locked(struct ipc_server_stereo_camera *m, uint64_t owner, const struct xrt_stereo_camera_event *ev) +{ + struct scam_evq *q = evq_for_owner_locked(m, owner, true); + if (q == NULL) { + return; + } + if (q->count == EVQ_CAP) { + q->head = (q->head + 1) % EVQ_CAP; // drop the oldest + q->count--; + } + q->ev[(q->head + q->count) % EVQ_CAP] = *ev; + q->count++; +} + +//! Queue a CAMERA_STATE event (effective state) to every connection with a stream on @p cam. +static void +notify_camera_state_locked(struct ipc_server_stereo_camera *m, const struct scam_camera *cam) +{ + struct xrt_stereo_camera_event ev = { + .kind = XRT_STEREO_CAMERA_EVENT_CAMERA_STATE, + .value = effective_state_locked(m, cam), + .camera_id = cam->camera_id, + }; + uint64_t done[MAX_STREAMS]; + uint32_t n = 0; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + const struct scam_stream *s = &m->streams[i]; + if (!s->used || s->camera != cam->index) { + continue; + } + bool dup = false; + for (uint32_t k = 0; k < n; k++) { + dup |= done[k] == s->owner; + } + if (!dup) { + done[n++] = s->owner; + evq_push_locked(m, s->owner, &ev); + } + } +} + +static void +notify_all_cameras_locked(struct ipc_server_stereo_camera *m) +{ + for (uint32_t i = 0; i < m->camera_count; i++) { + notify_camera_state_locked(m, &m->cams[i]); + } +} + +static void +indicator_changed(struct ipc_server_stereo_camera *m) +{ + (void)m; + ipc_server_stereo_camera_indicator_fn fn = g_indicator_fn; + if (fn != NULL) { + fn(g_indicator_ctx); + } } static bool @@ -477,8 +765,7 @@ set_state_locked(struct scam_camera *cam, uint32_t state) } } } - // TODO(R1 follow-up): queue XrEventDataStereoCameraStateChangedDXR to the - // owning connections (needs a server->client instance-event channel). + notify_camera_state_locked(cam->mgr, cam); } static void @@ -917,12 +1204,9 @@ stream_free_storage(struct scam_stream *s) ipc_shmem_destroy(&s->section, &map, (size_t)s->section_size); s->map = NULL; } + ipc_shmem_close_handle(&s->section_ro); wake_destroy(&s->wake); #ifdef XRT_OS_WINDOWS - if (s->sent_section_dup != NULL) { - CloseHandle(s->sent_section_dup); - s->sent_section_dup = NULL; - } if (s->sent_wake_dup != NULL) { CloseHandle(s->sent_wake_dup); s->sent_wake_dup = NULL; @@ -946,15 +1230,13 @@ stream_allocate_locked(struct scam_stream *s, const struct scam_camera *cam) s->section_size = s->slot_stride * U_STEREO_CAMERA_RING_SLOTS; void *map = NULL; s->section = XRT_SHMEM_HANDLE_INVALID; - xrt_result_t xret = ipc_shmem_create((size_t)s->section_size, &s->section, &map); + s->section_ro = XRT_SHMEM_HANDLE_INVALID; + xrt_result_t xret = ipc_shmem_create_with_readonly((size_t)s->section_size, &s->section, &map, &s->section_ro); if (xret != XRT_SUCCESS) { return xret; } s->map = map; memset(s->map, 0, (size_t)s->section_size); - // TODO(R3): hand POSIX consumers a read-only fd (Linux: reopen - // /proc/self/fd/N O_RDONLY; macOS: shm_open the name O_RDONLY before unlink; - // Android: an ASharedMemory with PROT_READ set via ASharedMemory_setProt). if (!wake_create(&s->wake)) { stream_free_storage(s); return XRT_ERROR_ALLOCATION; @@ -1029,7 +1311,10 @@ publish_locked(struct ipc_server_stereo_camera *m, if (!s->used || !s->started || !s->allocated || s->camera != cam->index) { continue; } - if (!client_visible_locked(s->ics)) { + // Spec §7.2: nothing while the OS session is locked / switched away, + // and nothing to a window-bearing client with no visible window. The + // pinned slot is cleared so the last image does not linger. + if (m->locked || !m->sharing_enabled || !client_visible_locked(s, now)) { u_stereo_camera_ring_clear(&s->ring); continue; } @@ -1244,14 +1529,33 @@ ipc_server_stereo_camera_create(struct xrt_instance *xinst) { struct ipc_server_stereo_camera *m = U_TYPED_CALLOC(struct ipc_server_stereo_camera); os_mutex_init(&m->lock); + os_mutex_init(&m->consent_lock); for (uint32_t i = 0; i < MAX_STREAMS; i++) { m->streams[i].section = XRT_SHMEM_HANDLE_INVALID; + m->streams[i].section_ro = XRT_SHMEM_HANDLE_INVALID; } + // R3: the consent policy over the real store; the prompt arrives later + // from the host (tray / menu bar) through set_prompt_provider. + u_camera_consent_init(&m->consent, u_camera_consent_store_default(), NULL, &env_ops, m); + m->consent.kill_switch = !debug_get_bool_option_stereo_camera_enabled(); + m->consent.dev_override = debug_get_bool_option_stereo_camera_dev_allow(); + m->consent.prompt_enabled = debug_get_bool_option_stereo_camera_prompt(); + m->consent.prompt_timeout_ms = PROMPT_TIMEOUT_MS; + m->sharing_enabled = u_camera_consent_sharing_enabled(&m->consent); + g_mgr = m; + if (!debug_get_bool_option_stereo_camera_enabled()) { U_LOG_W("stereo camera: disabled by DXR_STEREO_CAMERA=0 (kill switch) — zero cameras"); return m; } + if (!m->sharing_enabled) { + U_LOG_W("stereo camera: camera sharing is OFF (user toggle) — zero cameras until it is switched on"); + } + if (!m->consent.prompt_enabled) { + U_LOG_W("stereo camera: consent prompt disabled (DXR_STEREO_CAMERA_PROMPT=0) — an app without a stored " + "or delegated consent is refused"); + } const struct xrt_plugin_iface *iface = NULL; struct xrt_plugin_instance *inst = NULL; if (xinst == NULL || xinst->get_active_plugin == NULL || !xinst->get_active_plugin(xinst, &iface, &inst) || @@ -1352,6 +1656,10 @@ ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr) stream_free_storage(&m->streams[i]); } } + if (g_mgr == m) { + g_mgr = NULL; + } + os_mutex_destroy(&m->consent_lock); os_mutex_destroy(&m->lock); free(m); *mgr_ptr = NULL; @@ -1375,6 +1683,26 @@ stream_stop_locked(struct scam_stream *s, struct scam_camera *cam) (unsigned long long)cam->camera_id, (unsigned long long)s->id, s->consumer, cam->started_count); } +//! The SERVICE ends a started stream: stopped for good, the owner is told. +static void +stream_end_locked(struct ipc_server_stereo_camera *m, struct scam_stream *s, uint32_t reason) +{ + if (!s->used || s->ended) { + return; + } + stream_stop_locked(s, &m->cams[s->camera]); + s->ended = true; + s->end_reason = reason; + struct xrt_stereo_camera_event ev = { + .kind = XRT_STEREO_CAMERA_EVENT_STREAM_ENDED, + .value = reason, + .camera_id = m->cams[s->camera].camera_id, + .stream_id = s->id, + }; + evq_push_locked(m, s->owner, &ev); + wake_signal(&s->wake); // a consumer blocked on the wake handle re-polls and sees ENDED +} + static void stream_destroy_locked(struct ipc_server_stereo_camera *m, struct scam_stream *s) { @@ -1382,6 +1710,7 @@ stream_destroy_locked(struct ipc_server_stereo_camera *m, struct scam_stream *s) stream_free_storage(s); memset(s, 0, sizeof(*s)); s->section = XRT_SHMEM_HANDLE_INVALID; + s->section_ro = XRT_SHMEM_HANDLE_INVALID; #ifndef XRT_OS_WINDOWS s->wake.fds[0] = s->wake.fds[1] = -1; #endif @@ -1401,8 +1730,13 @@ ipc_server_client_stereo_camera_release(volatile struct ipc_client_state *ics) stream_destroy_locked(m, s); } } + struct scam_evq *q = evq_for_owner_locked(m, ics->stereo_camera_owner, false); + if (q != NULL) { + memset(q, 0, sizeof(*q)); + } os_mutex_unlock(&m->lock); ics->stereo_camera_owner = 0; + indicator_changed(m); } @@ -1421,7 +1755,8 @@ ipc_handle_stereo_camera_count(volatile struct ipc_client_state *ics, uint32_t * return XRT_SUCCESS; // zero cameras, indistinguishable from "none" on purpose } os_mutex_lock(&m->lock); - *out_count = m->camera_count; + // Spec §2 / §7.4: sharing off = zero cameras, indistinguishable from none. + *out_count = m->sharing_enabled ? m->camera_count : 0; os_mutex_unlock(&m->lock); return XRT_SUCCESS; } @@ -1450,12 +1785,14 @@ ipc_handle_stereo_camera_get_properties(volatile struct ipc_client_state *ics, } const struct scam_camera *cam = &m->cams[index]; out_props->camera_id = cam->camera_id; - u_stereo_camera_persistent_id(cam->info.device_identity, exe, out_props->persistent_id); + // Spec §7.5: keyed with the per-user secret — a page cannot derive the + // device from the id, and two installs never share one. + u_camera_consent_persistent_id(&m->consent, cam->info.device_identity, exe, out_props->persistent_id); snprintf(out_props->display_name, sizeof(out_props->display_name), "%s", cam->info.display_name); snprintf(out_props->platform_device_hint, sizeof(out_props->platform_device_hint), "%s", cam->info.platform_device_hint); out_props->flags = cam->info.flags; - out_props->state = cam->state; + out_props->state = effective_state_locked(m, cam); out_props->view_count = 2; out_props->eye_width = cam->info.eye_width; out_props->eye_height = cam->info.eye_height; @@ -1493,15 +1830,31 @@ ipc_handle_stereo_camera_get_calibration(volatile struct ipc_client_state *ics, if (m == NULL) { return XRT_ERROR_FEATURE_NOT_SUPPORTED; } + if (output != XRT_STEREO_CAMERA_OUTPUT_RAW && output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + char exe[512] = {0}; + ipc_server_peer_exe_path(ics->peer_pid, exe, sizeof(exe)); + os_mutex_lock(&m->lock); + bool known = camera_by_id_locked(m, camera_id) != NULL && m->sharing_enabled; + os_mutex_unlock(&m->lock); + if (!known) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + // Spec §7.5: calibration identifies the device — only after §7.1 passes. + // Evaluated with the manager lock released (it may prompt). + struct u_camera_consent_decision dec; + xrt_result_t xret = consent_evaluate(m, ics, exe, "stereo_camera_get_calibration", &dec); + if (xret != XRT_SUCCESS) { + return xret; + } os_mutex_lock(&m->lock); struct scam_camera *cam = camera_by_id_locked(m, camera_id); - if (cam == NULL || (output != XRT_STEREO_CAMERA_OUTPUT_RAW && output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED)) { + if (cam == NULL) { os_mutex_unlock(&m->lock); return XRT_ERROR_FEATURE_NOT_SUPPORTED; } - // Spec §7.5: calibration identifies the device — only after §7.1 passes. - xrt_result_t xret = authorise_locked(ics, cam, "stereo_camera_get_calibration"); - if (xret == XRT_SUCCESS && !output_allowed(ics, output)) { + if (!output_allowed(ics, dec.delegating, output)) { xret = XRT_ERROR_NOT_AUTHORIZED; } if (xret == XRT_SUCCESS && !cam->have_calib) { @@ -1600,19 +1953,20 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, (req->output != XRT_STEREO_CAMERA_OUTPUT_RAW && req->output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED)) { return XRT_ERROR_INPUT_UNSUPPORTED; } - if (!output_allowed(ics, req->output)) { - U_LOG_W("stereo_camera_stream_create: RAW refused for pid %ld (PRESENT_OWNER: rectified only)", - ics->peer_pid); + char exe[512] = {0}; + ipc_server_peer_exe_path(ics->peer_pid, exe, sizeof(exe)); + const bool delegating = exe_is_delegating(m, exe); + if (!output_allowed(ics, delegating, req->output)) { + U_LOG_W("stereo_camera_stream_create: RAW refused for pid %ld (%s: rectified only)", ics->peer_pid, + delegating ? "consent-delegating client" : "PRESENT_OWNER"); return XRT_ERROR_NOT_AUTHORIZED; } - char exe[260] = {0}; - ipc_server_peer_exe_path(ics->peer_pid, exe, sizeof(exe)); os_mutex_lock(&m->lock); - struct scam_camera *cam = camera_by_id_locked(m, req->camera_id); + struct scam_camera *cam = m->sharing_enabled ? camera_by_id_locked(m, req->camera_id) : NULL; if (cam == NULL) { os_mutex_unlock(&m->lock); - return XRT_ERROR_INPUT_UNSUPPORTED; + return m->sharing_enabled ? XRT_ERROR_INPUT_UNSUPPORTED : XRT_ERROR_STEREO_CAMERA_DISABLED; } bool calibrated = (cam->info.flags & (XRT_PLUGIN_STEREO_CAMERA_CALIBRATED | XRT_PLUGIN_STEREO_CAMERA_NATIVELY_RECTIFIED)) != 0; @@ -1623,7 +1977,7 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, // Raw frames never reach web pages: the browser gets truly rectified // frames or nothing, never the RAW-flagged fallback. if (req->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED && !camera_can_rectify(cam) && - !output_allowed(ics, XRT_STEREO_CAMERA_OUTPUT_RAW)) { + !output_allowed(ics, delegating, XRT_STEREO_CAMERA_OUTPUT_RAW)) { os_mutex_unlock(&m->lock); U_LOG_W( "stereo_camera_stream_create: camera %llu cannot be rectified; refused for pid %ld " @@ -1647,6 +2001,7 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, } memset(slot, 0, sizeof(*slot)); slot->section = XRT_SHMEM_HANDLE_INVALID; + slot->section_ro = XRT_SHMEM_HANDLE_INVALID; #ifndef XRT_OS_WINDOWS slot->wake.fds[0] = slot->wake.fds[1] = -1; #endif @@ -1663,6 +2018,10 @@ ipc_handle_stereo_camera_stream_create(volatile struct ipc_client_state *ics, ? XRT_STEREO_CAMERA_OUTPUT_RECTIFIED : XRT_STEREO_CAMERA_OUTPUT_RAW; snprintf(slot->consumer, sizeof(slot->consumer), "%s (pid %ld)", exe[0] ? exe : "?", ics->peer_pid); + snprintf(slot->exe, sizeof(slot->exe), "%s", exe); + slot->peer_pid = ics->peer_pid; + slot->client_class = ics->client_state.client_class; + slot->delegating = delegating; *out_stream_id = slot->id; os_mutex_unlock(&m->lock); return XRT_SUCCESS; @@ -1681,6 +2040,7 @@ ipc_handle_stereo_camera_stream_destroy(volatile struct ipc_client_state *ics, u stream_destroy_locked(m, s); } os_mutex_unlock(&m->lock); + indicator_changed(m); return XRT_SUCCESS; } @@ -1695,28 +2055,72 @@ ipc_handle_stereo_camera_stream_start(volatile struct ipc_client_state *ics, uin if (m == NULL) { return XRT_ERROR_FEATURE_NOT_SUPPORTED; } + // Pass 1 (locked): find the stream, copy what consent needs. + char exe[512]; os_mutex_lock(&m->lock); struct scam_stream *s = stream_by_id_locked(m, ics, stream_id); if (s == NULL) { os_mutex_unlock(&m->lock); return XRT_ERROR_INPUT_UNSUPPORTED; } + if (s->ended) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_STEREO_CAMERA_STREAM_ENDED; + } + if (s->started) { + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; + } + snprintf(exe, sizeof(exe), "%s", s->exe); + os_mutex_unlock(&m->lock); + + // Spec §7.1: start is THE authorisation point. Lock released — the tray + // prompt may block for up to a minute. + struct u_camera_consent_decision dec; + xrt_result_t xret = consent_evaluate(m, ics, exe, "stereo_camera_stream_start", &dec); + if (xret != XRT_SUCCESS) { + return xret; + } + + // Pass 2 (locked): the stream may have gone away while we asked. + os_mutex_lock(&m->lock); + s = stream_by_id_locked(m, ics, stream_id); + if (s == NULL) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_INPUT_UNSUPPORTED; + } + if (s->ended) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_STEREO_CAMERA_STREAM_ENDED; + } if (s->started) { os_mutex_unlock(&m->lock); return XRT_SUCCESS; } + if (!m->sharing_enabled) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_STEREO_CAMERA_DISABLED; + } struct scam_camera *cam = &m->cams[s->camera]; - // Spec §7.1: start is THE authorisation point. - xrt_result_t xret = authorise_locked(ics, cam, "stereo_camera_stream_start"); - if (xret == XRT_SUCCESS) { - xret = stream_allocate_locked(s, cam); + // BUSY: the plug-in could not open the source and its retry is still + // pending — distinct from "refused", so a browser maps it to NotReadableError. + if (cam->state == XRT_STEREO_CAMERA_STATE_UNAVAILABLE && cam->src == NULL && + os_monotonic_get_ns() < cam->retry_after_ns) { + os_mutex_unlock(&m->lock); + U_LOG_W("stereo camera %llu: stream %llu start refused — source unavailable (BUSY), retry later", + (unsigned long long)cam->camera_id, (unsigned long long)s->id); + return XRT_ERROR_STEREO_CAMERA_BUSY; } + s->delegating = dec.delegating || s->delegating; + s->consent_why = (uint32_t)dec.why; + xret = stream_allocate_locked(s, cam); if (xret != XRT_SUCCESS) { os_mutex_unlock(&m->lock); return xret; } u_stereo_camera_decimator_init(&s->dec, s->req.max_frame_rate, source_rate_hz(cam)); u_stereo_camera_ring_clear(&s->ring); + s->vis_checked_ns = 0; s->started = true; cam->started_count++; if (!cam->thread_started) { @@ -1730,12 +2134,12 @@ ipc_handle_stereo_camera_stream_start(volatile struct ipc_client_state *ics, uin cam->thread_started = true; } os_cond_signal(&cam->cond); - // TODO(R3): runtime-owned in-use indicator naming s->consumer. - U_LOG_W("stereo camera %llu: stream %llu started by %s — output %s, format %u, max %.1f Hz; %u started", + U_LOG_W("stereo camera %llu: stream %llu started by %s — output %s, format %u, max %.1f Hz; %u started%s", (unsigned long long)cam->camera_id, (unsigned long long)s->id, s->consumer, s->output == XRT_STEREO_CAMERA_OUTPUT_RECTIFIED ? "RECTIFIED" : "RAW", s->req.format, - s->req.max_frame_rate, cam->started_count); + s->req.max_frame_rate, cam->started_count, m->locked ? " (session locked: suspended)" : ""); os_mutex_unlock(&m->lock); + indicator_changed(m); // the tray shows "3D camera in use by " return XRT_SUCCESS; } @@ -1752,6 +2156,7 @@ ipc_handle_stereo_camera_stream_stop(volatile struct ipc_client_state *ics, uint stream_stop_locked(s, &m->cams[s->camera]); } os_mutex_unlock(&m->lock); + indicator_changed(m); return XRT_SUCCESS; } @@ -1785,21 +2190,10 @@ ipc_handle_stereo_camera_stream_get_section(volatile struct ipc_client_state *ic out_layout->section_size = s->section_size; out_layout->slot_stride = s->slot_stride; out_layout->slot_count = U_STEREO_CAMERA_RING_SLOTS; -#ifdef XRT_OS_WINDOWS - // A READ-ONLY duplicate: the consumer can map it but never write the ring. - if (s->sent_section_dup != NULL) { - CloseHandle(s->sent_section_dup); - s->sent_section_dup = NULL; - } - if (!DuplicateHandle(GetCurrentProcess(), s->section, GetCurrentProcess(), &s->sent_section_dup, FILE_MAP_READ, - FALSE, 0)) { - os_mutex_unlock(&m->lock); - return XRT_ERROR_IPC_FAILURE; - } - out_handles[0] = s->sent_section_dup; -#else - out_handles[0] = s->section; // SCM_RIGHTS installs a copy; ours stays open -#endif + // The READ-ONLY handle (FILE_MAP_READ duplicate on Windows, an O_RDONLY + // shm fd / PROT_READ ashmem on POSIX): the consumer can map the ring but + // never write it. The dispatch duplicates it into the peer; ours stays open. + out_handles[0] = s->section_ro; *out_handle_count = 1; ics->handles_sent = true; // browser#103 RC-1 os_mutex_unlock(&m->lock); @@ -1863,6 +2257,10 @@ ipc_handle_stereo_camera_acquire(volatile struct ipc_client_state *ics, os_mutex_unlock(&m->lock); return XRT_ERROR_INPUT_UNSUPPORTED; } + if (s->ended) { + os_mutex_unlock(&m->lock); + return XRT_ERROR_STEREO_CAMERA_STREAM_ENDED; + } int32_t slot = -1; uint64_t seq = 0; if (s->started && s->allocated && u_stereo_camera_ring_acquire(&s->ring, &slot, &seq)) { @@ -1915,3 +2313,233 @@ ipc_handle_stereo_camera_stream_stats(volatile struct ipc_client_state *ics, os_mutex_unlock(&m->lock); return XRT_SUCCESS; } + + +/* + * + * R3: events, control, and the service-UI entry points. + * + */ + +xrt_result_t +ipc_handle_stereo_camera_poll_event(volatile struct ipc_client_state *ics, + bool *out_has, + struct xrt_stereo_camera_event *out_event) +{ + *out_has = false; + U_ZERO(out_event); + struct ipc_server_stereo_camera *m = mgr_of(ics); + if (m == NULL || ics->stereo_camera_owner == 0) { + return XRT_SUCCESS; // never used a camera: nothing can be queued + } + os_mutex_lock(&m->lock); + struct scam_evq *q = evq_for_owner_locked(m, ics->stereo_camera_owner, false); + if (q != NULL && q->count > 0) { + *out_event = q->ev[q->head]; + q->head = (q->head + 1) % EVQ_CAP; + q->count--; + *out_has = true; + } + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; +} + +//! End every started stream (lock held). Returns how many were ended. +static uint32_t +stop_all_locked(struct ipc_server_stereo_camera *m, uint32_t reason) +{ + uint32_t n = 0; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + struct scam_stream *s = &m->streams[i]; + if (s->used && s->started) { + stream_end_locked(m, s, reason); + n++; + } + } + return n; +} + +static uint32_t +status_bits_locked(const struct ipc_server_stereo_camera *m) +{ + uint32_t started = 0; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + started += m->streams[i].used && m->streams[i].started ? 1u : 0u; + } + return (m->sharing_enabled ? 1u : 0u) | (m->locked ? 2u : 0u) | ((started & 0xffu) << 8); +} + +xrt_result_t +ipc_handle_stereo_camera_control(volatile struct ipc_client_state *ics, uint32_t op, uint32_t arg, uint32_t *out_value) +{ + *out_value = 0; + struct ipc_server_stereo_camera *m = mgr_of(ics); + xrt_result_t auth = require_diag_client(ics, "stereo_camera_control"); + if (auth != XRT_SUCCESS) { + return auth; + } + if (m == NULL) { + return XRT_ERROR_FEATURE_NOT_SUPPORTED; + } + char who[64]; + snprintf(who, sizeof(who), "diag client pid %ld", ics->peer_pid); + switch (op) { + case XRT_STEREO_CAMERA_CONTROL_STOP_ALL: *out_value = ipc_server_stereo_camera_stop_all(who); return XRT_SUCCESS; + case XRT_STEREO_CAMERA_CONTROL_SET_SHARING: + return ipc_server_stereo_camera_set_sharing(arg != 0, who) ? XRT_SUCCESS : XRT_ERROR_IPC_FAILURE; + case XRT_STEREO_CAMERA_CONTROL_SET_LOCKED: + ipc_server_stereo_camera_set_session_locked(arg != 0, who); + return XRT_SUCCESS; + case XRT_STEREO_CAMERA_CONTROL_STATUS: + os_mutex_lock(&m->lock); + *out_value = status_bits_locked(m); + os_mutex_unlock(&m->lock); + return XRT_SUCCESS; + default: return XRT_ERROR_INPUT_UNSUPPORTED; + } +} + +void +ipc_server_stereo_camera_set_prompt_provider(ipc_server_stereo_camera_prompt_fn fn, void *ctx) +{ + g_prompt_ctx = ctx; + g_prompt_fn = fn; +} + +void +ipc_server_stereo_camera_set_indicator_provider(ipc_server_stereo_camera_indicator_fn fn, void *ctx) +{ + g_indicator_ctx = ctx; + g_indicator_fn = fn; +} + +bool +ipc_server_stereo_camera_get_status(struct ipc_server_stereo_camera_status *out) +{ + memset(out, 0, sizeof(*out)); + struct ipc_server_stereo_camera *m = g_mgr; + if (m == NULL) { + return false; + } + os_mutex_lock(&m->lock); + out->camera_count = m->camera_count; + out->sharing_enabled = m->sharing_enabled; + out->locked = m->locked; + size_t n = 0; + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + const struct scam_stream *s = &m->streams[i]; + if (!s->used || !s->started) { + continue; + } + out->started_streams++; + // Base name of the executable, de-duplicated, comma separated. + const char *base = s->exe; + for (const char *p = s->exe; *p != '\0'; p++) { + if (*p == '\\' || *p == '/') { + base = p + 1; + } + } + if (base[0] == '\0') { + base = "unknown app"; + } + if (strstr(out->consumers, base) == NULL && n < sizeof(out->consumers) - 1) { + n += (size_t)snprintf(out->consumers + n, sizeof(out->consumers) - n, "%s%s", n ? ", " : "", base); + if (n >= sizeof(out->consumers)) { + n = sizeof(out->consumers) - 1; + } + } + } + os_mutex_unlock(&m->lock); + return true; +} + +uint32_t +ipc_server_stereo_camera_stop_all(const char *why) +{ + struct ipc_server_stereo_camera *m = g_mgr; + if (m == NULL) { + return 0; + } + os_mutex_lock(&m->lock); + uint32_t n = stop_all_locked(m, XRT_STEREO_CAMERA_END_USER_STOPPED); + os_mutex_unlock(&m->lock); + os_mutex_lock(&m->consent_lock); + u_camera_consent_forget_once(&m->consent); // "Allow once" grants die with the stop + os_mutex_unlock(&m->consent_lock); + U_LOG_W("stereo camera: STOP camera sharing (%s) — %u stream(s) ended", why != NULL ? why : "?", n); + indicator_changed(m); + return n; +} + +bool +ipc_server_stereo_camera_set_sharing(bool enabled, const char *why) +{ + struct ipc_server_stereo_camera *m = g_mgr; + if (m == NULL) { + return false; + } + const struct u_camera_consent_store_ops *st = m->consent.store; + bool persisted = st != NULL && st->set_sharing_enabled != NULL && st->set_sharing_enabled(m->consent.store_ctx, enabled); + if (!persisted) { + U_LOG_W("stereo camera: could not persist the sharing toggle (%s) — applied for this run only", + enabled ? "on" : "off"); + } + os_mutex_lock(&m->lock); + bool changed = m->sharing_enabled != enabled; + m->sharing_enabled = enabled; + if (changed) { + if (!enabled) { + (void)stop_all_locked(m, XRT_STEREO_CAMERA_END_DISABLED); + } + notify_all_cameras_locked(m); + } + os_mutex_unlock(&m->lock); + if (changed) { + U_LOG_W("stereo camera: sharing switched %s (%s)%s", enabled ? "ON" : "OFF", why != NULL ? why : "?", + enabled ? "" : " — every started stream ended, zero cameras enumerated"); + indicator_changed(m); + } + return true; +} + +void +ipc_server_stereo_camera_set_session_locked(bool locked, const char *why) +{ + struct ipc_server_stereo_camera *m = g_mgr; + if (m == NULL) { + return; + } + os_mutex_lock(&m->lock); + bool changed = m->locked != locked; + m->locked = locked; + if (changed) { + if (locked) { + // Spec §7.2: the last image must not linger on a locked desktop. + for (uint32_t i = 0; i < MAX_STREAMS; i++) { + if (m->streams[i].used) { + u_stereo_camera_ring_clear(&m->streams[i].ring); + } + } + } + notify_all_cameras_locked(m); + } + os_mutex_unlock(&m->lock); + if (changed) { + U_LOG_W("stereo camera: OS session %s (%s) — streams %s", locked ? "LOCKED" : "unlocked", + why != NULL ? why : "?", locked ? "suspended" : "resume"); + indicator_changed(m); + } +} + +bool +ipc_server_stereo_camera_is_session_locked(void) +{ + struct ipc_server_stereo_camera *m = g_mgr; + if (m == NULL) { + return false; + } + os_mutex_lock(&m->lock); + bool l = m->locked; + os_mutex_unlock(&m->lock); + return l; +} diff --git a/src/xrt/ipc/server/ipc_server_stereo_camera.h b/src/xrt/ipc/server/ipc_server_stereo_camera.h index 6d979be1c1..b51ddb6f3f 100644 --- a/src/xrt/ipc/server/ipc_server_stereo_camera.h +++ b/src/xrt/ipc/server/ipc_server_stereo_camera.h @@ -21,6 +21,10 @@ #pragma once #include "xrt/xrt_instance.h" +#include "util/u_camera_consent.h" + +#include +#include #ifdef __cplusplus extern "C" { @@ -45,6 +49,61 @@ ipc_server_stereo_camera_destroy(struct ipc_server_stereo_camera **mgr_ptr); void ipc_server_client_stereo_camera_release(volatile struct ipc_client_state *ics); + +/* + * + * R3: what the service's UI (tray / menu-bar status item) and platform hooks + * use. All of these address the process's one manager and are safe from any + * thread; they take the manager lock themselves. + * + */ + +/*! + * The consent prompt (" wants to use the 3D camera — Allow / Allow once / + * Deny"). Called on an IPC client thread with no manager lock held; it must + * BLOCK until the user answers or @p timeout_ms elapses, and return the + * u_camera_consent_prompt_answer value (UNAVAILABLE when there is no UI). + */ +typedef enum u_camera_consent_prompt_answer (*ipc_server_stereo_camera_prompt_fn)( + void *ctx, const char *exe, const char *app_name, long pid, uint32_t timeout_ms); + +void +ipc_server_stereo_camera_set_prompt_provider(ipc_server_stereo_camera_prompt_fn fn, void *ctx); + +//! Called (no lock held) whenever the in-use state may have changed. +typedef void (*ipc_server_stereo_camera_indicator_fn)(void *ctx); + +void +ipc_server_stereo_camera_set_indicator_provider(ipc_server_stereo_camera_indicator_fn fn, void *ctx); + +struct ipc_server_stereo_camera_status +{ + uint32_t camera_count; + uint32_t started_streams; //!< > 0 = "3D camera in use" + bool sharing_enabled; //!< the user's persistent toggle + bool locked; //!< OS session locked: every stream suspended + char consumers[256]; //!< "browser.exe, call-app.exe" (base names, de-duplicated) +}; + +//! false = no manager (service still starting). +bool +ipc_server_stereo_camera_get_status(struct ipc_server_stereo_camera_status *out); + +//! The user's "Stop camera sharing": ends every started stream now. Returns the count. +uint32_t +ipc_server_stereo_camera_stop_all(const char *why); + +//! The persistent "Share the 3D camera with apps" toggle (off ends every stream). +bool +ipc_server_stereo_camera_set_sharing(bool enabled, const char *why); + +//! Platform hook: the OS session is locked / switched away (true) or back (false). +void +ipc_server_stereo_camera_set_session_locked(bool locked, const char *why); + +bool +ipc_server_stereo_camera_is_session_locked(void); + #ifdef __cplusplus } #endif diff --git a/src/xrt/ipc/shared/ipc_shmem.c b/src/xrt/ipc/shared/ipc_shmem.c index 080ff0a093..74c023fd76 100644 --- a/src/xrt/ipc/shared/ipc_shmem.c +++ b/src/xrt/ipc/shared/ipc_shmem.c @@ -13,6 +13,8 @@ #include "shared/ipc_shmem.h" +#include + #if defined(XRT_OS_UNIX) #include #include @@ -49,6 +51,29 @@ ipc_shmem_create(size_t size, xrt_shmem_handle_t *out_handle, void **out_map) return XRT_SUCCESS; } +xrt_result_t +ipc_shmem_create_with_readonly(size_t size, + xrt_shmem_handle_t *out_handle, + void **out_map, + xrt_shmem_handle_t *out_ro) +{ + *out_ro = -1; + xrt_result_t result = ipc_shmem_create(size, out_handle, out_map); + if (result != XRT_SUCCESS) { + return result; + } + int ro = dup(*out_handle); + if (ro < 0 || ASharedMemory_setProt(ro, PROT_READ) != 0) { + if (ro >= 0) { + close(ro); + } + ipc_shmem_destroy(out_handle, out_map, size); + return XRT_ERROR_IPC_FAILURE; + } + *out_ro = ro; + return XRT_SUCCESS; +} + #elif defined(XRT_OS_UNIX) #define MONADO_SHMEM_NAME "/displayxr_shm" @@ -78,6 +103,46 @@ ipc_shmem_create(size_t size, xrt_shmem_handle_t *out_handle, void **out_map) return XRT_SUCCESS; } +xrt_result_t +ipc_shmem_create_with_readonly(size_t size, + xrt_shmem_handle_t *out_handle, + void **out_map, + xrt_shmem_handle_t *out_ro) +{ + *out_handle = -1; + *out_ro = -1; + // A unique name per call: the read-only fd must be opened by name before + // the name is unlinked, and two threads may be here at once. + static int counter = 0; + char name[48]; + int id = __sync_fetch_and_add(&counter, 1); + snprintf(name, sizeof(name), "/dxr_shm_%d_%d", (int)getpid(), id); + int fd = shm_open(name, O_CREAT | O_EXCL | O_RDWR, S_IRUSR | S_IWUSR); + if (fd < 0) { + return XRT_ERROR_IPC_FAILURE; + } + if (ftruncate(fd, (off_t)size) < 0) { + close(fd); + shm_unlink(name); + return XRT_ERROR_IPC_FAILURE; + } + int ro = shm_open(name, O_RDONLY, 0); + shm_unlink(name); + if (ro < 0) { + close(fd); + return XRT_ERROR_IPC_FAILURE; + } + xrt_result_t result = ipc_shmem_map(fd, size, out_map); + if (result != XRT_SUCCESS) { + close(fd); + close(ro); + return result; + } + *out_handle = fd; + *out_ro = ro; + return XRT_SUCCESS; +} + #elif defined(XRT_OS_WINDOWS) #include "util/u_windows.h" @@ -136,6 +201,26 @@ ipc_shmem_create(size_t size, xrt_shmem_handle_t *out_handle, void **out_map) return XRT_SUCCESS; } +xrt_result_t +ipc_shmem_create_with_readonly(size_t size, + xrt_shmem_handle_t *out_handle, + void **out_map, + xrt_shmem_handle_t *out_ro) +{ + *out_ro = NULL; + xrt_result_t result = ipc_shmem_create(size, out_handle, out_map); + if (result != XRT_SUCCESS) { + return result; + } + HANDLE ro = NULL; + if (!DuplicateHandle(GetCurrentProcess(), *out_handle, GetCurrentProcess(), &ro, FILE_MAP_READ, FALSE, 0)) { + ipc_shmem_destroy(out_handle, out_map, size); + return XRT_ERROR_IPC_FAILURE; + } + *out_ro = ro; + return XRT_SUCCESS; +} + #else #error "OS not yet supported" #endif @@ -159,6 +244,16 @@ ipc_shmem_destroy(xrt_shmem_handle_t *handle_ptr, void **map_ptr, size_t size) *handle_ptr = -1; } +void +ipc_shmem_close_handle(xrt_shmem_handle_t *handle_ptr) +{ + if (handle_ptr == NULL || *handle_ptr < 0) { + return; + } + close(*handle_ptr); + *handle_ptr = -1; +} + xrt_result_t ipc_shmem_map(xrt_shmem_handle_t handle, size_t size, void **out_map) { @@ -199,6 +294,16 @@ ipc_shmem_destroy(xrt_shmem_handle_t *handle_ptr, void **map_ptr, size_t size) *handle_ptr = NULL; } +void +ipc_shmem_close_handle(xrt_shmem_handle_t *handle_ptr) +{ + if (handle_ptr == NULL || *handle_ptr == NULL) { + return; + } + CloseHandle(*handle_ptr); + *handle_ptr = NULL; +} + xrt_result_t ipc_shmem_map(xrt_shmem_handle_t handle, size_t size, void **out_map) { diff --git a/src/xrt/ipc/shared/ipc_shmem.h b/src/xrt/ipc/shared/ipc_shmem.h index 0ede24c986..48b563e287 100644 --- a/src/xrt/ipc/shared/ipc_shmem.h +++ b/src/xrt/ipc/shared/ipc_shmem.h @@ -81,6 +81,24 @@ ipc_shmem_unmap(void **map_ptr, size_t size); void ipc_shmem_destroy(xrt_shmem_handle_t *handle_ptr, void **map_ptr, size_t size); +/*! + * Like @ref ipc_shmem_create, plus a READ-ONLY handle to the same memory in + * @p out_ro, for a consumer that must be able to map but never write (the + * stereo camera frame rings, ADR-043 R3). Windows: a FILE_MAP_READ duplicate; + * Linux / macOS: the section re-opened O_RDONLY before its name is unlinked; + * Android: a dup with ASharedMemory_setProt(PROT_READ). Close it with + * @ref ipc_shmem_close_handle. + */ +xrt_result_t +ipc_shmem_create_with_readonly(size_t size, + xrt_shmem_handle_t *out_handle, + void **out_map, + xrt_shmem_handle_t *out_ro); + +//! Close a bare handle (no mapping), e.g. the read-only one above. NULL/invalid-safe. +void +ipc_shmem_close_handle(xrt_shmem_handle_t *handle_ptr); + #ifdef __cplusplus } #endif diff --git a/src/xrt/ipc/shared/proto.json b/src/xrt/ipc/shared/proto.json index bcc283c338..9084763834 100644 --- a/src/xrt/ipc/shared/proto.json +++ b/src/xrt/ipc/shared/proto.json @@ -1282,5 +1282,22 @@ "out": [ {"name": "stats", "type": "struct xrt_stereo_camera_stream_stats"} ] + }, + + "stereo_camera_poll_event": { + "out": [ + {"name": "has", "type": "bool"}, + {"name": "event", "type": "struct xrt_stereo_camera_event"} + ] + }, + + "stereo_camera_control": { + "in": [ + {"name": "op", "type": "uint32_t"}, + {"name": "arg", "type": "uint32_t"} + ], + "out": [ + {"name": "value", "type": "uint32_t"} + ] } } diff --git a/src/xrt/state_trackers/oxr/oxr_event.c b/src/xrt/state_trackers/oxr/oxr_event.c index 470955d49b..c6a26ccfa0 100644 --- a/src/xrt/state_trackers/oxr/oxr_event.c +++ b/src/xrt/state_trackers/oxr/oxr_event.c @@ -836,6 +836,79 @@ oxr_event_push_XrEventDataMCPToolCall(struct oxr_logger *log, } #endif // OXR_HAVE_DXR_mcp_tools +#ifdef OXR_HAVE_DXR_stereo_camera +XrResult +oxr_event_push_XrEventDataStereoCameraStateChanged(struct oxr_logger *log, + struct oxr_instance *inst, + uint64_t cameraId, + XrStereoCameraStateDXR state) +{ + XrEventDataStereoCameraStateChangedDXR *changed; + struct oxr_event *event = NULL; + + ALLOC(log, inst, &event, &changed); + + changed->type = XR_TYPE_EVENT_DATA_STEREO_CAMERA_STATE_CHANGED_DXR; + changed->next = NULL; + changed->cameraId = cameraId; + changed->state = state; + event->result = XR_SUCCESS; + + U_LOG_I("OXR EVENT: Stereo camera %llu state -> %d", (unsigned long long)cameraId, (int)state); + + lock(inst); + push(inst, event); + unlock(inst); + + return XR_SUCCESS; +} + +XrResult +oxr_event_push_XrEventDataStereoCameraStreamEnded(struct oxr_logger *log, + struct oxr_instance *inst, + XrStereoCameraStreamDXR stream, + XrStereoCameraStreamEndReasonDXR reason) +{ + XrEventDataStereoCameraStreamEndedDXR *ended; + struct oxr_event *event = NULL; + + ALLOC(log, inst, &event, &ended); + + ended->type = XR_TYPE_EVENT_DATA_STEREO_CAMERA_STREAM_ENDED_DXR; + ended->next = NULL; + ended->stream = stream; + ended->reason = reason; + event->result = XR_SUCCESS; + + U_LOG_W("OXR EVENT: Stereo camera stream ended by the service (reason %d)", (int)reason); + + lock(inst); + push(inst, event); + unlock(inst); + + return XR_SUCCESS; +} + +XrResult +oxr_event_push_XrEventDataStereoCamerasChanged(struct oxr_logger *log, struct oxr_instance *inst) +{ + XrEventDataStereoCamerasChangedDXR *changed; + struct oxr_event *event = NULL; + + ALLOC(log, inst, &event, &changed); + + changed->type = XR_TYPE_EVENT_DATA_STEREO_CAMERAS_CHANGED_DXR; + changed->next = NULL; + event->result = XR_SUCCESS; + + lock(inst); + push(inst, event); + unlock(inst); + + return XR_SUCCESS; +} +#endif // OXR_HAVE_DXR_stereo_camera + XrResult oxr_event_remove_session_events(struct oxr_logger *log, struct oxr_session *sess) { @@ -901,6 +974,17 @@ oxr_poll_event(struct oxr_logger *log, struct oxr_instance *inst, XrEventDataBuf } os_mutex_unlock(&inst->sessions_mutex); +#ifdef OXR_HAVE_DXR_stereo_camera + // ADR-043 R3: instance-level camera events (no session needed) ride the + // same queue; the service queued them per connection. + if (inst->extensions.DXR_stereo_camera) { + ret = oxr_stereo_camera_poll_events(log, inst); + if (ret != XR_SUCCESS) { + return ret; + } + } +#endif + lock(inst); struct oxr_event *event = pop(inst); unlock(inst); diff --git a/src/xrt/state_trackers/oxr/oxr_objects.h b/src/xrt/state_trackers/oxr/oxr_objects.h index 7753ab179b..a51ad9b047 100644 --- a/src/xrt/state_trackers/oxr/oxr_objects.h +++ b/src/xrt/state_trackers/oxr/oxr_objects.h @@ -4375,6 +4375,25 @@ struct oxr_stereo_camera_stream_dxr uint64_t camera_id; //!< service-side camera id bool started; }; + +//! R3: drain the service's queued camera events into the instance queue (xrPollEvent). +XrResult +oxr_stereo_camera_poll_events(struct oxr_logger *log, struct oxr_instance *inst); + +XrResult +oxr_event_push_XrEventDataStereoCameraStateChanged(struct oxr_logger *log, + struct oxr_instance *inst, + uint64_t cameraId, + XrStereoCameraStateDXR state); + +XrResult +oxr_event_push_XrEventDataStereoCameraStreamEnded(struct oxr_logger *log, + struct oxr_instance *inst, + XrStereoCameraStreamDXR stream, + XrStereoCameraStreamEndReasonDXR reason); + +XrResult +oxr_event_push_XrEventDataStereoCamerasChanged(struct oxr_logger *log, struct oxr_instance *inst); #endif // OXR_HAVE_DXR_stereo_camera diff --git a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c index 5664d170c6..ab9bcc890a 100644 --- a/src/xrt/state_trackers/oxr/oxr_stereo_camera.c +++ b/src/xrt/state_trackers/oxr/oxr_stereo_camera.c @@ -14,8 +14,12 @@ * * Error contract: * - dead pipe (XRT_ERROR_IPC_FAILURE) -> XR_ERROR_INSTANCE_LOST - * - refused by consent (XRT_ERROR_NOT_AUTHORIZED) -> XR_ERROR_PERMISSION_INSUFFICIENT - * (retryable: the stream stays created) + * - OS camera switch / class (XRT_ERROR_NOT_AUTHORIZED) -> XR_ERROR_PERMISSION_INSUFFICIENT + * - user / stored consent refused -> XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR + * (retryable: the stream stays created; a browser maps it to NotAllowedError) + * - sharing off (toggle / kill switch) -> XR_ERROR_STEREO_CAMERA_DISABLED_DXR + * - source cannot be opened right now -> XR_ERROR_STEREO_CAMERA_BUSY_DXR + * - the service ended this stream -> XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR * - no such camera / bad format / output -> XR_ERROR_VALIDATION_FAILURE * - transport or output the runtime lacks -> XR_ERROR_FEATURE_UNSUPPORTED * - 8 streams on the camera already -> XR_ERROR_LIMIT_REACHED @@ -116,7 +120,25 @@ scam_xret(struct oxr_logger *log, xrt_result_t xret, const char *what) return oxr_error(log, XR_ERROR_INSTANCE_LOST, "%s: the runtime service connection is gone", what); case XRT_ERROR_NOT_AUTHORIZED: return oxr_error(log, XR_ERROR_PERMISSION_INSUFFICIENT, - "%s: the runtime refused camera access (consent; see XR_DXR_stereo_camera §7)", what); + "%s: the runtime refused camera access (OS camera privacy switch, or this client class " + "may not use the camera; see XR_DXR_stereo_camera §7)", + what); + case XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED: + return oxr_error(log, XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR, + "%s: the user (or the stored decision) refused this app the 3D camera; retry after " + "consent (XR_DXR_stereo_camera §7.1)", + what); + case XRT_ERROR_STEREO_CAMERA_DISABLED: + return oxr_error(log, XR_ERROR_STEREO_CAMERA_DISABLED_DXR, + "%s: 3D camera sharing is switched off (XR_DXR_stereo_camera §7.4)", what); + case XRT_ERROR_STEREO_CAMERA_BUSY: + return oxr_error(log, XR_ERROR_STEREO_CAMERA_BUSY_DXR, + "%s: the camera source cannot be opened right now; retry later", what); + case XRT_ERROR_STEREO_CAMERA_STREAM_ENDED: + return oxr_error(log, XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR, + "%s: the service ended this stream (user stop / sharing off / source lost); destroy it " + "and create a new one", + what); case XRT_ERROR_INPUT_UNSUPPORTED: return oxr_error(log, XR_ERROR_VALIDATION_FAILURE, "%s: unknown camera/stream or unsupported request", what); @@ -471,4 +493,67 @@ oxr_xrGetStereoCameraStreamStatsDXR(XrStereoCameraStreamDXR stream, XrStereoCame return XR_SUCCESS; } + +/* + * + * R3: events. The service queues them per connection; xrPollEvent drains + * them here into the instance's event queue, mapping a service stream id + * back to the handle the app holds. + * + */ + +static struct oxr_stereo_camera_stream_dxr * +stream_by_service_id(struct oxr_instance *inst, uint64_t id) +{ + struct oxr_stereo_camera_stream_dxr *found = NULL; + os_mutex_lock(&inst->handle.children_mutex); + for (uint32_t i = 0; i < XRT_MAX_HANDLE_CHILDREN; i++) { + struct oxr_handle_base *hb = inst->handle.children[i]; + if (hb != NULL && hb->debug == OXR_XR_DEBUG_STEREOCAMSTREAM) { + struct oxr_stereo_camera_stream_dxr *s = (struct oxr_stereo_camera_stream_dxr *)hb; + if (s->id == id) { + found = s; + break; + } + } + } + os_mutex_unlock(&inst->handle.children_mutex); + return found; +} + +XrResult +oxr_stereo_camera_poll_events(struct oxr_logger *log, struct oxr_instance *inst) +{ + struct xrt_stereo_camera_client *c = client_of(inst); + if (c == NULL || c->poll_event == NULL) { + return XR_SUCCESS; + } + for (int i = 0; i < 32; i++) { + bool has = false; + struct xrt_stereo_camera_event ev; + if (c->poll_event(c, &has, &ev) != XRT_SUCCESS || !has) { + return XR_SUCCESS; // a dead pipe surfaces on the next real call + } + switch (ev.kind) { + case XRT_STEREO_CAMERA_EVENT_CAMERA_STATE: + oxr_event_push_XrEventDataStereoCameraStateChanged(log, inst, ev.camera_id, + (XrStereoCameraStateDXR)ev.value); + break; + case XRT_STEREO_CAMERA_EVENT_STREAM_ENDED: { + struct oxr_stereo_camera_stream_dxr *s = stream_by_service_id(inst, ev.stream_id); + if (s != NULL) { + s->started = false; + oxr_event_push_XrEventDataStereoCameraStreamEnded( + log, inst, XRT_CAST_PTR_TO_OXR_HANDLE(XrStereoCameraStreamDXR, s), + (XrStereoCameraStreamEndReasonDXR)ev.value); + } + break; + } + case XRT_STEREO_CAMERA_EVENT_CAMERAS_CHANGED: oxr_event_push_XrEventDataStereoCamerasChanged(log, inst); break; + default: break; + } + } + return XR_SUCCESS; +} + #endif // OXR_HAVE_DXR_stereo_camera From 7f4678b3ab2f02a212db6f63f9a4c4656950696c Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:31 -0700 Subject: [PATCH 34/37] feat(service): Windows tray camera indicator, consent dialog, kill switch, WTS lock suspension Icon badge (red dot) + tooltip + menu line "3D camera in use by ", one balloon per transition, "Stop camera sharing", the "Share the 3D camera with apps" checkmark, a top-most consent dialog (Allow / Allow once / Deny) hosted on the tray thread for the manager's prompt provider, and WTSRegisterSessionNotification (lock / console or remote disconnect) plus SM_REMOTESESSION at start feeding the lock suspension. Written to the API; needs the Windows box to run. Co-Authored-By: Claude Fable 5.1 --- src/xrt/targets/service/CMakeLists.txt | 3 +- src/xrt/targets/service/service_tray_win.c | 379 ++++++++++++++++++++- 2 files changed, 377 insertions(+), 5 deletions(-) diff --git a/src/xrt/targets/service/CMakeLists.txt b/src/xrt/targets/service/CMakeLists.txt index b18327af89..d5edfde3d9 100644 --- a/src/xrt/targets/service/CMakeLists.txt +++ b/src/xrt/targets/service/CMakeLists.txt @@ -48,7 +48,8 @@ if(WIN32) service_tray_win.h service_resources.rc ) - target_link_libraries(displayxr-service PRIVATE shell32 ws2_32) + # wtsapi32: WTSRegisterSessionNotification — the stereo camera's lock-screen suspension (ADR-043 R3) + target_link_libraries(displayxr-service PRIVATE shell32 ws2_32 wtsapi32) # #1201 — embed the shared per-monitor-v2 DPI manifest. The service already # called SetProcessDpiAwarenessContext at the top of WinMain (now diff --git a/src/xrt/targets/service/service_tray_win.c b/src/xrt/targets/service/service_tray_win.c index 32fa03afb4..7b24717510 100644 --- a/src/xrt/targets/service/service_tray_win.c +++ b/src/xrt/targets/service/service_tray_win.c @@ -20,8 +20,11 @@ #include "util/u_logging.h" #include "server/ipc_server_interface.h" // ADR-045: ipc_server_request_display_reprobe +#include "server/ipc_server_stereo_camera.h" // ADR-043 R3: camera indicator, prompt, kill switch, lock #include "target_plugin_loader.h" // ADR-045: per-plug-in platform state for the tooltip +#include // WTSRegisterSessionNotification (lock-screen suspension) + #define IDI_DISPLAYXR_ICON_WHITE 101 #define IDI_DISPLAYXR_ICON_BLACK 102 #define WM_TRAYICON (WM_APP + 1) @@ -33,6 +36,12 @@ #define IDM_START_ON_LOGIN 1030 #define IDM_CONTROL_PANEL 1002 #define IDM_EXIT 1001 +// ADR-043 R3: stereo camera indicator / kill switch / sharing toggle +#define IDM_CAMERA_STOP 1060 +#define IDM_CAMERA_SHARING 1061 +#define WM_CAMERA_PROMPT (WM_APP + 2) //!< lParam = struct cam_prompt_req * (show) +#define WM_CAMERA_PROMPT_CLOSE (WM_APP + 3) //!< lParam = struct cam_prompt_req * (close, SendMessage) +#define WM_CAMERA_INDICATOR (WM_APP + 4) //!< refresh icon + tooltip now // ADR-045: tooltip refresh of the display-processor status line. #define TRAY_STATUS_TIMER_ID 1 @@ -62,6 +71,20 @@ static struct service_config s_config; static HANDLE s_reg_watch_thread = NULL; static HANDLE s_reg_watch_stop = NULL; +// ADR-043 R3: camera in-use icon (base icon + red dot) and the pending prompt. +static HICON s_icon_base = NULL; +static HICON s_icon_inuse = NULL; +static bool s_camera_in_use = false; + +struct cam_prompt_req +{ + wchar_t title[256]; + wchar_t body[512]; + HANDLE done; //!< signalled by a button / close + volatile LONG answer; //!< enum u_camera_consent_prompt_answer + HWND dlg; //!< tray thread only +}; + /* * @@ -287,6 +310,27 @@ show_context_menu(HWND hwnd) AppendMenuW(menu, MF_POPUP, (UINT_PTR)workspace_sub, name_wide); } + // ADR-043 R3: camera indicator line, kill switch, sharing toggle. + { + struct ipc_server_stereo_camera_status cst; + bool have = ipc_server_stereo_camera_get_status(&cst); + wchar_t line[256]; + if (!have || cst.camera_count == 0) { + wcscpy_s(line, ARRAYSIZE(line), (have && !cst.sharing_enabled) ? L"3D camera: sharing off" : L"3D camera: none"); + } else if (cst.started_streams > 0) { + wchar_t who[128]; + MultiByteToWideChar(CP_UTF8, 0, cst.consumers, -1, who, ARRAYSIZE(who)); + _snwprintf_s(line, ARRAYSIZE(line), _TRUNCATE, L"3D camera in use by %ls", who); + } else { + wcscpy_s(line, ARRAYSIZE(line), cst.sharing_enabled ? L"3D camera: not in use" : L"3D camera: sharing off"); + } + AppendMenuW(menu, MF_SEPARATOR, 0, NULL); + AppendMenuW(menu, MF_STRING | MF_GRAYED, 0, line); + AppendMenuW(menu, MF_STRING | ((have && cst.started_streams > 0) ? MF_ENABLED : MF_GRAYED), IDM_CAMERA_STOP, + L"Stop camera sharing"); + AppendMenuW(menu, MF_STRING | ((!have || cst.sharing_enabled) ? MF_CHECKED : MF_UNCHECKED), IDM_CAMERA_SHARING, + L"Share the 3D camera with apps"); + } AppendMenuW(menu, MF_SEPARATOR, 0, NULL); AppendMenuW(menu, MF_STRING | (s_config.start_on_login ? MF_CHECKED : MF_UNCHECKED), IDM_START_ON_LOGIN, L"Start on Windows login"); @@ -340,6 +384,255 @@ request_service_exit(void) * vendor-specific is known here. Never triggers discovery: it reads the * loader's records, which the IPC server's instance filled in. */ +/* + * + * ADR-043 R3: camera in-use badge, prompt dialog, indicator refresh. + * + */ + +//! @p base with a red "recording" dot bottom-right (32bpp DIB, alpha kept). +static HICON +make_badged_icon(HICON base) +{ + ICONINFO ii; + if (base == NULL || !GetIconInfo(base, &ii)) { + return NULL; + } + BITMAP bm; + HICON out = NULL; + if (ii.hbmColor != NULL && GetObject(ii.hbmColor, sizeof(bm), &bm) != 0 && bm.bmWidth > 0 && bm.bmHeight > 0) { + const int w = bm.bmWidth, h = bm.bmHeight; + BITMAPINFO bi; + ZeroMemory(&bi, sizeof(bi)); + bi.bmiHeader.biSize = sizeof(bi.bmiHeader); + bi.bmiHeader.biWidth = w; + bi.bmiHeader.biHeight = -h; // top-down + bi.bmiHeader.biPlanes = 1; + bi.bmiHeader.biBitCount = 32; + bi.bmiHeader.biCompression = BI_RGB; + HDC dc = GetDC(NULL); + uint32_t *px = NULL; + HBITMAP color = CreateDIBSection(dc, &bi, DIB_RGB_COLORS, (void **)&px, NULL, 0); + if (color != NULL && px != NULL) { + // Copy the icon's colour bits (with alpha) into the DIB. + HDC src = CreateCompatibleDC(dc); + HGDIOBJ old = SelectObject(src, ii.hbmColor); + GetDIBits(src, ii.hbmColor, 0, (UINT)h, px, &bi, DIB_RGB_COLORS); + SelectObject(src, old); + DeleteDC(src); + // Red dot, radius ~ 28 % of the icon, bottom-right, opaque. + const double r = w * 0.28, cx = w - r - 0.5, cy = h - r - 0.5; + for (int y = 0; y < h; y++) { + for (int x = 0; x < w; x++) { + double dx = x - cx, dy = y - cy; + if (dx * dx + dy * dy <= r * r) { + px[y * w + x] = 0xFFE03030u; // A R G B (BGRA in memory) + } + } + } + ICONINFO ni = ii; + ni.fIcon = TRUE; + ni.hbmColor = color; + out = CreateIconIndirect(&ni); + DeleteObject(color); + } + ReleaseDC(NULL, dc); + } + if (ii.hbmColor != NULL) { + DeleteObject(ii.hbmColor); + } + if (ii.hbmMask != NULL) { + DeleteObject(ii.hbmMask); + } + return out; +} + +//! Tray thread: re-read the camera status; swap icon + tooltip when it changed. +static void +update_camera_indicator(void) +{ + struct ipc_server_stereo_camera_status st; + bool in_use = ipc_server_stereo_camera_get_status(&st) && st.started_streams > 0; + if (in_use == s_camera_in_use) { + return; + } + s_camera_in_use = in_use; + if (s_icon_inuse == NULL && s_icon_base != NULL) { + s_icon_inuse = make_badged_icon(s_icon_base); + } + s_nid.hIcon = in_use && s_icon_inuse != NULL ? s_icon_inuse : s_icon_base; + s_nid.uFlags = NIF_ICON; + Shell_NotifyIconW(NIM_MODIFY, &s_nid); + s_nid.uFlags = NIF_ICON | NIF_TIP | NIF_MESSAGE; + if (in_use) { + // One balloon per transition: the user learns WHO started reading the camera. + wchar_t who[256]; + MultiByteToWideChar(CP_UTF8, 0, st.consumers, -1, who, ARRAYSIZE(who)); + NOTIFYICONDATAW n = s_nid; + n.uFlags = NIF_INFO; + n.dwInfoFlags = NIIF_INFO | NIIF_RESPECT_QUIET_TIME; + wcscpy_s(n.szInfoTitle, ARRAYSIZE(n.szInfoTitle), L"3D camera in use"); + _snwprintf_s(n.szInfo, ARRAYSIZE(n.szInfo), _TRUNCATE, L"%ls is receiving frames from the 3D camera.", who); + Shell_NotifyIconW(NIM_MODIFY, &n); + } +} + +//! Any thread (the manager's indicator provider): refresh on the tray thread. +static void +camera_indicator_changed(void *ctx) +{ + (void)ctx; + if (s_tray_hwnd != NULL) { + PostMessageW(s_tray_hwnd, WM_CAMERA_INDICATOR, 0, 0); + } +} + +#define CAM_BTN_ALLOW 1 +#define CAM_BTN_ONCE 2 +#define CAM_BTN_DENY 3 + +static LRESULT CALLBACK +cam_prompt_wnd_proc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) +{ + struct cam_prompt_req *req = (struct cam_prompt_req *)GetWindowLongPtrW(hwnd, GWLP_USERDATA); + switch (msg) { + case WM_COMMAND: { + if (req == NULL) { + return 0; + } + LONG a = -1; + switch (LOWORD(wParam)) { + case CAM_BTN_ALLOW: a = U_CAMERA_CONSENT_PROMPT_ALLOW; break; + case CAM_BTN_ONCE: a = U_CAMERA_CONSENT_PROMPT_ALLOW_ONCE; break; + case CAM_BTN_DENY: a = U_CAMERA_CONSENT_PROMPT_DENY; break; + default: return 0; + } + InterlockedExchange(&req->answer, a); + SetEvent(req->done); + DestroyWindow(hwnd); + return 0; + } + case WM_CLOSE: + // Dismissed without an answer: refused now, asked again next time. + if (req != NULL) { + InterlockedExchange(&req->answer, U_CAMERA_CONSENT_PROMPT_TIMEOUT); + SetEvent(req->done); + } + DestroyWindow(hwnd); + return 0; + case WM_DESTROY: + if (req != NULL) { + req->dlg = NULL; + } + return 0; + case WM_CTLCOLORSTATIC: return (LRESULT)GetSysColorBrush(COLOR_WINDOW); + default: return DefWindowProcW(hwnd, msg, wParam, lParam); + } +} + +//! Tray thread: show the consent dialog (" wants to use the 3D camera"). +static void +cam_prompt_show(struct cam_prompt_req *req) +{ + static bool registered = false; + HINSTANCE hi = GetModuleHandleW(NULL); + if (!registered) { + WNDCLASSEXW wc = {0}; + wc.cbSize = sizeof(wc); + wc.lpfnWndProc = cam_prompt_wnd_proc; + wc.hInstance = hi; + wc.hCursor = LoadCursorW(NULL, IDC_ARROW); + wc.hbrBackground = GetSysColorBrush(COLOR_WINDOW); + wc.lpszClassName = L"DisplayXRCameraConsent"; + wc.hIcon = s_icon_base; + RegisterClassExW(&wc); + registered = true; + } + UINT dpi = GetDpiForSystem(); +#define DP(v) MulDiv((v), (int)dpi, 96) + int w = DP(460), h = DP(190); + RECT wa; + SystemParametersInfoW(SPI_GETWORKAREA, 0, &wa, 0); + int x = wa.left + ((wa.right - wa.left) - w) / 2; + int y = wa.top + ((wa.bottom - wa.top) - h) / 2; + HWND dlg = CreateWindowExW(WS_EX_TOPMOST | WS_EX_DLGMODALFRAME, L"DisplayXRCameraConsent", L"DisplayXR", + WS_OVERLAPPED | WS_CAPTION | WS_SYSMENU, x, y, w, h, NULL, NULL, hi, NULL); + if (dlg == NULL) { + InterlockedExchange(&req->answer, U_CAMERA_CONSENT_PROMPT_UNAVAILABLE); + SetEvent(req->done); + return; + } + SetWindowLongPtrW(dlg, GWLP_USERDATA, (LONG_PTR)req); + HFONT font = (HFONT)GetStockObject(DEFAULT_GUI_FONT); + NONCLIENTMETRICSW ncm = {sizeof(ncm)}; + HFONT ui_font = NULL; + if (SystemParametersInfoW(SPI_GETNONCLIENTMETRICS, sizeof(ncm), &ncm, 0)) { + ui_font = CreateFontIndirectW(&ncm.lfMessageFont); + font = ui_font != NULL ? ui_font : font; + } + HWND t = CreateWindowExW(0, L"STATIC", req->title, WS_CHILD | WS_VISIBLE, DP(16), DP(14), DP(420), DP(22), dlg, + NULL, hi, NULL); + HWND b = CreateWindowExW(0, L"STATIC", req->body, WS_CHILD | WS_VISIBLE, DP(16), DP(42), DP(420), DP(56), dlg, + NULL, hi, NULL); + HWND deny = CreateWindowExW(0, L"BUTTON", L"Deny", WS_CHILD | WS_VISIBLE | WS_TABSTOP | BS_PUSHBUTTON, DP(16), + DP(108), DP(110), DP(30), dlg, (HMENU)(INT_PTR)CAM_BTN_DENY, hi, NULL); + HWND once = CreateWindowExW(0, L"BUTTON", L"Allow once", WS_CHILD | WS_VISIBLE | WS_TABSTOP | BS_PUSHBUTTON, + DP(206), DP(108), DP(110), DP(30), dlg, (HMENU)(INT_PTR)CAM_BTN_ONCE, hi, NULL); + HWND allow = CreateWindowExW(0, L"BUTTON", L"Allow", WS_CHILD | WS_VISIBLE | WS_TABSTOP | BS_DEFPUSHBUTTON, + DP(326), DP(108), DP(110), DP(30), dlg, (HMENU)(INT_PTR)CAM_BTN_ALLOW, hi, NULL); + HWND ctls[] = {t, b, deny, once, allow}; + for (size_t i = 0; i < ARRAYSIZE(ctls); i++) { + SendMessageW(ctls[i], WM_SETFONT, (WPARAM)font, TRUE); + } +#undef DP + req->dlg = dlg; + ShowWindow(dlg, SW_SHOWNORMAL); + SetForegroundWindow(dlg); + SetFocus(allow); +} + +/*! + * The manager's prompt provider (IPC client thread): hand the request to the + * tray thread, wait for the answer or the timeout, then close the dialog + * synchronously (SendMessage runs on the tray thread) before @p req dies. + */ +static enum u_camera_consent_prompt_answer +tray_camera_prompt(void *ctx, const char *exe, const char *app_name, long pid, uint32_t timeout_ms) +{ + (void)ctx; + if (s_tray_hwnd == NULL) { + return U_CAMERA_CONSENT_PROMPT_UNAVAILABLE; + } + struct cam_prompt_req req; + ZeroMemory(&req, sizeof(req)); + req.answer = U_CAMERA_CONSENT_PROMPT_TIMEOUT; + const char *base = exe != NULL ? exe : ""; + for (const char *p = base; *p != '\0'; p++) { + if (*p == '\\' || *p == '/') { + base = p + 1; + } + } + wchar_t wapp[128], wbase[256]; + MultiByteToWideChar(CP_UTF8, 0, app_name != NULL && app_name[0] ? app_name : "An app", -1, wapp, ARRAYSIZE(wapp)); + MultiByteToWideChar(CP_UTF8, 0, base[0] ? base : "the app", -1, wbase, ARRAYSIZE(wbase)); + _snwprintf_s(req.title, ARRAYSIZE(req.title), _TRUNCATE, L"\u201c%ls\u201d wants to use the 3D camera", wapp); + _snwprintf_s(req.body, ARRAYSIZE(req.body), _TRUNCATE, + L"%ls (pid %ld) will receive frames from the display\u2019s stereo camera. You can stop it at any " + L"time from the DisplayXR tray menu.", + wbase, pid); + req.done = CreateEventW(NULL, TRUE, FALSE, NULL); + if (req.done == NULL) { + return U_CAMERA_CONSENT_PROMPT_UNAVAILABLE; + } + PostMessageW(s_tray_hwnd, WM_CAMERA_PROMPT, 0, (LPARAM)&req); + WaitForSingleObject(req.done, timeout_ms); + // Close (or no-op) on the tray thread; returns only once the dialog no + // longer references req. + SendMessageW(s_tray_hwnd, WM_CAMERA_PROMPT_CLOSE, 0, (LPARAM)&req); + CloseHandle(req.done); + return (enum u_camera_consent_prompt_answer)InterlockedCompareExchange(&req.answer, 0, 0); +} + static void build_status_line(char *out, size_t cap) { @@ -397,8 +690,17 @@ update_status_tooltip(void) if (MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, ARRAYSIZE(wline)) == 0) { wline[0] = L'\0'; } + // ADR-043 R3 §7.3: the camera line names the consumers while any stream runs. + wchar_t wcam[128] = L""; + struct ipc_server_stereo_camera_status cst; + if (ipc_server_stereo_camera_get_status(&cst) && cst.started_streams > 0) { + wchar_t who[96]; + MultiByteToWideChar(CP_UTF8, 0, cst.consumers, -1, who, ARRAYSIZE(who)); + _snwprintf_s(wcam, ARRAYSIZE(wcam), _TRUNCATE, L"\n3D camera in use by %ls%ls", who, + cst.locked ? L" (suspended: locked)" : L""); + } wchar_t tip[ARRAYSIZE(s_nid.szTip)]; - _snwprintf_s(tip, ARRAYSIZE(tip), _TRUNCATE, L"DisplayXR Service\n%ls", wline); + _snwprintf_s(tip, ARRAYSIZE(tip), _TRUNCATE, L"DisplayXR Service\n%ls%ls", wline, wcam); if (wcscmp(tip, s_nid.szTip) == 0) { return; } @@ -608,6 +910,24 @@ session_wnd_proc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) } return TRUE; + // ADR-043 R3 §7.2: a locked / disconnected / switched-away session gets no + // camera frames. Registered with WTSRegisterSessionNotification below. + case WM_WTSSESSION_CHANGE: + switch (wParam) { + case WTS_SESSION_LOCK: ipc_server_stereo_camera_set_session_locked(true, "WTS_SESSION_LOCK"); break; + case WTS_CONSOLE_DISCONNECT: + ipc_server_stereo_camera_set_session_locked(true, "WTS_CONSOLE_DISCONNECT"); + break; + case WTS_REMOTE_DISCONNECT: + ipc_server_stereo_camera_set_session_locked(true, "WTS_REMOTE_DISCONNECT"); + break; + case WTS_SESSION_UNLOCK: ipc_server_stereo_camera_set_session_locked(false, "WTS_SESSION_UNLOCK"); break; + case WTS_CONSOLE_CONNECT: ipc_server_stereo_camera_set_session_locked(false, "WTS_CONSOLE_CONNECT"); break; + case WTS_REMOTE_CONNECT: ipc_server_stereo_camera_set_session_locked(false, "WTS_REMOTE_CONNECT"); break; + default: break; + } + return 0; + default: return DefWindowProcW(hwnd, msg, wParam, lParam); } } @@ -682,22 +1002,57 @@ tray_wnd_proc(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) config_changed(); break; + // ADR-043 R3: the user kill switch and the persistent sharing toggle. + case IDM_CAMERA_STOP: ipc_server_stereo_camera_stop_all("tray: Stop camera sharing"); break; + case IDM_CAMERA_SHARING: { + struct ipc_server_stereo_camera_status cst; + bool on = ipc_server_stereo_camera_get_status(&cst) ? cst.sharing_enabled : true; + ipc_server_stereo_camera_set_sharing(!on, "tray: Share the 3D camera with apps"); + break; + } + // Exit case IDM_EXIT: request_service_exit(); break; } return 0; } + case WM_CAMERA_PROMPT: cam_prompt_show((struct cam_prompt_req *)lParam); return 0; + + case WM_CAMERA_PROMPT_CLOSE: { + struct cam_prompt_req *req = (struct cam_prompt_req *)lParam; + if (req != NULL && req->dlg != NULL) { + HWND d = req->dlg; + SetWindowLongPtrW(d, GWLP_USERDATA, 0); + req->dlg = NULL; + DestroyWindow(d); + } + return 0; + } + + case WM_CAMERA_INDICATOR: + update_camera_indicator(); + update_status_tooltip(); + return 0; + case WM_TIMER: if (wParam == TRAY_STATUS_TIMER_ID) { + update_camera_indicator(); update_status_tooltip(); } return 0; case WM_SETTINGCHANGE: - // Windows theme changed — swap tray icon to match - s_nid.hIcon = load_theme_icon(); + // Windows theme changed — swap tray icon to match (and rebuild the badge) + s_icon_base = load_theme_icon(); + if (s_icon_inuse != NULL) { + DestroyIcon(s_icon_inuse); + s_icon_inuse = NULL; + } + s_camera_in_use = !s_camera_in_use; // force the swap below to re-pick + s_nid.hIcon = s_icon_base; Shell_NotifyIconW(NIM_MODIFY, &s_nid); + update_camera_indicator(); return 0; case WM_DESTROY: @@ -755,6 +1110,13 @@ tray_thread_body(LPVOID param) "Could not create the session-end window (error %lu); Restart Manager can only force-close " "the service.", GetLastError()); + } else if (!WTSRegisterSessionNotification(session_hwnd, NOTIFY_FOR_THIS_SESSION)) { + U_LOG_W("WTSRegisterSessionNotification failed (error %lu): the 3D camera will not suspend on lock", + GetLastError()); + } + // A remote (RDP) session has no local user in front of the camera. + if (GetSystemMetrics(SM_REMOTESESSION) != 0) { + ipc_server_stereo_camera_set_session_locked(true, "SM_REMOTESESSION at start"); } // Set up the tray icon @@ -767,7 +1129,8 @@ tray_thread_body(LPVOID param) wcscpy_s(s_nid.szTip, ARRAYSIZE(s_nid.szTip), L"DisplayXR Service"); // Load theme-appropriate icon (black for light taskbar, white for dark) - s_nid.hIcon = load_theme_icon(); + s_icon_base = load_theme_icon(); + s_nid.hIcon = s_icon_base; Shell_NotifyIconW(NIM_ADD, &s_nid); @@ -844,6 +1207,14 @@ service_tray_init(service_tray_shutdown_cb shutdown_cb, // ADR-045 R-c: plug-in registration changes re-probe selection. reg_watch_start(); + // ADR-043 R3: the stereo camera's consent prompt + in-use indicator live on + // this tray. Registered before ipc_server_main creates the camera manager + // (process-wide providers, not per manager). + if (s_tray_hwnd != NULL) { + ipc_server_stereo_camera_set_prompt_provider(tray_camera_prompt, NULL); + ipc_server_stereo_camera_set_indicator_provider(camera_indicator_changed, NULL); + } + return s_tray_hwnd != NULL; } From c3b6d4145f5df83813992ec57c48a55c42905729 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:32 -0700 Subject: [PATCH 35/37] feat(cli): camera consent/allow/deny/revoke/trust, status, stop-all, sharing, fake-lock; probe reports events list/calib/probe connect as CAMERA_CONSUMER (the browser utility's class); the control ops as DIAG. probe prints every event with its time since start and exits 4 / 6 / 7 / 8 for consent refused / sharing off / busy / ended by the service. Co-Authored-By: Claude Fable 5.1 --- src/xrt/targets/cli/cli_cmd_camera.c | 332 +++++++++++++++++++++++++-- 1 file changed, 312 insertions(+), 20 deletions(-) diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index cf5b71986c..b04031b687 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -10,9 +10,22 @@ * [--frames N] [--seconds S] [--out DIR] * [--max-disparity N] [--max-dy N] [--min-ncc C] * - * `probe` is a real consumer: it creates and starts a stream (so it goes - * through the service's authorisation — in R1, DXR_STEREO_CAMERA_DEV_ALLOW=1 - * on the SERVICE), maps the ring READ-ONLY, waits on the per-stream wake + * R3 consent / privacy (ADR-043 §7) — the store is edited LOCALLY (no service + * needed), the control ops go over IPC as a DIAG client: + * camera consent list the store (apps, delegating, sharing) + * camera allow|deny|revoke |--self stored per-app decision + * camera trust|untrust |--self user-level consent-DELEGATING list + * camera status sharing / locked / started streams (IPC) + * camera stop-all the user kill switch (IPC) + * camera sharing on|off the persistent sharing toggle (IPC) + * camera fake-lock on|off simulate the OS session lock (IPC, diagnostics) + * + * `list` / `calib` / `probe` connect as the CAMERA_CONSUMER class (ADR-043 R3) + * — the same class a browser's video-capture utility declares — so the CLI + * exercises exactly the consumer path: it is prompted / refused / allowed like + * any app. `probe` is a real consumer: it creates and starts a stream (so it + * goes through the service's consent decision: a refusal prints the distinct + * result and exits 4 / 6 / 7), maps the ring READ-ONLY, waits on the per-stream wake * handle, acquires, and reports the measured delivery rate, frame-index gaps, * the SBS layout, block-matched disparities of the last frame (the sim fake's * scene is 12 px background / 40 px bar), and the service's stream stats. @@ -51,6 +64,7 @@ #ifdef CLI_HAVE_IPC #include "xrt/xrt_instance.h" +#include "util/u_camera_consent.h" #include "xrt/xrt_plugin.h" #include "xrt/xrt_results.h" #include "xrt/xrt_stereo_camera.h" @@ -78,6 +92,9 @@ #include #include #endif +#ifdef XRT_OS_MACOS +#include +#endif static const char * state_str(uint32_t s) @@ -130,7 +147,15 @@ result_hint(xrt_result_t x) { switch (x) { case XRT_ERROR_NOT_AUTHORIZED: - return " = NOT_AUTHORIZED (R1: set DXR_STEREO_CAMERA_DEV_ALLOW=1 in the SERVICE environment)"; + return " = NOT_AUTHORIZED (OS camera privacy switch off, RAW to a delegating client, or wrong client class)"; + case XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED: + return " = CONSENT_REFUSED (no consent for this executable: the prompt was refused / unavailable, or a " + "stored Deny; `displayxr-cli camera allow --self` or `camera trust --self` grants it)"; + case XRT_ERROR_STEREO_CAMERA_DISABLED: + return " = DISABLED (camera sharing is off: tray toggle or DXR_STEREO_CAMERA=0)"; + case XRT_ERROR_STEREO_CAMERA_BUSY: return " = BUSY (the source cannot be opened right now; retry)"; + case XRT_ERROR_STEREO_CAMERA_STREAM_ENDED: + return " = STREAM_ENDED (the service ended this stream: user stop / sharing off)"; case XRT_ERROR_FEATURE_NOT_SUPPORTED: return " = FEATURE_NOT_SUPPORTED"; case XRT_ERROR_INPUT_UNSUPPORTED: return " = INPUT_UNSUPPORTED (bad id / format / output)"; case XRT_ERROR_CLIENT_LIMIT_REACHED: return " = LIMIT_REACHED (8 streams per camera)"; @@ -139,12 +164,24 @@ result_hint(xrt_result_t x) } } +//! Exit code for a refused start, so a script can tell the refusals apart. +static int +refusal_exit_code(xrt_result_t x) +{ + switch (x) { + case XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED: return 4; + case XRT_ERROR_STEREO_CAMERA_DISABLED: return 6; + case XRT_ERROR_STEREO_CAMERA_BUSY: return 7; + default: return 2; + } +} + static bool -cam_connect(struct ipc_connection *ipc_c) +cam_connect(struct ipc_connection *ipc_c, uint32_t client_class) { struct xrt_instance_info ii = {0}; snprintf(ii.app_info.application_name, sizeof(ii.app_info.application_name), "%s", "displayxr-cli"); - ii.app_info.declared_client_class = XRT_CLIENT_CLASS_DIAG; + ii.app_info.declared_client_class = client_class; xrt_result_t xret = ipc_client_connection_init(ipc_c, U_LOGGING_ERROR, &ii); if (xret != XRT_SUCCESS) { printf("displayxr-cli camera: not connected to the service (xrt_result=%d).\n", (int)xret); @@ -227,7 +264,7 @@ cmd_calib(struct ipc_connection *ipc_c, uint64_t id, uint32_t output, bool json) if (xret != XRT_SUCCESS) { printf("stereo_camera_get_calibration(%llu) failed: %d%s\n", (unsigned long long)id, (int)xret, result_hint(xret)); - return 2; + return refusal_exit_code(xret); } if (json) { printf("{\"output\": \"%s\", \"baselineMm\": %.3f, \"eyes\": [", @@ -658,7 +695,7 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) if (xret != XRT_SUCCESS) { printf("stream start failed: %d%s\n", (int)xret, result_hint(xret)); ipc_client_stereo_camera_stream_destroy(ipc_c, sid); - return 2; + return refusal_exit_code(xret); } struct xrt_stereo_camera_stream_layout lay; xrt_shmem_handle_t section; @@ -692,18 +729,52 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) int64_t deadline = seconds > 0.0f ? t_start + (int64_t)(seconds * 1e9) : t_start + 60ll * 1000000000; int got = 0, wakes = 0, not_ready = 0; bool rows_ok = true; + bool ended = false; + int events = 0; uint64_t first_index = 0, last_index = 0, gaps = 0; int64_t t_first = 0, t_last = 0; struct xrt_stereo_camera_frame_info last; memset(&last, 0, sizeof(last)); - while ((seconds > 0.0f || got < frames) && os_monotonic_get_ns() < deadline) { - if (!wait_wake(wake, 1000)) { + while ((seconds > 0.0f || got < frames) && os_monotonic_get_ns() < deadline && !ended) { + // R3: the service's events (state changes, a stream it ended) are + // polled like xrPollEvent would; printed with the time since start. + for (;;) { + bool has = false; + struct xrt_stereo_camera_event ev; + if (ipc_client_stereo_camera_poll_event(ipc_c, &has, &ev) != XRT_SUCCESS || !has) { + break; + } + events++; + double t = (double)(os_monotonic_get_ns() - t_start) * 1e-9; + if (ev.kind == XRT_STEREO_CAMERA_EVENT_CAMERA_STATE) { + printf("[%6.2f s] event: camera %llu state -> %s\n", t, (unsigned long long)ev.camera_id, + state_str(ev.value)); + } else if (ev.kind == XRT_STEREO_CAMERA_EVENT_STREAM_ENDED) { + printf("[%6.2f s] event: stream %llu ENDED by the service (reason %u = %s)\n", t, + (unsigned long long)ev.stream_id, ev.value, + ev.value == XRT_STEREO_CAMERA_END_USER_STOPPED ? "USER_STOPPED" + : ev.value == XRT_STEREO_CAMERA_END_DISABLED ? "DISABLED" + : "SOURCE_LOST"); + ended = ended || ev.stream_id == sid; + } else { + printf("[%6.2f s] event: cameras changed\n", t); + } + } + if (ended) { + break; + } + if (!wait_wake(wake, 250)) { continue; } wakes++; bool ready = false; struct xrt_stereo_camera_frame_info fi; xret = ipc_client_stereo_camera_acquire(ipc_c, sid, &ready, &fi); + if (xret == XRT_ERROR_STEREO_CAMERA_STREAM_ENDED) { + printf("acquire: %d%s\n", (int)xret, result_hint(xret)); + ended = true; + continue; // drain the ENDED event on the next pass + } if (xret != XRT_SUCCESS) { printf("acquire failed: %d%s\n", (int)xret, result_hint(xret)); break; @@ -724,13 +795,35 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) last = fi; got++; } + // One last drain: a stream the service ended queued its STREAM_ENDED event + // behind the acquire that reported it. + for (;;) { + bool has = false; + struct xrt_stereo_camera_event ev; + if (ipc_client_stereo_camera_poll_event(ipc_c, &has, &ev) != XRT_SUCCESS || !has) { + break; + } + events++; + double t = (double)(os_monotonic_get_ns() - t_start) * 1e-9; + if (ev.kind == XRT_STEREO_CAMERA_EVENT_STREAM_ENDED) { + printf("[%6.2f s] event: stream %llu ENDED by the service (reason %u = %s)\n", t, + (unsigned long long)ev.stream_id, ev.value, + ev.value == XRT_STEREO_CAMERA_END_USER_STOPPED ? "USER_STOPPED" + : ev.value == XRT_STEREO_CAMERA_END_DISABLED ? "DISABLED" + : "SOURCE_LOST"); + ended = ended || ev.stream_id == sid; + } else if (ev.kind == XRT_STEREO_CAMERA_EVENT_CAMERA_STATE) { + printf("[%6.2f s] event: camera %llu state -> %s\n", t, (unsigned long long)ev.camera_id, + state_str(ev.value)); + } + } double span = (double)(t_last - t_first) * 1e-9; double rate = (got > 1 && span > 0.0) ? (got - 1) / span : 0.0; printf( "received %d frames (index %llu..%llu, %llu source frames not delivered to this stream), %d wakes, " - "%d not-ready\n", + "%d not-ready, %d event(s)%s\n", got, (unsigned long long)first_index, (unsigned long long)last_index, (unsigned long long)gaps, wakes, - not_ready); + not_ready, events, ended ? " — STREAM ENDED BY THE SERVICE" : ""); printf("measured delivery rate: %.2f Hz over %.2f s\n", rate, span); if (got > 0) { @@ -876,8 +969,11 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) close_wake(wake); ipc_client_stereo_camera_stream_stop(ipc_c, sid); ipc_client_stereo_camera_stream_destroy(ipc_c, sid); + if (ended) { + return 8; // the service ended the stream (user stop / sharing off) + } if (got == 0) { - return 4; + return 9; } if (require_rectified) { if (last.output != XRT_STEREO_CAMERA_OUTPUT_RECTIFIED) { @@ -895,13 +991,214 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) return 0; } + +/* + * + * R3: consent store (local) and control ops (IPC, DIAG). + * + */ + +//! Absolute path of this executable — what the service sees as the peer. +static bool +self_exe_path(char *out, size_t cap) +{ + out[0] = '\0'; +#if defined(XRT_OS_WINDOWS) + wchar_t w[1024]; + DWORD n = GetModuleFileNameW(NULL, w, (DWORD)(sizeof(w) / sizeof(w[0]))); + if (n == 0) { + return false; + } + int m = WideCharToMultiByte(CP_UTF8, 0, w, (int)n, out, (int)cap - 1, NULL, NULL); + if (m <= 0) { + return false; + } + out[m] = '\0'; + return true; +#elif defined(XRT_OS_MACOS) + char tmp[1024]; + uint32_t sz = sizeof(tmp); + if (_NSGetExecutablePath(tmp, &sz) != 0) { + return false; + } + // proc_pidpath (what the service uses) returns the resolved path. + return realpath(tmp, out) != NULL || snprintf(out, cap, "%s", tmp) > 0; +#else + ssize_t n = readlink("/proc/self/exe", out, cap - 1); + if (n <= 0) { + return false; + } + out[n] = '\0'; + return true; +#endif +} + +//! "" or "--self" -> the path to act on. +static bool +resolve_exe_arg(int argc, const char **argv, int idx, char *out, size_t cap) +{ + if (idx >= argc) { + return false; + } + if (strcmp(argv[idx], "--self") == 0) { + return self_exe_path(out, cap); + } + snprintf(out, cap, "%s", argv[idx]); + return out[0] != '\0'; +} + +static void +consent_list_cb(void *ctx, char kind, const char *exe, const char *value) +{ + (void)ctx; + switch (kind) { + case 's': printf(" sharing: %s\n", value); break; + case 'a': printf(" app %-6s %s\n", value, exe); break; + case 'd': printf(" delegating %s (%s list)\n", exe, value); break; + default: break; + } +} + +static int +cmd_consent(void) +{ + char path[1024]; + if (u_camera_consent_store_path(path, sizeof(path))) { + printf("stereo camera consent store: %s\n", path); + } + u_camera_consent_store_list(consent_list_cb, NULL); + char self[1024]; + if (self_exe_path(self, sizeof(self))) { + printf(" (this CLI: %s)\n", self); + } + return 0; +} + +static int +cmd_store(const char *op, int argc, const char **argv) +{ + char exe[1024]; + if (!resolve_exe_arg(argc, argv, 3, exe, sizeof(exe))) { + printf("usage: displayxr-cli camera %s |--self\n", op); + return 1; + } + const struct u_camera_consent_store_ops *st = u_camera_consent_store_default(); + bool ok; + if (strcmp(op, "allow") == 0) { + ok = st->set(NULL, exe, U_CAMERA_CONSENT_STORED_ALLOW); + } else if (strcmp(op, "deny") == 0) { + ok = st->set(NULL, exe, U_CAMERA_CONSENT_STORED_DENY); + } else if (strcmp(op, "revoke") == 0) { + ok = st->set(NULL, exe, U_CAMERA_CONSENT_STORED_NONE); + } else if (strcmp(op, "trust") == 0) { + ok = u_camera_consent_store_set_delegating(exe, true); + } else { // untrust + ok = u_camera_consent_store_set_delegating(exe, false); + } + printf("%s %s: %s\n", op, exe, ok ? "ok (takes effect on the next stream start)" : "FAILED to write the store"); + return ok ? 0 : 2; +} + +static int +cmd_control(uint32_t op, uint32_t arg, const char *what) +{ + struct ipc_connection ipc_c = {0}; + if (!cam_connect(&ipc_c, XRT_CLIENT_CLASS_DIAG)) { + return 2; + } + uint32_t value = 0; + xrt_result_t xret = ipc_client_stereo_camera_control(&ipc_c, op, arg, &value); + if (xret != XRT_SUCCESS) { + printf("%s failed: %d%s\n", what, (int)xret, result_hint(xret)); + } else if (op == XRT_STEREO_CAMERA_CONTROL_STATUS) { + printf("stereo camera: sharing %s, session %s, %u started stream(s)\n", (value & 1u) ? "on" : "OFF", + (value & 2u) ? "LOCKED" : "unlocked", (value >> 8) & 0xffu); + } else if (op == XRT_STEREO_CAMERA_CONTROL_STOP_ALL) { + printf("%s: %u stream(s) ended\n", what, value); + } else { + printf("%s: ok\n", what); + } + ipc_client_connection_fini(&ipc_c); + return xret == XRT_SUCCESS ? 0 : 2; +} + +static bool +on_off_arg(int argc, const char **argv, int idx, uint32_t *out) +{ + if (idx >= argc) { + return false; + } + if (strcmp(argv[idx], "on") == 0 || strcmp(argv[idx], "1") == 0) { + *out = 1; + return true; + } + if (strcmp(argv[idx], "off") == 0 || strcmp(argv[idx], "0") == 0) { + *out = 0; + return true; + } + return false; +} + +static void +usage(void) +{ + printf( + "usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" + " probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] [--frames N] " + "[--seconds S] " + "[--out DIR]\n" + " [--max-disparity N] [--max-dy N (24)] [--min-ncc C (0.90)]\n" + " consent list the consent store\n" + " allow|deny|revoke |--self stored per-app decision (local)\n" + " trust|untrust |--self user-level consent-delegating list (local)\n" + " status | stop-all | sharing on|off | fake-lock on|off (over IPC, DIAG)\n" + "exit codes: 4 consent refused, 6 sharing off, 7 busy, 8 ended by the service, 5 rows misaligned\n"); +} + int cli_cmd_camera(int argc, const char **argv) { const char *sub = argc >= 3 ? argv[2] : "list"; bool json = cli_has_flag(argc, argv, "--json"); + + // Local store edits: no service needed. + if (strcmp(sub, "consent") == 0) { + return cmd_consent(); + } + if (strcmp(sub, "allow") == 0 || strcmp(sub, "deny") == 0 || strcmp(sub, "revoke") == 0 || + strcmp(sub, "trust") == 0 || strcmp(sub, "untrust") == 0) { + return cmd_store(sub, argc, argv); + } + // Control ops: DIAG class over IPC. + uint32_t onoff = 0; + if (strcmp(sub, "status") == 0) { + return cmd_control(XRT_STEREO_CAMERA_CONTROL_STATUS, 0, "status"); + } + if (strcmp(sub, "stop-all") == 0) { + return cmd_control(XRT_STEREO_CAMERA_CONTROL_STOP_ALL, 0, "stop-all"); + } + if (strcmp(sub, "sharing") == 0) { + if (!on_off_arg(argc, argv, 3, &onoff)) { + usage(); + return 1; + } + return cmd_control(XRT_STEREO_CAMERA_CONTROL_SET_SHARING, onoff, onoff ? "sharing on" : "sharing off"); + } + if (strcmp(sub, "fake-lock") == 0) { + if (!on_off_arg(argc, argv, 3, &onoff)) { + usage(); + return 1; + } + return cmd_control(XRT_STEREO_CAMERA_CONTROL_SET_LOCKED, onoff, onoff ? "fake-lock on" : "fake-lock off"); + } + if (strcmp(sub, "list") != 0 && strcmp(sub, "calib") != 0 && strcmp(sub, "probe") != 0) { + usage(); + return 1; + } + + // The consumer path: the same class a browser's capture utility declares. struct ipc_connection ipc_c = {0}; - if (!cam_connect(&ipc_c)) { + if (!cam_connect(&ipc_c, XRT_CLIENT_CLASS_CAMERA_CONSUMER)) { return 2; } int ret; @@ -914,12 +1211,7 @@ cli_cmd_camera(int argc, const char **argv) } else if (strcmp(sub, "probe") == 0) { ret = cmd_probe(&ipc_c, argc, argv); } else { - printf( - "usage: displayxr-cli camera list [--json] | calib [--raw|--rectified] [--json] |\n" - " probe [] [--raw|--rectified] [--format gray8|nv12|bgra8] [--fps F] [--frames N] " - "[--seconds S] " - "[--out DIR]\n" - " [--max-disparity N] [--max-dy N (24)] [--min-ncc C (0.90)]\n"); + usage(); ret = 1; } ipc_client_connection_fini(&ipc_c); From 19d2d6c02478f8e9d7223e0aee111d3419600b1f Mon Sep 17 00:00:00 2001 From: David Date: Sat, 3 Oct 2026 21:06:32 -0700 Subject: [PATCH 36/37] docs(stereo-camera): R3 consent and privacy as built MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spec §2a + §7 rewritten as built, §8 results, v2 history; roadmap R3 row and "R3 as built"; env census (DEV_ALLOW as the documented override, DXR_STEREO_CAMERA_PROMPT); docs/guides/stereo-camera-consent.md for app and browser developers (incl. the B1 migration); ADR-043 Amendment 1. Co-Authored-By: Claude Fable 5.1 --- docs/README.md | 1 + docs/adr/ADR-043-stereo-camera-source.md | 30 ++++ docs/guides/stereo-camera-consent.md | 145 ++++++++++++++++++ .../control-panel-performance-settings.md | 5 +- docs/roadmap/stereo-camera-source.md | 68 +++++++- docs/specs/extensions/XR_DXR_stereo_camera.md | 139 ++++++++++++----- 6 files changed, 345 insertions(+), 43 deletions(-) create mode 100644 docs/guides/stereo-camera-consent.md diff --git a/docs/README.md b/docs/README.md index 3a6cf764aa..12c4caf253 100644 --- a/docs/README.md +++ b/docs/README.md @@ -60,6 +60,7 @@ Contribute to the DisplayXR runtime — compositors, state tracker, auxiliary co - **[comp_multi One Pipeline (macOS/Linux/Android)](architecture/comp-multi-one-pipeline.md)** — #967 design note: bounded waits, shared-surface port to Android, issue split - **[In-Process vs Service](architecture/in-process-vs-service.md)** — compositor deployment modes - **[Implementing an Extension](guides/implementing-extension.md)** — how to add OpenXR extensions +- **[Stereo camera consent](guides/stereo-camera-consent.md)** — what gates `XR_DXR_stereo_camera` frames (consent prompt, delegating browsers, foreground rule, kill switches) and how an app or browser declares itself ### Internal Specs (`specs/runtime/`) diff --git a/docs/adr/ADR-043-stereo-camera-source.md b/docs/adr/ADR-043-stereo-camera-source.md index 56b1823eee..8ee8c63fb4 100644 --- a/docs/adr/ADR-043-stereo-camera-source.md +++ b/docs/adr/ADR-043-stereo-camera-source.md @@ -97,3 +97,33 @@ exactly so that knowledge lives in one vendor DLL and nowhere else. both hold Camera2. So the runtime (the vendor plug-in in the runtime APK, ADR-038) must own the front pair and run the vendor's in-app tracking from the same capture, on a worker thread (roadmap §C.1). That is more vendor code inside the runtime APK, and more CPU, than on Windows. + +## Amendment 1 (2026-10-03, R3 as built) — what changed from the decision text + +- **A camera-only client class.** The decision assumed the browser's capture component would + be classed as the browser (PRESENT_OWNER, by enabling `XR_DXR_weave`). In the field the + video-capture *utility* process is a separate OS process with no window and no weave, and + classing it PRESENT_OWNER spent a panel-owner slot. R3 adds `CAMERA_CONSUMER`, declared + explicitly (`XrStereoCameraClientInfoDXR`), session-less by contract and outside the + present-owner quota. Consent is still keyed by the verified executable, so the browser's + installer registration (delegating) covers the utility too. +- **Distinct refusal results instead of one `PERMISSION_INSUFFICIENT`.** A browser must map + "the user said no" to `NotAllowedError` and "the device cannot be read" to + `NotReadableError`; one code could not carry both. `PERMISSION_INSUFFICIENT` now means the OS + camera switch (or a class that may not use cameras); consent, sharing-off, busy and + service-ended each have their own `XrResult`. +- **The foreground rule is window visibility of the peer pid, not session state.** The spec + said "a session in VISIBLE / FOCUSED"; a camera consumer may have no session at all, and the + service already has the peer pid. Window-bearing classes need a visible top-level window; + delegating, camera-consumer and diagnostic clients are exempt (they were granted explicit + consent). The OS-lock suspension is unconditional. +- **The prompt blocks the caller (≤ 60 s) rather than returning "pending".** A capture + device's `AllocateAndStart` wants a final answer, and the stream handle stays valid either + way; an unanswered prompt is a refusal the app may retry. Unanswered is never persisted. +- **Two user kill switches, not one.** "Stop camera sharing" ends every started stream now + (apps see `STREAM_ENDED`); "Share the 3D camera with apps" is the persistent toggle the + decision described (zero cameras enumerated). +- **The dev override stayed** (`DXR_STEREO_CAMERA_DEV_ALLOW=1`) as a documented, loud + development switch below the sharing-off checks — the R1 text had it as a stand-in to be + deleted. + diff --git a/docs/guides/stereo-camera-consent.md b/docs/guides/stereo-camera-consent.md new file mode 100644 index 0000000000..bdacde6e06 --- /dev/null +++ b/docs/guides/stereo-camera-consent.md @@ -0,0 +1,145 @@ +# Stereo camera consent — for app and browser developers + +How the DisplayXR runtime decides whether your process may read the display's stereo camera +(`XR_DXR_stereo_camera`, [ADR-043](../adr/ADR-043-stereo-camera-source.md)), what the user sees, +and what you have to do. The normative text is the extension spec, +[§7 *Privacy*](../specs/extensions/XR_DXR_stereo_camera.md#7-privacy); this page is the +walkthrough. + +**Why the runtime does this at all.** The camera is the eye tracker's. The tracker opened the +device, so the OS camera permission model and its in-use light never see *you* as a consumer. The +runtime therefore enforces the equivalent itself: per-executable consent, a foreground rule, an +in-use indicator, kill switches, and an unlinkable device id. The vendor plug-in sees none of it. + +## 1. The three kinds of client + +| You are… | Declare | What gates frames | +|---|---|---| +| **A native app with a window** (a 3D call app) | nothing special — enable the extension | consent (§2) + your window must be visible (§3) | +| **A camera-only process** — a browser's video-capture utility, a capture helper, `displayxr-cli camera` | `XrStereoCameraClientInfoDXR` with `XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR` chained on `XrInstanceCreateInfo` | consent (§2); no window rule (you have none) | +| **A consent-delegating client** — a browser that shows its *own* per-origin camera prompt and in-use indicator | the camera-only declaration **and** a registration of your executable (§4) | your own prompt is the consent; your own visibility rule; never RAW frames | + +The camera-only declaration makes the service admit you as the `CAMERA_CONSUMER` class: you may +call only the camera entry points (`xrCreateSession` is refused), and you are **not a panel +owner** — a browser's capture utility no longer spends one of the two present-owner slots, even +though its executable is the browser's. Chain it like this: + +```c +XrStereoCameraClientInfoDXR consumer = { + .type = XR_TYPE_STEREO_CAMERA_CLIENT_INFO_DXR, + .flags = XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR, +}; +XrInstanceCreateInfo ci = {XR_TYPE_INSTANCE_CREATE_INFO, &consumer, /* … XR_DXR_stereo_camera in enabledExtensionNames */}; +``` + +## 2. Consent: what happens at `xrStartStereoCameraStreamDXR` + +The service looks at the **OS-derived executable path** of your process (never anything you +send) and walks this list; the first hit decides. The same check runs for +`xrGetStereoCameraCalibrationDXR` (calibration identifies the device). + +1. **Sharing off** — the user's tray / menu-bar toggle *"Share the 3D camera with apps"*, or + `DXR_STEREO_CAMERA=0` in the service's environment → `XR_ERROR_STEREO_CAMERA_DISABLED_DXR`. + (You also enumerate zero cameras.) +2. `DXR_STEREO_CAMERA_DEV_ALLOW=1` in the **service's** environment → allowed. A development + override; the service logs one WARN per run. Never on a user's machine. +3. Your executable path could not be verified → `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR`. +4. You are a **registered delegating client** → allowed, no prompt, nothing stored. +5. **Windows:** the OS camera privacy switch denies you (Settings → Privacy → Camera: globally, + for desktop apps, or for your app) → `XR_ERROR_PERMISSION_INSUFFICIENT`. +6. A **stored decision** for your executable (the user answered before, or an admin pre-seeded + it) → allowed, or `CONSENT_REFUSED`. +7. The user chose **Allow once** earlier for this very process → allowed. +8. The **prompt**: the service's tray (Windows) / menu-bar item (macOS) shows + + > **"" wants to use the 3D camera** + > *your-app.exe (pid 1234) will receive frames from the display's stereo camera. You can stop + > it at any time from the DisplayXR menu.* + > [ Deny ] [ Allow once ] [ Allow ] + + **Allow** and **Deny** are remembered for your executable; **Allow once** lasts for this + process. Your `xrStartStereoCameraStreamDXR` call **blocks** while the dialog is up (up to + 60 s). Unanswered, dismissed, or no prompt available (`DXR_STEREO_CAMERA_PROMPT=0`, a headless + Linux service) → `CONSENT_REFUSED`, nothing stored — you may try again later. + +A refused stream is not destroyed: call start again after the user has allowed you. Map the +results like this in a browser: + +| Result | Meaning | `getUserMedia` | +|---|---|---| +| `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR` | the user / policy said no | `NotAllowedError` | +| `XR_ERROR_PERMISSION_INSUFFICIENT` | the OS camera switch is off (or wrong client class / RAW asked) | `NotAllowedError` | +| `XR_ERROR_STEREO_CAMERA_DISABLED_DXR` | sharing is off | hide the device, or `NotReadableError` | +| `XR_ERROR_STEREO_CAMERA_BUSY_DXR` | the source cannot be opened right now | `NotReadableError` | +| `XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR` | the service ended your stream (see §5) | end the track | + +## 3. The foreground rule and the lock screen + +If you are a window-bearing client (an ordinary app, or a present-owner), frames are published +only while your process **owns a visible, non-minimised top-level window** (checked every +250 ms). Otherwise your stream is `SUSPENDED`: no new frames, and the frame you had pinned is +cleared so nothing lingers. Camera-only and delegating clients are exempt — the browser's own +tab rule applies to its pages. + +Every stream, of every client, is suspended while the **OS session is locked or switched away** +(Windows lock / disconnect, macOS screen lock / fast user switch) and resumes afterwards. You +see it as `XrEventDataStereoCameraStateChangedDXR` with `SUSPENDED` then `AVAILABLE` from +`xrPollEvent` (no session needed), and `xrEnumerateStereoCamerasDXR` reports `SUSPENDED` +meanwhile. Keep showing your last frame or a placeholder; do not tear the stream down. + +## 4. Being a consent-delegating client (browsers) + +A browser already asks the user per origin and shows its own in-use indicator, so the runtime +should not ask a second time per page. Register the browser's **executable path**: + +- **Installer (machine-wide):** Windows `HKLM\Software\DisplayXR\CameraConsent\Delegating`, a + `REG_SZ` value whose *data* is the full path (value name: anything, e.g. the file name); + Linux `/etc/displayxr/camera-delegating.json`; macOS + `/Library/Application Support/DisplayXR/camera-delegating.json` — both + `{"delegating": ["/full/path/to/browser", …]}`. +- **Per user / development:** `displayxr-cli camera trust ` (or `--self`), which writes the + user-level list (`HKCU\…\CameraConsent\Delegating`, `~/.config/…/camera_consent.json`). + +Obligations that come with it: prompt the user yourself before opening a stream, show your own +in-use indicator, stop the stream when your page stops capturing, and never expose raw +calibration or the raw frame — the runtime enforces the last one (RAW is refused to delegating +clients; `persistentId` is already a per-executable keyed hash, coarsen the rest yourself). + +An unsigned / development build that is not registered is simply a camera-only client: the +runtime prompts for it once (Allow is remembered for that path). + +## 5. What the user can do to you at any time + +- **"Stop camera sharing"** (tray / menu bar): every started stream ends. You receive + `XrEventDataStereoCameraStreamEndedDXR { stream, USER_STOPPED }`, your wake handle fires + once, and acquire / start on that handle return `XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR`. + Destroy it. You may create and start a new one — that goes through §2 again (and "Allow once" + grants were forgotten). +- **"Share the 3D camera with apps" off**: same, with reason `DISABLED`; `xrEnumerate…` then + returns zero cameras until it is switched back on. +- Change their answer: `displayxr-cli camera allow|deny|revoke ` edits the store; the next + start sees it. + +While your stream runs the user sees the runtime's icon badge and *"3D camera in use by +your-app.exe"* in the tray / menu bar — in addition to whatever you show. + +## 6. Developing and testing + +```sh +# the service, with the sim fake camera; prompts off = a headless CI service +SIM_DISPLAY_FAKE_STEREO_CAMERA=1 DXR_STEREO_CAMERA_PROMPT=0 displayxr-service + +displayxr-cli camera probe --seconds 2 # exit 4: CONSENT_REFUSED (no consent, no prompt) +displayxr-cli camera allow --self # or: camera trust --self (delegating) +displayxr-cli camera probe --seconds 2 # frames +displayxr-cli camera fake-lock on # every stream SUSPENDED (event printed by probe) +displayxr-cli camera fake-lock off +displayxr-cli camera stop-all # a running probe prints "ENDED … USER_STOPPED", exits 8 +displayxr-cli camera sharing off # zero cameras; start -> DISABLED (exit 6) +displayxr-cli camera consent # what is stored, and this CLI's own path +``` + +`DXR_STEREO_CAMERA_DEV_ALLOW=1` on the service skips consent for everything (loud WARN) — use +it on a dev box when you are iterating on frames, not on the consent path. The full env-var +census is in +[`docs/roadmap/control-panel-performance-settings.md`](../roadmap/control-panel-performance-settings.md). diff --git a/docs/roadmap/control-panel-performance-settings.md b/docs/roadmap/control-panel-performance-settings.md index e5b942bd89..9bee0a7fd2 100644 --- a/docs/roadmap/control-panel-performance-settings.md +++ b/docs/roadmap/control-panel-performance-settings.md @@ -547,7 +547,7 @@ as `docs/specs/vendor/oem-android-platform-requirements.md` §R6. | `DXR_DEVICE_REMOVED_EXIT_MS` | `compositor/d3d11_service/comp_d3d11_service.cpp:10279` | `getenv`, `static` cached | 2000 | Svc | 2 | Delay before the service exits on an unrecoverable `DEVICE_REMOVED` | | `DXR_HEALTH_MS` | `ipc/server/ipc_server_process.c:82` | `DEBUG_GET_ONCE_NUM` | 10000 | Svc | 2 | #951 `[HEALTH]` telemetry cadence; 0 disables | | `DXR_MAX_CLIENTS` | `ipc/server/ipc_server_process.c:87` | `DEBUG_GET_ONCE_NUM`, consumed `:541-551` | 0 = auto | Svc | 1 | #959 admitted-client cap. Read once at service start | -| `DXR_STEREO_CAMERA` | `ipc/server/ipc_server_stereo_camera.c` (`DEBUG_GET_ONCE_BOOL_OPTION`, read at camera-manager create) | on | Svc | 1 | ADR-043 admin/dev kill switch: `0` = the service enumerates zero stereo cameras (indistinguishable from "none", on purpose). The user-facing twin is the planned R3 "Share the 3D camera with apps" toggle — a privacy control, not a performance lever | +| `DXR_STEREO_CAMERA` | `ipc/server/ipc_server_stereo_camera.c` (`DEBUG_GET_ONCE_BOOL_OPTION`, read at camera-manager create) | on | Svc | 1 | ADR-043 admin/dev kill switch: `0` = the service enumerates zero stereo cameras (indistinguishable from "none", on purpose) and every start returns `XR_ERROR_STEREO_CAMERA_DISABLED_DXR`. The user-facing twin shipped in R3: the tray / menu-bar "Share the 3D camera with apps" toggle, persisted in the consent store — a privacy control, not a performance lever | | `DXR_ULTRALEAP_SETTLE_MS` | `drivers/ultraleap/ultraleap_provider.cpp:1023` | `getenv` at provider init | `UL_SETTLE_DEFAULT_MS` | Both | 1 | How long to wait for a definitive Leap "attached" answer before role arbitration | ### Test / dev — never exposed @@ -572,7 +572,8 @@ as `docs/specs/vendor/oem-android-platform-requirements.md` §R6. | `DXR_VIEW_CONFIG_LEGACY` | `state_trackers/oxr/oxr_system.c` (`oxr_system_fill_in`, `DEBUG_GET_ONCE_BOOL_OPTION`) | off | App | #1486 kill switch: restores the pre-#1486 single view configuration — `PRIMARY_STEREO` for any `view_count >= 2`, reporting the max across modes (4 on sim-display), and no `PRIMARY_MULTIVIEW_DXR`. Reintroduces the spec deviation and the CTS red on purpose, for one round of field bisection. Logs one `U_LOG_W` when active. 22 characters, so `debug.xrt.DXR_VIEW_CONFIG_LEGACY` is **stranded on Android** by the `PROP_NAME_MAX` trap above — env-only in practice. See [View-Configuration Model](../reference/view-configuration-model.md) | | `DXR_MODE_FLOOR` | `state_trackers/oxr/oxr_session.c` (`DEBUG_GET_ONCE_BOOL_OPTION`, one TU; `oxr_api_session.c` reads it through `oxr_session_mode_floor_enabled()`) | on | App | #1499 kill switch. `0` restores the pre-#1499 behaviour for EXTENSION sessions: `xrBeginSession` leaves the display in whatever rendering mode it found, and `xrRequestDisplayRenderingModeDXR` honours a request whose `viewCount` exceeds the session's view configuration — so the compositor's under-submit clamp paints the first tiles and the rest stay at the clear colour, which is exactly the state the issue objected to. It also silences the two #1499 log lines (the begin-time `UNFILLABLE` warning and the observation warning on a display-global mode change), because `0` promises pre-#1499 behaviour and a line the runtime never used to print is part of that promise. **One switch for BOTH halves on purpose**: a floor without the denial lets an app walk straight back into the mode the floor removed it from, and a denial without the floor refuses a mode the session is already in — half of this feature is not a state anyone should be able to bisect into. #1510's LEGACY floor is deliberately NOT affected (a legacy app was sized for its floored mode at `xrGetSystem`, so leaving it in a wider one would make the compromise view scale and the mode disagree). See [View-Configuration Model](../reference/view-configuration-model.md) | | `DXR_UNDER_SUBMIT` | `state_trackers/oxr/oxr_session_frame_end.c` (`DEBUG_GET_ONCE_NUM_OPTION`, one TU; the value→enum map is the pure `oxr_under_submit_from_setting()` in `oxr_view_config_rule.h`) | `1` | App | ADR-041 staging switch for the fixed-view-count submission contract. `0` = strict: every view configuration must submit exactly the count `xrLocateViews` returned. `1` (default) = strict **plus** the deprecated `PRIMARY_STEREO` 1-view arm (an `XR_DXR_display_info` instance submitting 1 while the active rendering mode is 1-view), which is accepted and logged **once per session** with a line naming the fix — released demos submit one view in 2D mode and need the compat window. `2` = kill switch: the pre-ADR-041 rules, `PRIMARY_MULTIVIEW_DXR` under-submit included. Out-of-range values **clamp to an end** rather than falling back to the default, so a typo cannot silently land on the default. The default flips to `0` in the first runtime release after `displayxr-common` and the five `displayxr-demo-*` demos ship the alias submission — the trigger is that shipment, not a date. Not a user-facing setting: it is a compat bisect knob with no performance dimension. Unreachable from a CTS run at any value (a conformance session never enables `XR_DXR_display_info`, which the compat arm requires). Distinct from `DXR_VIEW_CONFIG_LEGACY`, which restores the pre-#1486 *reporting*. See [ADR-041](../adr/ADR-041-fixed-view-count-with-per-frame-activity.md) and [View-Configuration Model](../reference/view-configuration-model.md) | -| `DXR_STEREO_CAMERA_DEV_ALLOW` | `ipc/server/ipc_server_stereo_camera.c` (`authorise_locked`, `DEBUG_GET_ONCE_BOOL_OPTION`) | off | Svc | ADR-043 **R1-only** stand-in for camera consent: stream start and calibration reads are REFUSED (`XR_ERROR_PERMISSION_INSUFFICIENT`) unless it is set in the SERVICE environment. Deleted when R3 lands the OS + DisplayXR consent path. Never a setting | +| `DXR_STEREO_CAMERA_DEV_ALLOW` | `ipc/server/ipc_server_stereo_camera.c` (fed into `u_camera_consent::dev_override`, `DEBUG_GET_ONCE_BOOL_OPTION`) | off | Svc | ADR-043 R3 **development override**: every stream start and calibration read is ALLOWED without consent — no prompt, no store entry — with one WARN per service run. Sharing-off (`DXR_STEREO_CAMERA=0` or the user toggle) still wins over it. Never on a user's machine; never a setting | +| `DXR_STEREO_CAMERA_PROMPT` | `ipc/server/ipc_server_stereo_camera.c` (`u_camera_consent::prompt_enabled`, `DEBUG_GET_ONCE_BOOL_OPTION`) | on | Svc | ADR-043 R3: `0` = never show the tray consent prompt; an app with no stored decision and no delegating registration is then refused with `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR` (headless / CI services, kiosks that pre-seed the store with `displayxr-cli camera allow`). Not a setting | | `DXR_IPC_FD` | `auxiliary/util/u_sandbox.c:181`; `ipc/client/ipc_client_connection.c:211, 234` | unset | Both / App | #1056 adopts an embedder-supplied service socket. Not a setting | | `DXR_IPC_HANDLE` | `ipc/client/ipc_client_connection.c:437, 456` | unset | App | Windows analogue of the above | | `DXR_WL_ORIGIN_STAMP` | `compositor/vk_native/comp_vk_native_compositor.c` (`vk_debug_origin_stamp`, Linux + Wayland) | off | App | #1748 measurement hook: writes a barcode of each frame's present origin into its top-left 352x8 device px — it **overwrites content**. Read back by the extension's `DISPLAYXR_STAMP_AUDIT=1` (shell environment) in `scripts/linux/move_sync/`. Never for a real session | diff --git a/docs/roadmap/stereo-camera-source.md b/docs/roadmap/stereo-camera-source.md index 859e3fbe4d..34d70442a1 100644 --- a/docs/roadmap/stereo-camera-source.md +++ b/docs/roadmap/stereo-camera-source.md @@ -1,8 +1,8 @@ # Stereo camera source — browser integration, vendor plug-in notes, phased plan **Status:** design (2026-09-25); **R1 implemented** on `feat/stereo-camera-r1` (2026-09-26), -**R2 implemented** on `feat/stereo-camera-r2` (2026-09-26, stacked) — see §E; maintainer decisions -for R1 in §G. **L1** (Leia provider) is on `displayxr-leia-plugin` `feat/stereo-camera-l1`. Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). +**R2 implemented** on `feat/stereo-camera-r2` (2026-09-26, stacked), **R3 implemented** on +`feat/stereo-camera-r3` (2026-10-03, stacked) — see §E; maintainer decisions for R1 in §G. **L1** (Leia provider) is on `displayxr-leia-plugin` `feat/stereo-camera-l1`. Decision: [ADR-043](../adr/ADR-043-stereo-camera-source.md). Extension: [`XR_DXR_stereo_camera`](../specs/extensions/XR_DXR_stereo_camera.md). Consumer driving it: the web SDK's 3D call module (`@displayxr/inline3d/call`, RFC 0002 on `displayxr-web` branch `feat/call-p1`, §4 *Capture*). @@ -196,7 +196,7 @@ The implementation belongs in `displayxr-leia-plugin` (`src/drv_leia/`), per ADR | **R0** | runtime | This design: ADR-043, spec, this page | Maintainer sign-off on the open questions below | | **R1** ✅ | runtime | **Implemented 2026-09-26 (see "R1 as built" below).** Header `XR_DXR_stereo_camera.h` + `index.json` note (together — the catalog lint requires both); `xrt_plugin_iface` slots + `XRT_PLUGIN_IFACE_HAS_STEREO_CAMERA` (appended after `create_dp_d3d11_lift`, so **lands after ADR-042's PR**); platform-neutral camera manager (thread, refcount + linger, 3-slot pinned ring, per-stream wake handles, decimation, NV12/BGRA conversion); IPC + OpenXR entry points; sim_display fake; `displayxr-cli camera list/calib/probe`; selftest check | `camera probe` on the sim fake writes frames on Windows + Linux CI; ring/pinning unit tests (mirror `tests_lift_mailbox`) | | **R2** ✅ | runtime | **Implemented 2026-09-26 (see "R2 as built" below).** `u_stereo_rectify` (Bouguet, valid-region crop, LUT remap) with golden tests against OpenCV-generated fixtures; RECTIFIED output | sim fake: row error < 0.5 px, disparity = f·B/Z within 0.5 px — **measured 0.061 px worst block, 0.063 px worst disparity error** | -| **R3** | runtime | Privacy: OS consent check, consent store + tray prompt, delegating-client registration, visibility/lock suspension, indicator, kill switches | Manual matrix: allow/deny/revoke, OS switch off, lock screen, background app — each blocks frames | +| **R3** ✅ | runtime | **Implemented 2026-10-03 (see "R3 as built" below).** Privacy: OS consent check, consent store + tray prompt, delegating-client registration, visibility/lock suspension, indicator, kill switches, camera-only client class, distinct refusal results, events delivered, keyed `persistentId`, read-only consumer section | Policy matrix unit-tested with fakes; sim-fake end to end on macOS: refused without consent → allowed after `camera allow --self` / `camera trust` → suspended on a (fake) lock → ended by the kill switch. **Windows tray + WTS path and the manual panel matrix (allow/deny/revoke, OS switch off, real lock screen, background app) still need the Windows box** | | **L1** | leia-plugin (Windows) | Slots over the SR raw-camera channel per §D; calibration by active serial; keep-alive; repin runtime (feature-macro repin, `downstream-pins.json` `features` track) | On a panel box, incl. the multi-folder box: `camera probe --rectified` rows aligned; the vendor call app running at the same time still gets every frame; tracking/weave unaffected (frame-time + tracking-state logs) | | **B1** | browser (Windows) | §B device factory + duplicate hiding + hint + delegating registration in the installer; web SDK: prefer the hint in `camera:'auto'`, fill `hello` from it | RFC 0002 P1 call between two panel laptops sends rectified SBS from the tracker camera **while both are tracking** | | **A1** | leia-plugin (Android) + runtime | §C.1: the runtime owns the front pair and runs CNSDK in-app tracking from the same capture (worker thread, latest-frame slot); per-eye AHB transport + per-eye frame layout; Android consent + visibility | `camera probe` on the tablet via the runtime app **while the weave tracks from the same capture**; pair at 30.00 fps (not 24) | @@ -289,6 +289,68 @@ against the sim fake). - **Not in R2:** the GPU backend (seam in place), and a rectifier for a camera whose calibration changes at runtime (the refinement's reset hook is there for it). +### R3 as built + +Spec §7 is the normative text; the developer-facing walkthrough is +[`docs/guides/stereo-camera-consent.md`](../guides/stereo-camera-consent.md). What landed, and +where: + +- **Camera-only client class.** `XRT_CLIENT_CLASS_CAMERA_CONSUMER` (value 6), declared with + `XrStereoCameraClientInfoDXR` + `CONSUMER_ONLY_BIT` on `XrInstanceCreateInfo` (spec §2a). + Verified by use (it can only call camera handlers; `session_create` refuses it), quota 4, + **never counted toward the PRESENT_OWNER quota** — the owner-counting rule is now the pure + `u_client_class_present_owner_count()` (`auxiliary/util/u_client_class.c`), tested host-side. + A hybrid runtime always routes a declared consumer to the service. `displayxr-cli camera + list|calib|probe` connect as this class; the control subcommands as `DIAG`. + **B1 migration:** the browser's video-capture utility must chain the struct instead of + enabling `XR_DXR_weave` to be classed as the browser (the weave-derived PRESENT_OWNER claim + is outranked by the declaration anyway); its other processes keep the sibling rule. +- **Consent policy + store** (`u_camera_consent.{h,c}`, `u_camera_consent_store.c`): the eight- + step decision tree of spec §7.1 over two injected vtables (store, environment), the real store + (Windows registry under `HKCU\Software\DisplayXR\CameraConsent`, POSIX + `camera_consent.json` mode 0600 in the user config dir, plus the installer-written delegating + lists), the Windows `ConsentStore\webcam` read, and the keyed `persistentId` + (HMAC-SHA-256 over a 32-byte per-user secret the store generates once; a compact + `u_sha256.c` pinned by FIPS / RFC 4231 vectors). `DXR_STEREO_CAMERA_DEV_ALLOW=1` stays as the + documented dev override (one WARN; sharing-off still beats it); `DXR_STEREO_CAMERA=0` the kill + switch; `DXR_STEREO_CAMERA_PROMPT=0` turns the prompt off (headless / CI: an app with no + stored or delegated consent is refused). +- **Service** (`ipc_server_stereo_camera.c`): consent evaluated at stream start and calibration + read with the manager lock released and one evaluation at a time (the prompt blocks the + client thread ≤ 60 s); distinct xrt results (`XRT_ERROR_STEREO_CAMERA_CONSENT_REFUSED` / + `_DISABLED` / `_BUSY` / `_STREAM_ENDED`, −45…−48) mapped 1:1 to the new `XrResult`s; the + foreground rule per published frame (250 ms cache; APP / PRESENT_OWNER need a visible + top-level window of the peer pid — `EnumWindows` on Windows, `NSRunningApplication` on macOS, + always visible on Linux; delegating / CAMERA_CONSUMER / DIAG exempt); session-lock + suspension (`ipc_server_stereo_camera_set_session_locked`, fed by WTS on Windows and + `screenIsLocked` + `sessionDidResignActive` on macOS; `camera fake-lock` for tests); + per-connection event queues drained by `stereo_camera_poll_event`; `stop_all`, `set_sharing` + and `get_status` for the UI; the read-only consumer section on every platform + (`ipc_shmem_create_with_readonly`: FILE_MAP_READ duplicate / O_RDONLY re-open before unlink / + `ASharedMemory_setProt(PROT_READ)`). +- **OpenXR:** `scam_xret` maps the four new results; `xrPollEvent` drains the camera events + into the instance queue (`oxr_stereo_camera_poll_events`), mapping a service stream id back + to the app's handle for `XrEventDataStereoCameraStreamEndedDXR`. +- **UI:** Windows tray — icon badge (red dot), tooltip + menu line "3D camera in use by ", + one balloon per transition, "Stop camera sharing", the "Share the 3D camera with apps" + checkmark, the consent dialog (top-most, Allow / Allow once / Deny) on the tray thread, WTS + session-change registration + `SM_REMOTESESSION`. macOS menu bar — the same five pieces as + an `NSStatusItem` badge, menu lines and a floating `NSPanel` (not a modal: a modal would park + the service's run loop) driven from the main-thread pump. +- **CLI:** `camera consent | allow | deny | revoke | trust | untrust (|--self)` edit the + store locally; `camera status | stop-all | sharing on|off | fake-lock on|off` over IPC; + `probe` prints every event with its time since start and exits 4 / 6 / 7 / 8 for consent + refused / sharing off / busy / ended by the service. +- **Tests:** `tests_camera_consent` (policy decisions for every rule with fake store + + environment; Allow-once bound to the pid; prompt timeout / unavailable retried; keyed id + stable per key and unlinkable across keys, consumers and devices; SHA-256 / HMAC vectors; + the PRESENT_OWNER owner count with a CAMERA_CONSUMER sibling). End to end on the macOS sim + fake, see the guide. +- **Not in R3:** Android consent (A1); a signer check on the delegating entry (the registered + path is trusted as the installer wrote it); the macOS OS-camera (TCC) switch — not readable + for a consumer the OS cannot see; the Windows tray + WTS + dialog code is written to the API + but has only been compile-reasoned on macOS: it needs the Windows box. + ## F. Risks - **Undocumented vendor channel.** The raw-camera shared memory is a reader-defined contract with diff --git a/docs/specs/extensions/XR_DXR_stereo_camera.md b/docs/specs/extensions/XR_DXR_stereo_camera.md index 7773f530d4..6e6e8b0f43 100644 --- a/docs/specs/extensions/XR_DXR_stereo_camera.md +++ b/docs/specs/extensions/XR_DXR_stereo_camera.md @@ -62,16 +62,32 @@ if (xrAcquireStereoCameraFrameDXR(stream, &f) == XR_SUCCESS) { /* f.slot is pinn | Service with a camera source | one entry per camera, with its `state` (§3) | | `DXR_STEREO_CAMERA=0` in the **service's** environment, or the user kill switch (§7.4) | count 0 — indistinguishable from "no camera", on purpose | -**R1 dev gate.** Until the consent path (§7.1, R3) exists, the service refuses every -`xrStartStereoCameraStreamDXR` and `xrGetStereoCameraCalibrationDXR` with -`XR_ERROR_PERMISSION_INSUFFICIENT` **unless `DXR_STEREO_CAMERA_DEV_ALLOW=1` is set in the -service's environment** — deny by default. Enumeration works without it (it reveals no -calibration and no serial). - Instance-level, not session-level: a capture component (a browser's video-capture service) has no compositor session and needs none. Calls travel on the instance's IPC connection; streams belong to that connection and die with it. +### 2a. Declaring a camera-only client (spec v2, R3) + +A process that will only ever read a camera — a browser's video-capture utility, `displayxr-cli +camera` — chains `XrStereoCameraClientInfoDXR` with +`XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR` on `XrInstanceCreateInfo`: + +```c +XrStereoCameraClientInfoDXR ci = {XR_TYPE_STEREO_CAMERA_CLIENT_INFO_DXR, NULL, + XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR}; +XrInstanceCreateInfo ii = {XR_TYPE_INSTANCE_CREATE_INFO, &ci, ...}; // enabledExtensionNames includes XR_DXR_stereo_camera +``` + +The service admits such an instance as the **`CAMERA_CONSUMER` client class** (quota 4): it may +call only the camera entry points — `xrCreateSession` is refused with +`XR_ERROR_PERMISSION_INSUFFICIENT` — and it is **not a panel owner**, so it never counts toward +the `PRESENT_OWNER` quota even when its executable is the browser's (the quota counts distinct +owner *executables*; a camera-only sibling of an admitted owner is simply not in the count). The +declaration outranks the `XR_DXR_weave`-derived present-owner claim: a capture utility that +enables both is still a camera consumer. A hybrid runtime always routes a declared consumer to +the service (an in-process instance has no cameras). Without the declaration the instance is +classed as before (APP, or PRESENT_OWNER when it enables `XR_DXR_weave`). + ## 3. Enumerating cameras ```c @@ -316,14 +332,12 @@ When `SHARED_WITH_EYE_TRACKING` is set: ## 7. Privacy -> **R1 status.** Only the hook points exist, all deny-by-default, in -> `src/xrt/ipc/server/ipc_server_stereo_camera.c`: `authorise_locked()` (§7.1 — R1 allows only -> with `DXR_STEREO_CAMERA_DEV_ALLOW=1` in the service environment), `client_visible_locked()` -> (§7.2 — always true in R1), `output_allowed()` (RAW refused to `PRESENT_OWNER`, i.e. the -> browser, per the R1 decision), the `DXR_STEREO_CAMERA=0` kill switch (§7.4), and one WARN per -> stream start/stop naming the peer executable in place of the indicator (§7.3). `persistentId` -> (§7.5) is an unkeyed 128-bit hash of (device identity, peer executable) in R1; R3 keys it with a -> persisted per-user secret. +> **R3 status: as built** (`src/xrt/ipc/server/ipc_server_stereo_camera.c`, policy in +> `src/xrt/auxiliary/util/u_camera_consent.{h,c}` + the platform store +> `u_camera_consent_store.c`, UI in `targets/service/service_tray_win.c` and +> `ipc/server/ipc_server_macos_appkit.m`). Developer guide: +> [`docs/guides/stereo-camera-consent.md`](../../guides/stereo-camera-consent.md). The vendor +> plug-in never sees any of this. The frames bypass the OS camera stack (the tracker, not the consumer, opened the device), so OS camera permissions and in-use indicators do not see this consumer. The runtime therefore enforces @@ -332,44 +346,88 @@ the equivalent itself. All checks are made on the **OS-derived peer** of the con [service-architecture §4.1a](../../architecture/service-architecture.md#41a-declared-peer-identity-for-a-brokered-connection-browser103-rc-1)), never on client-asserted fields. -### 7.1 Authorisation (at `xrStartStereoCameraStreamDXR`) +### 7.1 Authorisation (at `xrStartStereoCameraStreamDXR` and `xrGetStereoCameraCalibrationDXR`) + +The decision tree, first hit wins, evaluated with the manager lock released and one evaluation +at a time (so two apps never race two prompts): -| Check | Windows | Android | +| # | Check | Outcome | |---|---|---| -| OS camera privacy | the peer executable's webcam consent in `CapabilityAccessManager\ConsentStore\webcam` (global + per-app desktop entry) must not be Deny | the peer uid must hold `android.permission.CAMERA` (granted, not just declared) | -| DisplayXR consent | the peer executable (path + Authenticode signer) is in the user's allow list (`HKCU\Software\DisplayXR\StereoCamera\Consent`), else the service raises a first-use prompt from its tray ("*App* wants to use the 3D camera — Allow / Deny"); a Deny is remembered | the runtime app's consent activity, first use per package | -| Consent-delegating clients | a registered browser that shows its own per-origin permission prompt and in-use indicator (roadmap §B) is marked **delegating** at install time; the runtime's check is then per executable, the browser's per origin | same, per package | +| 1 | Sharing off: `DXR_STEREO_CAMERA=0` in the service environment, or the user's **"Share the 3D camera with apps"** toggle (tray / menu bar, persisted in the store) | `XR_ERROR_STEREO_CAMERA_DISABLED_DXR` (and zero cameras enumerated) | +| 2 | `DXR_STEREO_CAMERA_DEV_ALLOW=1` in the service environment — development override, one WARN per service run; **never** on a user's machine | allowed | +| 3 | The peer executable could not be verified (`""`) | `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR` | +| 4 | The executable is a registered **consent-delegating** client (a browser that shows its own per-origin prompt + in-use indicator): machine list written by its installer (`HKLM\Software\DisplayXR\CameraConsent\Delegating`, `/etc/displayxr/camera-delegating.json`, `/Library/Application Support/DisplayXR/camera-delegating.json`) or the user list (`displayxr-cli camera trust `) | allowed, no prompt, no store entry; never RAW (§7.5) | +| 5 | The OS camera privacy switch denies this executable (Windows `CapabilityAccessManager\ConsentStore\webcam`: machine policy, user global, desktop-app class, per-app entry; no equivalent is readable on macOS / Linux) | `XR_ERROR_PERMISSION_INSUFFICIENT` | +| 6 | The stored per-app decision (`HKCU\Software\DisplayXR\CameraConsent\Apps\`, `camera_consent.json` `apps`) | Allow → allowed; Deny → `CONSENT_REFUSED` | +| 7 | "Allow once" already granted to this same process (executable + pid) | allowed | +| 8 | The **tray prompt**: *" wants to use the 3D camera — Allow / Allow once / Deny"* (Windows: a top-most dialog from the tray thread; macOS: a floating panel from the menu-bar item — never a modal that would park the service's run loop). **Allow** and **Deny** are written to the store; **Allow once** is remembered for the process. The call blocks for up to 60 s; unanswered / dismissed, or no prompt available (`DXR_STEREO_CAMERA_PROMPT=0`, headless Linux service) | `CONSENT_REFUSED` — retryable, asked again next time | + +A refused stream stays created and may be started again (e.g. after `displayxr-cli camera allow`). +The policy is pure (two injected vtables) and unit-tested with fakes in `tests_camera_consent`. -Failure is `XR_ERROR_PERMISSION_INSUFFICIENT`; the stream stays created and may be started again -(e.g. after the user allows it). +**Android** (A1, not built): CAMERA permission of the peer uid + the runtime app's consent +activity per package; `android_package_is_visible` for §7.2. ### 7.2 Foreground rule -Frames are published to a stream only while its client is **visible**: for an OpenXR app, a -session in `VISIBLE` or `FOCUSED`; on Android, the peer package visible to the user (the plug-in -host's `android_package_is_visible`). Otherwise the stream is `SUSPENDED` for that client — it -receives no new frames, and its pinned slot is cleared on suspension so the last image does not -linger. A **delegating** client (the browser's capture component) has no session; its visibility -rule is the browser's own (a capturing tab keeps its indicator; closing it stops the track), and -the runtime additionally suspends every stream while the OS session is locked or switched away. +Frames are published to a stream only while its client is **visible**, checked per published +frame with a 250 ms cache: + +- **Window-bearing classes (APP, PRESENT_OWNER):** the verified peer pid must own a visible, + non-minimised top-level window (Windows `EnumWindows`; macOS `NSRunningApplication` not hidden + and not activation-prohibited; desktop Linux has no display connection in the service and is + always visible — the OS lock below is the gate there). Otherwise the stream is `SUSPENDED` for + that client: no new frames, pinned slot cleared so the last image does not linger. +- **Delegating clients** follow their own visibility rule (a capturing tab keeps its indicator; + closing it stops the track). **`CAMERA_CONSUMER` and `DIAG`** have no window by contract and + are exempt — they were granted explicit consent instead. +- **Session lock:** every stream is suspended while the OS session is locked or switched away + (Windows `WM_WTSSESSION_CHANGE` lock / console or remote disconnect, plus `SM_REMOTESESSION` at + start; macOS `com.apple.screenIsLocked` and `NSWorkspaceSessionDidResignActive`), and resumes on + unlock. Enumerated and event-reported state is `SUSPENDED` meanwhile (§7.6). ### 7.3 Indicator -While any stream is started, the service shows an in-use indicator (Windows tray badge + tooltip -naming the consumer executables; Android an ongoing notification from the runtime app), in -addition to whatever the consumer shows. The indicator is owned by the runtime because the camera -LED, if any, is lit by tracking and says nothing about who is receiving frames. +While any stream is started, the service shows an in-use indicator: Windows tray icon with a red +dot, tooltip and menu line *"3D camera in use by "* (executable base names), one balloon on +the transition; macOS menu-bar icon with a red dot, tooltip and menu line. The indicator is owned +by the runtime because the camera LED, if any, is lit by tracking and says nothing about who is +receiving frames. ### 7.4 Kill switches -User: a tray / runtime-app toggle "Share the 3D camera with apps" (off ⇒ zero cameras -enumerated). Admin/dev: `DXR_STEREO_CAMERA=0` in the service environment. +- **User, immediate:** tray / menu-bar **"Stop camera sharing"** ends every started stream now + (each gets `XrEventDataStereoCameraStreamEndedDXR` with `USER_STOPPED`; acquire and start on it + return `XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR`; "Allow once" grants are forgotten). The app + may create and start a new stream, which goes through §7.1 again. +- **User, persistent:** **"Share the 3D camera with apps"** (checkmark) — off ⇒ zero cameras + enumerated, every stream ended with `DISABLED`, start refused with + `XR_ERROR_STEREO_CAMERA_DISABLED_DXR`. Stored with the consent. +- **Admin / dev:** `DXR_STEREO_CAMERA=0` in the service environment (zero cameras). +- `displayxr-cli camera stop-all | sharing on|off` drive the same two switches over IPC. ### 7.5 Fingerprinting -Calibration and serials identify a device uniquely. `persistentId` is a keyed hash of (device -identity, peer executable), never the serial; calibration is returned only to a client that has -passed §7.1 for that camera. A browser must coarsen what it gives pages (roadmap §B.3). +Calibration and serials identify a device uniquely. `persistentId` is +**HMAC-SHA-256(secret, device identity ‖ consumer executable)** rendered as `dxrcam-` + 32 hex +chars, where the secret is 32 random bytes generated once per user and kept in the consent +store (`Secret` / `"secret"`). The same device gives every executable a different id, the same +executable a different id on every install, and nothing about the serial is derivable from it. +Calibration is returned only to a client that has passed §7.1 for that camera; RAW output is +refused to `PRESENT_OWNER` and to every delegating client (both expose cameras to third-party +content). A browser must still coarsen what it gives pages (roadmap §B.3). + +### 7.6 Events + +The service queues, per connection (16 deep, oldest dropped), and the client drains from +`xrPollEvent` (no session needed): + +- `XrEventDataStereoCameraStateChangedDXR` — the camera's **effective** state to every connection + with a stream on it: the source state, or `SUSPENDED` while the session is locked / sharing is + off. +- `XrEventDataStereoCameraStreamEndedDXR` — a stream the service ended (§7.4), with the handle + and the reason. +- `XrEventDataStereoCamerasChangedDXR` — reserved (the camera set is fixed for a service run). ## 8. Errors @@ -377,7 +435,11 @@ passed §7.1 for that camera. A browser must coarsen what it gives pages (roadma |---|---| | `XR_STEREO_CAMERA_FRAME_NOT_READY_DXR` (success) | no frame newer than the last acquired | | `XR_ERROR_VALIDATION_FAILURE` | struct out of contract (unknown camera, format/transport not in the supported bits, RECTIFIED on an uncalibrated camera) | -| `XR_ERROR_PERMISSION_INSUFFICIENT` | §7.1 refused; retryable after consent | +| `XR_ERROR_PERMISSION_INSUFFICIENT` | the OS camera privacy switch denies the app (§7.1 step 5); RAW asked by a browser / delegating client; a client class that may not use cameras (or `xrCreateSession` from a `CAMERA_CONSUMER`) | +| `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR` | §7.1: the user / the stored decision refused, the prompt was unanswered or unavailable; retryable after consent. A browser maps it to `NotAllowedError` | +| `XR_ERROR_STEREO_CAMERA_DISABLED_DXR` | §7.4: sharing is off (user toggle or `DXR_STEREO_CAMERA=0`) | +| `XR_ERROR_STEREO_CAMERA_BUSY_DXR` | the plug-in could not open the source and its retry is pending; retry later. A browser maps it to `NotReadableError` | +| `XR_ERROR_STEREO_CAMERA_STREAM_ENDED_DXR` | start / acquire on a stream the service ended (§7.4); destroy it and create a new one | | `XR_ERROR_FEATURE_UNSUPPORTED` | GPU transport requested on a platform without it | | `XR_ERROR_LIMIT_REACHED` | more than 8 streams per camera service-wide | | `XR_ERROR_RUNTIME_FAILURE` | transient service refusal on a healthy pipe; retry | @@ -591,5 +653,6 @@ camera state change and per stream start/stop with the peer executable; INFO sta | Version | Change | |---|---| +| 2 (R3) | Consent and privacy as built (§7): `XrStereoCameraClientInfoDXR` + `XR_STEREO_CAMERA_CLIENT_CONSUMER_ONLY_BIT_DXR` (the `CAMERA_CONSUMER` class, §2a); distinct results `XR_ERROR_STEREO_CAMERA_CONSENT_REFUSED_DXR` / `_DISABLED_DXR` / `_BUSY_DXR` / `_STREAM_ENDED_DXR`; `XrEventDataStereoCameraStreamEndedDXR`; the state-changed event is now delivered; `persistentId` keyed with a per-user secret; read-only consumer section on every platform; `DXR_STEREO_CAMERA_DEV_ALLOW` becomes a documented dev override, `DXR_STEREO_CAMERA_PROMPT=0` disables the prompt. Everything appended at `1004999311–316`; nothing renumbered | | 1 (R2) | No API change. RECTIFIED output and rectified calibration now come from the service's rectifier for any CALIBRATED, non-natively-rectified source; the browser is refused a camera it cannot rectify; sim_display `_DISTORT=1` fake; `camera probe` row alignment + `--rectified` | | 1 | R1 implements: enumerate, calibration (RAW; RECTIFIED for natively rectified sources), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, stream stats, plug-in slots, sim_display fake, CLI, selftest. Deferred: state-change events (structs defined, not delivered), GPU transports, rectifier (R2), consent/indicator (R3). Design scope: Enumerate + state events, calibration (raw / rectified), streams with start/stop, latest-wins acquire over a pinned 3-slot shared-memory ring with per-stream wake handles, optional GPU transports, runtime-enforced consent / foreground / indicator, plug-in iface slots, sim_display fake, CLI. | From fa524ced753f463de142dbb732ea02541bf71004 Mon Sep 17 00:00:00 2001 From: David Date: Tue, 6 Oct 2026 00:30:09 -0700 Subject: [PATCH 37/37] fix(stereo-camera): probe exits 6 when camera sharing is off; tray prompt cursor type Two findings from the R3 hardware run on the SR laptop (#1819): - 'camera sharing off' then 'camera probe' exited 3 ('the service exposes no stereo camera') instead of the documented 6. While sharing is off the service enumerates no camera, so the probe never reached the stream-create refusal that maps to 6. The probe now asks the service's status (a DIAG-only control, so over a second short-lived connection) when the enumeration is empty, and reports DISABLED / exit 6 when sharing is off. A machine with no stereo camera still exits 3. - service_tray_win.c: LoadCursorW(NULL, IDC_ARROW) is MSVC C4133 in this non-UNICODE build (IDC_ARROW is the ANSI-typed MAKEINTRESOURCE); use MAKEINTRESOURCEW(32512). Compiled on macOS (displayxr-cli, displayxr-service). NOT run: the exit code needs a service with the sim or SR camera and a verified DIAG CLI; the tray file is Windows-only. Co-Authored-By: Claude Fable 5.1 --- src/xrt/targets/cli/cli_cmd_camera.c | 27 ++++++++++++++++++++++ src/xrt/targets/service/service_tray_win.c | 3 ++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/xrt/targets/cli/cli_cmd_camera.c b/src/xrt/targets/cli/cli_cmd_camera.c index b04031b687..059a33627e 100644 --- a/src/xrt/targets/cli/cli_cmd_camera.c +++ b/src/xrt/targets/cli/cli_cmd_camera.c @@ -192,6 +192,29 @@ cam_connect(struct ipc_connection *ipc_c, uint32_t client_class) } +/*! + * True when the service reports camera sharing OFF. While sharing is off the service enumerates + * no camera at all, so "this machine has no stereo camera" and "sharing is off" look identical + * from the enumeration; only the status control tells them apart, and that control is DIAG-only, + * hence a second, short-lived connection. + */ +static bool +cam_sharing_is_off(void) +{ + struct ipc_connection diag = {0}; + struct xrt_instance_info ii = {0}; + snprintf(ii.app_info.application_name, sizeof(ii.app_info.application_name), "%s", "displayxr-cli"); + ii.app_info.declared_client_class = XRT_CLIENT_CLASS_DIAG; + if (ipc_client_connection_init(&diag, U_LOGGING_ERROR, &ii) != XRT_SUCCESS) { + return false; // cannot tell: report what the enumeration said + } + uint32_t value = 0; + xrt_result_t xret = ipc_client_stereo_camera_control(&diag, XRT_STEREO_CAMERA_CONTROL_STATUS, 0, &value); + ipc_client_connection_fini(&diag); + return xret == XRT_SUCCESS && (value & 1u) == 0; +} + + /* * * list / calib. @@ -669,6 +692,10 @@ cmd_probe(struct ipc_connection *ipc_c, int argc, const char **argv) } } if (!found && id == 0) { + if (cam_sharing_is_off()) { + printf("camera probe: refused%s\n", result_hint(XRT_ERROR_STEREO_CAMERA_DISABLED)); + return refusal_exit_code(XRT_ERROR_STEREO_CAMERA_DISABLED); // 6, as documented + } printf("camera probe: the service exposes no stereo camera.\n"); return 3; } diff --git a/src/xrt/targets/service/service_tray_win.c b/src/xrt/targets/service/service_tray_win.c index 7b24717510..f6b1f8361e 100644 --- a/src/xrt/targets/service/service_tray_win.c +++ b/src/xrt/targets/service/service_tray_win.c @@ -541,7 +541,8 @@ cam_prompt_show(struct cam_prompt_req *req) wc.cbSize = sizeof(wc); wc.lpfnWndProc = cam_prompt_wnd_proc; wc.hInstance = hi; - wc.hCursor = LoadCursorW(NULL, IDC_ARROW); + // IDC_ARROW is the ANSI-typed MAKEINTRESOURCE in this non-UNICODE build (MSVC C4133 with the W call). + wc.hCursor = LoadCursorW(NULL, MAKEINTRESOURCEW(32512)); // IDC_ARROW wc.hbrBackground = GetSysColorBrush(COLOR_WINDOW); wc.lpszClassName = L"DisplayXRCameraConsent"; wc.hIcon = s_icon_base;