desktop-avatar: TigerFaceViewer — head-tracked billboard + A/B probe (plugin #236)

samples/desktop-avatar/Assets/TigerFaceViewer.cs

@@ -0,0 +1,261 @@
+// Copyright 2024-2026, DisplayXR contributors
+// SPDX-License-Identifier: Apache-2.0
+
+// Optional head-tracked billboard for the tiger (yaw-only) + A/B probe for the
+// ways an app can ask "where is the viewer?" (plugin issue #236).
+//
+// The yaw math mirrors the native reference implementation in
+// displayxr-demo-avatar (windows/main.cpp, "Face-the-viewer billboard") 1:1,
+// because that one is viewer-confirmed on hardware:
+//
+//     targetYaw = FACE_YAW_SIGN * atan2(hx, |hz|)
+//
+// with (hx, hz) = the head centroid in RAW DISPLAY SPACE — metres, origin at the
+// physical panel centre, viewer at +Z — from displayxr_get_eye_positions. That is
+// deliberately NOT the world-space render-ready eyes: physical space is
+// independent of where the rig, camera and model sit in the scene, and immune to
+// the display rig's scale-as-zoom (which inflates world-space distances, e.g. a
+// 0.06 m IPD reads as ~1.2 world units here). Two earlier attempts to derive the
+// angle in world space failed exactly there — one silently rotated the wrong
+// object, the next found its reference vector degenerate because the tiger's root
+// sits at the camera's x/z.
+//
+// Smoothing is the reference's time-based exponential (tau), and the heading only
+// chases while eye tracking is LOCKED — warmup positions jitter the heading.
+//
+// The once-per-second log still prints the world-space provider API
+// (DisplayXRProvider.TryGetViewerEyes, new in v2.9.1) next to the raw physical
+// eyes and to Camera.GetStereoViewMatrix, so the sources stay comparable — that
+// A/B is the point of the probe for issue #236.
+//
+// Auto-installs via RuntimeInitializeOnLoadMethod (the KooimaProbe pattern), so
+// it needs no scene edit. Off by default — press F.
+//
+// Not ported from the reference: the zone-canvas rebase (it offsets hx by the
+// window's centre on the panel for off-centre windows). Without it the heading is
+// referenced to the PANEL centre rather than the window centre — a constant bias
+// when the window is off-centre, not a tracking failure.
+
+using System;
+using DisplayXR;
+using UnityEngine;
+using UnityEngine.InputSystem;
+
+[DisallowMultipleComponent]
+public class TigerFaceViewer : MonoBehaviour
+{
+    [Tooltip("Toggles head-tracked billboarding on/off. On by default, so F " +
+             "turns it OFF and restores the tiger's authored rotation.")]
+    public Key toggleKey = Key.F;
+
+    [Tooltip("Billboard from startup. The tiger's DragRotateCube is wired by " +
+             "TransparentAutoSetup at AfterSceneLoad, same phase as this " +
+             "component, so enabling retries until the target exists.")]
+    public bool startEnabled = true;
+
+    [Tooltip("Yaw direction. -1 matches the viewer-confirmed native reference " +
+             "(displayxr-demo-avatar FACE_YAW_SIGN); flip to +1 if the tiger " +
+             "turns away from you instead of toward you.")]
+    public float yawSign = -1f;
+
+    [Tooltip("Multiplies the viewer's off-axis angle. 1 = physically faithful " +
+             "(what the native reference does); >1 exaggerates for a demo.")]
+    public float gain = 1f;
+
+    [Tooltip("Smoothing time constant in seconds (settle ~= 3x tau). Matches the " +
+             "reference's 0.04. Time-based, so it is frame-rate independent.")]
+    public float tau = 0.04f;
+
+    [Tooltip("Clamp on the applied yaw (degrees).")]
+    public float maxYawDegrees = 60f;
+
+    [Tooltip("Seconds between A/B probe log lines. 0 = don't log.")]
+    public float logInterval = 1f;
+
+    private Transform m_Tiger;
+    private Quaternion m_BaseRotation;
+    private bool m_Enabled;
+    private float m_Timer;
+    private float m_Yaw;             // smoothed, applied
+    private float m_TargetYaw;       // pre-smoothing, for the log
+    private bool m_UserTurnedOff;    // an explicit F press wins over startEnabled
+    private float m_RetryTimer;
+
+    private static bool s_RawMissing;
+
+    [RuntimeInitializeOnLoadMethod(RuntimeInitializeLoadType.AfterSceneLoad)]
+    static void AutoInstall()
+    {
+        if (FindAnyObjectByType<TigerFaceViewer>() != null) return;
+        var go = new GameObject("TigerFaceViewer");
+        DontDestroyOnLoad(go);
+        go.AddComponent<TigerFaceViewer>();
+        Debug.Log("[TigerFaceViewer] installed — head-tracked billboard ON by default (press F to turn it off).");
+    }
+
+    void Update()
+    {
+        var kb = Keyboard.current;
+        if (kb != null && toggleKey != Key.None && kb[toggleKey].wasPressedThisFrame)
+        {
+            m_UserTurnedOff = m_Enabled;   // F while on = the user wants it off
+            Toggle();
+        }
+
+        // Auto-enable at startup. TransparentAutoSetup adds the tiger's
+        // DragRotateCube in the same AfterSceneLoad phase we install in, and the
+        // order between them is undefined — so retry rather than assume the
+        // target exists on frame one. Throttled: FindObjectsByType is not free.
+        if (startEnabled && !m_Enabled && !m_UserTurnedOff)
+        {
+            m_RetryTimer -= Time.deltaTime;
+            if (m_RetryTimer <= 0f)
+            {
+                m_RetryTimer = 0.25f;
+                if (FindTiger() != null) Toggle();
+            }
+        }
+
+        if (logInterval > 0f && m_Enabled)
+        {
+            m_Timer += Time.deltaTime;
+            if (m_Timer >= logInterval) { m_Timer = 0f; LogSources(); }
+        }
+    }
+
+    void Toggle()
+    {
+        if (m_Enabled)
+        {
+            m_Enabled = false;
+            if (m_Tiger != null) m_Tiger.rotation = m_BaseRotation;
+            Debug.Log("[TigerFaceViewer] OFF (rotation restored)");
+            return;
+        }
+
+        m_Tiger = FindTiger();
+        if (m_Tiger == null)
+        {
+            Debug.LogWarning("[TigerFaceViewer] no DragRotateCube target found — staying off.");
+            return;
+        }
+
+        m_BaseRotation = m_Tiger.rotation;
+        m_Yaw = 0f;
+        m_Enabled = true;
+        m_Timer = logInterval; // log immediately
+
+        var rend = m_Tiger.GetComponentInChildren<Renderer>(true);
+        bool raw = TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked);
+        Debug.Log($"[TigerFaceViewer] ON  target={m_Tiger.name} " +
+                  $"renderer={(rend != null ? rend.GetType().Name + "/" + rend.name : "<none>")} " +
+                  $"pos={Fmt(m_Tiger.position)} yawSign={yawSign} gain={gain:F1} " +
+                  $"| rawHead ok={raw} ({hx:F3},{hy:F3},{hz:F3}) tracked={tracked}");
+    }
+
+    void LateUpdate()
+    {
+        // LateUpdate so we win over the Animator (which runs earlier in the frame)
+        // and over DragRotateCube's drag rotation while this mode is on.
+        if (!m_Enabled || m_Tiger == null) return;
+        if (!TryGetRawHead(out float hx, out float _, out float hz, out bool tracked)) return;
+
+        // Reference math, 1:1: yaw from the head's lateral offset over its
+        // distance to the panel. |hz| guarded so a viewer in the panel plane
+        // can't blow the angle up.
+        float hzAbs = Mathf.Max(Mathf.Abs(hz), 1e-3f);
+        m_TargetYaw = Mathf.Clamp(yawSign * Mathf.Atan2(hx, hzAbs) * Mathf.Rad2Deg * gain,
+                                  -maxYawDegrees, maxYawDegrees);
+
+        // Time-based exponential smoothing (frame-rate independent). Only chase
+        // while tracking is locked — warmup positions jitter the heading.
+        if (tracked)
+        {
+            float a = Mathf.Clamp01(1f - Mathf.Exp(-Time.deltaTime / Mathf.Max(tau, 1e-4f)));
+            m_Yaw = Mathf.LerpAngle(m_Yaw, m_TargetYaw, a);
+        }
+
+        m_Tiger.rotation = Quaternion.AngleAxis(m_Yaw, Vector3.up) * m_BaseRotation;
+    }
+
+    // Print every viewer-position source together: the raw physical eyes that
+    // drive the billboard, the world-space provider API (new in v2.9.1), and the
+    // Camera.GetStereoViewMatrix readback that issue #236 was filed about.
+    void LogSources()
+    {
+        bool raw = TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked);
+
+        bool okProv = DisplayXRProvider.TryGetViewerEyes(out Vector3 pl, out Vector3 pr);
+        Vector3 pHead = okProv ? (pl + pr) * 0.5f : Vector3.zero;
+
+        var cam = ActiveCamera();
+        bool stereo = cam != null && cam.stereoEnabled;
+        Vector3 sl = Vector3.zero, sr = Vector3.zero;
+        if (stereo)
+        {
+            sl = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Left).inverse.GetColumn(3);
+            sr = cam.GetStereoViewMatrix(Camera.StereoscopicEye.Right).inverse.GetColumn(3);
+        }
+        bool camEyesEqual = stereo && (sl - sr).sqrMagnitude < 1e-10f;
+
+        Debug.Log($"[TigerFaceViewer] rawHead ok={raw} ({hx:F3},{hy:F3},{hz:F3})m tracked={tracked} " +
+                  $"-> targetYaw={m_TargetYaw:F1} yaw={m_Yaw:F1}deg on {(m_Tiger != null ? m_Tiger.name : "<none>")} " +
+                  $"| provider(world) ok={okProv} head={Fmt(pHead)} ipd={(pr - pl).magnitude:F4} " +
+                  $"| GetStereoViewMatrix stereo={stereo} eyesEqual={camEyesEqual} " +
+                  $"| eyeTracked={DisplayXRProvider.IsEyeTracked}");
+    }
+
+    // Head centroid in RAW DISPLAY SPACE: metres, origin at the physical panel
+    // centre, +X right, +Y up, viewer at +Z. Same channel the Scene-view eye
+    // gizmo uses (DisplayXRGizmoHelpers.TryGetLiveRawEyes) and the same one the
+    // native reference billboards from.
+    static bool TryGetRawHead(out float hx, out float hy, out float hz, out bool tracked)
+    {
+        hx = 0f; hy = 0f; hz = 0f; tracked = false;
+        if (s_RawMissing) return false;
+        try
+        {
+            DisplayXRNative.displayxr_get_eye_positions(
+                out float lx, out float ly, out float lz,
+                out float rx, out float ry, out float rz,
+                out int isTracked);
+            hx = (lx + rx) * 0.5f;
+            hy = (ly + ry) * 0.5f;
+            hz = (lz + rz) * 0.5f;
+            tracked = isTracked != 0;
+            // All-zero = the runtime hasn't filled the raw channel yet.
+            if (hx == 0f && hy == 0f && hz == 0f) return false;
+            return true;
+        }
+        catch (DllNotFoundException) { s_RawMissing = true; return false; }
+        catch (EntryPointNotFoundException) { s_RawMissing = true; return false; }
+    }
+
+    static string Fmt(Vector3 v) => $"({v.x:F3},{v.y:F3},{v.z:F3})";
+
+    static Camera ActiveCamera()
+    {
+        var cam = DisplayXRRigManager.ActiveCamera;
+        return cam != null ? cam : Camera.main;
+    }
+
+    // DragRotateCube owns each target's drag rotation, so it is the handle we
+    // want — but TransparentAutoSetup wires one onto EVERY target root, which is
+    // the tiger AND the legacy fallback cube. FindAnyObjectByType returns an
+    // arbitrary one, and picking the cube silently rotates an invisible fixture
+    // while the tiger sits still (looks exactly like "head tracking is dead").
+    // Prefer the skinned mesh — that's the tiger, and it needs no name match.
+    static Transform FindTiger()
+    {
+        var drags = FindObjectsByType<DragRotateCube>();
+        if (drags == null || drags.Length == 0) return null;
+        foreach (var d in drags)
+            if (d.target is SkinnedMeshRenderer) return d.transform;
+        // No skinned mesh (cube-only fallback scene): take the first target that
+        // is actually being drawn.
+        foreach (var d in drags)
+            if (d.target != null && d.target.enabled && d.target.gameObject.activeInHierarchy)
+                return d.transform;
+        return drags[0].transform;
+    }
+}

samples/desktop-avatar/CLAUDE.md

@@ -44,6 +44,7 @@ All test-project components are runtime-wired by `TransparentAutoSetup` — ther
 | `WheelZoomVHeight` | `Assets/TransparentAutoSetup.cs` (nested) | Scroll-wheel → `DisplayXRDisplay.virtualDisplayHeight`. Display-centric only. Active-rig gated (only the focused rig drains the wheel accumulator). |
 | `LockToForwardAxis` | `Assets/LockToForwardAxis.cs` | **Tiger-branch tweak.** Locks the rig camera's world X/Y to its startup values each `Update`, after the plugin's `DisplayXRInputController` has moved it. Net effect: AQDE keys become no-ops, only W/S still push the camera in/out (so only the in/out-of-display-plane axis is user-controllable). Uses `[DefaultExecutionOrder(int.MaxValue)]` to run after the plugin's input controller. |
 | `ClipAtDisplayPlane` | `Assets/TransparentAutoSetup.cs` (nested) | **Work in progress.** Currently hooks `Camera.onPreCull` and rewrites the per-eye stereo projection's `m22`/`m23` (near/far elements) to clip at raw eye-Z. Tiger renders but the clip doesn't visibly take effect. See `docs~/handoff-foreground-clipping.md` and issue #2. |
+| `TigerFaceViewer` | `Assets/TigerFaceViewer.cs` | **Head-tracked billboard (yaw-only), ON by default** — the tiger turns to face the tracked viewer; press **F** to turn it off and restore its authored rotation. Yaw math is ported 1:1 from the native reference (see below). The once-per-second log prints every viewer-position source side by side, which is also the A/B probe for plugin #236. Self-installs via `[RuntimeInitializeOnLoadMethod]` — no scene wiring. |
 | `AutoBoxColliderFromRenderer` | `Assets/TransparentAutoSetup.cs` (nested) | Deferred BoxCollider sizing for **non-SMR** clickables (the cube fallback). Waits for `renderer.bounds` to be valid (a few frames), then sizes the box. Skipped for `SkinnedMeshRenderer` — the plugin handles those per-triangle. |
 
 ## How transparency + clickthrough work
@@ -56,6 +57,58 @@ The mechanism splits across plugin (most of it) and test-project (small bootstra
 4. **Per-pixel click-through** — `WM_NCHITTEST` in the native overlay reads `s_hit_active` (set by the C# polling code each frame). When the cyclopean ray hits the tiger silhouette → `s_hit_active=1` → `HTCLIENT` → overlay captures. Otherwise `HTTRANSPARENT` → click forwards to the underlying app via `forward_click_to_underlying_window` (`SetForegroundWindow` + `PostMessage`).
 5. **Hit testing** uses **per-triangle ray-tri** (Möller-Trumbore) against `SkinnedMeshRenderer.BakeMesh()` output, transformed via `Matrix4x4.TRS(smr.position, smr.rotation, Vector3.one)` — position + rotation only, no scale (BakeMesh's output is already in world units). 8-frame hysteresis smooths over silhouette-edge sub-pixel jitter. Active-rig gate prevents the two rigs from flapping `s_hit_active`. Implemented entirely in the plugin; the test project just sets `clickableRenderers` to the SMR.
 
+## Head-coupled effects: work in PHYSICAL space, not world space
+
+**Billboard yaw is computed in raw display space, not Unity world space.** This is
+the single most important lesson from building `TigerFaceViewer`, and it mirrors the
+viewer-confirmed native reference in
+[`displayxr-demo-avatar`](https://github.com/DisplayXR/displayxr-demo-avatar)
+(`windows/main.cpp`, "Face-the-viewer billboard"):
+
+```csharp
+// head centroid in metres, origin = physical panel centre, viewer at +Z
+float hzAbs    = Mathf.Max(Mathf.Abs(hz), 1e-3f);
+float targetYaw = FACE_YAW_SIGN * Mathf.Atan2(hx, hzAbs) * Mathf.Rad2Deg;  // FACE_YAW_SIGN = -1
+```
+
+Typical live values: `hz` ~0.5 m, `hx` ±0.12 m → ±14° of yaw.
+
+Why not world space: on the **display-centric rig, scale-as-zoom inflates world
+units**, so the same head reads as `ipd≈1.2` world units against a physical 0.06 m —
+and any reference vector you build from scene objects inherits the scene's layout.
+Two world-space attempts failed here: one keyed off `(camera − tiger)`, which
+collapses because the tiger's root sits at the camera's x/z; the other picked its
+target with `FindAnyObjectByType<DragRotateCube>()` and silently rotated the
+**invisible fallback cube** (`TransparentAutoSetup` wires one onto every target
+root — the tiger *and* `Cube`). Physical space has neither failure mode.
+
+Smoothing is time-based exponential (`tau = 0.04 s`), and the heading only chases
+while eye tracking is **locked** — warmup positions jitter it.
+
+Viewer-position sources, all live per-frame:
+
+| API | Space | Use for |
+|-----|-------|---------|
+| `DisplayXRNative.displayxr_get_eye_positions` | **physical metres**, panel-centre origin, viewer at +Z | **head-coupled effects — start here** (what the billboard uses) |
+| `DisplayXRProvider.TryGetViewerHead` / `TryGetViewerEyes` | Unity **world** (v2.9.1+) | when the effect genuinely needs scene-space coordinates |
+| `DisplayXRNative.displayxr_get_stereo_matrices` | per-eye view + proj | cyclopean hit-test, off-axis probes (`KooimaProbe`) |
+
+Not ported from the reference: the **zone-canvas rebase**, which offsets `hx` by the
+window's centre on the panel. Without it the heading references the *panel* centre
+rather than the *window* centre — a constant bias when the window is off-centre, not
+a tracking failure.
+
+### On `Camera.GetStereoViewMatrix` (plugin #236)
+
+[Plugin #236](https://github.com/DisplayXR/displayxr-unity/issues/236) reports it
+returning the **same matrix for both eyes** under the provider, which silently
+freezes anything built on it. The plugin never writes that cache
+(`Camera.SetStereoViewMatrix` went away with the #166 provider migration). But note:
+**it does NOT reproduce in a built player here** — `TigerFaceViewer`'s log reads
+`eyesEqual=False` on Windows/D3D12, with values matching the provider's exactly. So
+the failure is configuration-specific (editor vs player, Unity version), not
+universal. Use the physical or provider APIs above and the question doesn't arise.
+
 ## Tiger asset facts
 
 - **Rig type: Generic** (not Legacy). Required for `Animator` instead of the deprecated `Animation` component.
@@ -90,6 +143,13 @@ After Build And Run for Windows:
 6. **W / S** keys → camera (and therefore the tiger relative to the display) push in/out of the display plane.
 7. **A / Q / D / E** keys → no effect (locked by `LockToForwardAxis`).
 8. Tab → cycles between Main Camera (display rig) and Cam Centric (camera rig); `DragRotateCube` rebinds its listeners automatically.
+9. **Head-tracked billboard is ON at startup** (`TigerFaceViewer`). Move your head
+   left/right in front of the panel: the tiger turns to follow you. The
+   once-per-second log should read like
+   `rawHead (-0.115,0.111,0.509)m tracked=True -> targetYaw=12.4 yaw=11.8deg on
+   cartoon tiger…` — i.e. `hz` ~0.5 m, `hx` ±0.12 m, yaw ±14°, on the **tiger**
+   (if the target name is `Cube`, it grabbed the invisible fallback fixture).
+   Press **F** to turn it off and restore the authored rotation.
 
 ## Open issues
 

samples/desktop-avatar/Packages/manifest.json

@@ -1,6 +1,6 @@
 {
   "dependencies": {
-    "com.displayxr.unity": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.0",
+    "com.displayxr.unity": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.1",
     "com.unity.multiplayer.center": "1.0.1",
     "com.unity.render-pipelines.universal": "17.0.4",
     "com.unity.ugui": "2.0.0",

samples/desktop-avatar/Packages/packages-lock.json

@@ -1,13 +1,13 @@
 {
   "dependencies": {
     "com.displayxr.unity": {
-      "version": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.0",
+      "version": "https://github.com/DisplayXR/displayxr-unity.git#upm/v2.9.1",
       "depth": 0,
       "source": "git",
       "dependencies": {
         "com.unity.xr.management": "4.4.0"
       },
-      "hash": "a21c59c2925339f36feb4989fbeae1031f5cc929"
+      "hash": "416bf4e8018e71afb626d6c8b423cd258b16bd56"
     },
     "com.unity.burst": {
       "version": "1.8.28",