diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index 13dfbd13b1..21a5161249 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -208,6 +208,16 @@ interface CloudXRComponentProps { /** Callback fired with the resolved server address after proxy configuration is applied. */ onServerAddress?: (address: string) => void; + /** + * Callback fired with the reference space poses are actually streamed in: the one + * `config.referenceSpaceType` resolved to (including the 'auto' fallback order and the + * WebXRManager last resort), with `config.xrOffset*` already applied. Anything that + * needs to express poses in the same frame as the stream should take this rather than + * request its own space, which would miss the offset and could resolve differently. + * Fired on each session creation; not fired when no space could be resolved. + */ + onReferenceSpace?: (space: XRReferenceSpace) => void; + /** * Callback fired with render performance metrics ({@link CloudXR.MetricsCadence.PerRender}). * Payloads are partial - only the fields sampled on this tick are present. @@ -303,6 +313,7 @@ export default function CloudXRComponent({ warmupBeginTimeoutMs, warmupEndTimeoutMs, onServerAddress, + onReferenceSpace, onRenderPerformanceMetrics, onStreamingPerformanceMetrics, onNetworkPerformanceMetrics, @@ -720,6 +731,11 @@ export default function CloudXRComponent({ referenceSpace = referenceSpace.getOffsetReferenceSpace(offsetTransform); } + // Resolved only now: this is the space the session streams in, after both + // the fallback chain and the offset. Published per CloudXR session rather + // than here, so a reconnect does not leave the parent without one. + const streamingReferenceSpace = referenceSpace; + // Fill in CloudXR session options. const cloudXROptions: CloudXR.SessionOptions = { serverAddress: connectionConfig.serverIP, @@ -1085,6 +1101,9 @@ export default function CloudXRComponent({ // Notify parent that session is ready onSessionReady?.(cxrSession); + // Republished on every attempt, reconnects included: the parent drops its + // copy whenever onSessionReady(null) fires, so the two have to stay paired. + onReferenceSpace?.(streamingReferenceSpace); // Start session (synchronous call that initiates connection) try { diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index 3da3cfbd80..34220d6060 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -71,10 +71,10 @@ import { RecorderProvider, useRecorder } from './RecorderContext'; import { SuppressWebGLRendererWhenHeadless } from './SuppressWebGLRendererWhenHeadless'; import { TraceVisualization } from './TraceVisualization'; import { - SystemNotice, formatSystemNotice, formatSystemNoticeBody, isSystemNoticeMessage, + SystemNotice, } from './types/serverMessages'; // Performance metrics signals - raw numeric data backing the in-XR HUD. @@ -212,6 +212,10 @@ function AppContent() { const [errorMessage, setErrorMessage] = useState(''); // CloudXR session reference const [cloudXRSession, setCloudXRSession] = useState(null); + // The space CloudXR resolved for the current session, so the recorder can express + // poses in the same frame the stream uses. Cleared with the session: a space from an + // ended session yields no poses. + const [streamingSpace, setStreamingSpace] = useState(null); // XR mode state for UI visibility const [isXRMode, setIsXRMode] = useState(false); // Server address being used for connection @@ -701,6 +705,7 @@ function AppContent() { stopStreamTestCountdown(); streamTest.value = null; setCloudXRSession(session); + if (!session) setStreamingSpace(null); }; const systemNoticeTimerRef = useRef(null); @@ -1193,12 +1198,17 @@ function AppContent() { {cloudXR2DUI && config && ( <> - + (null); @@ -512,24 +521,31 @@ export default function CloudXR3DUI({ useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { - if (event.key.toLowerCase() !== RESET_PANEL_KEY) { - return; - } // Unlike a function key, a plain letter can be typed into any of the 2D settings form's - // text inputs - don't reset the panel out from under someone typing a server IP. - const target = event.target as HTMLElement | null; + // text inputs - don't reset the panel out from under someone typing a server IP. A held + // key or a browser chord (Ctrl+R to reload) is not a shortcut press either. if ( - target?.tagName === 'INPUT' || - target?.tagName === 'TEXTAREA' || - target?.isContentEditable + event.repeat || + event.ctrlKey || + event.altKey || + event.metaKey || + isEditableKeyboardTarget(event.target) ) { return; } + // The recorder keys get first refusal, then the panel key; one listener, so the + // two sets cannot both fire on the same press. + if (handleTeleopShortcut(event)) { + return; + } + if (event.key.toLowerCase() !== RESET_PANEL_KEY) { + return; + } resetPanelRelativeToHead(camera); }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); - }, [camera, resetPanelRelativeToHead]); + }, [camera, resetPanelRelativeToHead, handleTeleopShortcut]); const isCompact = minimizeOnPlay && playInProgress; const isMinimizedLayout = isCompact || panelHidden; @@ -875,6 +891,12 @@ export default function CloudXR3DUI({ ? 'Replaying' : 'Recording'} + {recorder.mode === 'idle' && recorder.recordingInterrupted && ( + + Recording stopped because the tracking origin changed. Save it or start a + new recording. + + )} {recorder.mode !== 'replaying' && ( { @@ -50,11 +67,13 @@ export function RecorderComponent({ isConnected, showTrace }: RecorderComponentP recorder.beginFrame( xrFrame, - state.gl.xr.getReferenceSpace(), + streamingSpace ?? state.gl.xr.getReferenceSpace(), isConnected && isVisible, showTrace && isVisible ); + onFrameState(); + if (recorder.mode === 'recording') { tickRef.current++; if (tickRef.current % 30 === 0) { diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx new file mode 100644 index 0000000000..b7c67cbd3c --- /dev/null +++ b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx @@ -0,0 +1,71 @@ +/** @jest-environment jsdom */ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; + +import { type RecorderContextValue, RecorderProvider, useRecorder } from './RecorderContext'; + +let current: RecorderContextValue; +let root: Root; + +function Probe() { + current = useRecorder(); + return null; +} + +function frame(session: XRSession): XRFrame { + return { + session, + predictedDisplayTime: 0, + getViewerPose: () => ({ + transform: { position: { x: 0, y: 0, z: 0 }, orientation: { x: 0, y: 0, z: 0, w: 1 } }, + }), + } as unknown as XRFrame; +} + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + root = createRoot(document.createElement('div')); + act(() => + root.render( + + + + ) + ); +}); + +afterEach(() => act(() => root.unmount())); + +test('saves interrupted recording and updates the UI when the tracking origin changes', () => { + const session = { inputSources: [] } as unknown as XRSession; + const space = new EventTarget() as XRReferenceSpace; + act(() => current.startRecord()); + act(() => current.recorder.beginFrame(frame(session), space)); + act(() => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform: null })); + current.onFrameState(); + }); + expect(current.mode).toBe('idle'); + expect(current.recordingInterrupted).toBe(true); + expect(current.savedRecording?.frames).toHaveLength(1); + act(() => current.startRecord()); + expect(current.recordingInterrupted).toBe(false); +}); diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 8db8fae7b9..8660520b5d 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -23,7 +23,15 @@ * to descendant components. */ -import React, { createContext, useCallback, useContext, useMemo, useRef, useState } from 'react'; +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { type Recording, type ReplayPacing, XRInputRecorder } from './xrInputRecorder'; @@ -37,6 +45,8 @@ export interface RecorderContextValue { stopRecord: () => void; startReplay: () => void; stopReplay: () => void; + recordingInterrupted: boolean; + onFrameState: () => void; setReplayPacing: (pacing: ReplayPacing) => void; onSaveRecording: () => void; onLoadRecording: () => void; @@ -59,6 +69,7 @@ function setLoadStatus(message: string, type: 'success' | 'error'): void { export function RecorderProvider({ children }: { children: React.ReactNode }) { const recorder = useMemo(() => new XRInputRecorder(), []); + useEffect(() => () => recorder.dispose(), [recorder]); const [mode, setMode] = useState<'idle' | 'recording' | 'replaying'>('idle'); const [savedRecording, setSavedRecordingState] = useState(null); const [recordedFrameCount, setRecordedFrameCount] = useState(0); @@ -93,6 +104,16 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { setMode('idle'); }, [recorder]); + const onFrameState = useCallback(() => { + if (mode !== recorder.mode) { + if (mode === 'recording') { + setSavedRecordingState(recorder.getRecording()); + setRecordedFrameCount(recorder.recordedFrameCount); + } + setMode(recorder.mode); + } + }, [mode, recorder]); + const setReplayPacing = useCallback((pacing: ReplayPacing) => { setReplayPacingState(pacing); }, []); @@ -155,6 +176,8 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { stopRecord, startReplay, stopReplay, + recordingInterrupted: recorder.recordingInterrupted, + onFrameState, setReplayPacing, onSaveRecording, onLoadRecording, @@ -180,3 +203,115 @@ export function useRecorder(): RecorderContextValue { if (!ctx) throw new Error('useRecorder must be used inside RecorderProvider'); return ctx; } + +/** + * True when a plain letter is being typed rather than used as a shortcut. + * + * Shared with CloudXRUI's panel-reset key, which runs off the same listener: + * one definition means the two cannot disagree about what counts as typing. + */ +export function isEditableKeyboardTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false; + if (target.isContentEditable) return true; + if (target instanceof HTMLInputElement) return target.type !== 'file'; + return target instanceof HTMLTextAreaElement || target instanceof HTMLSelectElement; +} + +interface TeleopKeyboardShortcutOptions { + recordingEnabled: boolean; + startTeleop?: () => void; + teleopStartDisabled?: boolean; + /** Reset Teleop. Only reachable as a key because its button lives in the XR canvas. */ + resetTeleop?: () => void; +} + +/** + * Build the keydown handler for the Teleop Play and recording controls exposed by the XR UI. + * + * Returns a handler rather than registering a listener: CloudXRUI already owns a + * window keydown listener for the panel-reset key, and a second one would mean two + * places deciding what a letter means. The caller reports whether the key was + * consumed, so its own keys can be skipped when this one handles the event. + */ +export function useTeleopShortcutHandler({ + recordingEnabled, + startTeleop, + teleopStartDisabled = false, + resetTeleop, +}: TeleopKeyboardShortcutOptions): (event: KeyboardEvent) => boolean { + const { + recorder, + savedRecording, + startRecord, + stopRecord, + startReplay, + stopReplay, + onSaveRecording, + } = useRecorder(); + + return useCallback( + (event: KeyboardEvent): boolean => { + if (!recordingEnabled && !startTeleop && !resetTeleop) return false; + + const key = + event.code === 'KeyT' + ? 't' + : event.code === 'KeyC' + ? 'c' + : event.code === 'KeyP' + ? 'p' + : event.code === 'KeyS' + ? 's' + : event.code === 'KeyX' + ? 'x' + : event.key.toLowerCase(); + + if (key === 't') { + if (!startTeleop || teleopStartDisabled) return false; + startTeleop(); + } else if (key === 'c') { + if (!recordingEnabled) return false; + if (recorder.mode === 'recording') { + stopRecord(); + } else if (recorder.mode === 'idle') { + startRecord(); + } else { + return false; + } + } else if (key === 'p') { + if (!recordingEnabled) return false; + if (recorder.mode === 'replaying') { + stopReplay(); + } else if (recorder.mode === 'idle' && savedRecording) { + startReplay(); + } else { + return false; + } + } else if (key === 's') { + if (!recordingEnabled || recorder.mode !== 'idle' || !savedRecording) return false; + onSaveRecording(); + } else if (key === 'x') { + if (!resetTeleop) return false; + resetTeleop(); + } else { + return false; + } + + event.preventDefault(); + return true; + }, + [ + recorder, + onSaveRecording, + recordingEnabled, + savedRecording, + startRecord, + startReplay, + startTeleop, + stopRecord, + stopReplay, + teleopStartDisabled, + resetTeleop, + ] + ); +} diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts new file mode 100644 index 0000000000..c5e1344cc1 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ReactElement } from 'react'; +import { type Points, Vector3 } from 'three'; + +import { TraceVisualization } from './TraceVisualization'; +import type { RecordedFrame } from './xrInputRecorder'; + +let mockFrame: () => void; +const sample: RecordedFrame = { + timeMs: 0, + poses: { + leftGrip: { px: 1, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, + leftAim: null, + rightGrip: null, + rightAim: null, + }, + gamepads: { left: null, right: null }, + handJoints: { left: {}, right: {} }, +}; +const mockRecorder = { + currentFrame: sample as RecordedFrame | null, + replaySceneAlignment: { px: 10, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, +}; + +jest.mock('@react-three/fiber', () => ({ + useFrame: (callback: () => void) => { + mockFrame = callback; + }, +})); +jest.mock('./RecorderContext', () => ({ + useRecorder: () => ({ recorder: mockRecorder, mode: 'replaying' }), +})); +jest.mock('react', () => ({ + ...jest.requireActual('react'), + useRef: () => ({ current: null }), + useEffect: jest.fn(), +})); + +test('places the whole replay trail in aligned scene space and hides it while paused', () => { + const tree = TraceVisualization({ showTrace: true }); + const points = (tree.props.children as ReactElement<{ object: Points }>[])[0].props.object; + const worldPoint = (index: number) => { + points.updateMatrixWorld(); + return new Vector3() + .fromBufferAttribute(points.geometry.getAttribute('position'), index) + .applyMatrix4(points.matrixWorld); + }; + mockFrame(); + expect(worldPoint(0).x).toBeCloseTo(11); + + // A reset changes the placement of earlier samples as well as the newest one. + mockRecorder.replaySceneAlignment = { + px: 100, + py: 0, + pz: 0, + ox: 0, + oy: Math.SQRT1_2, + oz: 0, + ow: Math.SQRT1_2, + }; + mockFrame(); + for (const index of [0, 1]) { + expect(worldPoint(index).x).toBeCloseTo(100); + expect(worldPoint(index).z).toBeCloseTo(-1); + } + mockRecorder.currentFrame = null; + mockFrame(); + expect(points.visible).toBe(false); + for (const { props } of tree.props.children as ReactElement<{ object: Points }>[]) { + props.object.geometry.dispose(); + } +}); diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx index 9d5d7f38f7..1df584ce25 100644 --- a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx @@ -128,15 +128,25 @@ export function TraceVisualization({ showTrace }: { showTrace: boolean }) { ); useFrame(() => { - if (!showTrace) { + const frame = recorder.currentFrame; + if (!showTrace || !frame) { channels.forEach(channel => { channel.points.visible = false; }); return; } - const frame = recorder.currentFrame; - if (!frame) return; + // Transform the whole trail so origin resets also move its earlier samples. + const alignment = recorder.replaySceneAlignment; + for (const { points } of channels) { + points.position.set(alignment?.px ?? 0, alignment?.py ?? 0, alignment?.pz ?? 0); + points.quaternion.set( + alignment?.ox ?? 0, + alignment?.oy ?? 0, + alignment?.oz ?? 0, + alignment?.ow ?? 1 + ); + } updateChannel(channels[0], frame.poses.leftGrip); updateChannel(channels[1], frame.poses.rightGrip); updateChannel(channels[2], frame.handJoints.left.wrist); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index ca967ab17d..348aa715fc 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -45,7 +45,7 @@ function pose(x: number, y = 0, z = 0, orientation: DOMPointInit = { w: 1 }): XR emulatedPosition: false, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function jointPose(x: number, radius = 0.01): XRJointPose { @@ -56,9 +56,19 @@ function gamepad(axis: number): Gamepad { return { axes: [axis], buttons: [{ value: axis, pressed: axis > 0, touched: true }], - } as Gamepad; + } as unknown as Gamepad; } +function makeSession(inputSources: XRInputSource[] = []): XRSession { + return Object.assign(new EventTarget(), { inputSources }) as unknown as XRSession; +} + +// Frames in a test share a session unless the test explicitly starts another. +let defaultSession: XRSession; +beforeEach(() => { + defaultSession = makeSession(); +}); + type PoseResolver = (space: XRSpace, baseSpace: XRSpace) => XRPose | null; type JointResolver = (joint: XRJointSpace, baseSpace: XRSpace) => XRJointPose | null; @@ -66,15 +76,18 @@ function makeFrame( inputSources: XRInputSource[] = [], getPose: PoseResolver = () => null, getJointPose: JointResolver = () => null, - predictedDisplayTime = 0 + predictedDisplayTime = 0, + viewerPose: XRPose | null = pose(0), + session: XRSession = defaultSession ): XRFrame { - const session = { inputSources } as XRSession; + Object.assign(session, { inputSources }); return { session, predictedDisplayTime, getPose, getJointPose, - } as XRFrame; + getViewerPose: () => viewerPose, + } as unknown as XRFrame; } function frameData(x = 0): RecordedFrame { @@ -249,6 +262,7 @@ describe('lifecycle and frame advancement', () => { recorder.beginFrame(makeFrame(), sceneSpace, true, true); expect(recorder.currentFrame).toEqual({ timeMs: 0, + controllerProfiles: { left: null, right: null }, poses: { leftGrip: null, leftAim: null, rightGrip: null, rightAim: null }, gamepads: { left: null, right: null }, handJoints: { left: {}, right: {} }, @@ -256,6 +270,25 @@ describe('lifecycle and frame advancement', () => { }); }); +test.each([1, -1])('interpolates the shortest rotation for quaternion sign %s', sign => { + const first = timedFrame(0, 0); + const last = timedFrame(100, 4); + last.poses.leftGrip = { + ...last.poses.leftGrip!, + oy: sign * Math.sin(Math.PI / 3), + ow: sign * Math.cos(Math.PI / 3), + }; + const recorder = new XRInputRecorder(); + recorder.startReplay(recording(first, last), false); + recorder.beginFrame(makeFrame(), sceneSpace); + recorder.beginFrame(makeFrame([], undefined, undefined, 50), sceneSpace); + const result = recorder.currentFrame!.poses.leftGrip!; + expect(result.px).toBeCloseTo(2); + expect(result.oy).toBeCloseTo(Math.sin(Math.PI / 6)); + expect(result.ow).toBeCloseTo(Math.cos(Math.PI / 6)); + expect(Math.hypot(result.ox, result.oy, result.oz, result.ow)).toBeCloseTo(1); +}); + describe('canonical scene-space capture', () => { test('captures grip, aim, gamepad, and joints by WebXR joint name', () => { const grip = {} as XRSpace; @@ -289,15 +322,27 @@ describe('canonical scene-space capture', () => { expect(captured.handJoints.left.wrist?.px).toBe(3); expect(captured.handJoints.left['index-finger-tip']?.px).toBe(4); }); + + test('waits for a viewer pose before recording the first frame', () => { + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(makeFrame([], undefined, undefined, 0, null), sceneSpace); + expect(recorder.recordedFrameCount).toBe(0); + + recorder.beginFrame(makeFrame([], undefined, undefined, 1, pose(0)), sceneSpace); + expect(recorder.recordedFrameCount).toBe(1); + }); }); describe('scoped CloudXR replay frame', () => { - test('does not alter global prototypes and returns real frames outside replay', () => { + test('preserves live tracking outside replay without changing browser frames', () => { const recorder = new XRInputRecorder(); const frame = makeFrame(); const originalGetPose = frame.getPose; recorder.startRecording(); - expect(recorder.adaptTrackingFrame(frame)).toBe(frame); + const adapted = recorder.adaptTrackingFrame(frame); + expect(adapted.session.inputSources).toBe(frame.session.inputSources); + expect(adapted.getViewerPose(sceneSpace)).toEqual(frame.getViewerPose(sceneSpace)); expect(frame.getPose).toBe(originalGetPose); recorder.stopRecording(); }); @@ -360,8 +405,8 @@ describe('scoped CloudXR replay frame', () => { expect(adapted.session.inputSources).toBe(replaySources); expect(replaySources[0].gamepad).toBe(replaySources[0].gamepad); expect(replaySources[0].gamepad?.axes).toEqual([3]); - expect(adapted.getJointPose(wrist, cloudSpace)?.transform.position.x).toBe(15); - expect(adapted.getJointPose(wrist, cloudSpace)?.radius).toBe(0.02); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.transform.position.x).toBe(15); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.radius).toBe(0.02); }); test('delegates unknown spaces and joints to the real frame', () => { @@ -378,17 +423,470 @@ describe('scoped CloudXR replay frame', () => { const adapted = recorder.adaptTrackingFrame(frame); expect(adapted.getPose(unknownSpace, sceneSpace)?.transform.position.x).toBe(7); - expect(adapted.getJointPose(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + expect(adapted.getJointPose?.(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + }); + + test('keeps in-memory recording and replay in the same reference space unchanged', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const recordingFrame = makeFrame( + [source], + space => (space === grip ? pose(2) : null), + undefined, + 0, + pose(1), + session + ); + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(recordingFrame, sceneSpace); + recorder.stopRecording(); + const saved = recorder.getRecording(); + + const replayFrame = makeFrame([source], undefined, undefined, 1, pose(11), session); + recorder.startReplay(saved, true, 'frame'); + recorder.beginFrame(replayFrame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(replayFrame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(2); + }); +}); + +describe('reference-space resets during replay', () => { + function setup(pacing: 'frame' | 'time' = 'frame') { + const grip = {} as XRSpace; + const wrist = {} as XRJointSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + gamepad: gamepad(1), + hand: new Map([['wrist', wrist]]), + } as unknown as XRInputSource; + const session = makeSession([source]); + const space = new EventTarget() as XRReferenceSpace; + const recorder = new XRInputRecorder(); + const saved = recording(timedFrame(0, 1), timedFrame(100, 3)); + const frame = (viewerX: number, time = 0, viewer: XRPose | null = pose(viewerX)) => + makeFrame( + [source], + () => pose(999), + () => jointPose(999), + time, + viewer, + session + ); + const advance = (viewerX: number, time = 0) => { + const current = frame(viewerX, time); + recorder.beginFrame(current, space); + return recorder.adaptTrackingFrame(current); + }; + const reset = (transform: XRRigidTransform | null) => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform })); + }; + recorder.startReplay(saved, false, pacing); + return { recorder, grip, wrist, source, session, space, saved, frame, advance, reset }; + } + + test.each(['frame', 'time'] as const)( + '%s replay preserves world placement after headset motion and repeated translation resets', + pacing => { + const { recorder, grip, wrist, space, advance, reset } = setup(pacing); + // Replay is not viewer-aligned: these are recorded coordinates shifted by + // the resets that have accumulated, and the viewer's own x is irrelevant. + advance(10); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 1 : 3 + ); + reset(new XRRigidTransform({ x: -100 })); + const firstReset = advance(112, pacing === 'time' ? 50 : 1); + expect(firstReset.getPose(grip, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 102 : 103 + ); + expect(firstReset.getJointPose?.(wrist, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 104 : 105 + ); + reset(new XRRigidTransform({ x: -20 })); + expect(advance(132, 100).getPose(grip, space)?.transform.position.x).toBeCloseTo(123); + } + ); + + test('uses the inverse reset rotation for position and orientation', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + saved.frames = [saved.frames[0]]; + recorder.stopReplay(); + recorder.startReplay(saved, false, 'frame'); + advance(10); + advance(10); + const q = Math.sqrt(0.5); + reset(new XRRigidTransform({}, { x: 0, y: q, z: 0, w: q })); + const result = advance(999).getPose(grip, space)!; + expect(result.transform.position.x).toBeCloseTo(0); + // The quarter turn maps the recorded x = 1 onto z; no viewer term is added. + expect(result.transform.position.z).toBeCloseTo(1); + expect(result.transform.orientation.y).toBeCloseTo(-q); + expect(result.transform.orientation.w).toBeCloseTo(q); + }); + + test('preserves cached placement when a known reset occurs while replay is stopped', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + advance(10); + advance(10); + recorder.stopReplay(); + reset(new XRRigidTransform({ x: -100 })); + recorder.startReplay(saved, false, 'frame'); + // Recorded x = 1 plus the +100 the reset accumulated while replay was stopped. + expect(advance(112).getPose(grip, space)?.transform.position.x).toBeCloseTo(101); + }); + + test.each([true, false])('stops recording on reset (known transform: %s)', known => { + const { recorder, grip, space, frame, reset } = setup(); + recorder.stopReplay(); + recorder.startRecording(); + recorder.beginFrame(frame(0), space); + reset(known ? new XRRigidTransform({ x: -100 }) : null); + expect(recorder.mode).toBe('idle'); + expect(recorder.recordingInterrupted).toBe(true); + recorder.beginFrame(frame(100), space); + expect(recorder.getRecording().frames).toHaveLength(1); + const saved = recorder.getRecording(); + recorder.startReplay(saved, false, 'frame'); + recorder.beginFrame(frame(100), space); + if (known) { + expect( + recorder.adaptTrackingFrame(frame(100)).getPose(grip, space)?.transform.position.x + ).toBeCloseTo(1099); + } + }); +}); + +describe('recorded hands without live tracking', () => { + function benchmark(pacing: 'frame' | 'time') { + const session = makeSession(); + const recorder = new XRInputRecorder(); + const frame = (sources: XRInputSource[] = [], time = 0) => + makeFrame( + sources, + () => null, + () => null, + time, + pose(0), + session + ); + // CloudXR initializes its active-hand set on the first tracking frame and + // subsequently refreshes it from inputsourceschange, not from joint poses. + const active = new Set(); + const refresh = (current: XRSession) => { + active.clear(); + for (const source of current.inputSources) { + if (source.hand) active.add(source.handedness); + } + }; + const idle = recorder.adaptTrackingFrame(frame()); + refresh(idle.session); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => refresh(event.session)); + idle.session.addEventListener('inputsourceschange', changed); + const read = (current: XRFrame) => { + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = Array.from(adapted.session.inputSources).find(s => s.handedness === 'left'); + if (!source?.hand || !active.has(source.handedness)) return null; + return adapted.getJointPose?.(source.hand.get('wrist')!, sceneSpace)?.transform.position.x; + }; + recorder.startReplay(recording(timedFrame(0, 1), timedFrame(100, 3)), false, pacing); + return { recorder, session, frame, active, changed, read, idle }; + } + + test.each(['frame', 'time'] as const)( + '%s replay advances when hands were never detected', + pacing => { + const { read, frame, active, changed, session } = benchmark(pacing); + expect(read(frame())).toBeCloseTo(3); + expect(active.has('left')).toBe(true); + expect(read(frame([], pacing === 'time' ? 50 : 1))).toBeCloseTo(pacing === 'time' ? 4 : 5); + expect(changed).toHaveBeenCalledTimes(1); + expect(session.inputSources).toHaveLength(0); + } + ); + + test('live hands appearing and disappearing do not gate or replace recorded hands', () => { + const { read, frame, changed, session, active, recorder } = benchmark('time'); + expect(read(frame())).toBeCloseTo(3); + const live = { + handedness: 'left', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const visible = frame([live], 25); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [live], + removed: [], + }) + ); + expect(read(visible)).toBeCloseTo(3.5); + const absent = frame([], 50); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [live], + }) + ); + expect(active.has('left')).toBe(true); + expect(read(absent)).toBeCloseTo(4); + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(absent); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + }); + + test('does not expose an unrecorded live hand to the SDK during replay', () => { + const { recorder, read, frame, active } = benchmark('frame'); + const liveRight = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const current = frame([liveRight]); + expect(read(current)).toBeCloseTo(3); + expect([...active]).toEqual(['left']); + expect(recorder.adaptTrackingFrame(current).session.inputSources).toHaveLength(1); + expect(current.session.inputSources).toEqual([liveRight]); + }); + + test('recorded tracking loss removes the source, and recorded recovery restores it', () => { + const { recorder, read, frame, active } = benchmark('frame'); + recorder.stopReplay(); + const lost = timedFrame(50, 2); + lost.handJoints.left = { wrist: null }; + recorder.startReplay(recording(timedFrame(0, 1), lost, timedFrame(100, 3)), false, 'frame'); + expect(read(frame())).toBeCloseTo(3); + expect(read(frame())).toBeNull(); + expect(active.size).toBe(0); + expect(read(frame())).toBeCloseTo(5); + }); + + test('forwards native events outside replay and supports listener cleanup', () => { + const { recorder, session, frame, changed, idle } = benchmark('frame'); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame()); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + expect(changed.mock.calls[0][0].session).toBe(idle.session); + idle.session.removeEventListener('inputsourceschange', changed); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + recorder.dispose(); }); }); +describe('controller replay without live hardware', () => { + function controllerFrame(timeMs: number, x: number, profiles?: string[]): RecordedFrame { + const sample = timedFrame(timeMs, x); + sample.handJoints.left = {}; + if (profiles) sample.controllerProfiles = { left: profiles, right: null }; + return sample; + } + + test.each(['time', 'frame'] as const)( + 'replays poses and controls without live sources (%s)', + pacing => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const frame = (time: number) => makeFrame([], undefined, undefined, time, pose(0), session); + const scoped = recorder.adaptTrackingFrame(frame(0)).session; + const active = new Set(); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => { + active.clear(); + for (const source of event.session.inputSources) { + if (source.gamepad && !source.hand) active.add(source.handedness); + } + }); + scoped.addEventListener('inputsourceschange', changed); + recorder.startReplay( + recording(controllerFrame(0, 0), controllerFrame(100, 1)), + false, + pacing + ); + let firstSource: XRInputSource | undefined; + for (const [time, expected] of [ + [0, 0], + [50, pacing === 'time' ? 0.5 : 1], + [100, 1], + ]) { + const current = frame(time); + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = adapted.session.inputSources[0]; + firstSource ??= source; + expect(source).toBe(firstSource); + expect(active.has('left')).toBe(true); + expect(source.hand).toBeUndefined(); + expect(source.profiles).toEqual(['generic-trigger-squeeze-thumbstick']); + expect(adapted.getPose(source.gripSpace!, sceneSpace)?.transform.position.x).toBe(expected); + expect(adapted.getPose(source.targetRaySpace, sceneSpace)?.transform.position.x).toBe( + expected + 1 + ); + expect(source.gamepad?.axes).toEqual([expected]); + expect(source.gamepad?.buttons[0]).toEqual({ + value: expected, + pressed: true, + touched: true, + }); + expect(source.gamepad).toBe(source.gamepad); + expect(session.inputSources).toHaveLength(0); + } + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame(100)); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + recorder.dispose(); + } + ); + + test('keeps recorded profiles and controls through live controller connection changes', () => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const changed = jest.fn(); + const initial = makeFrame([], undefined, undefined, 0, pose(0), session); + recorder.adaptTrackingFrame(initial).session.addEventListener('inputsourceschange', changed); + const profiles = ['meta-quest-touch-plus', 'oculus-touch-v3']; + recorder.startReplay( + recording(controllerFrame(0, 1, profiles), controllerFrame(100, 3, profiles)), + false + ); + recorder.beginFrame(initial, sceneSpace); + const source = recorder.adaptTrackingFrame(initial).session.inputSources[0]; + const live = { + handedness: 'left', + profiles: ['pico-4u'], + gamepad: gamepad(99), + } as XRInputSource; + for (const [time, inputs] of [ + [50, [live]], + [100, []], + ] as const) { + const frame = makeFrame([...inputs], undefined, undefined, time, pose(0), session); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session, added: inputs, removed: [] }) + ); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources).toEqual([source]); + expect(source.profiles).toEqual(profiles); + expect(source.gamepad?.axes[0]).toBe(time === 50 ? 2 : 3); + } + expect(changed).toHaveBeenCalledTimes(1); + }); + + test('round-trips profiles and replays a controller alongside a recorded hand', () => { + const recorder = new XRInputRecorder(); + const profiles = ['pico-4u', 'oculus-touch-v2']; + const controller = { + handedness: 'left', + profiles, + gamepad: gamepad(0.75), + gripSpace: {}, + targetRaySpace: {}, + } as XRInputSource; + const hand = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + recorder.startRecording(); + recorder.beginFrame( + makeFrame( + [controller, hand], + () => pose(2), + () => jointPose(3) + ), + sceneSpace + ); + recorder.stopRecording(); + const saved = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(saved.frames[0].controllerProfiles).toEqual({ left: profiles, right: null }); + recorder.startReplay(saved); + const frame = makeFrame(); + recorder.beginFrame(frame, sceneSpace); + const sources = Array.from(recorder.adaptTrackingFrame(frame).session.inputSources); + expect(sources.find(source => source.handedness === 'left')?.profiles).toEqual(profiles); + expect(sources.find(source => source.handedness === 'right')?.hand).toBeDefined(); + }); + + test('preserves gaps and device switches at recorded timestamps', () => { + const recorder = new XRInputRecorder(); + const first = controllerFrame(0, 1, ['meta-quest-touch-plus']); + const other = controllerFrame(100, 9, ['pico-4u']); + const lost = controllerFrame(200, 10); + lost.gamepads.left = null; + lost.poses.leftGrip = lost.poses.leftAim = null; + recorder.startReplay( + recording(first, other, lost, timedFrame(300, 20), controllerFrame(400, 30)), + false + ); + const sources: XRInputSource[] = []; + for (const time of [0, 50, 100, 200, 300, 400]) { + const frame = makeFrame([], undefined, undefined, time); + recorder.beginFrame(frame, sceneSpace); + const adapted = recorder.adaptTrackingFrame(frame); + const source = adapted.session.inputSources[0]; + if (time === 200) { + expect(adapted.session.inputSources).toHaveLength(0); + } else { + sources.push(source); + expect(adapted.session.inputSources).toHaveLength(1); + expect(!!source.hand).toBe(time === 300); + expect(source.gamepad?.axes[0]).toBe( + time < 100 ? 1 : time === 100 ? 9 : time === 300 ? 20 : 30 + ); + } + } + expect(sources[0]).toBe(sources[1]); + expect(sources[2]).not.toBe(sources[0]); + expect(sources[2].profiles).toEqual(['pico-4u']); + }); + + test.each([null, {}, { left: 'quest', right: null }, { left: [42], right: null }])( + 'rejects malformed controller profiles (%j)', + controllerProfiles => { + expect(() => + XRInputRecorder.importJSON( + JSON.stringify({ version: 1, frames: [{ ...frameData(), controllerProfiles }] }) + ) + ).toThrow('controllerProfiles is invalid'); + } + ); +}); + describe('serialization', () => { test('round-trips version 1 recordings', () => { const recorder = new XRInputRecorder(); recorder.startRecording(); recorder.beginFrame(makeFrame(), sceneSpace); recorder.stopRecording(); - expect(XRInputRecorder.importJSON(recorder.exportJSON()).frames).toHaveLength(1); + const imported = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(imported.frames).toHaveLength(1); expect(typeof recorder.getRecording().recordedAt).toBe('number'); }); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 8838d38ba5..ba5eaf729a 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -22,6 +22,10 @@ * UI pointers, and CloudXR rendering/reprojection. */ +import { Quaternion, Vector3 } from 'three'; + +import { XRReplaySession } from './xrReplaySession'; + type PoseData = { px: number; py: number; @@ -54,6 +58,8 @@ export type RecordedFrame = { rightAim: SerializedPose; }; gamepads: Hands; + /** Controller profiles select SDK button mappings; absent in legacy recordings. */ + controllerProfiles?: Hands; /** Joint poses keyed by XRHandJoint name, also in scene space. */ handJoints: Hands; }; @@ -66,7 +72,7 @@ export type Recording = { export type ReplayPacing = 'frame' | 'time'; -type Handedness = 'left' | 'right'; +const IDENTITY_POSE: PoseData = { px: 0, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }; function emptyFrame(timeMs = 0): RecordedFrame { return { @@ -79,10 +85,22 @@ function emptyFrame(timeMs = 0): RecordedFrame { function serializePose(pose: XRPose | null | undefined): SerializedPose { if (!pose) return null; - const { position: p, orientation: o } = pose.transform; + return serializeTransform(pose.transform); +} + +function serializeTransform(transform: XRRigidTransform): PoseData { + const { position: p, orientation: o } = transform; return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; } +function isPoseData(value: unknown): value is PoseData { + if (typeof value !== 'object' || value === null) return false; + const pose = value as Partial; + return [pose.px, pose.py, pose.pz, pose.ox, pose.oy, pose.oz, pose.ow].every(component => + Number.isFinite(component) + ); +} + function serializeJoint(pose: XRJointPose | null | undefined): SerializedJoint { const serialized = serializePose(pose); return serialized ? { ...serialized, radius: pose?.radius ?? 0.005 } : null; @@ -102,12 +120,16 @@ function serializeGamepad(gamepad: Gamepad | null | undefined): SerializedGamepa function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = 0): RecordedFrame { const captured = emptyFrame(timeMs); + captured.controllerProfiles = { left: null, right: null }; for (const source of frame.session.inputSources) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; captured.gamepads[hand] = serializeGamepad(source.gamepad); + if (!source.hand && source.gamepad) { + captured.controllerProfiles[hand] = Array.from(source.profiles ?? []); + } captured.poses[`${hand}Grip`] = source.gripSpace ? serializePose(frame.getPose(source.gripSpace, referenceSpace)) : null; @@ -126,6 +148,7 @@ function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = } function makePose(pose: PoseData): XRPose { + // WebXR permits null velocities; @types/webxr currently excludes null. return { transform: new XRRigidTransform( { x: pose.px, y: pose.py, z: pose.pz, w: 1 }, @@ -134,7 +157,7 @@ function makePose(pose: PoseData): XRPose { emulatedPosition: true, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function makeJointPose(joint: Exclude): XRJointPose { @@ -164,64 +187,23 @@ function bindWebXRMember(target: object, property: PropertyKey): unknown { return typeof value === 'function' ? value.bind(target) : value; } -function proxyInputSource(source: XRInputSource, gamepad: SerializedGamepad | null): XRInputSource { - const replayGamepad = gamepad ? makeGamepad(gamepad) : null; - return new Proxy(source, { - get(target, property) { - if (property === 'gamepad') return replayGamepad; - return bindWebXRMember(target, property); - }, - }); -} - function lerp(from: number, to: number, alpha: number): number { return from + (to - from) * alpha; } function interpolatePose(from: T, to: T, alpha: number): T { - let tox = to.ox; - let toy = to.oy; - let toz = to.oz; - let tow = to.ow; - const dot = from.ox * tox + from.oy * toy + from.oz * toz + from.ow * tow; - if (dot < 0) { - tox = -tox; - toy = -toy; - toz = -toz; - tow = -tow; - } - - const shortestDot = Math.abs(dot); - let fromScale = 1 - alpha; - let toScale = alpha; - if (shortestDot < 0.9995) { - const angle = Math.acos(Math.min(1, shortestDot)); - const sinAngle = Math.sin(angle); - fromScale = Math.sin((1 - alpha) * angle) / sinAngle; - toScale = Math.sin(alpha * angle) / sinAngle; - } - - let ox = from.ox * fromScale + tox * toScale; - let oy = from.oy * fromScale + toy * toScale; - let oz = from.oz * fromScale + toz * toScale; - let ow = from.ow * fromScale + tow * toScale; - const magnitude = Math.hypot(ox, oy, oz, ow); - if (magnitude > 0) { - ox /= magnitude; - oy /= magnitude; - oz /= magnitude; - ow /= magnitude; - } + baseOrientation.set(to.ox, to.oy, to.oz, to.ow); + poseOrientation.set(from.ox, from.oy, from.oz, from.ow).slerp(baseOrientation, alpha).normalize(); return { ...from, px: lerp(from.px, to.px, alpha), py: lerp(from.py, to.py, alpha), pz: lerp(from.pz, to.pz, alpha), - ox, - oy, - oz, - ow, + ox: poseOrientation.x, + oy: poseOrientation.y, + oz: poseOrientation.z, + ow: poseOrientation.w, }; } @@ -277,24 +259,39 @@ function interpolateJoints(from: JointPoses, to: JointPoses, alpha: number): Joi return result; } +function recordedControllerProfiles(frame: RecordedFrame, hand: 'left' | 'right'): string[] | null { + if (!frame.gamepads[hand]) return null; + if (frame.controllerProfiles !== undefined) return frame.controllerProfiles[hand]; + // Hand sources can expose gamepads too. Even null joint samples identify a hand. + // Legacy controller recordings have no profile, so use the SDK's generic mapping. + return Object.keys(frame.handJoints[hand]).length ? null : ['generic-trigger-squeeze-thumbstick']; +} + function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number): RecordedFrame { const interval = to.timeMs - from.timeMs; const alpha = interval > 0 ? (timeMs - from.timeMs) / interval : 0; + // Device switches are discrete even when two controllers have the same layout. + const sameDevice = (hand: 'left' | 'right') => + JSON.stringify(recordedControllerProfiles(from, hand)) === + JSON.stringify(recordedControllerProfiles(to, hand)); + const leftTo = sameDevice('left') ? to : from; + const rightTo = sameDevice('right') ? to : from; return { timeMs, + controllerProfiles: from.controllerProfiles, poses: { - leftGrip: interpolateOptionalPose(from.poses.leftGrip, to.poses.leftGrip, alpha), - leftAim: interpolateOptionalPose(from.poses.leftAim, to.poses.leftAim, alpha), - rightGrip: interpolateOptionalPose(from.poses.rightGrip, to.poses.rightGrip, alpha), - rightAim: interpolateOptionalPose(from.poses.rightAim, to.poses.rightAim, alpha), + leftGrip: interpolateOptionalPose(from.poses.leftGrip, leftTo.poses.leftGrip, alpha), + leftAim: interpolateOptionalPose(from.poses.leftAim, leftTo.poses.leftAim, alpha), + rightGrip: interpolateOptionalPose(from.poses.rightGrip, rightTo.poses.rightGrip, alpha), + rightAim: interpolateOptionalPose(from.poses.rightAim, rightTo.poses.rightAim, alpha), }, gamepads: { - left: interpolateGamepad(from.gamepads.left, to.gamepads.left, alpha), - right: interpolateGamepad(from.gamepads.right, to.gamepads.right, alpha), + left: interpolateGamepad(from.gamepads.left, leftTo.gamepads.left, alpha), + right: interpolateGamepad(from.gamepads.right, rightTo.gamepads.right, alpha), }, handJoints: { - left: interpolateJoints(from.handJoints.left, to.handJoints.left, alpha), - right: interpolateJoints(from.handJoints.right, to.handJoints.right, alpha), + left: interpolateJoints(from.handJoints.left, leftTo.handJoints.left, alpha), + right: interpolateJoints(from.handJoints.right, rightTo.handJoints.right, alpha), }, }; } @@ -334,28 +331,44 @@ function replayDuration(frames: RecordedFrame[]): number { return 0; } -/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ -function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - const p = baseFromScene.position; - const q = baseFromScene.orientation; - - // Rotate the recorded translation by q using v' = v + qw*t + cross(q.xyz, t). - const tx = 2 * (q.y * pose.pz - q.z * pose.py); - const ty = 2 * (q.z * pose.px - q.x * pose.pz); - const tz = 2 * (q.x * pose.py - q.y * pose.px); +// Reused only by synchronous pose math; no per-joint scratch allocations. +const posePosition = new Vector3(); +const poseOrientation = new Quaternion(); +const baseOrientation = new Quaternion(); +/** Apply a baseSpace <- sceneSpace transform to a pose expressed in sceneSpace. */ +function transformByPose(pose: T, baseFromScene: PoseData): T { + baseOrientation.set(baseFromScene.ox, baseFromScene.oy, baseFromScene.oz, baseFromScene.ow); + posePosition.set(pose.px, pose.py, pose.pz).applyQuaternion(baseOrientation); + poseOrientation.set(pose.ox, pose.oy, pose.oz, pose.ow).premultiply(baseOrientation); return { ...pose, - px: p.x + pose.px + q.w * tx + q.y * tz - q.z * ty, - py: p.y + pose.py + q.w * ty + q.z * tx - q.x * tz, - pz: p.z + pose.pz + q.w * tz + q.x * ty - q.y * tx, - ox: q.w * pose.ox + q.x * pose.ow + q.y * pose.oz - q.z * pose.oy, - oy: q.w * pose.oy - q.x * pose.oz + q.y * pose.ow + q.z * pose.ox, - oz: q.w * pose.oz + q.x * pose.oy - q.y * pose.ox + q.z * pose.ow, - ow: q.w * pose.ow - q.x * pose.ox - q.y * pose.oy - q.z * pose.oz, + px: posePosition.x + baseFromScene.px, + py: posePosition.y + baseFromScene.py, + pz: posePosition.z + baseFromScene.pz, + ox: poseOrientation.x, + oy: poseOrientation.y, + oz: poseOrientation.z, + ow: poseOrientation.w, }; } +function inversePose(pose: PoseData): PoseData { + const magnitudeSquared = + pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; + const scale = magnitudeSquared > 0 ? 1 / magnitudeSquared : 1; + return transformByPose( + { ...IDENTITY_POSE, px: -pose.px, py: -pose.py, pz: -pose.pz }, + { + ...IDENTITY_POSE, + ox: -pose.ox * scale, + oy: -pose.oy * scale, + oz: -pose.oz * scale, + ow: pose.ow * scale, + } + ); +} + /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -371,7 +384,6 @@ function frameTimestampMs(frame: XRFrame): number { export class XRInputRecorder { private _mode: 'idle' | 'recording' | 'replaying' = 'idle'; private _frames: RecordedFrame[] = []; - private _replayFrames: RecordedFrame[] = []; private _replayIndex = 0; private _loopReplay = true; private _replayPacing: ReplayPacing = 'time'; @@ -381,6 +393,16 @@ export class XRInputRecorder { private _sceneReferenceSpace: XRReferenceSpace | null = null; private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; + private _recordingAlignment: PoseData | null = null; + private _alignmentCache = new WeakMap(); + private _replayRecording: Recording | null = null; + private _replaySceneAlignment: PoseData | null = null; + private _observedSession: XRSession | null = null; + // Cached alignments use the origin observed at session/reference-space entry. + private _currentFromInitial: PoseData = IDENTITY_POSE; + private _recordingInterrupted = false; + private _trackingSession: XRReplaySession | null = null; + private _replaySources = new Map(); get mode() { return this._mode; @@ -398,12 +420,23 @@ export class XRInputRecorder { return this._currentFrame; } + /** Places recording-space traces in the same scene coordinates as SDK replay. */ + get replaySceneAlignment(): Readonly | null { + return this._replaySceneAlignment; + } + + get recordingInterrupted(): boolean { + return this._recordingInterrupted; + } + startRecording(): void { this._assertIdle(); this._frames = []; this._currentFrame = null; this._recordingStartTime = null; this._recordedAt = Date.now(); + this._recordingAlignment = null; + this._recordingInterrupted = false; this._mode = 'recording'; } @@ -415,13 +448,15 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); - this._replayFrames = recording.frames; + this._replaySources.clear(); this._replayIndex = 0; this._loopReplay = loop; this._replayPacing = pacing; this._replayElapsedMs = 0; this._lastReplayDisplayTime = null; this._currentFrame = null; + this._replayRecording = recording; + this._replaySceneAlignment = null; this._mode = 'replaying'; } @@ -429,6 +464,9 @@ export class XRInputRecorder { if (this._mode !== 'replaying') return; this._currentFrame = null; this._lastReplayDisplayTime = null; + this._replayRecording = null; + this._replaySources.clear(); + this._replaySceneAlignment = null; this._mode = 'idle'; } @@ -443,7 +481,7 @@ export class XRInputRecorder { connected = true, captureLive = false ): void { - this._sceneReferenceSpace = sceneReferenceSpace; + this._observeReferenceSpace(frame.session, sceneReferenceSpace); if (this._mode === 'idle') { this._currentFrame = @@ -458,6 +496,13 @@ export class XRInputRecorder { if (this._mode === 'recording') { if (!sceneReferenceSpace) return; + if (!this._recordingAlignment) { + // Hold the first frames until the runtime reports a viewer pose: while + // tracking is still settling there is no origin for the recorded + // coordinates to be relative to. + if (!frame.getViewerPose(sceneReferenceSpace)) return; + this._recordingAlignment = inversePose(this._currentFromInitial); + } const now = frameTimestampMs(frame); this._recordingStartTime ??= now; const timeMs = Math.max(0, now - this._recordingStartTime); @@ -466,7 +511,14 @@ export class XRInputRecorder { return; } - if (this._replayFrames.length === 0) { + const frames = this._replayRecording!.frames; + if (frames.length === 0) { + this._currentFrame = null; + return; + } + + if (!sceneReferenceSpace || !this._prepareReplayAlignment(frame, sceneReferenceSpace)) { + this._lastReplayDisplayTime = null; this._currentFrame = null; return; } @@ -476,8 +528,8 @@ export class XRInputRecorder { return; } - this._currentFrame = this._replayFrames[this._replayIndex]; - if (this._replayIndex < this._replayFrames.length - 1) { + this._currentFrame = frames[this._replayIndex]; + if (this._replayIndex < frames.length - 1) { this._replayIndex++; } else if (this._loopReplay) { this._replayIndex = 0; @@ -490,13 +542,14 @@ export class XRInputRecorder { } this._lastReplayDisplayTime = displayTime; - const firstTime = this._replayFrames[0].timeMs; - const lastTime = this._replayFrames[this._replayFrames.length - 1].timeMs; - const duration = replayDuration(this._replayFrames); + const frames = this._replayRecording!.frames; + const firstTime = frames[0].timeMs; + const lastTime = frames[frames.length - 1].timeMs; + const duration = replayDuration(frames); const elapsed = this._loopReplay && duration > 0 ? this._replayElapsedMs % duration : this._replayElapsedMs; const sampleTime = Math.min(firstTime + elapsed, lastTime); - const sample = sampleAtTime(this._replayFrames, sampleTime); + const sample = sampleAtTime(frames, sampleTime); this._currentFrame = sample.frame; this._replayIndex = sample.index; } @@ -506,18 +559,23 @@ export class XRInputRecorder { * objects are modified, and callers retain the original frame for rendering. */ adaptTrackingFrame = (frame: XRFrame): XRFrame => { - const replay = this._currentFrame; - if (this._mode !== 'replaying' || !replay) return frame; + if (this._trackingSession?.nativeSession !== frame.session) { + this._trackingSession?.dispose(); + this._trackingSession = new XRReplaySession(frame.session); + } + const replaying = this._mode === 'replaying'; + this._trackingSession.setInputSources(replaying ? this._replayInputSources() : null); + const session = this._trackingSession.session; + const replay = this._currentFrame ?? emptyFrame(0); - const session = this._proxySession(frame.session, replay); return new Proxy(frame, { get: (target, property) => { if (property === 'session') return session; - if (property === 'getPose') { + if (replaying && property === 'getPose') { return (space: XRSpace, baseSpace: XRSpace) => this._replayPose(target, replay, space, baseSpace); } - if (property === 'getJointPose') { + if (replaying && property === 'getJointPose') { return (joint: XRJointSpace, baseSpace: XRSpace) => this._replayJoint(target, replay, joint, baseSpace); } @@ -552,17 +610,35 @@ export class XRInputRecorder { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { throw new Error('Malformed recording: frame timeMs must be finite and monotonic'); } + if (frame.controllerProfiles !== undefined) { + const profiles = frame.controllerProfiles; + if ( + !profiles || + !(['left', 'right'] as const).every( + hand => + profiles[hand] === null || + (Array.isArray(profiles[hand]) && + profiles[hand]!.every(profile => typeof profile === 'string')) + ) + ) { + throw new Error('Malformed recording: controllerProfiles is invalid'); + } + } previousTime = frame.timeMs; } return recording; } getRecording(): Recording { - return { + const recording: Recording = { version: 1, recordedAt: this._recordedAt, frames: [...this._frames], }; + if (this._recordingAlignment) { + this._alignmentCache.set(recording, this._recordingAlignment); + } + return recording; } private _assertIdle(): void { @@ -571,19 +647,117 @@ export class XRInputRecorder { } } - private _proxySession(session: XRSession, replay: RecordedFrame): XRSession { - const inputSources = Array.from(session.inputSources, source => { - const hand = source.handedness; - return hand === 'left' || hand === 'right' - ? proxyInputSource(source, replay.gamepads[hand]) - : source; - }); + private _observeReferenceSpace( + session: XRSession, + referenceSpace: XRReferenceSpace | null + ): void { + if (session === this._observedSession && referenceSpace === this._sceneReferenceSpace) { + return; + } - return new Proxy(session, { - get(target, property) { - if (property === 'inputSources') return inputSources; - return bindWebXRMember(target, property); - }, + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._interruptRecording(); + this._invalidateAlignments(); + this._observedSession = session; + this._sceneReferenceSpace = referenceSpace; + referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); + } + + private _interruptRecording(): void { + if (this._mode === 'recording' && this._frames.length > 0) { + // One recording must not mix coordinates from different reference-space origins. + this.stopRecording(); + this._recordingInterrupted = true; + } + } + + private _invalidateAlignments(): void { + this._alignmentCache = new WeakMap(); + this._recordingAlignment = null; + this._currentFromInitial = IDENTITY_POSE; + this._replaySceneAlignment = null; + this._lastReplayDisplayTime = null; + this._currentFrame = null; + } + + private _onReferenceSpaceReset = (event: XRReferenceSpaceEvent): void => { + this._interruptRecording(); + if (!event.transform) { + this._invalidateAlignments(); + return; + } + + // The event gives oldScene <- newScene. Preserve world placement using its + // inverse; the current viewer pose includes operator motion, not just drift. + const oldFromNew = serializeTransform(event.transform); + this._currentFromInitial = transformByPose(this._currentFromInitial, inversePose(oldFromNew)); + this._replaySceneAlignment = null; + this._currentFrame = null; + }; + + private _prepareReplayAlignment(frame: XRFrame, referenceSpace: XRReferenceSpace): boolean { + if (this._replaySceneAlignment) return true; + const recording = this._replayRecording; + if (!recording) return false; + + let initialFromRecorded = this._alignmentCache.get(recording); + if (!initialFromRecorded) { + // Recording and streaming share one reference space (see RecorderComponent), + // so recorded poses are already in the frame the scene is anchored to and + // need no cross-session transform. The mapping is still cached against the + // origin as it stood at entry, so a later reference-space reset shifts + // replay with the scene instead of stranding it. + initialFromRecorded = inversePose(this._currentFromInitial); + this._alignmentCache.set(recording, initialFromRecorded); + } + + this._replaySceneAlignment = transformByPose(initialFromRecorded, this._currentFromInitial); + return true; + } + + dispose(): void { + this._trackingSession?.dispose(); + this._trackingSession = null; + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._sceneReferenceSpace = null; + this._observedSession = null; + this._invalidateAlignments(); + } + + private _replayInputSources(): XRInputSource[] { + const replay = this._currentFrame; + if (!replay) return []; + // Only recorded devices belong in the SDK view; live sources can otherwise + // activate a hand/controller that is absent from the recording. + return (['left', 'right'] as const).flatMap(handedness => { + const joints = replay.handJoints[handedness]; + const profiles = recordedControllerProfiles(replay, handedness); + if (!profiles && !Object.values(joints).some(Boolean)) return []; + + const key = JSON.stringify([handedness, profiles]); + let source = this._replaySources.get(key); + if (!source) { + source = { + handedness, + targetRayMode: 'tracked-pointer', + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: profiles ? [...profiles] : ['generic-hand-select'], + // @types/webxr still requires obsolete numeric constants on XRHand. + hand: profiles ? undefined : (new Map() as XRHand), + }; + this._replaySources.set(key, source); + } + if (source.hand) { + const hand = source.hand as Map; + for (const name of Object.keys(joints) as XRHandJoint[]) { + if (!hand.has(name)) + hand.set(name, Object.assign(new EventTarget(), { jointName: name })); + } + } + const gamepad = replay.gamepads[handedness]; + Object.assign(source, { gamepad: gamepad ? makeGamepad(gamepad) : null }); + return [source]; }); } @@ -593,7 +767,7 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { @@ -612,7 +786,7 @@ export class XRInputRecorder { joint: XRJointSpace, baseSpace: XRSpace ): XRJointPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if ((hand !== 'left' && hand !== 'right') || !source.hand) continue; for (const [name, candidate] of source.hand.entries()) { @@ -622,7 +796,7 @@ export class XRInputRecorder { return transformed ? makeJointPose(transformed) : undefined; } } - return frame.getJointPose(joint, baseSpace) ?? undefined; + return frame.getJointPose?.(joint, baseSpace) ?? undefined; } private _poseInBase( @@ -640,8 +814,12 @@ export class XRInputRecorder { baseSpace: XRSpace ): T | null { if (!pose || !this._sceneReferenceSpace) return null; - if (baseSpace === this._sceneReferenceSpace) return pose; + if (!this._replaySceneAlignment) return null; + const currentScenePose = transformByPose(pose, this._replaySceneAlignment); + if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); - return relation ? transformFromScene(pose, relation.transform) : null; + return relation + ? transformByPose(currentScenePose, serializeTransform(relation.transform)) + : null; } } diff --git a/deps/cloudxr/webxr_client/src/xrReplaySession.ts b/deps/cloudxr/webxr_client/src/xrReplaySession.ts new file mode 100644 index 0000000000..7fd94e8cd6 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/xrReplaySession.ts @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** CloudXR's input-source events must describe the same sources as its tracking frame. */ +export class XRReplaySession { + private _sources: XRInputSource[] | null = null; + private readonly _events = new EventTarget(); + readonly session: XRSession; + + constructor(readonly nativeSession: XRSession) { + this.session = new Proxy(nativeSession, { + get: (target, property) => { + if (property === 'inputSources') return this._sources ?? target.inputSources; + if (property === 'addEventListener' || property === 'removeEventListener') { + return ( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions + ) => { + const owner = type === 'inputsourceschange' ? this._events : target; + owner[property](type, listener, options); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + nativeSession.addEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + setInputSources(sources: XRInputSource[] | null): void { + const previous = Array.from(this._sources ?? this.nativeSession.inputSources); + this._sources = sources; + const next = Array.from(sources ?? this.nativeSession.inputSources); + const added = next.filter(source => !previous.includes(source)); + const removed = previous.filter(source => !next.includes(source)); + if (added.length || removed.length) this._dispatch(added, removed); + } + + dispose(): void { + this.nativeSession.removeEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + private _onNativeSourcesChange = (event: XRInputSourcesChangeEvent): void => { + // Recorded device presence is independent of live tracking and controller disconnects. + if (this._sources === null) this._dispatch(Array.from(event.added), Array.from(event.removed)); + }; + + private _dispatch(added: XRInputSource[], removed: XRInputSource[]): void { + this._events.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session: this.session, added, removed }) + ); + } +}