From 3005987ab75e0ce1ae156352c3acafa74afd3dda Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Sat, 29 Aug 2026 15:33:46 -0700 Subject: [PATCH 01/12] fix(webxr): align replays across XR sessions Signed-off-by: Yanzi Zhu --- .../webxr_client/src/xrInputRecorder.test.ts | 182 ++++++++++++- .../webxr_client/src/xrInputRecorder.ts | 248 +++++++++++++++++- 2 files changed, 411 insertions(+), 19 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index ca967ab17d..ea50ce8651 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -45,7 +45,7 @@ function pose(x: number, y = 0, z = 0, orientation: DOMPointInit = { w: 1 }): XR emulatedPosition: false, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function jointPose(x: number, radius = 0.01): XRJointPose { @@ -56,7 +56,11 @@ function gamepad(axis: number): Gamepad { return { axes: [axis], buttons: [{ value: axis, pressed: axis > 0, touched: true }], - } as Gamepad; + } as unknown as Gamepad; +} + +function makeSession(inputSources: XRInputSource[] = []): XRSession { + return { inputSources } as unknown as XRSession; } type PoseResolver = (space: XRSpace, baseSpace: XRSpace) => XRPose | null; @@ -66,15 +70,17 @@ function makeFrame( inputSources: XRInputSource[] = [], getPose: PoseResolver = () => null, getJointPose: JointResolver = () => null, - predictedDisplayTime = 0 + predictedDisplayTime = 0, + viewerPose: XRPose | null = pose(0), + session: XRSession = makeSession(inputSources) ): XRFrame { - const session = { inputSources } as XRSession; return { session, predictedDisplayTime, getPose, getJointPose, - } as XRFrame; + getViewerPose: () => viewerPose, + } as unknown as XRFrame; } function frameData(x = 0): RecordedFrame { @@ -103,6 +109,17 @@ function recording(...frames: RecordedFrame[]): Recording { return { version: 1, frames }; } +function calibratedRecording(viewerX: number, ...frames: RecordedFrame[]): Recording { + return { + version: 1, + calibration: { + mode: 'viewer-start-yaw', + pose: { px: viewerX, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, + }, + frames, + }; +} + function timedFrame(timeMs: number, x: number): RecordedFrame { return { ...frameData(x), timeMs }; } @@ -289,6 +306,33 @@ describe('canonical scene-space capture', () => { expect(captured.handJoints.left.wrist?.px).toBe(3); expect(captured.handJoints.left['index-finger-tip']?.px).toBe(4); }); + + test('captures a gravity-aligned viewer pose for cross-session calibration', () => { + const quarterTurn = Math.sqrt(0.5); + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame( + makeFrame([], undefined, undefined, 0, pose(1, 2, 3, { y: quarterTurn, w: quarterTurn })), + sceneSpace + ); + recorder.stopRecording(); + + const calibration = recorder.getRecording().calibration; + expect(calibration?.mode).toBe('viewer-start-yaw'); + expect(calibration?.pose).toMatchObject({ px: 1, py: 2, pz: 3, ox: 0, oz: 0 }); + expect(calibration?.pose.oy).toBeCloseTo(quarterTurn); + expect(calibration?.pose.ow).toBeCloseTo(quarterTurn); + }); + + test('waits for a viewer pose before recording the first frame', () => { + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(makeFrame([], undefined, undefined, 0, null), sceneSpace); + expect(recorder.recordedFrameCount).toBe(0); + + recorder.beginFrame(makeFrame([], undefined, undefined, 1, pose(0)), sceneSpace); + expect(recorder.recordedFrameCount).toBe(1); + }); }); describe('scoped CloudXR replay frame', () => { @@ -360,8 +404,8 @@ describe('scoped CloudXR replay frame', () => { expect(adapted.session.inputSources).toBe(replaySources); expect(replaySources[0].gamepad).toBe(replaySources[0].gamepad); expect(replaySources[0].gamepad?.axes).toEqual([3]); - expect(adapted.getJointPose(wrist, cloudSpace)?.transform.position.x).toBe(15); - expect(adapted.getJointPose(wrist, cloudSpace)?.radius).toBe(0.02); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.transform.position.x).toBe(15); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.radius).toBe(0.02); }); test('delegates unknown spaces and joints to the real frame', () => { @@ -378,7 +422,123 @@ describe('scoped CloudXR replay frame', () => { const adapted = recorder.adaptTrackingFrame(frame); expect(adapted.getPose(unknownSpace, sceneSpace)?.transform.position.x).toBe(7); - expect(adapted.getJointPose(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + expect(adapted.getJointPose?.(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + }); + + test('aligns loaded poses from the recorded viewer origin to the current XR session', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); + const frame = makeFrame([source], undefined, undefined, 0, pose(11, 5), session); + const recorder = new XRInputRecorder(); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(frame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(12); + expect(replayed?.transform.position.y).toBeCloseTo(5); + }); + + test('applies the calibrated heading to replayed poses', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const sample = frameData(); + sample.poses.leftGrip = { px: 0, py: 0, pz: -1, ox: 0, oy: 0, oz: 0, ow: 1 }; + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(0, sample))); + const quarterTurn = Math.sqrt(0.5); + const frame = makeFrame( + [source], + undefined, + undefined, + 0, + pose(0, 0, 0, { y: quarterTurn, w: quarterTurn }), + session + ); + const recorder = new XRInputRecorder(); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(frame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(-1); + expect(replayed?.transform.position.z).toBeCloseTo(0); + expect(replayed?.transform.orientation.y).toBeCloseTo(quarterTurn); + expect(replayed?.transform.orientation.w).toBeCloseTo(quarterTurn); + }); + + test('freezes calibration within a session and recalibrates for a new session', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const firstSession = makeSession([source]); + const secondSession = makeSession([source]); + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); + const recorder = new XRInputRecorder(); + + const firstFrame = makeFrame([source], undefined, undefined, 0, pose(11), firstSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(firstFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(firstFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(12); + recorder.stopReplay(); + + const movedViewerFrame = makeFrame([source], undefined, undefined, 1, pose(21), firstSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(movedViewerFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(movedViewerFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(12); + recorder.stopReplay(); + + const newSessionFrame = makeFrame([source], undefined, undefined, 2, pose(21), secondSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(newSessionFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(newSessionFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(22); + }); + + test('keeps in-memory recording and replay in the same reference space unchanged', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const recordingFrame = makeFrame( + [source], + space => (space === grip ? pose(2) : null), + undefined, + 0, + pose(1), + session + ); + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(recordingFrame, sceneSpace); + recorder.stopRecording(); + const saved = recorder.getRecording(); + + const replayFrame = makeFrame([source], undefined, undefined, 1, pose(11), session); + recorder.startReplay(saved, true, 'frame'); + recorder.beginFrame(replayFrame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(replayFrame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(2); }); }); @@ -388,7 +548,9 @@ describe('serialization', () => { recorder.startRecording(); recorder.beginFrame(makeFrame(), sceneSpace); recorder.stopRecording(); - expect(XRInputRecorder.importJSON(recorder.exportJSON()).frames).toHaveLength(1); + const imported = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(imported.frames).toHaveLength(1); + expect(imported.calibration?.mode).toBe('viewer-start-yaw'); expect(typeof recorder.getRecording().recordedAt).toBe('number'); }); @@ -423,6 +585,8 @@ describe('serialization', () => { { version: 1, frames: null }, { version: 1, frames: [{ ...frameData(), timeMs: undefined }] }, { version: 1, frames: [frameData(2), frameData(1)] }, + { version: 1, calibration: { mode: 'viewer-start-yaw', pose: {} }, frames: [] }, + { version: 1, calibration: { mode: 'unknown', pose: {} }, frames: [] }, null, 42, [], diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 8838d38ba5..07863cca08 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -61,6 +61,12 @@ export type RecordedFrame = { export type Recording = { version: 1; recordedAt?: number; + /** Optional cross-session origin calibration. Absent in legacy recordings. */ + calibration?: { + mode: 'viewer-start-yaw'; + /** Gravity-aligned viewer pose in the recording's scene reference space. */ + pose: PoseData; + }; frames: RecordedFrame[]; }; @@ -83,6 +89,14 @@ function serializePose(pose: XRPose | null | undefined): SerializedPose { return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; } +function isPoseData(value: unknown): value is PoseData { + if (typeof value !== 'object' || value === null) return false; + const pose = value as Partial; + return [pose.px, pose.py, pose.pz, pose.ox, pose.oy, pose.oz, pose.ow].every(component => + Number.isFinite(component) + ); +} + function serializeJoint(pose: XRJointPose | null | undefined): SerializedJoint { const serialized = serializePose(pose); return serialized ? { ...serialized, radius: pose?.radius ?? 0.005 } : null; @@ -134,7 +148,7 @@ function makePose(pose: PoseData): XRPose { emulatedPosition: true, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function makeJointPose(joint: Exclude): XRJointPose { @@ -334,10 +348,15 @@ function replayDuration(frames: RecordedFrame[]): number { return 0; } -/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ -function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - const p = baseFromScene.position; - const q = baseFromScene.orientation; +/** Apply a baseSpace <- sceneSpace transform to a pose expressed in sceneSpace. */ +function transformByPose(pose: T, baseFromScene: PoseData): T { + const p = { x: baseFromScene.px, y: baseFromScene.py, z: baseFromScene.pz }; + const q = { + x: baseFromScene.ox, + y: baseFromScene.oy, + z: baseFromScene.oz, + w: baseFromScene.ow, + }; // Rotate the recorded translation by q using v' = v + qw*t + cross(q.xyz, t). const tx = 2 * (q.y * pose.pz - q.z * pose.py); @@ -356,6 +375,87 @@ function transformFromScene(pose: T, baseFromScene: XRRigidT }; } +/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ +function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { + const { position: p, orientation: q } = baseFromScene; + return transformByPose(pose, { + px: p.x, + py: p.y, + pz: p.z, + ox: q.x, + oy: q.y, + oz: q.z, + ow: q.w, + }); +} + +function inversePose(pose: PoseData): PoseData { + const magnitudeSquared = + pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; + const scale = magnitudeSquared > 0 ? 1 / magnitudeSquared : 1; + const inverseOrientation = { + ox: -pose.ox * scale, + oy: -pose.oy * scale, + oz: -pose.oz * scale, + ow: pose.ow * scale, + }; + const inverseTranslation = transformByPose( + { px: -pose.px, py: -pose.py, pz: -pose.pz, ox: 0, oy: 0, oz: 0, ow: 1 }, + { px: 0, py: 0, pz: 0, ...inverseOrientation } + ); + return { + px: inverseTranslation.px, + py: inverseTranslation.py, + pz: inverseTranslation.pz, + ...inverseOrientation, + }; +} + +/** Keep translation and heading while removing viewer pitch and roll. */ +function gravityAlignedViewerPose(pose: PoseData): PoseData { + const yaw = Math.atan2( + 2 * (pose.ox * pose.oz + pose.ow * pose.oy), + 1 - 2 * (pose.ox * pose.ox + pose.oy * pose.oy) + ); + const halfYaw = yaw / 2; + return { + px: pose.px, + py: pose.py, + pz: pose.pz, + ox: 0, + oy: Math.sin(halfYaw), + oz: 0, + ow: Math.cos(halfYaw), + }; +} + +function captureViewerCalibration( + frame: XRFrame, + referenceSpace: XRReferenceSpace +): Recording['calibration'] | undefined { + const viewerPose = serializePose(frame.getViewerPose(referenceSpace)); + return viewerPose + ? { mode: 'viewer-start-yaw', pose: gravityAlignedViewerPose(viewerPose) } + : undefined; +} + +function sceneAlignment(recordedViewer: PoseData, currentViewer: PoseData): PoseData { + // currentScene <- recordedScene = currentScene <- viewer <- recordedScene. + return transformByPose(inversePose(recordedViewer), currentViewer); +} + +type RecordingContext = { + session: XRSession; + referenceSpace: XRReferenceSpace; + referenceSpaceEpoch: number; +}; + +type CachedAlignment = { + referenceSpace: XRReferenceSpace; + referenceSpaceEpoch: number; + transform: PoseData | null; +}; + /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -381,6 +481,17 @@ export class XRInputRecorder { private _sceneReferenceSpace: XRReferenceSpace | null = null; private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; + private _recordingCalibration: Recording['calibration']; + private _recordingContext: RecordingContext | null = null; + private _recordingContexts = new WeakMap(); + private _alignmentCache = new WeakMap>(); + private _replayRecording: Recording | null = null; + private _replaySourceContext: RecordingContext | null = null; + private _replaySceneAlignment: PoseData | null = null; + private _replayAlignmentReady = false; + private _observedSession: XRSession | null = null; + private _observedReferenceSpace: XRReferenceSpace | null = null; + private _referenceSpaceEpochs = new WeakMap(); get mode() { return this._mode; @@ -404,6 +515,8 @@ export class XRInputRecorder { this._currentFrame = null; this._recordingStartTime = null; this._recordedAt = Date.now(); + this._recordingCalibration = undefined; + this._recordingContext = null; this._mode = 'recording'; } @@ -422,6 +535,10 @@ export class XRInputRecorder { this._replayElapsedMs = 0; this._lastReplayDisplayTime = null; this._currentFrame = null; + this._replayRecording = recording; + this._replaySourceContext = this._recordingContexts.get(recording) ?? null; + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; this._mode = 'replaying'; } @@ -429,6 +546,10 @@ export class XRInputRecorder { if (this._mode !== 'replaying') return; this._currentFrame = null; this._lastReplayDisplayTime = null; + this._replayRecording = null; + this._replaySourceContext = null; + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; this._mode = 'idle'; } @@ -443,7 +564,7 @@ export class XRInputRecorder { connected = true, captureLive = false ): void { - this._sceneReferenceSpace = sceneReferenceSpace; + this._observeReferenceSpace(frame.session, sceneReferenceSpace); if (this._mode === 'idle') { this._currentFrame = @@ -458,6 +579,15 @@ export class XRInputRecorder { if (this._mode === 'recording') { if (!sceneReferenceSpace) return; + if (!this._recordingCalibration) { + this._recordingCalibration = captureViewerCalibration(frame, sceneReferenceSpace); + if (!this._recordingCalibration) return; + this._recordingContext = { + session: frame.session, + referenceSpace: sceneReferenceSpace, + referenceSpaceEpoch: this._referenceSpaceEpoch(frame.session), + }; + } const now = frameTimestampMs(frame); this._recordingStartTime ??= now; const timeMs = Math.max(0, now - this._recordingStartTime); @@ -471,6 +601,11 @@ export class XRInputRecorder { return; } + if (!sceneReferenceSpace || !this._prepareReplayAlignment(frame, sceneReferenceSpace)) { + this._currentFrame = null; + return; + } + if (this._replayPacing === 'time') { this._advanceTimedReplay(frameTimestampMs(frame)); return; @@ -547,6 +682,15 @@ export class XRInputRecorder { if (!Array.isArray(recording.frames)) { throw new Error('Malformed recording: frames is not an array'); } + if ( + recording.calibration !== undefined && + (typeof recording.calibration !== 'object' || + recording.calibration === null || + recording.calibration.mode !== 'viewer-start-yaw' || + !isPoseData(recording.calibration.pose)) + ) { + throw new Error('Malformed recording: calibration is invalid'); + } let previousTime = -1; for (const frame of recording.frames) { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { @@ -558,11 +702,16 @@ export class XRInputRecorder { } getRecording(): Recording { - return { + const recording: Recording = { version: 1, recordedAt: this._recordedAt, + calibration: this._recordingCalibration, frames: [...this._frames], }; + if (this._recordingContext) { + this._recordingContexts.set(recording, this._recordingContext); + } + return recording; } private _assertIdle(): void { @@ -571,6 +720,81 @@ export class XRInputRecorder { } } + private _referenceSpaceEpoch(session: XRSession): number { + return this._referenceSpaceEpochs.get(session) ?? 0; + } + + private _observeReferenceSpace( + session: XRSession, + referenceSpace: XRReferenceSpace | null + ): void { + if (session === this._observedSession && referenceSpace === this._observedReferenceSpace) { + return; + } + + this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + if (session === this._observedSession && this._observedReferenceSpace !== null) { + this._referenceSpaceEpochs.set(session, this._referenceSpaceEpoch(session) + 1); + } else if (!this._referenceSpaceEpochs.has(session)) { + this._referenceSpaceEpochs.set(session, 0); + } + + this._observedSession = session; + this._observedReferenceSpace = referenceSpace; + this._sceneReferenceSpace = referenceSpace; + referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); + this._replayAlignmentReady = false; + } + + private _onReferenceSpaceReset = (): void => { + if (!this._observedSession) return; + this._referenceSpaceEpochs.set( + this._observedSession, + this._referenceSpaceEpoch(this._observedSession) + 1 + ); + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; + this._currentFrame = null; + }; + + private _prepareReplayAlignment(frame: XRFrame, referenceSpace: XRReferenceSpace): boolean { + if (this._replayAlignmentReady) return true; + const recording = this._replayRecording; + if (!recording) return false; + + const referenceSpaceEpoch = this._referenceSpaceEpoch(frame.session); + let sessionCache = this._alignmentCache.get(recording); + const cached = sessionCache?.get(frame.session); + if ( + cached && + cached.referenceSpace === referenceSpace && + cached.referenceSpaceEpoch === referenceSpaceEpoch + ) { + this._replaySceneAlignment = cached.transform; + this._replayAlignmentReady = true; + return true; + } + + let transform: PoseData | null = null; + const source = this._replaySourceContext; + const sameReferenceSpace = + source?.session === frame.session && + source.referenceSpace === referenceSpace && + source.referenceSpaceEpoch === referenceSpaceEpoch; + if (!sameReferenceSpace && recording.calibration) { + const currentCalibration = captureViewerCalibration(frame, referenceSpace); + if (!currentCalibration) return false; + transform = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + } + + sessionCache ??= new WeakMap(); + sessionCache.set(frame.session, { referenceSpace, referenceSpaceEpoch, transform }); + this._alignmentCache.set(recording, sessionCache); + this._replaySceneAlignment = transform; + this._replayAlignmentReady = true; + return true; + } + private _proxySession(session: XRSession, replay: RecordedFrame): XRSession { const inputSources = Array.from(session.inputSources, source => { const hand = source.handedness; @@ -622,7 +846,7 @@ export class XRInputRecorder { return transformed ? makeJointPose(transformed) : undefined; } } - return frame.getJointPose(joint, baseSpace) ?? undefined; + return frame.getJointPose?.(joint, baseSpace) ?? undefined; } private _poseInBase( @@ -640,8 +864,12 @@ export class XRInputRecorder { baseSpace: XRSpace ): T | null { if (!pose || !this._sceneReferenceSpace) return null; - if (baseSpace === this._sceneReferenceSpace) return pose; + if (!this._replayAlignmentReady) return null; + const currentScenePose = this._replaySceneAlignment + ? transformByPose(pose, this._replaySceneAlignment) + : pose; + if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); - return relation ? transformFromScene(pose, relation.transform) : null; + return relation ? transformFromScene(currentScenePose, relation.transform) : null; } } From 97a37ae647a35b13f907b989f97036f45756d4c2 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Fri, 11 Sep 2026 15:49:55 -0700 Subject: [PATCH 02/12] fix(webxr): replay hands without live tracking and preserve alignment Signed-off-by: Yanzi Zhu --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 29 +- .../webxr_client/src/RecorderComponent.tsx | 4 +- .../webxr_client/src/RecorderContext.test.tsx | 95 ++++++ .../webxr_client/src/RecorderContext.tsx | 36 +- .../webxr_client/src/xrInputRecorder.test.ts | 317 +++++++++++++++++- .../webxr_client/src/xrInputRecorder.ts | 300 ++++++++++------- .../webxr_client/src/xrReplaySession.ts | 68 ++++ 7 files changed, 720 insertions(+), 129 deletions(-) create mode 100644 deps/cloudxr/webxr_client/src/RecorderContext.test.tsx create mode 100644 deps/cloudxr/webxr_client/src/xrReplaySession.ts diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 8355e0e32c..988640cb4b 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -603,10 +603,37 @@ export default function CloudXR3DUI({ {recorder.mode === 'recording' ? `REC ${recorder.recordedFrameCount} frames` : recorder.mode === 'replaying' - ? 'Replaying' + ? recorder.replayNeedsCalibration + ? 'Replay paused' + : 'Replaying' : 'Recording'} + {recorder.replayNeedsCalibration && recorder.mode === 'replaying' && ( + + Return your headset to its recording-start position and heading, then + calibrate. + + )} + {recorder.mode === 'idle' && recorder.recordingInterrupted && ( + + Recording stopped because the tracking origin changed. Save it or start a + new recording. + + )} + {recorder.mode === 'idle' && !recorder.recordingInterrupted && ( + + Note your headset position and heading when starting a recording. Use the + same pose to calibrate replay in a new session. + + )} + {recorder.mode === 'replaying' && recorder.replayNeedsCalibration && ( + + )} {recorder.mode !== 'replaying' && ( { @@ -55,6 +55,8 @@ export function RecorderComponent({ isConnected, showTrace }: RecorderComponentP showTrace && isVisible ); + onFrameState(); + if (recorder.mode === 'recording') { tickRef.current++; if (tickRef.current % 30 === 0) { diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx new file mode 100644 index 0000000000..8ec67646f7 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx @@ -0,0 +1,95 @@ +/** @jest-environment jsdom */ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; + +import { type RecorderContextValue, RecorderProvider, useRecorder } from './RecorderContext'; + +let current: RecorderContextValue; +let root: Root; + +function Probe() { + current = useRecorder(); + return null; +} + +function frame(session: XRSession): XRFrame { + return { + session, + predictedDisplayTime: 0, + getViewerPose: () => ({ + transform: { position: { x: 0, y: 0, z: 0 }, orientation: { x: 0, y: 0, z: 0, w: 1 } }, + }), + } as unknown as XRFrame; +} + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + root = createRoot(document.createElement('div')); + act(() => + root.render( + + + + ) + ); +}); + +afterEach(() => act(() => root.unmount())); + +test('exposes cross-session calibration and resumes after the operator calibrates', () => { + const space = new EventTarget() as XRReferenceSpace; + const original = { inputSources: [] } as unknown as XRSession; + act(() => current.startRecord()); + act(() => current.recorder.beginFrame(frame(original), space)); + act(() => current.stopRecord()); + expect(current.savedRecording?.frames).toHaveLength(1); + act(() => current.startReplay()); + const next = { inputSources: [] } as unknown as XRSession; + act(() => { + current.recorder.beginFrame(frame(next), space); + current.onFrameState(); + }); + expect(current.mode).toBe('replaying'); + expect(current.replayNeedsCalibration).toBe(true); + act(() => current.calibrateReplay()); + act(() => { + current.recorder.beginFrame(frame(next), space); + current.onFrameState(); + }); + expect(current.replayNeedsCalibration).toBe(false); + expect(current.recorder.currentFrame).not.toBeNull(); +}); + +test('saves interrupted recording and updates the UI when the tracking origin changes', () => { + const session = { inputSources: [] } as unknown as XRSession; + const space = new EventTarget() as XRReferenceSpace; + act(() => current.startRecord()); + act(() => current.recorder.beginFrame(frame(session), space)); + act(() => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform: null })); + current.onFrameState(); + }); + expect(current.mode).toBe('idle'); + expect(current.recordingInterrupted).toBe(true); + expect(current.savedRecording?.frames).toHaveLength(1); + act(() => current.startRecord()); + expect(current.recordingInterrupted).toBe(false); +}); diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 8db8fae7b9..4381a72192 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -23,7 +23,15 @@ * to descendant components. */ -import React, { createContext, useCallback, useContext, useMemo, useRef, useState } from 'react'; +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { type Recording, type ReplayPacing, XRInputRecorder } from './xrInputRecorder'; @@ -37,6 +45,10 @@ export interface RecorderContextValue { stopRecord: () => void; startReplay: () => void; stopReplay: () => void; + calibrateReplay: () => void; + replayNeedsCalibration: boolean; + recordingInterrupted: boolean; + onFrameState: () => void; setReplayPacing: (pacing: ReplayPacing) => void; onSaveRecording: () => void; onLoadRecording: () => void; @@ -59,9 +71,11 @@ function setLoadStatus(message: string, type: 'success' | 'error'): void { export function RecorderProvider({ children }: { children: React.ReactNode }) { const recorder = useMemo(() => new XRInputRecorder(), []); + useEffect(() => () => recorder.dispose(), [recorder]); const [mode, setMode] = useState<'idle' | 'recording' | 'replaying'>('idle'); const [savedRecording, setSavedRecordingState] = useState(null); const [recordedFrameCount, setRecordedFrameCount] = useState(0); + const [replayNeedsCalibration, setReplayNeedsCalibration] = useState(false); const [replayPacing, setReplayPacingState] = useState('time'); const fileInputRef = useRef(null); @@ -85,6 +99,7 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { if (recorder.mode !== 'idle' || !savedRecording) return; recorder.startReplay(savedRecording, true, replayPacing); setMode('replaying'); + setReplayNeedsCalibration(recorder.replayNeedsCalibration); }, [recorder, replayPacing, savedRecording]); const stopReplay = useCallback(() => { @@ -93,6 +108,21 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { setMode('idle'); }, [recorder]); + const calibrateReplay = useCallback(() => recorder.calibrateReplay(), [recorder]); + + const onFrameState = useCallback(() => { + if (mode !== recorder.mode) { + if (mode === 'recording') { + setSavedRecordingState(recorder.getRecording()); + setRecordedFrameCount(recorder.recordedFrameCount); + } + setMode(recorder.mode); + } + if (replayNeedsCalibration !== recorder.replayNeedsCalibration) { + setReplayNeedsCalibration(recorder.replayNeedsCalibration); + } + }, [mode, recorder, replayNeedsCalibration]); + const setReplayPacing = useCallback((pacing: ReplayPacing) => { setReplayPacingState(pacing); }, []); @@ -155,6 +185,10 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { stopRecord, startReplay, stopReplay, + calibrateReplay, + replayNeedsCalibration, + recordingInterrupted: recorder.recordingInterrupted, + onFrameState, setReplayPacing, onSaveRecording, onLoadRecording, diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index ea50ce8651..551d32c506 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -60,9 +60,15 @@ function gamepad(axis: number): Gamepad { } function makeSession(inputSources: XRInputSource[] = []): XRSession { - return { inputSources } as unknown as XRSession; + return Object.assign(new EventTarget(), { inputSources }) as unknown as XRSession; } +// Frames in a test share a session unless the test explicitly starts another. +let defaultSession: XRSession; +beforeEach(() => { + defaultSession = makeSession(); +}); + type PoseResolver = (space: XRSpace, baseSpace: XRSpace) => XRPose | null; type JointResolver = (joint: XRJointSpace, baseSpace: XRSpace) => XRJointPose | null; @@ -72,8 +78,9 @@ function makeFrame( getJointPose: JointResolver = () => null, predictedDisplayTime = 0, viewerPose: XRPose | null = pose(0), - session: XRSession = makeSession(inputSources) + session: XRSession = defaultSession ): XRFrame { + Object.assign(session, { inputSources }); return { session, predictedDisplayTime, @@ -336,12 +343,14 @@ describe('canonical scene-space capture', () => { }); describe('scoped CloudXR replay frame', () => { - test('does not alter global prototypes and returns real frames outside replay', () => { + test('preserves live tracking outside replay without changing browser frames', () => { const recorder = new XRInputRecorder(); const frame = makeFrame(); const originalGetPose = frame.getPose; recorder.startRecording(); - expect(recorder.adaptTrackingFrame(frame)).toBe(frame); + const adapted = recorder.adaptTrackingFrame(frame); + expect(adapted.session.inputSources).toBe(frame.session.inputSources); + expect(adapted.getViewerPose(sceneSpace)).toEqual(frame.getViewerPose(sceneSpace)); expect(frame.getPose).toBe(originalGetPose); recorder.stopRecording(); }); @@ -438,6 +447,9 @@ describe('scoped CloudXR replay frame', () => { const recorder = new XRInputRecorder(); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(frame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); expect(replayed?.transform.position.x).toBeCloseTo(12); @@ -467,6 +479,9 @@ describe('scoped CloudXR replay frame', () => { const recorder = new XRInputRecorder(); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(frame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); expect(replayed?.transform.position.x).toBeCloseTo(-1); @@ -490,6 +505,9 @@ describe('scoped CloudXR replay frame', () => { const firstFrame = makeFrame([source], undefined, undefined, 0, pose(11), firstSession); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(firstFrame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(firstFrame, sceneSpace); expect( recorder.adaptTrackingFrame(firstFrame).getPose(grip, sceneSpace)?.transform.position.x ).toBeCloseTo(12); @@ -506,6 +524,9 @@ describe('scoped CloudXR replay frame', () => { const newSessionFrame = makeFrame([source], undefined, undefined, 2, pose(21), secondSession); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(newSessionFrame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(newSessionFrame, sceneSpace); expect( recorder.adaptTrackingFrame(newSessionFrame).getPose(grip, sceneSpace)?.transform.position.x ).toBeCloseTo(22); @@ -542,6 +563,294 @@ describe('scoped CloudXR replay frame', () => { }); }); +describe('explicit replay calibration and reference-space resets', () => { + function setup(pacing: 'frame' | 'time' = 'frame') { + const grip = {} as XRSpace; + const wrist = {} as XRJointSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + gamepad: gamepad(1), + hand: new Map([['wrist', wrist]]), + } as unknown as XRInputSource; + const session = makeSession([source]); + const space = new EventTarget() as XRReferenceSpace; + const recorder = new XRInputRecorder(); + const saved = calibratedRecording(0, timedFrame(0, 1), timedFrame(100, 3)); + const frame = (viewerX: number, time = 0, viewer: XRPose | null = pose(viewerX)) => + makeFrame( + [source], + () => pose(999), + () => jointPose(999), + time, + viewer, + session + ); + const advance = (viewerX: number, time = 0) => { + const current = frame(viewerX, time); + recorder.beginFrame(current, space); + return recorder.adaptTrackingFrame(current); + }; + const reset = (transform: XRRigidTransform | null) => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform })); + }; + recorder.startReplay(saved, false, pacing); + return { recorder, grip, wrist, source, session, space, saved, frame, advance, reset }; + } + + test('suppresses live input and waits for a deliberate calibration with a valid viewer pose', () => { + const { recorder, grip, wrist, space, frame, advance } = setup(); + const waiting = advance(10); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.currentFrame).toBeNull(); + expect(waiting.getPose(grip, space)).toBeUndefined(); + expect(waiting.getJointPose?.(wrist, space)).toBeUndefined(); + expect(waiting.session.inputSources).toHaveLength(0); + + recorder.calibrateReplay(); + recorder.beginFrame(frame(10, 0, null), space); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.replayFrameIndex).toBe(0); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); + expect(recorder.replayNeedsCalibration).toBe(false); + }); + + test.each(['frame', 'time'] as const)( + '%s replay preserves world placement after headset motion and repeated translation resets', + pacing => { + const { recorder, grip, wrist, space, advance, reset } = setup(pacing); + advance(10); + recorder.calibrateReplay(); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); + reset(new XRRigidTransform({ x: -100 })); + const firstReset = advance(112, pacing === 'time' ? 50 : 1); + expect(recorder.replayNeedsCalibration).toBe(false); + expect(firstReset.getPose(grip, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 112 : 113 + ); + expect(firstReset.getJointPose?.(wrist, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 114 : 115 + ); + reset(new XRRigidTransform({ x: -20 })); + expect(advance(132, 100).getPose(grip, space)?.transform.position.x).toBeCloseTo(133); + } + ); + + test('uses the inverse reset rotation for position and orientation', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + saved.frames = [saved.frames[0]]; + recorder.stopReplay(); + recorder.startReplay(saved, false, 'frame'); + advance(10); + recorder.calibrateReplay(); + advance(10); + const q = Math.sqrt(0.5); + reset(new XRRigidTransform({}, { x: 0, y: q, z: 0, w: q })); + const result = advance(999).getPose(grip, space)!; + expect(result.transform.position.x).toBeCloseTo(0); + expect(result.transform.position.z).toBeCloseTo(11); + expect(result.transform.orientation.y).toBeCloseTo(-q); + expect(result.transform.orientation.w).toBeCloseTo(q); + }); + + test('unknown reset pauses the replay clock and requires another explicit calibration', () => { + const { recorder, grip, space, advance, reset } = setup('time'); + advance(10, 1000); + recorder.calibrateReplay(); + advance(10, 1000); + expect(advance(12, 1050).getPose(grip, space)?.transform.position.x).toBeCloseTo(12); + reset(null); + expect(advance(112, 2000).getPose(grip, space)).toBeUndefined(); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.currentFrame).toBeNull(); + recorder.calibrateReplay(); + expect(advance(110, 5000).getPose(grip, space)?.transform.position.x).toBeCloseTo(112); + expect(recorder.currentFrame?.timeMs).toBe(50); + }); + + test('preserves cached placement when a known reset occurs while replay is stopped', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + advance(10); + recorder.calibrateReplay(); + advance(10); + recorder.stopReplay(); + reset(new XRRigidTransform({ x: -100 })); + recorder.startReplay(saved, false, 'frame'); + expect(advance(112).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + expect(recorder.replayNeedsCalibration).toBe(false); + }); + + test('calibration after a reset is stored in the original reference frame', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + advance(10); + reset(new XRRigidTransform({ x: -100 })); + recorder.calibrateReplay(); + expect(advance(110).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + recorder.stopReplay(); + recorder.startReplay(saved, false, 'frame'); + expect(advance(115).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + }); + + test('reference-space replacement invalidates placement and pending calibration', () => { + const { recorder, grip, frame, advance } = setup(); + advance(10); + recorder.calibrateReplay(); + const replacement = new EventTarget() as XRReferenceSpace; + const current = frame(20); + recorder.beginFrame(current, replacement); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.adaptTrackingFrame(current).getPose(grip, replacement)).toBeUndefined(); + recorder.calibrateReplay(); + recorder.beginFrame(current, replacement); + expect( + recorder.adaptTrackingFrame(current).getPose(grip, replacement)?.transform.position.x + ).toBeCloseTo(21); + }); + + test.each([true, false])('stops recording on reset (known transform: %s)', known => { + const { recorder, grip, space, frame, reset } = setup(); + recorder.stopReplay(); + recorder.startRecording(); + recorder.beginFrame(frame(0), space); + reset(known ? new XRRigidTransform({ x: -100 }) : null); + expect(recorder.mode).toBe('idle'); + expect(recorder.recordingInterrupted).toBe(true); + recorder.beginFrame(frame(100), space); + expect(recorder.getRecording().frames).toHaveLength(1); + const saved = recorder.getRecording(); + recorder.startReplay(saved, false, 'frame'); + recorder.beginFrame(frame(100), space); + expect(recorder.replayNeedsCalibration).toBe(!known); + if (known) { + expect( + recorder.adaptTrackingFrame(frame(100)).getPose(grip, space)?.transform.position.x + ).toBeCloseTo(1099); + } + }); +}); + +describe('recorded hands without live tracking', () => { + function benchmark(pacing: 'frame' | 'time') { + const session = makeSession(); + const recorder = new XRInputRecorder(); + const frame = (sources: XRInputSource[] = [], time = 0) => + makeFrame( + sources, + () => null, + () => null, + time, + pose(0), + session + ); + // CloudXR initializes its active-hand set on the first tracking frame and + // subsequently refreshes it from inputsourceschange, not from joint poses. + const active = new Set(); + const refresh = (current: XRSession) => { + active.clear(); + for (const source of current.inputSources) { + if (source.hand) active.add(source.handedness); + } + }; + const idle = recorder.adaptTrackingFrame(frame()); + refresh(idle.session); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => refresh(event.session)); + idle.session.addEventListener('inputsourceschange', changed); + const read = (current: XRFrame) => { + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = Array.from(adapted.session.inputSources).find(s => s.handedness === 'left'); + if (!source?.hand || !active.has(source.handedness)) return null; + return adapted.getJointPose?.(source.hand.get('wrist')!, sceneSpace)?.transform.position.x; + }; + recorder.startReplay(recording(timedFrame(0, 1), timedFrame(100, 3)), false, pacing); + return { recorder, session, frame, active, changed, read, idle }; + } + + test.each(['frame', 'time'] as const)( + '%s replay advances when hands were never detected', + pacing => { + const { read, frame, active, changed, session } = benchmark(pacing); + expect(read(frame())).toBeCloseTo(3); + expect(active.has('left')).toBe(true); + expect(read(frame([], pacing === 'time' ? 50 : 1))).toBeCloseTo(pacing === 'time' ? 4 : 5); + expect(changed).toHaveBeenCalledTimes(1); + expect(session.inputSources).toHaveLength(0); + } + ); + + test('live hands appearing and disappearing do not gate or replace recorded hands', () => { + const { read, frame, changed, session, active, recorder } = benchmark('time'); + expect(read(frame())).toBeCloseTo(3); + const live = { + handedness: 'left', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const visible = frame([live], 25); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [live], + removed: [], + }) + ); + expect(read(visible)).toBeCloseTo(3.5); + const absent = frame([], 50); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [live], + }) + ); + expect(active.has('left')).toBe(true); + expect(read(absent)).toBeCloseTo(4); + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(absent); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + }); + + test('recorded tracking loss removes the source, and recorded recovery restores it', () => { + const { recorder, read, frame, active } = benchmark('frame'); + recorder.stopReplay(); + const lost = timedFrame(50, 2); + lost.handJoints.left = { wrist: null }; + recorder.startReplay(recording(timedFrame(0, 1), lost, timedFrame(100, 3)), false, 'frame'); + expect(read(frame())).toBeCloseTo(3); + expect(read(frame())).toBeNull(); + expect(active.size).toBe(0); + expect(read(frame())).toBeCloseTo(5); + }); + + test('forwards native events outside replay and supports listener cleanup', () => { + const { recorder, session, frame, changed, idle } = benchmark('frame'); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame()); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + expect(changed.mock.calls[0][0].session).toBe(idle.session); + idle.session.removeEventListener('inputsourceschange', changed); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + recorder.dispose(); + }); +}); + describe('serialization', () => { test('round-trips version 1 recordings', () => { const recorder = new XRInputRecorder(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 07863cca08..8bf4072485 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -22,6 +22,8 @@ * UI pointers, and CloudXR rendering/reprojection. */ +import { XRReplaySession } from './xrReplaySession'; + type PoseData = { px: number; py: number; @@ -72,7 +74,7 @@ export type Recording = { export type ReplayPacing = 'frame' | 'time'; -type Handedness = 'left' | 'right'; +const IDENTITY_POSE: PoseData = { px: 0, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }; function emptyFrame(timeMs = 0): RecordedFrame { return { @@ -85,7 +87,11 @@ function emptyFrame(timeMs = 0): RecordedFrame { function serializePose(pose: XRPose | null | undefined): SerializedPose { if (!pose) return null; - const { position: p, orientation: o } = pose.transform; + return serializeTransform(pose.transform); +} + +function serializeTransform(transform: XRRigidTransform): PoseData { + const { position: p, orientation: o } = transform; return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; } @@ -140,6 +146,7 @@ function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = } function makePose(pose: PoseData): XRPose { + // WebXR permits null velocities; @types/webxr currently excludes null. return { transform: new XRRigidTransform( { x: pose.px, y: pose.py, z: pose.pz, w: 1 }, @@ -178,16 +185,6 @@ function bindWebXRMember(target: object, property: PropertyKey): unknown { return typeof value === 'function' ? value.bind(target) : value; } -function proxyInputSource(source: XRInputSource, gamepad: SerializedGamepad | null): XRInputSource { - const replayGamepad = gamepad ? makeGamepad(gamepad) : null; - return new Proxy(source, { - get(target, property) { - if (property === 'gamepad') return replayGamepad; - return bindWebXRMember(target, property); - }, - }); -} - function lerp(from: number, to: number, alpha: number): number { return from + (to - from) * alpha; } @@ -377,16 +374,7 @@ function transformByPose(pose: T, baseFromScene: PoseData): /** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - const { position: p, orientation: q } = baseFromScene; - return transformByPose(pose, { - px: p.x, - py: p.y, - pz: p.z, - ox: q.x, - oy: q.y, - oz: q.z, - ow: q.w, - }); + return transformByPose(pose, serializeTransform(baseFromScene)); } function inversePose(pose: PoseData): PoseData { @@ -444,18 +432,6 @@ function sceneAlignment(recordedViewer: PoseData, currentViewer: PoseData): Pose return transformByPose(inversePose(recordedViewer), currentViewer); } -type RecordingContext = { - session: XRSession; - referenceSpace: XRReferenceSpace; - referenceSpaceEpoch: number; -}; - -type CachedAlignment = { - referenceSpace: XRReferenceSpace; - referenceSpaceEpoch: number; - transform: PoseData | null; -}; - /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -482,16 +458,19 @@ export class XRInputRecorder { private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; private _recordingCalibration: Recording['calibration']; - private _recordingContext: RecordingContext | null = null; - private _recordingContexts = new WeakMap(); - private _alignmentCache = new WeakMap>(); + private _recordingAlignment: PoseData | null = null; + private _alignmentCache = new WeakMap(); private _replayRecording: Recording | null = null; - private _replaySourceContext: RecordingContext | null = null; private _replaySceneAlignment: PoseData | null = null; - private _replayAlignmentReady = false; + private _calibrationRequested = false; private _observedSession: XRSession | null = null; private _observedReferenceSpace: XRReferenceSpace | null = null; - private _referenceSpaceEpochs = new WeakMap(); + // Cached alignments use the origin observed at session/reference-space entry. + private _currentFromInitial: PoseData = IDENTITY_POSE; + private _recordingInterrupted = false; + private _trackingSession: XRReplaySession | null = null; + private _recordedHandSources: XRInputSource[] = []; + private _sourceProxies = new WeakMap(); get mode() { return this._mode; @@ -509,6 +488,25 @@ export class XRInputRecorder { return this._currentFrame; } + get replayNeedsCalibration(): boolean { + return ( + this._mode === 'replaying' && + !!this._replayRecording?.calibration && + !this._alignmentCache.has(this._replayRecording) + ); + } + + get recordingInterrupted(): boolean { + return this._recordingInterrupted; + } + + /** Call after the operator returns to the recording's physical starting pose. */ + calibrateReplay(): void { + if (this.replayNeedsCalibration && this._observedReferenceSpace) { + this._calibrationRequested = true; + } + } + startRecording(): void { this._assertIdle(); this._frames = []; @@ -516,7 +514,8 @@ export class XRInputRecorder { this._recordingStartTime = null; this._recordedAt = Date.now(); this._recordingCalibration = undefined; - this._recordingContext = null; + this._recordingAlignment = null; + this._recordingInterrupted = false; this._mode = 'recording'; } @@ -529,6 +528,31 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); this._replayFrames = recording.frames; + this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { + const names = new Set(); + for (const frame of recording.frames) { + for (const name of Object.keys(frame.handJoints[handedness])) names.add(name); + } + if (!names.size) return []; + const hand = new Map(); + for (const name of names) { + hand.set( + name as XRHandJoint, + Object.assign(new EventTarget(), { jointName: name as XRHandJoint }) + ); + } + return [ + { + handedness, + // @types/webxr still requires obsolete numeric constants on XRHand. + hand: hand as XRHand, + targetRayMode: 'tracked-pointer' as const, + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: ['generic-hand-select'], + }, + ]; + }); this._replayIndex = 0; this._loopReplay = loop; this._replayPacing = pacing; @@ -536,9 +560,8 @@ export class XRInputRecorder { this._lastReplayDisplayTime = null; this._currentFrame = null; this._replayRecording = recording; - this._replaySourceContext = this._recordingContexts.get(recording) ?? null; this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._mode = 'replaying'; } @@ -547,9 +570,9 @@ export class XRInputRecorder { this._currentFrame = null; this._lastReplayDisplayTime = null; this._replayRecording = null; - this._replaySourceContext = null; + this._recordedHandSources = []; this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._mode = 'idle'; } @@ -582,11 +605,7 @@ export class XRInputRecorder { if (!this._recordingCalibration) { this._recordingCalibration = captureViewerCalibration(frame, sceneReferenceSpace); if (!this._recordingCalibration) return; - this._recordingContext = { - session: frame.session, - referenceSpace: sceneReferenceSpace, - referenceSpaceEpoch: this._referenceSpaceEpoch(frame.session), - }; + this._recordingAlignment = inversePose(this._currentFromInitial); } const now = frameTimestampMs(frame); this._recordingStartTime ??= now; @@ -602,6 +621,7 @@ export class XRInputRecorder { } if (!sceneReferenceSpace || !this._prepareReplayAlignment(frame, sceneReferenceSpace)) { + this._lastReplayDisplayTime = null; this._currentFrame = null; return; } @@ -641,18 +661,26 @@ export class XRInputRecorder { * objects are modified, and callers retain the original frame for rendering. */ adaptTrackingFrame = (frame: XRFrame): XRFrame => { - const replay = this._currentFrame; - if (this._mode !== 'replaying' || !replay) return frame; + if (this._trackingSession?.nativeSession !== frame.session) { + this._trackingSession?.dispose(); + this._trackingSession = new XRReplaySession(frame.session); + } + const replaying = this._mode === 'replaying'; + this._trackingSession.setInputSources( + replaying ? this._replayInputSources(frame.session) : null + ); + const session = this._trackingSession.session; + // Waiting for calibration must not silently substitute live robot inputs. + const replay = this._currentFrame ?? emptyFrame(0); - const session = this._proxySession(frame.session, replay); return new Proxy(frame, { get: (target, property) => { if (property === 'session') return session; - if (property === 'getPose') { + if (replaying && property === 'getPose') { return (space: XRSpace, baseSpace: XRSpace) => this._replayPose(target, replay, space, baseSpace); } - if (property === 'getJointPose') { + if (replaying && property === 'getJointPose') { return (joint: XRJointSpace, baseSpace: XRSpace) => this._replayJoint(target, replay, joint, baseSpace); } @@ -708,8 +736,8 @@ export class XRInputRecorder { calibration: this._recordingCalibration, frames: [...this._frames], }; - if (this._recordingContext) { - this._recordingContexts.set(recording, this._recordingContext); + if (this._recordingAlignment) { + this._alignmentCache.set(recording, this._recordingAlignment); } return recording; } @@ -720,10 +748,6 @@ export class XRInputRecorder { } } - private _referenceSpaceEpoch(session: XRSession): number { - return this._referenceSpaceEpochs.get(session) ?? 0; - } - private _observeReferenceSpace( session: XRSession, referenceSpace: XRReferenceSpace | null @@ -733,81 +757,115 @@ export class XRInputRecorder { } this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); - if (session === this._observedSession && this._observedReferenceSpace !== null) { - this._referenceSpaceEpochs.set(session, this._referenceSpaceEpoch(session) + 1); - } else if (!this._referenceSpaceEpochs.has(session)) { - this._referenceSpaceEpochs.set(session, 0); - } - + this._interruptRecording(); + this._invalidateAlignments(); this._observedSession = session; this._observedReferenceSpace = referenceSpace; this._sceneReferenceSpace = referenceSpace; referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); - this._replayAlignmentReady = false; } - private _onReferenceSpaceReset = (): void => { - if (!this._observedSession) return; - this._referenceSpaceEpochs.set( - this._observedSession, - this._referenceSpaceEpoch(this._observedSession) + 1 - ); + private _interruptRecording(): void { + if (this._mode === 'recording' && this._frames.length > 0) { + // One recording must not mix coordinates from different reference-space origins. + this.stopRecording(); + this._recordingInterrupted = true; + } + } + + private _invalidateAlignments(): void { + this._alignmentCache = new WeakMap(); + this._recordingAlignment = null; + this._currentFromInitial = IDENTITY_POSE; + this._replaySceneAlignment = null; + this._calibrationRequested = false; + this._lastReplayDisplayTime = null; + this._currentFrame = null; + } + + private _onReferenceSpaceReset = (event: XRReferenceSpaceEvent): void => { + this._interruptRecording(); + if (!event.transform) { + this._invalidateAlignments(); + return; + } + + // The event gives oldScene <- newScene. Preserve world placement using its + // inverse; the current viewer pose includes operator motion, not just drift. + const oldFromNew = serializeTransform(event.transform); + this._currentFromInitial = transformByPose(this._currentFromInitial, inversePose(oldFromNew)); this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._currentFrame = null; }; private _prepareReplayAlignment(frame: XRFrame, referenceSpace: XRReferenceSpace): boolean { - if (this._replayAlignmentReady) return true; + if (this._replaySceneAlignment) return true; const recording = this._replayRecording; if (!recording) return false; - const referenceSpaceEpoch = this._referenceSpaceEpoch(frame.session); - let sessionCache = this._alignmentCache.get(recording); - const cached = sessionCache?.get(frame.session); - if ( - cached && - cached.referenceSpace === referenceSpace && - cached.referenceSpaceEpoch === referenceSpaceEpoch - ) { - this._replaySceneAlignment = cached.transform; - this._replayAlignmentReady = true; - return true; - } - - let transform: PoseData | null = null; - const source = this._replaySourceContext; - const sameReferenceSpace = - source?.session === frame.session && - source.referenceSpace === referenceSpace && - source.referenceSpaceEpoch === referenceSpaceEpoch; - if (!sameReferenceSpace && recording.calibration) { - const currentCalibration = captureViewerCalibration(frame, referenceSpace); - if (!currentCalibration) return false; - transform = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + let initialFromRecorded = this._alignmentCache.get(recording); + if (!initialFromRecorded) { + let currentFromRecorded = IDENTITY_POSE; + if (recording.calibration) { + if (!this._calibrationRequested) return false; + const currentCalibration = captureViewerCalibration(frame, referenceSpace); + if (!currentCalibration) return false; + currentFromRecorded = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + } + initialFromRecorded = transformByPose( + currentFromRecorded, + inversePose(this._currentFromInitial) + ); + this._alignmentCache.set(recording, initialFromRecorded); + this._calibrationRequested = false; } - sessionCache ??= new WeakMap(); - sessionCache.set(frame.session, { referenceSpace, referenceSpaceEpoch, transform }); - this._alignmentCache.set(recording, sessionCache); - this._replaySceneAlignment = transform; - this._replayAlignmentReady = true; + this._replaySceneAlignment = transformByPose(initialFromRecorded, this._currentFromInitial); return true; } - private _proxySession(session: XRSession, replay: RecordedFrame): XRSession { - const inputSources = Array.from(session.inputSources, source => { - const hand = source.handedness; - return hand === 'left' || hand === 'right' - ? proxyInputSource(source, replay.gamepads[hand]) - : source; - }); + dispose(): void { + this._trackingSession?.dispose(); + this._trackingSession = null; + this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._observedReferenceSpace = null; + this._observedSession = null; + this._invalidateAlignments(); + } - return new Proxy(session, { - get(target, property) { - if (property === 'inputSources') return inputSources; - return bindWebXRMember(target, property); - }, + private _replayInputSources(session: XRSession): XRInputSource[] { + const recordedHands = new Set(this._recordedHandSources.map(source => source.handedness)); + const sources = [ + ...this._recordedHandSources.filter(source => { + const hand = source.handedness as 'left' | 'right'; + return Object.values(this._currentFrame?.handJoints[hand] ?? {}).some(Boolean); + }), + ...Array.from(session.inputSources).filter(source => !recordedHands.has(source.handedness)), + ]; + return sources.map(source => { + let proxy = this._sourceProxies.get(source); + if (!proxy) { + let lastGamepad: SerializedGamepad | null | undefined; + let replayGamepad: Gamepad | null = null; + proxy = new Proxy(source, { + get: (target, property) => { + const hand = target.handedness; + if (property === 'gamepad' && (hand === 'left' || hand === 'right')) { + const gamepad = this._currentFrame?.gamepads[hand]; + // WebXR gamepad identity is stable for repeated reads of one frame. + if (gamepad !== lastGamepad) { + replayGamepad = gamepad ? makeGamepad(gamepad) : null; + lastGamepad = gamepad; + } + return replayGamepad; + } + return bindWebXRMember(target, property); + }, + }); + this._sourceProxies.set(source, proxy); + } + return proxy; }); } @@ -817,7 +875,7 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { @@ -836,7 +894,7 @@ export class XRInputRecorder { joint: XRJointSpace, baseSpace: XRSpace ): XRJointPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { const hand = source.handedness; if ((hand !== 'left' && hand !== 'right') || !source.hand) continue; for (const [name, candidate] of source.hand.entries()) { @@ -864,10 +922,8 @@ export class XRInputRecorder { baseSpace: XRSpace ): T | null { if (!pose || !this._sceneReferenceSpace) return null; - if (!this._replayAlignmentReady) return null; - const currentScenePose = this._replaySceneAlignment - ? transformByPose(pose, this._replaySceneAlignment) - : pose; + if (!this._replaySceneAlignment) return null; + const currentScenePose = transformByPose(pose, this._replaySceneAlignment); if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); return relation ? transformFromScene(currentScenePose, relation.transform) : null; diff --git a/deps/cloudxr/webxr_client/src/xrReplaySession.ts b/deps/cloudxr/webxr_client/src/xrReplaySession.ts new file mode 100644 index 0000000000..b375c45b96 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/xrReplaySession.ts @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** CloudXR's input-source events must describe the same sources as its tracking frame. */ +export class XRReplaySession { + private _sources: XRInputSource[] | null = null; + private readonly _events = new EventTarget(); + readonly session: XRSession; + + constructor(readonly nativeSession: XRSession) { + this.session = new Proxy(nativeSession, { + get: (target, property) => { + if (property === 'inputSources') return this._sources ?? target.inputSources; + if (property === 'addEventListener' || property === 'removeEventListener') { + return ( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions + ) => { + const owner = type === 'inputsourceschange' ? this._events : target; + owner[property](type, listener, options); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + nativeSession.addEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + setInputSources(sources: XRInputSource[] | null): void { + const previous = Array.from(this._sources ?? this.nativeSession.inputSources); + this._sources = sources; + const next = Array.from(sources ?? this.nativeSession.inputSources); + const added = next.filter(source => !previous.includes(source)); + const removed = previous.filter(source => !next.includes(source)); + if (added.length || removed.length) this._dispatch(added, removed); + } + + dispose(): void { + this.nativeSession.removeEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + private _onNativeSourcesChange = (event: XRInputSourcesChangeEvent): void => { + // Recorded sources do not disappear when the browser loses sight of a hand. + if (this._sources === null) this._dispatch(Array.from(event.added), Array.from(event.removed)); + }; + + private _dispatch(added: XRInputSource[], removed: XRInputSource[]): void { + this._events.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session: this.session, added, removed }) + ); + } +} From 3b09d494d1cdfab903a5cce28be7f7dab1d381dd Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Fri, 11 Sep 2026 20:26:58 -0700 Subject: [PATCH 03/12] fix(webxr): replay controllers without live hardware Signed-off-by: Yanzi Zhu --- .../webxr_client/src/xrInputRecorder.test.ts | 195 ++++++++++++++++++ .../webxr_client/src/xrInputRecorder.ts | 104 ++++++++-- .../webxr_client/src/xrReplaySession.ts | 2 +- 3 files changed, 288 insertions(+), 13 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index 551d32c506..f43fed9871 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -273,6 +273,7 @@ describe('lifecycle and frame advancement', () => { recorder.beginFrame(makeFrame(), sceneSpace, true, true); expect(recorder.currentFrame).toEqual({ timeMs: 0, + controllerProfiles: { left: null, right: null }, poses: { leftGrip: null, leftAim: null, rightGrip: null, rightAim: null }, gamepads: { left: null, right: null }, handJoints: { left: {}, right: {} }, @@ -851,6 +852,200 @@ describe('recorded hands without live tracking', () => { }); }); +describe('controller replay without live hardware', () => { + function controllerFrame(timeMs: number, x: number, profiles?: string[]): RecordedFrame { + const sample = timedFrame(timeMs, x); + sample.handJoints.left = {}; + if (profiles) sample.controllerProfiles = { left: profiles, right: null }; + return sample; + } + + test.each(['time', 'frame'] as const)( + 'replays poses and controls without live sources (%s)', + pacing => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const frame = (time: number) => makeFrame([], undefined, undefined, time, pose(0), session); + const scoped = recorder.adaptTrackingFrame(frame(0)).session; + const active = new Set(); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => { + active.clear(); + for (const source of event.session.inputSources) { + if (source.gamepad && !source.hand) active.add(source.handedness); + } + }); + scoped.addEventListener('inputsourceschange', changed); + recorder.startReplay( + recording(controllerFrame(0, 0), controllerFrame(100, 1)), + false, + pacing + ); + let firstSource: XRInputSource | undefined; + for (const [time, expected] of [ + [0, 0], + [50, pacing === 'time' ? 0.5 : 1], + [100, 1], + ]) { + const current = frame(time); + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = adapted.session.inputSources[0]; + firstSource ??= source; + expect(source).toBe(firstSource); + expect(active.has('left')).toBe(true); + expect(source.hand).toBeUndefined(); + expect(source.profiles).toEqual(['generic-trigger-squeeze-thumbstick']); + expect(adapted.getPose(source.gripSpace!, sceneSpace)?.transform.position.x).toBe(expected); + expect(adapted.getPose(source.targetRaySpace, sceneSpace)?.transform.position.x).toBe( + expected + 1 + ); + expect(source.gamepad?.axes).toEqual([expected]); + expect(source.gamepad?.buttons[0]).toEqual({ + value: expected, + pressed: true, + touched: true, + }); + expect(source.gamepad).toBe(source.gamepad); + expect(session.inputSources).toHaveLength(0); + } + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame(100)); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + recorder.dispose(); + } + ); + + test('keeps recorded profiles and controls through live controller connection changes', () => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const changed = jest.fn(); + const initial = makeFrame([], undefined, undefined, 0, pose(0), session); + recorder.adaptTrackingFrame(initial).session.addEventListener('inputsourceschange', changed); + const profiles = ['meta-quest-touch-plus', 'oculus-touch-v3']; + recorder.startReplay( + recording(controllerFrame(0, 1, profiles), controllerFrame(100, 3, profiles)), + false + ); + recorder.beginFrame(initial, sceneSpace); + const source = recorder.adaptTrackingFrame(initial).session.inputSources[0]; + const live = { + handedness: 'left', + profiles: ['pico-4u'], + gamepad: gamepad(99), + } as XRInputSource; + for (const [time, inputs] of [ + [50, [live]], + [100, []], + ] as const) { + const frame = makeFrame([...inputs], undefined, undefined, time, pose(0), session); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session, added: inputs, removed: [] }) + ); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources).toEqual([source]); + expect(source.profiles).toEqual(profiles); + expect(source.gamepad?.axes[0]).toBe(time === 50 ? 2 : 3); + } + expect(changed).toHaveBeenCalledTimes(1); + }); + + test('round-trips profiles and replays a controller alongside a recorded hand', () => { + const recorder = new XRInputRecorder(); + const profiles = ['pico-4u', 'oculus-touch-v2']; + const controller = { + handedness: 'left', + profiles, + gamepad: gamepad(0.75), + gripSpace: {}, + targetRaySpace: {}, + } as XRInputSource; + const hand = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + recorder.startRecording(); + recorder.beginFrame( + makeFrame( + [controller, hand], + () => pose(2), + () => jointPose(3) + ), + sceneSpace + ); + recorder.stopRecording(); + const saved = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(saved.frames[0].controllerProfiles).toEqual({ left: profiles, right: null }); + delete saved.calibration; + recorder.startReplay(saved); + const frame = makeFrame(); + recorder.beginFrame(frame, sceneSpace); + const sources = Array.from(recorder.adaptTrackingFrame(frame).session.inputSources); + expect(sources.find(source => source.handedness === 'left')?.profiles).toEqual(profiles); + expect(sources.find(source => source.handedness === 'right')?.hand).toBeDefined(); + }); + + test('preserves gaps and device switches at recorded timestamps', () => { + const recorder = new XRInputRecorder(); + const first = controllerFrame(0, 1, ['meta-quest-touch-plus']); + const other = controllerFrame(100, 9, ['pico-4u']); + const lost = controllerFrame(200, 10); + lost.gamepads.left = null; + lost.poses.leftGrip = lost.poses.leftAim = null; + recorder.startReplay( + recording(first, other, lost, timedFrame(300, 20), controllerFrame(400, 30)), + false + ); + const sources: XRInputSource[] = []; + for (const time of [0, 50, 100, 200, 300, 400]) { + const frame = makeFrame([], undefined, undefined, time); + recorder.beginFrame(frame, sceneSpace); + const adapted = recorder.adaptTrackingFrame(frame); + const source = adapted.session.inputSources[0]; + if (time === 200) { + expect(adapted.session.inputSources).toHaveLength(0); + } else { + sources.push(source); + expect(adapted.session.inputSources).toHaveLength(1); + expect(!!source.hand).toBe(time === 300); + expect(source.gamepad?.axes[0]).toBe( + time < 100 ? 1 : time === 100 ? 9 : time === 300 ? 20 : 30 + ); + } + } + expect(sources[0]).toBe(sources[1]); + expect(sources[2]).not.toBe(sources[0]); + expect(sources[2].profiles).toEqual(['pico-4u']); + }); + + test('waits for calibration and restores the live controller on stop', () => { + const recorder = new XRInputRecorder(); + const live = { handedness: 'left', gamepad: gamepad(99) } as XRInputSource; + const frame = makeFrame([live]); + recorder.startReplay(calibratedRecording(0, controllerFrame(0, 1))); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources).toHaveLength(0); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources[0].gamepad?.axes).toEqual([1]); + recorder.stopReplay(); + expect(recorder.adaptTrackingFrame(frame).session.inputSources[0]).toBe(live); + }); + + test.each([null, {}, { left: 'quest', right: null }, { left: [42], right: null }])( + 'rejects malformed controller profiles (%j)', + controllerProfiles => { + expect(() => + XRInputRecorder.importJSON( + JSON.stringify({ version: 1, frames: [{ ...frameData(), controllerProfiles }] }) + ) + ).toThrow('controllerProfiles is invalid'); + } + ); +}); + describe('serialization', () => { test('round-trips version 1 recordings', () => { const recorder = new XRInputRecorder(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 8bf4072485..03dea38ed8 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -56,6 +56,8 @@ export type RecordedFrame = { rightAim: SerializedPose; }; gamepads: Hands; + /** Controller profiles select SDK button mappings; absent in legacy recordings. */ + controllerProfiles?: Hands; /** Joint poses keyed by XRHandJoint name, also in scene space. */ handJoints: Hands; }; @@ -122,12 +124,16 @@ function serializeGamepad(gamepad: Gamepad | null | undefined): SerializedGamepa function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = 0): RecordedFrame { const captured = emptyFrame(timeMs); + captured.controllerProfiles = { left: null, right: null }; for (const source of frame.session.inputSources) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; captured.gamepads[hand] = serializeGamepad(source.gamepad); + if (!source.hand && source.gamepad) { + captured.controllerProfiles[hand] = Array.from(source.profiles ?? []); + } captured.poses[`${hand}Grip`] = source.gripSpace ? serializePose(frame.getPose(source.gripSpace, referenceSpace)) : null; @@ -288,24 +294,45 @@ function interpolateJoints(from: JointPoses, to: JointPoses, alpha: number): Joi return result; } +function recordedControllerProfiles(frame: RecordedFrame, hand: 'left' | 'right'): string[] | null { + if (!frame.gamepads[hand]) return null; + if (frame.controllerProfiles !== undefined) return frame.controllerProfiles[hand]; + // Hand sources can expose gamepads too. Even null joint samples identify a hand. + // Legacy controller recordings have no profile, so use the SDK's generic mapping. + return Object.keys(frame.handJoints[hand]).length ? null : ['generic-trigger-squeeze-thumbstick']; +} + +function controllerKey(hand: string, profiles: string[]): string { + return JSON.stringify([hand, profiles]); +} + function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number): RecordedFrame { const interval = to.timeMs - from.timeMs; const alpha = interval > 0 ? (timeMs - from.timeMs) / interval : 0; + // Device switches are discrete even when two controllers have the same layout. + const sameDevice = (hand: 'left' | 'right') => + JSON.stringify(recordedControllerProfiles(from, hand)) === + JSON.stringify(recordedControllerProfiles(to, hand)); + const leftTo = sameDevice('left') ? to : from; + const rightTo = sameDevice('right') ? to : from; return { timeMs, + ...(from.controllerProfiles !== undefined + ? { controllerProfiles: from.controllerProfiles } + : {}), poses: { - leftGrip: interpolateOptionalPose(from.poses.leftGrip, to.poses.leftGrip, alpha), - leftAim: interpolateOptionalPose(from.poses.leftAim, to.poses.leftAim, alpha), - rightGrip: interpolateOptionalPose(from.poses.rightGrip, to.poses.rightGrip, alpha), - rightAim: interpolateOptionalPose(from.poses.rightAim, to.poses.rightAim, alpha), + leftGrip: interpolateOptionalPose(from.poses.leftGrip, leftTo.poses.leftGrip, alpha), + leftAim: interpolateOptionalPose(from.poses.leftAim, leftTo.poses.leftAim, alpha), + rightGrip: interpolateOptionalPose(from.poses.rightGrip, rightTo.poses.rightGrip, alpha), + rightAim: interpolateOptionalPose(from.poses.rightAim, rightTo.poses.rightAim, alpha), }, gamepads: { - left: interpolateGamepad(from.gamepads.left, to.gamepads.left, alpha), - right: interpolateGamepad(from.gamepads.right, to.gamepads.right, alpha), + left: interpolateGamepad(from.gamepads.left, leftTo.gamepads.left, alpha), + right: interpolateGamepad(from.gamepads.right, rightTo.gamepads.right, alpha), }, handJoints: { - left: interpolateJoints(from.handJoints.left, to.handJoints.left, alpha), - right: interpolateJoints(from.handJoints.right, to.handJoints.right, alpha), + left: interpolateJoints(from.handJoints.left, leftTo.handJoints.left, alpha), + right: interpolateJoints(from.handJoints.right, rightTo.handJoints.right, alpha), }, }; } @@ -470,6 +497,7 @@ export class XRInputRecorder { private _recordingInterrupted = false; private _trackingSession: XRReplaySession | null = null; private _recordedHandSources: XRInputSource[] = []; + private _recordedControllerSources = new Map(); private _sourceProxies = new WeakMap(); get mode() { @@ -528,6 +556,24 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); this._replayFrames = recording.frames; + this._recordedControllerSources.clear(); + // Replay owns device presence as well as poses, for both hands and controllers. + for (const frame of recording.frames) { + for (const handedness of ['left', 'right'] as const) { + const profiles = recordedControllerProfiles(frame, handedness); + if (!profiles) continue; + const key = controllerKey(handedness, profiles); + if (!this._recordedControllerSources.has(key)) { + this._recordedControllerSources.set(key, { + handedness, + targetRayMode: 'tracked-pointer', + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: [...profiles], + }); + } + } + } this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { const names = new Set(); for (const frame of recording.frames) { @@ -571,6 +617,7 @@ export class XRInputRecorder { this._lastReplayDisplayTime = null; this._replayRecording = null; this._recordedHandSources = []; + this._recordedControllerSources.clear(); this._replaySceneAlignment = null; this._calibrationRequested = false; this._mode = 'idle'; @@ -724,6 +771,20 @@ export class XRInputRecorder { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { throw new Error('Malformed recording: frame timeMs must be finite and monotonic'); } + if (frame.controllerProfiles !== undefined) { + const profiles = frame.controllerProfiles; + if ( + !profiles || + !(['left', 'right'] as const).every( + hand => + profiles[hand] === null || + (Array.isArray(profiles[hand]) && + profiles[hand]!.every(profile => typeof profile === 'string')) + ) + ) { + throw new Error('Malformed recording: controllerProfiles is invalid'); + } + } previousTime = frame.timeMs; } return recording; @@ -835,13 +896,28 @@ export class XRInputRecorder { } private _replayInputSources(session: XRSession): XRInputSource[] { - const recordedHands = new Set(this._recordedHandSources.map(source => source.handedness)); + const recordedSources = [ + ...this._recordedHandSources, + ...this._recordedControllerSources.values(), + ]; + const recordedSides = new Set(recordedSources.map(source => source.handedness)); + const replay = this._currentFrame; const sources = [ ...this._recordedHandSources.filter(source => { const hand = source.handedness as 'left' | 'right'; - return Object.values(this._currentFrame?.handJoints[hand] ?? {}).some(Boolean); + return ( + replay && + !recordedControllerProfiles(replay, hand) && + Object.values(replay.handJoints[hand]).some(Boolean) + ); + }), + ...(['left', 'right'] as const).flatMap(hand => { + const profiles = replay && recordedControllerProfiles(replay, hand); + const source = + profiles && this._recordedControllerSources.get(controllerKey(hand, profiles)); + return source ? [source] : []; }), - ...Array.from(session.inputSources).filter(source => !recordedHands.has(source.handedness)), + ...Array.from(session.inputSources).filter(source => !recordedSides.has(source.handedness)), ]; return sources.map(source => { let proxy = this._sourceProxies.get(source); @@ -875,7 +951,11 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { + for (const source of [ + ...this._recordedHandSources, + ...this._recordedControllerSources.values(), + ...frame.session.inputSources, + ]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { diff --git a/deps/cloudxr/webxr_client/src/xrReplaySession.ts b/deps/cloudxr/webxr_client/src/xrReplaySession.ts index b375c45b96..7fd94e8cd6 100644 --- a/deps/cloudxr/webxr_client/src/xrReplaySession.ts +++ b/deps/cloudxr/webxr_client/src/xrReplaySession.ts @@ -56,7 +56,7 @@ export class XRReplaySession { } private _onNativeSourcesChange = (event: XRInputSourcesChangeEvent): void => { - // Recorded sources do not disappear when the browser loses sight of a hand. + // Recorded device presence is independent of live tracking and controller disconnects. if (this._sources === null) this._dispatch(Array.from(event.added), Array.from(event.removed)); }; From dab67865ea9d88d95a0cabe3be786b6df3de525f Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Sat, 12 Sep 2026 09:08:49 -0700 Subject: [PATCH 04/12] fix(webxr): simplify replay sources and align traces Signed-off-by: Yanzi Zhu --- .../src/TraceVisualization.test.ts | 88 ++++++++ .../webxr_client/src/TraceVisualization.tsx | 16 +- .../webxr_client/src/xrInputRecorder.test.ts | 14 ++ .../webxr_client/src/xrInputRecorder.ts | 188 ++++++------------ 4 files changed, 175 insertions(+), 131 deletions(-) create mode 100644 deps/cloudxr/webxr_client/src/TraceVisualization.test.ts diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts new file mode 100644 index 0000000000..9f84dcaa09 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ReactElement } from 'react'; +import { type Points, Vector3 } from 'three'; + +import { TraceVisualization } from './TraceVisualization'; +import type { RecordedFrame } from './xrInputRecorder'; + +let mockFrame: () => void; +const sample: RecordedFrame = { + timeMs: 0, + poses: { + leftGrip: { px: 1, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, + leftAim: null, + rightGrip: null, + rightAim: null, + }, + gamepads: { left: null, right: null }, + handJoints: { left: {}, right: {} }, +}; +const mockRecorder = { + currentFrame: sample as RecordedFrame | null, + replaySceneAlignment: { px: 10, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, +}; + +jest.mock('@react-three/fiber', () => ({ + useFrame: (callback: () => void) => { + mockFrame = callback; + }, +})); +jest.mock('./RecorderContext', () => ({ + useRecorder: () => ({ recorder: mockRecorder, mode: 'replaying' }), +})); +jest.mock('react', () => ({ + ...jest.requireActual('react'), + useRef: () => ({ current: null }), + useEffect: jest.fn(), +})); + +test('places the whole replay trail in calibrated scene space and hides it while paused', () => { + const tree = TraceVisualization({ showTrace: true }); + const points = (tree.props.children as ReactElement<{ object: Points }>[])[0].props.object; + const worldPoint = (index: number) => { + points.updateMatrixWorld(); + return new Vector3() + .fromBufferAttribute(points.geometry.getAttribute('position'), index) + .applyMatrix4(points.matrixWorld); + }; + mockFrame(); + expect(worldPoint(0).x).toBeCloseTo(11); + + // A reset changes the placement of earlier samples as well as the newest one. + mockRecorder.replaySceneAlignment = { + px: 100, + py: 0, + pz: 0, + ox: 0, + oy: Math.SQRT1_2, + oz: 0, + ow: Math.SQRT1_2, + }; + mockFrame(); + for (const index of [0, 1]) { + expect(worldPoint(index).x).toBeCloseTo(100); + expect(worldPoint(index).z).toBeCloseTo(-1); + } + mockRecorder.currentFrame = null; + mockFrame(); + expect(points.visible).toBe(false); + for (const { props } of tree.props.children as ReactElement<{ object: Points }>[]) { + props.object.geometry.dispose(); + } +}); diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx index 9d5d7f38f7..1df584ce25 100644 --- a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx @@ -128,15 +128,25 @@ export function TraceVisualization({ showTrace }: { showTrace: boolean }) { ); useFrame(() => { - if (!showTrace) { + const frame = recorder.currentFrame; + if (!showTrace || !frame) { channels.forEach(channel => { channel.points.visible = false; }); return; } - const frame = recorder.currentFrame; - if (!frame) return; + // Transform the whole trail so origin resets also move its earlier samples. + const alignment = recorder.replaySceneAlignment; + for (const { points } of channels) { + points.position.set(alignment?.px ?? 0, alignment?.py ?? 0, alignment?.pz ?? 0); + points.quaternion.set( + alignment?.ox ?? 0, + alignment?.oy ?? 0, + alignment?.oz ?? 0, + alignment?.ow ?? 1 + ); + } updateChannel(channels[0], frame.poses.leftGrip); updateChannel(channels[1], frame.poses.rightGrip); updateChannel(channels[2], frame.handJoints.left.wrist); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index f43fed9871..fe85337f77 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -814,6 +814,20 @@ describe('recorded hands without live tracking', () => { expect(changed).toHaveBeenCalledTimes(2); }); + test('does not expose an unrecorded live hand to the SDK during replay', () => { + const { recorder, read, frame, active } = benchmark('frame'); + const liveRight = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const current = frame([liveRight]); + expect(read(current)).toBeCloseTo(3); + expect([...active]).toEqual(['left']); + expect(recorder.adaptTrackingFrame(current).session.inputSources).toHaveLength(1); + expect(current.session.inputSources).toEqual([liveRight]); + }); + test('recorded tracking loss removes the source, and recorded recovery restores it', () => { const { recorder, read, frame, active } = benchmark('frame'); recorder.stopReplay(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 03dea38ed8..c06c1b15f6 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -302,10 +302,6 @@ function recordedControllerProfiles(frame: RecordedFrame, hand: 'left' | 'right' return Object.keys(frame.handJoints[hand]).length ? null : ['generic-trigger-squeeze-thumbstick']; } -function controllerKey(hand: string, profiles: string[]): string { - return JSON.stringify([hand, profiles]); -} - function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number): RecordedFrame { const interval = to.timeMs - from.timeMs; const alpha = interval > 0 ? (timeMs - from.timeMs) / interval : 0; @@ -399,11 +395,6 @@ function transformByPose(pose: T, baseFromScene: PoseData): }; } -/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ -function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - return transformByPose(pose, serializeTransform(baseFromScene)); -} - function inversePose(pose: PoseData): PoseData { const magnitudeSquared = pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; @@ -474,7 +465,6 @@ function frameTimestampMs(frame: XRFrame): number { export class XRInputRecorder { private _mode: 'idle' | 'recording' | 'replaying' = 'idle'; private _frames: RecordedFrame[] = []; - private _replayFrames: RecordedFrame[] = []; private _replayIndex = 0; private _loopReplay = true; private _replayPacing: ReplayPacing = 'time'; @@ -491,14 +481,11 @@ export class XRInputRecorder { private _replaySceneAlignment: PoseData | null = null; private _calibrationRequested = false; private _observedSession: XRSession | null = null; - private _observedReferenceSpace: XRReferenceSpace | null = null; // Cached alignments use the origin observed at session/reference-space entry. private _currentFromInitial: PoseData = IDENTITY_POSE; private _recordingInterrupted = false; private _trackingSession: XRReplaySession | null = null; - private _recordedHandSources: XRInputSource[] = []; - private _recordedControllerSources = new Map(); - private _sourceProxies = new WeakMap(); + private _replaySources = new Map(); get mode() { return this._mode; @@ -516,6 +503,11 @@ export class XRInputRecorder { return this._currentFrame; } + /** Places recording-space traces in the same scene coordinates as SDK replay. */ + get replaySceneAlignment(): Readonly | null { + return this._replaySceneAlignment; + } + get replayNeedsCalibration(): boolean { return ( this._mode === 'replaying' && @@ -530,7 +522,7 @@ export class XRInputRecorder { /** Call after the operator returns to the recording's physical starting pose. */ calibrateReplay(): void { - if (this.replayNeedsCalibration && this._observedReferenceSpace) { + if (this.replayNeedsCalibration && this._sceneReferenceSpace) { this._calibrationRequested = true; } } @@ -555,50 +547,7 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); - this._replayFrames = recording.frames; - this._recordedControllerSources.clear(); - // Replay owns device presence as well as poses, for both hands and controllers. - for (const frame of recording.frames) { - for (const handedness of ['left', 'right'] as const) { - const profiles = recordedControllerProfiles(frame, handedness); - if (!profiles) continue; - const key = controllerKey(handedness, profiles); - if (!this._recordedControllerSources.has(key)) { - this._recordedControllerSources.set(key, { - handedness, - targetRayMode: 'tracked-pointer', - targetRaySpace: new EventTarget(), - gripSpace: new EventTarget(), - profiles: [...profiles], - }); - } - } - } - this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { - const names = new Set(); - for (const frame of recording.frames) { - for (const name of Object.keys(frame.handJoints[handedness])) names.add(name); - } - if (!names.size) return []; - const hand = new Map(); - for (const name of names) { - hand.set( - name as XRHandJoint, - Object.assign(new EventTarget(), { jointName: name as XRHandJoint }) - ); - } - return [ - { - handedness, - // @types/webxr still requires obsolete numeric constants on XRHand. - hand: hand as XRHand, - targetRayMode: 'tracked-pointer' as const, - targetRaySpace: new EventTarget(), - gripSpace: new EventTarget(), - profiles: ['generic-hand-select'], - }, - ]; - }); + this._replaySources.clear(); this._replayIndex = 0; this._loopReplay = loop; this._replayPacing = pacing; @@ -616,8 +565,7 @@ export class XRInputRecorder { this._currentFrame = null; this._lastReplayDisplayTime = null; this._replayRecording = null; - this._recordedHandSources = []; - this._recordedControllerSources.clear(); + this._replaySources.clear(); this._replaySceneAlignment = null; this._calibrationRequested = false; this._mode = 'idle'; @@ -662,7 +610,8 @@ export class XRInputRecorder { return; } - if (this._replayFrames.length === 0) { + const frames = this._replayRecording!.frames; + if (frames.length === 0) { this._currentFrame = null; return; } @@ -678,8 +627,8 @@ export class XRInputRecorder { return; } - this._currentFrame = this._replayFrames[this._replayIndex]; - if (this._replayIndex < this._replayFrames.length - 1) { + this._currentFrame = frames[this._replayIndex]; + if (this._replayIndex < frames.length - 1) { this._replayIndex++; } else if (this._loopReplay) { this._replayIndex = 0; @@ -692,13 +641,14 @@ export class XRInputRecorder { } this._lastReplayDisplayTime = displayTime; - const firstTime = this._replayFrames[0].timeMs; - const lastTime = this._replayFrames[this._replayFrames.length - 1].timeMs; - const duration = replayDuration(this._replayFrames); + const frames = this._replayRecording!.frames; + const firstTime = frames[0].timeMs; + const lastTime = frames[frames.length - 1].timeMs; + const duration = replayDuration(frames); const elapsed = this._loopReplay && duration > 0 ? this._replayElapsedMs % duration : this._replayElapsedMs; const sampleTime = Math.min(firstTime + elapsed, lastTime); - const sample = sampleAtTime(this._replayFrames, sampleTime); + const sample = sampleAtTime(frames, sampleTime); this._currentFrame = sample.frame; this._replayIndex = sample.index; } @@ -713,9 +663,7 @@ export class XRInputRecorder { this._trackingSession = new XRReplaySession(frame.session); } const replaying = this._mode === 'replaying'; - this._trackingSession.setInputSources( - replaying ? this._replayInputSources(frame.session) : null - ); + this._trackingSession.setInputSources(replaying ? this._replayInputSources() : null); const session = this._trackingSession.session; // Waiting for calibration must not silently substitute live robot inputs. const replay = this._currentFrame ?? emptyFrame(0); @@ -813,15 +761,14 @@ export class XRInputRecorder { session: XRSession, referenceSpace: XRReferenceSpace | null ): void { - if (session === this._observedSession && referenceSpace === this._observedReferenceSpace) { + if (session === this._observedSession && referenceSpace === this._sceneReferenceSpace) { return; } - this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); this._interruptRecording(); this._invalidateAlignments(); this._observedSession = session; - this._observedReferenceSpace = referenceSpace; this._sceneReferenceSpace = referenceSpace; referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); } @@ -889,59 +836,46 @@ export class XRInputRecorder { dispose(): void { this._trackingSession?.dispose(); this._trackingSession = null; - this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); - this._observedReferenceSpace = null; + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._sceneReferenceSpace = null; this._observedSession = null; this._invalidateAlignments(); } - private _replayInputSources(session: XRSession): XRInputSource[] { - const recordedSources = [ - ...this._recordedHandSources, - ...this._recordedControllerSources.values(), - ]; - const recordedSides = new Set(recordedSources.map(source => source.handedness)); + private _replayInputSources(): XRInputSource[] { const replay = this._currentFrame; - const sources = [ - ...this._recordedHandSources.filter(source => { - const hand = source.handedness as 'left' | 'right'; - return ( - replay && - !recordedControllerProfiles(replay, hand) && - Object.values(replay.handJoints[hand]).some(Boolean) - ); - }), - ...(['left', 'right'] as const).flatMap(hand => { - const profiles = replay && recordedControllerProfiles(replay, hand); - const source = - profiles && this._recordedControllerSources.get(controllerKey(hand, profiles)); - return source ? [source] : []; - }), - ...Array.from(session.inputSources).filter(source => !recordedSides.has(source.handedness)), - ]; - return sources.map(source => { - let proxy = this._sourceProxies.get(source); - if (!proxy) { - let lastGamepad: SerializedGamepad | null | undefined; - let replayGamepad: Gamepad | null = null; - proxy = new Proxy(source, { - get: (target, property) => { - const hand = target.handedness; - if (property === 'gamepad' && (hand === 'left' || hand === 'right')) { - const gamepad = this._currentFrame?.gamepads[hand]; - // WebXR gamepad identity is stable for repeated reads of one frame. - if (gamepad !== lastGamepad) { - replayGamepad = gamepad ? makeGamepad(gamepad) : null; - lastGamepad = gamepad; - } - return replayGamepad; - } - return bindWebXRMember(target, property); - }, - }); - this._sourceProxies.set(source, proxy); + if (!replay) return []; + // Only recorded devices belong in the SDK view; live sources can otherwise + // activate a hand/controller that is absent from the recording. + return (['left', 'right'] as const).flatMap(handedness => { + const joints = replay.handJoints[handedness]; + const profiles = recordedControllerProfiles(replay, handedness); + if (!profiles && !Object.values(joints).some(Boolean)) return []; + + const key = JSON.stringify([handedness, profiles]); + let source = this._replaySources.get(key); + if (!source) { + source = { + handedness, + targetRayMode: 'tracked-pointer', + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: profiles ? [...profiles] : ['generic-hand-select'], + // @types/webxr still requires obsolete numeric constants on XRHand. + hand: profiles ? undefined : (new Map() as XRHand), + }; + this._replaySources.set(key, source); } - return proxy; + if (source.hand) { + const hand = source.hand as Map; + for (const name of Object.keys(joints) as XRHandJoint[]) { + if (!hand.has(name)) + hand.set(name, Object.assign(new EventTarget(), { jointName: name })); + } + } + const gamepad = replay.gamepads[handedness]; + Object.assign(source, { gamepad: gamepad ? makeGamepad(gamepad) : null }); + return [source]; }); } @@ -951,11 +885,7 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of [ - ...this._recordedHandSources, - ...this._recordedControllerSources.values(), - ...frame.session.inputSources, - ]) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { @@ -974,7 +904,7 @@ export class XRInputRecorder { joint: XRJointSpace, baseSpace: XRSpace ): XRJointPose | undefined { - for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if ((hand !== 'left' && hand !== 'right') || !source.hand) continue; for (const [name, candidate] of source.hand.entries()) { @@ -1006,6 +936,8 @@ export class XRInputRecorder { const currentScenePose = transformByPose(pose, this._replaySceneAlignment); if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); - return relation ? transformFromScene(currentScenePose, relation.transform) : null; + return relation + ? transformByPose(currentScenePose, serializeTransform(relation.transform)) + : null; } } From 277846ce374d6271bdccb2bd08f9887673df1e51 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Sun, 20 Sep 2026 20:17:00 -0700 Subject: [PATCH 05/12] refactor(webxr): reuse Three.js replay pose math Signed-off-by: Yanzi Zhu --- deps/cloudxr/webxr_client/src/App.tsx | 2 +- .../webxr_client/src/xrInputRecorder.test.ts | 19 +++ .../webxr_client/src/xrInputRecorder.ts | 108 ++++++------------ 3 files changed, 53 insertions(+), 76 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index 717c774ce7..6f53743a3e 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -71,10 +71,10 @@ import { RecorderProvider, useRecorder } from './RecorderContext'; import { SuppressWebGLRendererWhenHeadless } from './SuppressWebGLRendererWhenHeadless'; import { TraceVisualization } from './TraceVisualization'; import { - SystemNotice, formatSystemNotice, formatSystemNoticeBody, isSystemNoticeMessage, + SystemNotice, } from './types/serverMessages'; // Performance metrics signals - raw numeric data backing the in-XR HUD. diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index fe85337f77..b93f3d92cc 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -281,6 +281,25 @@ describe('lifecycle and frame advancement', () => { }); }); +test.each([1, -1])('interpolates the shortest rotation for quaternion sign %s', sign => { + const first = timedFrame(0, 0); + const last = timedFrame(100, 4); + last.poses.leftGrip = { + ...last.poses.leftGrip!, + oy: sign * Math.sin(Math.PI / 3), + ow: sign * Math.cos(Math.PI / 3), + }; + const recorder = new XRInputRecorder(); + recorder.startReplay(recording(first, last), false); + recorder.beginFrame(makeFrame(), sceneSpace); + recorder.beginFrame(makeFrame([], undefined, undefined, 50), sceneSpace); + const result = recorder.currentFrame!.poses.leftGrip!; + expect(result.px).toBeCloseTo(2); + expect(result.oy).toBeCloseTo(Math.sin(Math.PI / 6)); + expect(result.ow).toBeCloseTo(Math.cos(Math.PI / 6)); + expect(Math.hypot(result.ox, result.oy, result.oz, result.ow)).toBeCloseTo(1); +}); + describe('canonical scene-space capture', () => { test('captures grip, aim, gamepad, and joints by WebXR joint name', () => { const grip = {} as XRSpace; diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index c06c1b15f6..46aa35ddb1 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -22,6 +22,8 @@ * UI pointers, and CloudXR rendering/reprojection. */ +import { Quaternion, Vector3 } from 'three'; + import { XRReplaySession } from './xrReplaySession'; type PoseData = { @@ -196,49 +198,18 @@ function lerp(from: number, to: number, alpha: number): number { } function interpolatePose(from: T, to: T, alpha: number): T { - let tox = to.ox; - let toy = to.oy; - let toz = to.oz; - let tow = to.ow; - const dot = from.ox * tox + from.oy * toy + from.oz * toz + from.ow * tow; - if (dot < 0) { - tox = -tox; - toy = -toy; - toz = -toz; - tow = -tow; - } - - const shortestDot = Math.abs(dot); - let fromScale = 1 - alpha; - let toScale = alpha; - if (shortestDot < 0.9995) { - const angle = Math.acos(Math.min(1, shortestDot)); - const sinAngle = Math.sin(angle); - fromScale = Math.sin((1 - alpha) * angle) / sinAngle; - toScale = Math.sin(alpha * angle) / sinAngle; - } - - let ox = from.ox * fromScale + tox * toScale; - let oy = from.oy * fromScale + toy * toScale; - let oz = from.oz * fromScale + toz * toScale; - let ow = from.ow * fromScale + tow * toScale; - const magnitude = Math.hypot(ox, oy, oz, ow); - if (magnitude > 0) { - ox /= magnitude; - oy /= magnitude; - oz /= magnitude; - ow /= magnitude; - } + baseOrientation.set(to.ox, to.oy, to.oz, to.ow); + poseOrientation.set(from.ox, from.oy, from.oz, from.ow).slerp(baseOrientation, alpha).normalize(); return { ...from, px: lerp(from.px, to.px, alpha), py: lerp(from.py, to.py, alpha), pz: lerp(from.pz, to.pz, alpha), - ox, - oy, - oz, - ow, + ox: poseOrientation.x, + oy: poseOrientation.y, + oz: poseOrientation.z, + ow: poseOrientation.w, }; } @@ -313,9 +284,7 @@ function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number const rightTo = sameDevice('right') ? to : from; return { timeMs, - ...(from.controllerProfiles !== undefined - ? { controllerProfiles: from.controllerProfiles } - : {}), + controllerProfiles: from.controllerProfiles, poses: { leftGrip: interpolateOptionalPose(from.poses.leftGrip, leftTo.poses.leftGrip, alpha), leftAim: interpolateOptionalPose(from.poses.leftAim, leftTo.poses.leftAim, alpha), @@ -368,30 +337,25 @@ function replayDuration(frames: RecordedFrame[]): number { return 0; } +// Reused only by synchronous pose math; no per-joint scratch allocations. +const posePosition = new Vector3(); +const poseOrientation = new Quaternion(); +const baseOrientation = new Quaternion(); + /** Apply a baseSpace <- sceneSpace transform to a pose expressed in sceneSpace. */ function transformByPose(pose: T, baseFromScene: PoseData): T { - const p = { x: baseFromScene.px, y: baseFromScene.py, z: baseFromScene.pz }; - const q = { - x: baseFromScene.ox, - y: baseFromScene.oy, - z: baseFromScene.oz, - w: baseFromScene.ow, - }; - - // Rotate the recorded translation by q using v' = v + qw*t + cross(q.xyz, t). - const tx = 2 * (q.y * pose.pz - q.z * pose.py); - const ty = 2 * (q.z * pose.px - q.x * pose.pz); - const tz = 2 * (q.x * pose.py - q.y * pose.px); - + baseOrientation.set(baseFromScene.ox, baseFromScene.oy, baseFromScene.oz, baseFromScene.ow); + posePosition.set(pose.px, pose.py, pose.pz).applyQuaternion(baseOrientation); + poseOrientation.set(pose.ox, pose.oy, pose.oz, pose.ow).premultiply(baseOrientation); return { ...pose, - px: p.x + pose.px + q.w * tx + q.y * tz - q.z * ty, - py: p.y + pose.py + q.w * ty + q.z * tx - q.x * tz, - pz: p.z + pose.pz + q.w * tz + q.x * ty - q.y * tx, - ox: q.w * pose.ox + q.x * pose.ow + q.y * pose.oz - q.z * pose.oy, - oy: q.w * pose.oy - q.x * pose.oz + q.y * pose.ow + q.z * pose.ox, - oz: q.w * pose.oz + q.x * pose.oy - q.y * pose.ox + q.z * pose.ow, - ow: q.w * pose.ow - q.x * pose.ox - q.y * pose.oy - q.z * pose.oz, + px: posePosition.x + baseFromScene.px, + py: posePosition.y + baseFromScene.py, + pz: posePosition.z + baseFromScene.pz, + ox: poseOrientation.x, + oy: poseOrientation.y, + oz: poseOrientation.z, + ow: poseOrientation.w, }; } @@ -399,22 +363,16 @@ function inversePose(pose: PoseData): PoseData { const magnitudeSquared = pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; const scale = magnitudeSquared > 0 ? 1 / magnitudeSquared : 1; - const inverseOrientation = { - ox: -pose.ox * scale, - oy: -pose.oy * scale, - oz: -pose.oz * scale, - ow: pose.ow * scale, - }; - const inverseTranslation = transformByPose( - { px: -pose.px, py: -pose.py, pz: -pose.pz, ox: 0, oy: 0, oz: 0, ow: 1 }, - { px: 0, py: 0, pz: 0, ...inverseOrientation } + return transformByPose( + { ...IDENTITY_POSE, px: -pose.px, py: -pose.py, pz: -pose.pz }, + { + ...IDENTITY_POSE, + ox: -pose.ox * scale, + oy: -pose.oy * scale, + oz: -pose.oz * scale, + ow: pose.ow * scale, + } ); - return { - px: inverseTranslation.px, - py: inverseTranslation.py, - pz: inverseTranslation.pz, - ...inverseOrientation, - }; } /** Keep translation and heading while removing viewer pitch and roll. */ From 303086750ca838148a0d019c5963bc07309cc511 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Tue, 20 Oct 2026 19:47:44 -0700 Subject: [PATCH 06/12] feat(webxr): add keyboard shortcuts for the recorder and teleop controls The Start Teleop, record, replay, save and reset controls are xrButtons drawn into the WebXR canvas, so nothing outside the headset can reach them: there is no DOM node for a host-side automation driver to click. That makes an unattended benchmark run impossible to drive from the machine running the sim. Bind them to keys instead, which a driver can synthesise over CDP: T Start Teleop C Record / Stop Record P Play / Stop Replay S Save Recording X Reset Teleop Record is on C, not the more obvious R, because CloudXRUI already binds 'r' on window to resetPanelRelativeToHead - the only way to recover the in-headset panel once it and its drag handle are left behind out of reach. Both listeners would fire on one keypress. Panel recovery is the more important of the two, so it keeps the letter. The handler keys off event.code so the bindings survive non-QWERTY layouts, and bails out on repeats, modifier chords and edits in text inputs. preventDefault runs only when a binding actually handled the event, leaving every other key to the page. Signed-off-by: Emily Ma --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 11 +- .../webxr_client/src/RecorderContext.tsx | 110 ++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 5581f3f165..4cb2200946 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -64,7 +64,7 @@ import { import arrowLeftStartOnRectangleSvg from './icons/arrow-left-start-on-rectangle.svg'; import arrowUturnLeftSvg from './icons/arrow-uturn-left.svg'; import playCircleSvg from './icons/play-circle.svg'; -import { useRecorder } from './RecorderContext'; +import { useRecorder, useTeleopKeyboardShortcuts } from './RecorderContext'; // Face-camera rotation constants const FACE_CAMERA_DAMPING = 10; // Higher = faster rotation toward camera @@ -297,6 +297,15 @@ export default function CloudXR3DUI({ onDismissSystemNotice, }: CloudXRUIProps) { const recorder = useRecorder(); + // T/C/P/S/X. Delivered from a real keyboard, or synthesised over CDP so a host + // can drive an unattended run. Record is on C because 'r' is already taken by + // the panel reset below. + useTeleopKeyboardShortcuts({ + recordingEnabled: showRecordingControls, + startTeleop: onStartTeleop, + teleopStartDisabled: playInProgress, + resetTeleop: onResetTeleop, + }); const MINIMIZE_ON_PLAY_KEY = 'cxr.isaac.minimizeOnPlay'; const groupRef = useRef(null); diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 4381a72192..0ec3bdfc67 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -214,3 +214,113 @@ export function useRecorder(): RecorderContextValue { if (!ctx) throw new Error('useRecorder must be used inside RecorderProvider'); return ctx; } + +function isEditableKeyboardTarget(target: EventTarget | null): boolean { + if (!(target instanceof HTMLElement)) return false; + if (target.isContentEditable) return true; + if (target instanceof HTMLInputElement) return target.type !== 'file'; + return target instanceof HTMLTextAreaElement || target instanceof HTMLSelectElement; +} + +interface TeleopKeyboardShortcutOptions { + recordingEnabled: boolean; + startTeleop?: () => void; + teleopStartDisabled?: boolean; + /** Reset Teleop. Only reachable as a key because its button lives in the XR canvas. */ + resetTeleop?: () => void; +} + +/** Enable keyboard/ADB shortcuts for the Teleop Play and recording controls exposed by the XR UI. */ +export function useTeleopKeyboardShortcuts({ + recordingEnabled, + startTeleop, + teleopStartDisabled = false, + resetTeleop, +}: TeleopKeyboardShortcutOptions): void { + const { + recorder, + savedRecording, + startRecord, + stopRecord, + startReplay, + stopReplay, + onSaveRecording, + } = useRecorder(); + + useEffect(() => { + if (!recordingEnabled && !startTeleop && !resetTeleop) return; + + const onKeyDown = (event: KeyboardEvent) => { + if ( + event.repeat || + event.ctrlKey || + event.altKey || + event.metaKey || + isEditableKeyboardTarget(event.target) + ) { + return; + } + + const key = + event.code === 'KeyT' + ? 't' + : event.code === 'KeyC' + ? 'c' + : event.code === 'KeyP' + ? 'p' + : event.code === 'KeyS' + ? 's' + : event.code === 'KeyX' + ? 'x' + : event.key.toLowerCase(); + + if (key === 't') { + if (!startTeleop || teleopStartDisabled) return; + startTeleop(); + } else if (key === 'c') { + if (!recordingEnabled) return; + if (recorder.mode === 'recording') { + stopRecord(); + } else if (recorder.mode === 'idle') { + startRecord(); + } else { + return; + } + } else if (key === 'p') { + if (!recordingEnabled) return; + if (recorder.mode === 'replaying') { + stopReplay(); + } else if (recorder.mode === 'idle' && savedRecording) { + startReplay(); + } else { + return; + } + } else if (key === 's') { + if (!recordingEnabled || recorder.mode !== 'idle' || !savedRecording) return; + onSaveRecording(); + } else if (key === 'x') { + if (!resetTeleop) return; + resetTeleop(); + } else { + return; + } + + event.preventDefault(); + }; + + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [ + recorder, + onSaveRecording, + recordingEnabled, + savedRecording, + startRecord, + startReplay, + startTeleop, + stopRecord, + stopReplay, + teleopStartDisabled, + resetTeleop, + ]); +} From 5cec40acb370452d74317ab3f75dc9dc3882803e Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Tue, 20 Oct 2026 19:47:44 -0700 Subject: [PATCH 07/12] fix(webxr): record input in the reference space CloudXR streams in three.js pins WebXRManager to 'local-floor', whose origin is on the floor, while CloudXR requests config.referenceSpaceType, which defaults to 'local' and puts the origin at the headset. Recording in one frame and streaming in the other makes recorded hand poses absolute heights above the floor while the scene stays anchored to the headset, so entering a session at a different height moves the table out from under a replayed trajectory and the grasp misses. Resolve the streaming space the same way CloudXRComponent does, including the 'auto' fallback order, and record in it. Recorded poses are then already expressed in the frame the scene is anchored to. Note this is the client half of a contract with the server: XrCfg.anchor_pos is defined as "this position appears at the origin of the XR device's local frame", so the two sides have to agree on which space that is. Forcing the client to 'local-floor' instead raises the viewpoint by roughly one eye height. Signed-off-by: Emily Ma --- deps/cloudxr/webxr_client/src/App.tsx | 6 +- .../webxr_client/src/RecorderComponent.tsx | 72 +++++++++++++++++-- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index 6e2352b987..8d6cf14a9a 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -1193,7 +1193,11 @@ function AppContent() { {cloudXR2DUI && config && ( <> - + { + const types: XRReferenceSpaceType[] = + referenceSpaceType === 'auto' ? AUTO_FALLBACKS : [referenceSpaceType as XRReferenceSpaceType]; + for (const type of types) { + try { + const space = await session.requestReferenceSpace(type); + if (space) return space; + } catch { + // Try the next candidate; CloudXR tolerates the same failures. + } + } + return null; +} + + interface RecorderComponentProps { /** True when the CloudXR session is in the Connected state — gates replay frame advancement. */ isConnected: boolean; /** Capture live input while idle so the trace can render it. */ showTrace: boolean; + /** `config.referenceSpaceType`, so recording uses the frame CloudXR streams in. */ + referenceSpaceType: string; } -export function RecorderComponent({ isConnected, showTrace }: RecorderComponentProps) { +export function RecorderComponent({ + isConnected, + showTrace, + referenceSpaceType, +}: RecorderComponentProps) { const { recorder, onFrameRecord, onFrameState } = useRecorder(); + const gl = useThree(state => state.gl); const tickRef = useRef(0); + const [streamingSpace, setStreamingSpace] = useState(null); + + useEffect(() => { + const webXRManager = gl.xr; + if (!webXRManager) return; + let cancelled = false; + + const onSessionStart = () => { + const session = webXRManager.getSession() as XRSession | null; + if (!session) return; + void resolveStreamingSpace(session, referenceSpaceType).then(space => { + if (!cancelled) setStreamingSpace(space); + }); + }; + const onSessionEnd = () => { + if (!cancelled) setStreamingSpace(null); + }; + + if (webXRManager.getSession()) onSessionStart(); + webXRManager.addEventListener('sessionstart', onSessionStart); + webXRManager.addEventListener('sessionend', onSessionEnd); + return () => { + cancelled = true; + webXRManager.removeEventListener('sessionstart', onSessionStart); + webXRManager.removeEventListener('sessionend', onSessionEnd); + }; + }, [gl, referenceSpaceType]); useFrame(state => { const xrFrame = state.gl.xr.getFrame() as XRFrame | null; @@ -50,7 +114,7 @@ export function RecorderComponent({ isConnected, showTrace }: RecorderComponentP recorder.beginFrame( xrFrame, - state.gl.xr.getReferenceSpace(), + streamingSpace ?? state.gl.xr.getReferenceSpace(), isConnected && isVisible, showTrace && isVisible ); From f7bbfe57a69b66a16959a406282e50a383b69f19 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Tue, 20 Oct 2026 19:47:44 -0700 Subject: [PATCH 08/12] fix(webxr): remove cross-session replay calibration Calibration holds replay until an operator physically returns to the recording's starting pose and confirms. That puts a human back in the loop, which is exactly what unattended replay exists to avoid, so the feature comes out rather than being short-circuited: a bypassed gate leaves unreachable code and tests that assert behaviour the product no longer has. Replayed poses are now the recorded coordinates themselves. That is sound because of the previous commit: recording and streaming share one reference space, so a recording is already expressed in the frame the scene is anchored to and needs no cross-session transform. Removed: Recording.calibration and its import validation, captureViewerCalibration, gravityAlignedViewerPose, sceneAlignment, replayNeedsCalibration, calibrateReplay, the context fields that surfaced them and the in-headset Calibrate button. Deliberately kept: * _replaySceneAlignment / _currentFromInitial / _alignmentCache. These are not calibration - they preserve world placement across *within-session* reference-space resets, which still happen and still matter. The alignment is still cached against the origin observed at entry so a later reset shifts replay with the scene instead of stranding it. * "Wait for a viewer pose before the first recorded frame". The behaviour was expressed through calibration capture; it is now explicit, because a recording that starts before tracking settles has no origin to be relative to. The reference-space reset tests stay and keep their coverage; their expected positions lose the +10 viewer term that the removed alignment used to add, and are re-derived from recorded coordinates plus accumulated reset translation. Signed-off-by: Emily Ma --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 23 +- .../webxr_client/src/RecorderContext.test.tsx | 24 -- .../webxr_client/src/RecorderContext.tsx | 13 +- .../src/TraceVisualization.test.ts | 2 +- .../webxr_client/src/xrInputRecorder.test.ts | 231 ++---------------- .../webxr_client/src/xrInputRecorder.ts | 98 +------- 6 files changed, 28 insertions(+), 363 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 4cb2200946..3654c4417e 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -881,37 +881,16 @@ export default function CloudXR3DUI({ {recorder.mode === 'recording' ? `REC ${recorder.recordedFrameCount} frames` : recorder.mode === 'replaying' - ? recorder.replayNeedsCalibration - ? 'Replay paused' - : 'Replaying' + ? 'Replaying' : 'Recording'} - {recorder.replayNeedsCalibration && recorder.mode === 'replaying' && ( - - Return your headset to its recording-start position and heading, then - calibrate. - - )} {recorder.mode === 'idle' && recorder.recordingInterrupted && ( Recording stopped because the tracking origin changed. Save it or start a new recording. )} - {recorder.mode === 'idle' && !recorder.recordingInterrupted && ( - - Note your headset position and heading when starting a recording. Use the - same pose to calibrate replay in a new session. - - )} - {recorder.mode === 'replaying' && recorder.replayNeedsCalibration && ( - - )} {recorder.mode !== 'replaying' && ( { afterEach(() => act(() => root.unmount())); -test('exposes cross-session calibration and resumes after the operator calibrates', () => { - const space = new EventTarget() as XRReferenceSpace; - const original = { inputSources: [] } as unknown as XRSession; - act(() => current.startRecord()); - act(() => current.recorder.beginFrame(frame(original), space)); - act(() => current.stopRecord()); - expect(current.savedRecording?.frames).toHaveLength(1); - act(() => current.startReplay()); - const next = { inputSources: [] } as unknown as XRSession; - act(() => { - current.recorder.beginFrame(frame(next), space); - current.onFrameState(); - }); - expect(current.mode).toBe('replaying'); - expect(current.replayNeedsCalibration).toBe(true); - act(() => current.calibrateReplay()); - act(() => { - current.recorder.beginFrame(frame(next), space); - current.onFrameState(); - }); - expect(current.replayNeedsCalibration).toBe(false); - expect(current.recorder.currentFrame).not.toBeNull(); -}); - test('saves interrupted recording and updates the UI when the tracking origin changes', () => { const session = { inputSources: [] } as unknown as XRSession; const space = new EventTarget() as XRReferenceSpace; diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 0ec3bdfc67..50c5dcb933 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -45,8 +45,6 @@ export interface RecorderContextValue { stopRecord: () => void; startReplay: () => void; stopReplay: () => void; - calibrateReplay: () => void; - replayNeedsCalibration: boolean; recordingInterrupted: boolean; onFrameState: () => void; setReplayPacing: (pacing: ReplayPacing) => void; @@ -75,7 +73,6 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { const [mode, setMode] = useState<'idle' | 'recording' | 'replaying'>('idle'); const [savedRecording, setSavedRecordingState] = useState(null); const [recordedFrameCount, setRecordedFrameCount] = useState(0); - const [replayNeedsCalibration, setReplayNeedsCalibration] = useState(false); const [replayPacing, setReplayPacingState] = useState('time'); const fileInputRef = useRef(null); @@ -99,7 +96,6 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { if (recorder.mode !== 'idle' || !savedRecording) return; recorder.startReplay(savedRecording, true, replayPacing); setMode('replaying'); - setReplayNeedsCalibration(recorder.replayNeedsCalibration); }, [recorder, replayPacing, savedRecording]); const stopReplay = useCallback(() => { @@ -108,8 +104,6 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { setMode('idle'); }, [recorder]); - const calibrateReplay = useCallback(() => recorder.calibrateReplay(), [recorder]); - const onFrameState = useCallback(() => { if (mode !== recorder.mode) { if (mode === 'recording') { @@ -118,10 +112,7 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { } setMode(recorder.mode); } - if (replayNeedsCalibration !== recorder.replayNeedsCalibration) { - setReplayNeedsCalibration(recorder.replayNeedsCalibration); - } - }, [mode, recorder, replayNeedsCalibration]); + }, [mode, recorder]); const setReplayPacing = useCallback((pacing: ReplayPacing) => { setReplayPacingState(pacing); @@ -185,8 +176,6 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { stopRecord, startReplay, stopReplay, - calibrateReplay, - replayNeedsCalibration, recordingInterrupted: recorder.recordingInterrupted, onFrameState, setReplayPacing, diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts index 9f84dcaa09..c5e1344cc1 100644 --- a/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts @@ -52,7 +52,7 @@ jest.mock('react', () => ({ useEffect: jest.fn(), })); -test('places the whole replay trail in calibrated scene space and hides it while paused', () => { +test('places the whole replay trail in aligned scene space and hides it while paused', () => { const tree = TraceVisualization({ showTrace: true }); const points = (tree.props.children as ReactElement<{ object: Points }>[])[0].props.object; const worldPoint = (index: number) => { diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index b93f3d92cc..348aa715fc 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -116,17 +116,6 @@ function recording(...frames: RecordedFrame[]): Recording { return { version: 1, frames }; } -function calibratedRecording(viewerX: number, ...frames: RecordedFrame[]): Recording { - return { - version: 1, - calibration: { - mode: 'viewer-start-yaw', - pose: { px: viewerX, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, - }, - frames, - }; -} - function timedFrame(timeMs: number, x: number): RecordedFrame { return { ...frameData(x), timeMs }; } @@ -334,23 +323,6 @@ describe('canonical scene-space capture', () => { expect(captured.handJoints.left['index-finger-tip']?.px).toBe(4); }); - test('captures a gravity-aligned viewer pose for cross-session calibration', () => { - const quarterTurn = Math.sqrt(0.5); - const recorder = new XRInputRecorder(); - recorder.startRecording(); - recorder.beginFrame( - makeFrame([], undefined, undefined, 0, pose(1, 2, 3, { y: quarterTurn, w: quarterTurn })), - sceneSpace - ); - recorder.stopRecording(); - - const calibration = recorder.getRecording().calibration; - expect(calibration?.mode).toBe('viewer-start-yaw'); - expect(calibration?.pose).toMatchObject({ px: 1, py: 2, pz: 3, ox: 0, oz: 0 }); - expect(calibration?.pose.oy).toBeCloseTo(quarterTurn); - expect(calibration?.pose.ow).toBeCloseTo(quarterTurn); - }); - test('waits for a viewer pose before recording the first frame', () => { const recorder = new XRInputRecorder(); recorder.startRecording(); @@ -454,104 +426,6 @@ describe('scoped CloudXR replay frame', () => { expect(adapted.getJointPose?.(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); }); - test('aligns loaded poses from the recorded viewer origin to the current XR session', () => { - const grip = {} as XRSpace; - const source = { - handedness: 'left', - gripSpace: grip, - targetRaySpace: {} as XRSpace, - } as XRInputSource; - const session = makeSession([source]); - const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); - const frame = makeFrame([source], undefined, undefined, 0, pose(11, 5), session); - const recorder = new XRInputRecorder(); - recorder.startReplay(loaded, true, 'frame'); - recorder.beginFrame(frame, sceneSpace); - expect(recorder.replayNeedsCalibration).toBe(true); - recorder.calibrateReplay(); - recorder.beginFrame(frame, sceneSpace); - - const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); - expect(replayed?.transform.position.x).toBeCloseTo(12); - expect(replayed?.transform.position.y).toBeCloseTo(5); - }); - - test('applies the calibrated heading to replayed poses', () => { - const grip = {} as XRSpace; - const source = { - handedness: 'left', - gripSpace: grip, - targetRaySpace: {} as XRSpace, - } as XRInputSource; - const session = makeSession([source]); - const sample = frameData(); - sample.poses.leftGrip = { px: 0, py: 0, pz: -1, ox: 0, oy: 0, oz: 0, ow: 1 }; - const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(0, sample))); - const quarterTurn = Math.sqrt(0.5); - const frame = makeFrame( - [source], - undefined, - undefined, - 0, - pose(0, 0, 0, { y: quarterTurn, w: quarterTurn }), - session - ); - const recorder = new XRInputRecorder(); - recorder.startReplay(loaded, true, 'frame'); - recorder.beginFrame(frame, sceneSpace); - expect(recorder.replayNeedsCalibration).toBe(true); - recorder.calibrateReplay(); - recorder.beginFrame(frame, sceneSpace); - - const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); - expect(replayed?.transform.position.x).toBeCloseTo(-1); - expect(replayed?.transform.position.z).toBeCloseTo(0); - expect(replayed?.transform.orientation.y).toBeCloseTo(quarterTurn); - expect(replayed?.transform.orientation.w).toBeCloseTo(quarterTurn); - }); - - test('freezes calibration within a session and recalibrates for a new session', () => { - const grip = {} as XRSpace; - const source = { - handedness: 'left', - gripSpace: grip, - targetRaySpace: {} as XRSpace, - } as XRInputSource; - const firstSession = makeSession([source]); - const secondSession = makeSession([source]); - const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); - const recorder = new XRInputRecorder(); - - const firstFrame = makeFrame([source], undefined, undefined, 0, pose(11), firstSession); - recorder.startReplay(loaded, true, 'frame'); - recorder.beginFrame(firstFrame, sceneSpace); - expect(recorder.replayNeedsCalibration).toBe(true); - recorder.calibrateReplay(); - recorder.beginFrame(firstFrame, sceneSpace); - expect( - recorder.adaptTrackingFrame(firstFrame).getPose(grip, sceneSpace)?.transform.position.x - ).toBeCloseTo(12); - recorder.stopReplay(); - - const movedViewerFrame = makeFrame([source], undefined, undefined, 1, pose(21), firstSession); - recorder.startReplay(loaded, true, 'frame'); - recorder.beginFrame(movedViewerFrame, sceneSpace); - expect( - recorder.adaptTrackingFrame(movedViewerFrame).getPose(grip, sceneSpace)?.transform.position.x - ).toBeCloseTo(12); - recorder.stopReplay(); - - const newSessionFrame = makeFrame([source], undefined, undefined, 2, pose(21), secondSession); - recorder.startReplay(loaded, true, 'frame'); - recorder.beginFrame(newSessionFrame, sceneSpace); - expect(recorder.replayNeedsCalibration).toBe(true); - recorder.calibrateReplay(); - recorder.beginFrame(newSessionFrame, sceneSpace); - expect( - recorder.adaptTrackingFrame(newSessionFrame).getPose(grip, sceneSpace)?.transform.position.x - ).toBeCloseTo(22); - }); - test('keeps in-memory recording and replay in the same reference space unchanged', () => { const grip = {} as XRSpace; const source = { @@ -583,7 +457,7 @@ describe('scoped CloudXR replay frame', () => { }); }); -describe('explicit replay calibration and reference-space resets', () => { +describe('reference-space resets during replay', () => { function setup(pacing: 'frame' | 'time' = 'frame') { const grip = {} as XRSpace; const wrist = {} as XRJointSpace; @@ -597,7 +471,7 @@ describe('explicit replay calibration and reference-space resets', () => { const session = makeSession([source]); const space = new EventTarget() as XRReferenceSpace; const recorder = new XRInputRecorder(); - const saved = calibratedRecording(0, timedFrame(0, 1), timedFrame(100, 3)); + const saved = recording(timedFrame(0, 1), timedFrame(100, 3)); const frame = (viewerX: number, time = 0, viewer: XRPose | null = pose(viewerX)) => makeFrame( [source], @@ -619,41 +493,26 @@ describe('explicit replay calibration and reference-space resets', () => { return { recorder, grip, wrist, source, session, space, saved, frame, advance, reset }; } - test('suppresses live input and waits for a deliberate calibration with a valid viewer pose', () => { - const { recorder, grip, wrist, space, frame, advance } = setup(); - const waiting = advance(10); - expect(recorder.replayNeedsCalibration).toBe(true); - expect(recorder.currentFrame).toBeNull(); - expect(waiting.getPose(grip, space)).toBeUndefined(); - expect(waiting.getJointPose?.(wrist, space)).toBeUndefined(); - expect(waiting.session.inputSources).toHaveLength(0); - - recorder.calibrateReplay(); - recorder.beginFrame(frame(10, 0, null), space); - expect(recorder.replayNeedsCalibration).toBe(true); - expect(recorder.replayFrameIndex).toBe(0); - expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); - expect(recorder.replayNeedsCalibration).toBe(false); - }); - test.each(['frame', 'time'] as const)( '%s replay preserves world placement after headset motion and repeated translation resets', pacing => { const { recorder, grip, wrist, space, advance, reset } = setup(pacing); + // Replay is not viewer-aligned: these are recorded coordinates shifted by + // the resets that have accumulated, and the viewer's own x is irrelevant. advance(10); - recorder.calibrateReplay(); - expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 1 : 3 + ); reset(new XRRigidTransform({ x: -100 })); const firstReset = advance(112, pacing === 'time' ? 50 : 1); - expect(recorder.replayNeedsCalibration).toBe(false); expect(firstReset.getPose(grip, space)?.transform.position.x).toBeCloseTo( - pacing === 'time' ? 112 : 113 + pacing === 'time' ? 102 : 103 ); expect(firstReset.getJointPose?.(wrist, space)?.transform.position.x).toBeCloseTo( - pacing === 'time' ? 114 : 115 + pacing === 'time' ? 104 : 105 ); reset(new XRRigidTransform({ x: -20 })); - expect(advance(132, 100).getPose(grip, space)?.transform.position.x).toBeCloseTo(133); + expect(advance(132, 100).getPose(grip, space)?.transform.position.x).toBeCloseTo(123); } ); @@ -663,69 +522,26 @@ describe('explicit replay calibration and reference-space resets', () => { recorder.stopReplay(); recorder.startReplay(saved, false, 'frame'); advance(10); - recorder.calibrateReplay(); advance(10); const q = Math.sqrt(0.5); reset(new XRRigidTransform({}, { x: 0, y: q, z: 0, w: q })); const result = advance(999).getPose(grip, space)!; expect(result.transform.position.x).toBeCloseTo(0); - expect(result.transform.position.z).toBeCloseTo(11); + // The quarter turn maps the recorded x = 1 onto z; no viewer term is added. + expect(result.transform.position.z).toBeCloseTo(1); expect(result.transform.orientation.y).toBeCloseTo(-q); expect(result.transform.orientation.w).toBeCloseTo(q); }); - test('unknown reset pauses the replay clock and requires another explicit calibration', () => { - const { recorder, grip, space, advance, reset } = setup('time'); - advance(10, 1000); - recorder.calibrateReplay(); - advance(10, 1000); - expect(advance(12, 1050).getPose(grip, space)?.transform.position.x).toBeCloseTo(12); - reset(null); - expect(advance(112, 2000).getPose(grip, space)).toBeUndefined(); - expect(recorder.replayNeedsCalibration).toBe(true); - expect(recorder.currentFrame).toBeNull(); - recorder.calibrateReplay(); - expect(advance(110, 5000).getPose(grip, space)?.transform.position.x).toBeCloseTo(112); - expect(recorder.currentFrame?.timeMs).toBe(50); - }); - test('preserves cached placement when a known reset occurs while replay is stopped', () => { const { recorder, grip, space, saved, advance, reset } = setup(); advance(10); - recorder.calibrateReplay(); advance(10); recorder.stopReplay(); reset(new XRRigidTransform({ x: -100 })); recorder.startReplay(saved, false, 'frame'); - expect(advance(112).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); - expect(recorder.replayNeedsCalibration).toBe(false); - }); - - test('calibration after a reset is stored in the original reference frame', () => { - const { recorder, grip, space, saved, advance, reset } = setup(); - advance(10); - reset(new XRRigidTransform({ x: -100 })); - recorder.calibrateReplay(); - expect(advance(110).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); - recorder.stopReplay(); - recorder.startReplay(saved, false, 'frame'); - expect(advance(115).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); - }); - - test('reference-space replacement invalidates placement and pending calibration', () => { - const { recorder, grip, frame, advance } = setup(); - advance(10); - recorder.calibrateReplay(); - const replacement = new EventTarget() as XRReferenceSpace; - const current = frame(20); - recorder.beginFrame(current, replacement); - expect(recorder.replayNeedsCalibration).toBe(true); - expect(recorder.adaptTrackingFrame(current).getPose(grip, replacement)).toBeUndefined(); - recorder.calibrateReplay(); - recorder.beginFrame(current, replacement); - expect( - recorder.adaptTrackingFrame(current).getPose(grip, replacement)?.transform.position.x - ).toBeCloseTo(21); + // Recorded x = 1 plus the +100 the reset accumulated while replay was stopped. + expect(advance(112).getPose(grip, space)?.transform.position.x).toBeCloseTo(101); }); test.each([true, false])('stops recording on reset (known transform: %s)', known => { @@ -741,7 +557,6 @@ describe('explicit replay calibration and reference-space resets', () => { const saved = recorder.getRecording(); recorder.startReplay(saved, false, 'frame'); recorder.beginFrame(frame(100), space); - expect(recorder.replayNeedsCalibration).toBe(!known); if (known) { expect( recorder.adaptTrackingFrame(frame(100)).getPose(grip, space)?.transform.position.x @@ -1011,7 +826,6 @@ describe('controller replay without live hardware', () => { recorder.stopRecording(); const saved = XRInputRecorder.importJSON(recorder.exportJSON()); expect(saved.frames[0].controllerProfiles).toEqual({ left: profiles, right: null }); - delete saved.calibration; recorder.startReplay(saved); const frame = makeFrame(); recorder.beginFrame(frame, sceneSpace); @@ -1053,20 +867,6 @@ describe('controller replay without live hardware', () => { expect(sources[2].profiles).toEqual(['pico-4u']); }); - test('waits for calibration and restores the live controller on stop', () => { - const recorder = new XRInputRecorder(); - const live = { handedness: 'left', gamepad: gamepad(99) } as XRInputSource; - const frame = makeFrame([live]); - recorder.startReplay(calibratedRecording(0, controllerFrame(0, 1))); - recorder.beginFrame(frame, sceneSpace); - expect(recorder.adaptTrackingFrame(frame).session.inputSources).toHaveLength(0); - recorder.calibrateReplay(); - recorder.beginFrame(frame, sceneSpace); - expect(recorder.adaptTrackingFrame(frame).session.inputSources[0].gamepad?.axes).toEqual([1]); - recorder.stopReplay(); - expect(recorder.adaptTrackingFrame(frame).session.inputSources[0]).toBe(live); - }); - test.each([null, {}, { left: 'quest', right: null }, { left: [42], right: null }])( 'rejects malformed controller profiles (%j)', controllerProfiles => { @@ -1087,7 +887,6 @@ describe('serialization', () => { recorder.stopRecording(); const imported = XRInputRecorder.importJSON(recorder.exportJSON()); expect(imported.frames).toHaveLength(1); - expect(imported.calibration?.mode).toBe('viewer-start-yaw'); expect(typeof recorder.getRecording().recordedAt).toBe('number'); }); @@ -1122,8 +921,6 @@ describe('serialization', () => { { version: 1, frames: null }, { version: 1, frames: [{ ...frameData(), timeMs: undefined }] }, { version: 1, frames: [frameData(2), frameData(1)] }, - { version: 1, calibration: { mode: 'viewer-start-yaw', pose: {} }, frames: [] }, - { version: 1, calibration: { mode: 'unknown', pose: {} }, frames: [] }, null, 42, [], diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 46aa35ddb1..ba5eaf729a 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -67,12 +67,6 @@ export type RecordedFrame = { export type Recording = { version: 1; recordedAt?: number; - /** Optional cross-session origin calibration. Absent in legacy recordings. */ - calibration?: { - mode: 'viewer-start-yaw'; - /** Gravity-aligned viewer pose in the recording's scene reference space. */ - pose: PoseData; - }; frames: RecordedFrame[]; }; @@ -375,39 +369,6 @@ function inversePose(pose: PoseData): PoseData { ); } -/** Keep translation and heading while removing viewer pitch and roll. */ -function gravityAlignedViewerPose(pose: PoseData): PoseData { - const yaw = Math.atan2( - 2 * (pose.ox * pose.oz + pose.ow * pose.oy), - 1 - 2 * (pose.ox * pose.ox + pose.oy * pose.oy) - ); - const halfYaw = yaw / 2; - return { - px: pose.px, - py: pose.py, - pz: pose.pz, - ox: 0, - oy: Math.sin(halfYaw), - oz: 0, - ow: Math.cos(halfYaw), - }; -} - -function captureViewerCalibration( - frame: XRFrame, - referenceSpace: XRReferenceSpace -): Recording['calibration'] | undefined { - const viewerPose = serializePose(frame.getViewerPose(referenceSpace)); - return viewerPose - ? { mode: 'viewer-start-yaw', pose: gravityAlignedViewerPose(viewerPose) } - : undefined; -} - -function sceneAlignment(recordedViewer: PoseData, currentViewer: PoseData): PoseData { - // currentScene <- recordedScene = currentScene <- viewer <- recordedScene. - return transformByPose(inversePose(recordedViewer), currentViewer); -} - /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -432,12 +393,10 @@ export class XRInputRecorder { private _sceneReferenceSpace: XRReferenceSpace | null = null; private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; - private _recordingCalibration: Recording['calibration']; private _recordingAlignment: PoseData | null = null; private _alignmentCache = new WeakMap(); private _replayRecording: Recording | null = null; private _replaySceneAlignment: PoseData | null = null; - private _calibrationRequested = false; private _observedSession: XRSession | null = null; // Cached alignments use the origin observed at session/reference-space entry. private _currentFromInitial: PoseData = IDENTITY_POSE; @@ -466,32 +425,16 @@ export class XRInputRecorder { return this._replaySceneAlignment; } - get replayNeedsCalibration(): boolean { - return ( - this._mode === 'replaying' && - !!this._replayRecording?.calibration && - !this._alignmentCache.has(this._replayRecording) - ); - } - get recordingInterrupted(): boolean { return this._recordingInterrupted; } - /** Call after the operator returns to the recording's physical starting pose. */ - calibrateReplay(): void { - if (this.replayNeedsCalibration && this._sceneReferenceSpace) { - this._calibrationRequested = true; - } - } - startRecording(): void { this._assertIdle(); this._frames = []; this._currentFrame = null; this._recordingStartTime = null; this._recordedAt = Date.now(); - this._recordingCalibration = undefined; this._recordingAlignment = null; this._recordingInterrupted = false; this._mode = 'recording'; @@ -514,7 +457,6 @@ export class XRInputRecorder { this._currentFrame = null; this._replayRecording = recording; this._replaySceneAlignment = null; - this._calibrationRequested = false; this._mode = 'replaying'; } @@ -525,7 +467,6 @@ export class XRInputRecorder { this._replayRecording = null; this._replaySources.clear(); this._replaySceneAlignment = null; - this._calibrationRequested = false; this._mode = 'idle'; } @@ -555,9 +496,11 @@ export class XRInputRecorder { if (this._mode === 'recording') { if (!sceneReferenceSpace) return; - if (!this._recordingCalibration) { - this._recordingCalibration = captureViewerCalibration(frame, sceneReferenceSpace); - if (!this._recordingCalibration) return; + if (!this._recordingAlignment) { + // Hold the first frames until the runtime reports a viewer pose: while + // tracking is still settling there is no origin for the recorded + // coordinates to be relative to. + if (!frame.getViewerPose(sceneReferenceSpace)) return; this._recordingAlignment = inversePose(this._currentFromInitial); } const now = frameTimestampMs(frame); @@ -623,7 +566,6 @@ export class XRInputRecorder { const replaying = this._mode === 'replaying'; this._trackingSession.setInputSources(replaying ? this._replayInputSources() : null); const session = this._trackingSession.session; - // Waiting for calibration must not silently substitute live robot inputs. const replay = this._currentFrame ?? emptyFrame(0); return new Proxy(frame, { @@ -663,15 +605,6 @@ export class XRInputRecorder { if (!Array.isArray(recording.frames)) { throw new Error('Malformed recording: frames is not an array'); } - if ( - recording.calibration !== undefined && - (typeof recording.calibration !== 'object' || - recording.calibration === null || - recording.calibration.mode !== 'viewer-start-yaw' || - !isPoseData(recording.calibration.pose)) - ) { - throw new Error('Malformed recording: calibration is invalid'); - } let previousTime = -1; for (const frame of recording.frames) { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { @@ -700,7 +633,6 @@ export class XRInputRecorder { const recording: Recording = { version: 1, recordedAt: this._recordedAt, - calibration: this._recordingCalibration, frames: [...this._frames], }; if (this._recordingAlignment) { @@ -744,7 +676,6 @@ export class XRInputRecorder { this._recordingAlignment = null; this._currentFromInitial = IDENTITY_POSE; this._replaySceneAlignment = null; - this._calibrationRequested = false; this._lastReplayDisplayTime = null; this._currentFrame = null; } @@ -761,7 +692,6 @@ export class XRInputRecorder { const oldFromNew = serializeTransform(event.transform); this._currentFromInitial = transformByPose(this._currentFromInitial, inversePose(oldFromNew)); this._replaySceneAlignment = null; - this._calibrationRequested = false; this._currentFrame = null; }; @@ -772,19 +702,13 @@ export class XRInputRecorder { let initialFromRecorded = this._alignmentCache.get(recording); if (!initialFromRecorded) { - let currentFromRecorded = IDENTITY_POSE; - if (recording.calibration) { - if (!this._calibrationRequested) return false; - const currentCalibration = captureViewerCalibration(frame, referenceSpace); - if (!currentCalibration) return false; - currentFromRecorded = sceneAlignment(recording.calibration.pose, currentCalibration.pose); - } - initialFromRecorded = transformByPose( - currentFromRecorded, - inversePose(this._currentFromInitial) - ); + // Recording and streaming share one reference space (see RecorderComponent), + // so recorded poses are already in the frame the scene is anchored to and + // need no cross-session transform. The mapping is still cached against the + // origin as it stood at entry, so a later reference-space reset shifts + // replay with the scene instead of stranding it. + initialFromRecorded = inversePose(this._currentFromInitial); this._alignmentCache.set(recording, initialFromRecorded); - this._calibrationRequested = false; } this._replaySceneAlignment = transformByPose(initialFromRecorded, this._currentFromInitial); From 659c3aaad2ee220a01f84b90331d6e3aef325b8c Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Tue, 20 Oct 2026 19:47:44 -0700 Subject: [PATCH 09/12] refactor(webxr): fold the recorder shortcuts into CloudXRUI's keydown listener The shortcuts arrived as a second window keydown listener alongside the one CloudXRUI already installs for the panel-reset key, each with its own copy of the "is the operator typing?" guard. Two listeners deciding what a letter means is a standing invitation for them to disagree. useTeleopKeyboardShortcuts becomes useTeleopShortcutHandler: it returns a handler instead of registering a listener, and reports whether it consumed the event so CloudXRUI can skip its own key. isEditableKeyboardTarget is exported and now guards both, replacing the inline tagName check - one definition, so the two cannot drift apart. The shared guard is the stricter of the two, which changes the panel key slightly: a held key and a modifier chord no longer trigger it, so Ctrl+R reloads the page without also moving the panel. RESET_PANEL_KEY keeps its name and value; oob_teleop_adb.py synthesises that exact key over CDP and the two are documented as staying in sync. Signed-off-by: Emily Ma --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 31 ++++--- .../webxr_client/src/RecorderContext.tsx | 86 ++++++++++--------- 2 files changed, 63 insertions(+), 54 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 3654c4417e..e033077db4 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -64,7 +64,7 @@ import { import arrowLeftStartOnRectangleSvg from './icons/arrow-left-start-on-rectangle.svg'; import arrowUturnLeftSvg from './icons/arrow-uturn-left.svg'; import playCircleSvg from './icons/play-circle.svg'; -import { useRecorder, useTeleopKeyboardShortcuts } from './RecorderContext'; +import { isEditableKeyboardTarget, useRecorder, useTeleopShortcutHandler } from './RecorderContext'; // Face-camera rotation constants const FACE_CAMERA_DAMPING = 10; // Higher = faster rotation toward camera @@ -299,8 +299,8 @@ export default function CloudXR3DUI({ const recorder = useRecorder(); // T/C/P/S/X. Delivered from a real keyboard, or synthesised over CDP so a host // can drive an unattended run. Record is on C because 'r' is already taken by - // the panel reset below. - useTeleopKeyboardShortcuts({ + // the panel reset below, which shares this component's one keydown listener. + const handleTeleopShortcut = useTeleopShortcutHandler({ recordingEnabled: showRecordingControls, startTeleop: onStartTeleop, teleopStartDisabled: playInProgress, @@ -521,24 +521,31 @@ export default function CloudXR3DUI({ useEffect(() => { const handleKeyDown = (event: KeyboardEvent) => { - if (event.key.toLowerCase() !== RESET_PANEL_KEY) { - return; - } // Unlike a function key, a plain letter can be typed into any of the 2D settings form's - // text inputs - don't reset the panel out from under someone typing a server IP. - const target = event.target as HTMLElement | null; + // text inputs - don't reset the panel out from under someone typing a server IP. A held + // key or a browser chord (Ctrl+R to reload) is not a shortcut press either. if ( - target?.tagName === 'INPUT' || - target?.tagName === 'TEXTAREA' || - target?.isContentEditable + event.repeat || + event.ctrlKey || + event.altKey || + event.metaKey || + isEditableKeyboardTarget(event.target) ) { return; } + // The recorder keys get first refusal, then the panel key; one listener, so the + // two sets cannot both fire on the same press. + if (handleTeleopShortcut(event)) { + return; + } + if (event.key.toLowerCase() !== RESET_PANEL_KEY) { + return; + } resetPanelRelativeToHead(camera); }; window.addEventListener('keydown', handleKeyDown); return () => window.removeEventListener('keydown', handleKeyDown); - }, [camera, resetPanelRelativeToHead]); + }, [camera, resetPanelRelativeToHead, handleTeleopShortcut]); const isCompact = minimizeOnPlay && playInProgress; const isMinimizedLayout = isCompact || panelHidden; diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 50c5dcb933..8660520b5d 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -204,7 +204,13 @@ export function useRecorder(): RecorderContextValue { return ctx; } -function isEditableKeyboardTarget(target: EventTarget | null): boolean { +/** + * True when a plain letter is being typed rather than used as a shortcut. + * + * Shared with CloudXRUI's panel-reset key, which runs off the same listener: + * one definition means the two cannot disagree about what counts as typing. + */ +export function isEditableKeyboardTarget(target: EventTarget | null): boolean { if (!(target instanceof HTMLElement)) return false; if (target.isContentEditable) return true; if (target instanceof HTMLInputElement) return target.type !== 'file'; @@ -219,13 +225,20 @@ interface TeleopKeyboardShortcutOptions { resetTeleop?: () => void; } -/** Enable keyboard/ADB shortcuts for the Teleop Play and recording controls exposed by the XR UI. */ -export function useTeleopKeyboardShortcuts({ +/** + * Build the keydown handler for the Teleop Play and recording controls exposed by the XR UI. + * + * Returns a handler rather than registering a listener: CloudXRUI already owns a + * window keydown listener for the panel-reset key, and a second one would mean two + * places deciding what a letter means. The caller reports whether the key was + * consumed, so its own keys can be skipped when this one handles the event. + */ +export function useTeleopShortcutHandler({ recordingEnabled, startTeleop, teleopStartDisabled = false, resetTeleop, -}: TeleopKeyboardShortcutOptions): void { +}: TeleopKeyboardShortcutOptions): (event: KeyboardEvent) => boolean { const { recorder, savedRecording, @@ -236,19 +249,9 @@ export function useTeleopKeyboardShortcuts({ onSaveRecording, } = useRecorder(); - useEffect(() => { - if (!recordingEnabled && !startTeleop && !resetTeleop) return; - - const onKeyDown = (event: KeyboardEvent) => { - if ( - event.repeat || - event.ctrlKey || - event.altKey || - event.metaKey || - isEditableKeyboardTarget(event.target) - ) { - return; - } + return useCallback( + (event: KeyboardEvent): boolean => { + if (!recordingEnabled && !startTeleop && !resetTeleop) return false; const key = event.code === 'KeyT' @@ -264,52 +267,51 @@ export function useTeleopKeyboardShortcuts({ : event.key.toLowerCase(); if (key === 't') { - if (!startTeleop || teleopStartDisabled) return; + if (!startTeleop || teleopStartDisabled) return false; startTeleop(); } else if (key === 'c') { - if (!recordingEnabled) return; + if (!recordingEnabled) return false; if (recorder.mode === 'recording') { stopRecord(); } else if (recorder.mode === 'idle') { startRecord(); } else { - return; + return false; } } else if (key === 'p') { - if (!recordingEnabled) return; + if (!recordingEnabled) return false; if (recorder.mode === 'replaying') { stopReplay(); } else if (recorder.mode === 'idle' && savedRecording) { startReplay(); } else { - return; + return false; } } else if (key === 's') { - if (!recordingEnabled || recorder.mode !== 'idle' || !savedRecording) return; + if (!recordingEnabled || recorder.mode !== 'idle' || !savedRecording) return false; onSaveRecording(); } else if (key === 'x') { - if (!resetTeleop) return; + if (!resetTeleop) return false; resetTeleop(); } else { - return; + return false; } event.preventDefault(); - }; - - window.addEventListener('keydown', onKeyDown); - return () => window.removeEventListener('keydown', onKeyDown); - }, [ - recorder, - onSaveRecording, - recordingEnabled, - savedRecording, - startRecord, - startReplay, - startTeleop, - stopRecord, - stopReplay, - teleopStartDisabled, - resetTeleop, - ]); + return true; + }, + [ + recorder, + onSaveRecording, + recordingEnabled, + savedRecording, + startRecord, + startReplay, + startTeleop, + stopRecord, + stopReplay, + teleopStartDisabled, + resetTeleop, + ] + ); } From b680c26f6af0ec001121a544b2208921e2c37b8b Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Tue, 20 Oct 2026 19:47:44 -0700 Subject: [PATCH 10/12] fix(webxr): take the streaming reference space from CloudXRComponent RecorderComponent resolved its own reference space by copying CloudXRComponent's request - the same 'auto' fallback order, re-implemented. The copy was already incomplete: CloudXRComponent applies config.xrOffset* through getOffsetReferenceSpace after resolving, and falls back to the WebXRManager space when every request fails. Neither was mirrored, so any non-zero XR offset put recording back into a different frame from streaming - exactly the mismatch recording in the streaming space was meant to remove. CloudXRComponent gains an onReferenceSpace callback, fired once the space is final, and RecorderComponent takes it as a prop. The duplicated resolution, its session listeners and its local state go away. Deriving it twice could only ever agree by coincidence; there is one space the session streams in, and this is now the one the recorder uses. Signed-off-by: Emily Ma --- .../helpers/react/CloudXRComponent.tsx | 15 ++++ deps/cloudxr/webxr_client/src/App.tsx | 8 +- .../webxr_client/src/RecorderComponent.tsx | 81 ++++--------------- 3 files changed, 39 insertions(+), 65 deletions(-) diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index 13dfbd13b1..ccb2a869a5 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -208,6 +208,16 @@ interface CloudXRComponentProps { /** Callback fired with the resolved server address after proxy configuration is applied. */ onServerAddress?: (address: string) => void; + /** + * Callback fired with the reference space poses are actually streamed in: the one + * `config.referenceSpaceType` resolved to (including the 'auto' fallback order and the + * WebXRManager last resort), with `config.xrOffset*` already applied. Anything that + * needs to express poses in the same frame as the stream should take this rather than + * request its own space, which would miss the offset and could resolve differently. + * Fired on each session creation; not fired when no space could be resolved. + */ + onReferenceSpace?: (space: XRReferenceSpace) => void; + /** * Callback fired with render performance metrics ({@link CloudXR.MetricsCadence.PerRender}). * Payloads are partial - only the fields sampled on this tick are present. @@ -303,6 +313,7 @@ export default function CloudXRComponent({ warmupBeginTimeoutMs, warmupEndTimeoutMs, onServerAddress, + onReferenceSpace, onRenderPerformanceMetrics, onStreamingPerformanceMetrics, onNetworkPerformanceMetrics, @@ -720,6 +731,10 @@ export default function CloudXRComponent({ referenceSpace = referenceSpace.getOffsetReferenceSpace(offsetTransform); } + // Published only now: this is the space the session is about to stream in, + // after both the fallback chain and the offset. + onReferenceSpace?.(referenceSpace); + // Fill in CloudXR session options. const cloudXROptions: CloudXR.SessionOptions = { serverAddress: connectionConfig.serverIP, diff --git a/deps/cloudxr/webxr_client/src/App.tsx b/deps/cloudxr/webxr_client/src/App.tsx index 8d6cf14a9a..34220d6060 100644 --- a/deps/cloudxr/webxr_client/src/App.tsx +++ b/deps/cloudxr/webxr_client/src/App.tsx @@ -212,6 +212,10 @@ function AppContent() { const [errorMessage, setErrorMessage] = useState(''); // CloudXR session reference const [cloudXRSession, setCloudXRSession] = useState(null); + // The space CloudXR resolved for the current session, so the recorder can express + // poses in the same frame the stream uses. Cleared with the session: a space from an + // ended session yields no poses. + const [streamingSpace, setStreamingSpace] = useState(null); // XR mode state for UI visibility const [isXRMode, setIsXRMode] = useState(false); // Server address being used for connection @@ -701,6 +705,7 @@ function AppContent() { stopStreamTestCountdown(); streamTest.value = null; setCloudXRSession(session); + if (!session) setStreamingSpace(null); }; const systemNoticeTimerRef = useRef(null); @@ -1196,13 +1201,14 @@ function AppContent() { { - const types: XRReferenceSpaceType[] = - referenceSpaceType === 'auto' ? AUTO_FALLBACKS : [referenceSpaceType as XRReferenceSpaceType]; - for (const type of types) { - try { - const space = await session.requestReferenceSpace(type); - if (space) return space; - } catch { - // Try the next candidate; CloudXR tolerates the same failures. - } - } - return null; -} - - interface RecorderComponentProps { - /** True when the CloudXR session is in the Connected state — gates replay frame advancement. */ + /** True when the CloudXR session is in the Connected state - gates replay frame advancement. */ isConnected: boolean; /** Capture live input while idle so the trace can render it. */ showTrace: boolean; - /** `config.referenceSpaceType`, so recording uses the frame CloudXR streams in. */ - referenceSpaceType: string; + /** + * The space CloudXR streams poses in, as reported by CloudXRComponent's onReferenceSpace. + * + * three.js pins WebXRManager to 'local-floor' (origin on the floor) while CloudXR requests + * `config.referenceSpaceType` ('local' by default, origin at the headset). Recording in one + * frame and streaming in the other makes hand poses absolute above the floor while the scene + * stays anchored to the headset, so entering at a different height moves the table but not + * the trajectory. Taken from CloudXRComponent rather than resolved again here so the two + * cannot disagree, and so `config.xrOffset*` is included. + * + * Null before a session exists; the WebXRManager space is used until then. + */ + streamingSpace: XRReferenceSpace | null; } export function RecorderComponent({ isConnected, showTrace, - referenceSpaceType, + streamingSpace, }: RecorderComponentProps) { const { recorder, onFrameRecord, onFrameState } = useRecorder(); - const gl = useThree(state => state.gl); const tickRef = useRef(0); - const [streamingSpace, setStreamingSpace] = useState(null); - - useEffect(() => { - const webXRManager = gl.xr; - if (!webXRManager) return; - let cancelled = false; - - const onSessionStart = () => { - const session = webXRManager.getSession() as XRSession | null; - if (!session) return; - void resolveStreamingSpace(session, referenceSpaceType).then(space => { - if (!cancelled) setStreamingSpace(space); - }); - }; - const onSessionEnd = () => { - if (!cancelled) setStreamingSpace(null); - }; - - if (webXRManager.getSession()) onSessionStart(); - webXRManager.addEventListener('sessionstart', onSessionStart); - webXRManager.addEventListener('sessionend', onSessionEnd); - return () => { - cancelled = true; - webXRManager.removeEventListener('sessionstart', onSessionStart); - webXRManager.removeEventListener('sessionend', onSessionEnd); - }; - }, [gl, referenceSpaceType]); useFrame(state => { const xrFrame = state.gl.xr.getFrame() as XRFrame | null; From 6d4998b5b5ebe3d54b1ae3b5cf581617b5a590c3 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Wed, 21 Oct 2026 12:57:15 +0800 Subject: [PATCH 11/12] fix(webxr): republish the streaming reference space on reconnect onReferenceSpace fired once, from handleSessionStart. The reconnect path re-enters establishSession only, after clearing the parent's copy with onSessionReady(null), so from the first recoverable stream error onwards the recorder fell back to three.js' local-floor space while CloudXR kept streaming in the configured one - silently, and for the rest of the WebXR session. Publish per CloudXR session instead, right after onSessionReady(cxrSession), so the clear and the republish stay paired. Initial connect is unchanged. Signed-off-by: Emily Ma --- .../webxr_client/helpers/react/CloudXRComponent.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx index ccb2a869a5..21a5161249 100644 --- a/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx +++ b/deps/cloudxr/webxr_client/helpers/react/CloudXRComponent.tsx @@ -731,9 +731,10 @@ export default function CloudXRComponent({ referenceSpace = referenceSpace.getOffsetReferenceSpace(offsetTransform); } - // Published only now: this is the space the session is about to stream in, - // after both the fallback chain and the offset. - onReferenceSpace?.(referenceSpace); + // Resolved only now: this is the space the session streams in, after both + // the fallback chain and the offset. Published per CloudXR session rather + // than here, so a reconnect does not leave the parent without one. + const streamingReferenceSpace = referenceSpace; // Fill in CloudXR session options. const cloudXROptions: CloudXR.SessionOptions = { @@ -1100,6 +1101,9 @@ export default function CloudXRComponent({ // Notify parent that session is ready onSessionReady?.(cxrSession); + // Republished on every attempt, reconnects included: the parent drops its + // copy whenever onSessionReady(null) fires, so the two have to stay paired. + onReferenceSpace?.(streamingReferenceSpace); // Start session (synchronous call that initiates connection) try { From 09c679fb67344c843f74a8c35a86f0626281833e Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Wed, 28 Oct 2026 11:07:37 +0800 Subject: [PATCH 12/12] fix(webxr): enable the keyboard shortcuts only with recording controls T and X were live whenever the page had focus, while C, P and S needed showRecordingControls. Host keyboard teleop binds the same letters (X is arm roll and base yaw, T arm pitch, C arm yaw, S arm X), so a browser on the same keyboard could turn a motion key into a reset. Gate all five on showRecordingControls. It is off by default, so the letters stay with keyboard teleop unless recording controls are asked for. Signed-off-by: Emily Ma --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 5 +++-- deps/cloudxr/webxr_client/src/RecorderContext.tsx | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index e033077db4..880c2f3d28 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -298,8 +298,9 @@ export default function CloudXR3DUI({ }: CloudXRUIProps) { const recorder = useRecorder(); // T/C/P/S/X. Delivered from a real keyboard, or synthesised over CDP so a host - // can drive an unattended run. Record is on C because 'r' is already taken by - // the panel reset below, which shares this component's one keydown listener. + // can drive an unattended run. Only active with showRecordingControls. Record is + // on C because 'r' is already taken by the panel reset below, which shares this + // component's one keydown listener. const handleTeleopShortcut = useTeleopShortcutHandler({ recordingEnabled: showRecordingControls, startTeleop: onStartTeleop, diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 8660520b5d..e7d49db2dd 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -218,6 +218,10 @@ export function isEditableKeyboardTarget(target: EventTarget | null): boolean { } interface TeleopKeyboardShortcutOptions { + /** + * showRecordingControls. Gates every shortcut, T and X included: it is off by + * default, which leaves these letters to host keyboard teleop. + */ recordingEnabled: boolean; startTeleop?: () => void; teleopStartDisabled?: boolean; @@ -251,7 +255,7 @@ export function useTeleopShortcutHandler({ return useCallback( (event: KeyboardEvent): boolean => { - if (!recordingEnabled && !startTeleop && !resetTeleop) return false; + if (!recordingEnabled) return false; const key = event.code === 'KeyT' @@ -270,7 +274,6 @@ export function useTeleopShortcutHandler({ if (!startTeleop || teleopStartDisabled) return false; startTeleop(); } else if (key === 'c') { - if (!recordingEnabled) return false; if (recorder.mode === 'recording') { stopRecord(); } else if (recorder.mode === 'idle') { @@ -279,7 +282,6 @@ export function useTeleopShortcutHandler({ return false; } } else if (key === 'p') { - if (!recordingEnabled) return false; if (recorder.mode === 'replaying') { stopReplay(); } else if (recorder.mode === 'idle' && savedRecording) { @@ -288,7 +290,7 @@ export function useTeleopShortcutHandler({ return false; } } else if (key === 's') { - if (!recordingEnabled || recorder.mode !== 'idle' || !savedRecording) return false; + if (recorder.mode !== 'idle' || !savedRecording) return false; onSaveRecording(); } else if (key === 'x') { if (!resetTeleop) return false;