From 52f38410df2b92b201f1609592e5db4f4331f257 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Sat, 29 Aug 2026 15:33:46 -0700 Subject: [PATCH 1/4] fix(webxr): align replays across XR sessions Signed-off-by: Yanzi Zhu --- .../webxr_client/src/xrInputRecorder.test.ts | 182 ++++++++++++- .../webxr_client/src/xrInputRecorder.ts | 248 +++++++++++++++++- 2 files changed, 411 insertions(+), 19 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index ca967ab17d..ea50ce8651 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -45,7 +45,7 @@ function pose(x: number, y = 0, z = 0, orientation: DOMPointInit = { w: 1 }): XR emulatedPosition: false, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function jointPose(x: number, radius = 0.01): XRJointPose { @@ -56,7 +56,11 @@ function gamepad(axis: number): Gamepad { return { axes: [axis], buttons: [{ value: axis, pressed: axis > 0, touched: true }], - } as Gamepad; + } as unknown as Gamepad; +} + +function makeSession(inputSources: XRInputSource[] = []): XRSession { + return { inputSources } as unknown as XRSession; } type PoseResolver = (space: XRSpace, baseSpace: XRSpace) => XRPose | null; @@ -66,15 +70,17 @@ function makeFrame( inputSources: XRInputSource[] = [], getPose: PoseResolver = () => null, getJointPose: JointResolver = () => null, - predictedDisplayTime = 0 + predictedDisplayTime = 0, + viewerPose: XRPose | null = pose(0), + session: XRSession = makeSession(inputSources) ): XRFrame { - const session = { inputSources } as XRSession; return { session, predictedDisplayTime, getPose, getJointPose, - } as XRFrame; + getViewerPose: () => viewerPose, + } as unknown as XRFrame; } function frameData(x = 0): RecordedFrame { @@ -103,6 +109,17 @@ function recording(...frames: RecordedFrame[]): Recording { return { version: 1, frames }; } +function calibratedRecording(viewerX: number, ...frames: RecordedFrame[]): Recording { + return { + version: 1, + calibration: { + mode: 'viewer-start-yaw', + pose: { px: viewerX, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, + }, + frames, + }; +} + function timedFrame(timeMs: number, x: number): RecordedFrame { return { ...frameData(x), timeMs }; } @@ -289,6 +306,33 @@ describe('canonical scene-space capture', () => { expect(captured.handJoints.left.wrist?.px).toBe(3); expect(captured.handJoints.left['index-finger-tip']?.px).toBe(4); }); + + test('captures a gravity-aligned viewer pose for cross-session calibration', () => { + const quarterTurn = Math.sqrt(0.5); + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame( + makeFrame([], undefined, undefined, 0, pose(1, 2, 3, { y: quarterTurn, w: quarterTurn })), + sceneSpace + ); + recorder.stopRecording(); + + const calibration = recorder.getRecording().calibration; + expect(calibration?.mode).toBe('viewer-start-yaw'); + expect(calibration?.pose).toMatchObject({ px: 1, py: 2, pz: 3, ox: 0, oz: 0 }); + expect(calibration?.pose.oy).toBeCloseTo(quarterTurn); + expect(calibration?.pose.ow).toBeCloseTo(quarterTurn); + }); + + test('waits for a viewer pose before recording the first frame', () => { + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(makeFrame([], undefined, undefined, 0, null), sceneSpace); + expect(recorder.recordedFrameCount).toBe(0); + + recorder.beginFrame(makeFrame([], undefined, undefined, 1, pose(0)), sceneSpace); + expect(recorder.recordedFrameCount).toBe(1); + }); }); describe('scoped CloudXR replay frame', () => { @@ -360,8 +404,8 @@ describe('scoped CloudXR replay frame', () => { expect(adapted.session.inputSources).toBe(replaySources); expect(replaySources[0].gamepad).toBe(replaySources[0].gamepad); expect(replaySources[0].gamepad?.axes).toEqual([3]); - expect(adapted.getJointPose(wrist, cloudSpace)?.transform.position.x).toBe(15); - expect(adapted.getJointPose(wrist, cloudSpace)?.radius).toBe(0.02); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.transform.position.x).toBe(15); + expect(adapted.getJointPose?.(wrist, cloudSpace)?.radius).toBe(0.02); }); test('delegates unknown spaces and joints to the real frame', () => { @@ -378,7 +422,123 @@ describe('scoped CloudXR replay frame', () => { const adapted = recorder.adaptTrackingFrame(frame); expect(adapted.getPose(unknownSpace, sceneSpace)?.transform.position.x).toBe(7); - expect(adapted.getJointPose(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + expect(adapted.getJointPose?.(unknownJoint, sceneSpace)?.transform.position.x).toBe(8); + }); + + test('aligns loaded poses from the recorded viewer origin to the current XR session', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); + const frame = makeFrame([source], undefined, undefined, 0, pose(11, 5), session); + const recorder = new XRInputRecorder(); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(frame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(12); + expect(replayed?.transform.position.y).toBeCloseTo(5); + }); + + test('applies the calibrated heading to replayed poses', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const sample = frameData(); + sample.poses.leftGrip = { px: 0, py: 0, pz: -1, ox: 0, oy: 0, oz: 0, ow: 1 }; + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(0, sample))); + const quarterTurn = Math.sqrt(0.5); + const frame = makeFrame( + [source], + undefined, + undefined, + 0, + pose(0, 0, 0, { y: quarterTurn, w: quarterTurn }), + session + ); + const recorder = new XRInputRecorder(); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(frame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(-1); + expect(replayed?.transform.position.z).toBeCloseTo(0); + expect(replayed?.transform.orientation.y).toBeCloseTo(quarterTurn); + expect(replayed?.transform.orientation.w).toBeCloseTo(quarterTurn); + }); + + test('freezes calibration within a session and recalibrates for a new session', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const firstSession = makeSession([source]); + const secondSession = makeSession([source]); + const loaded = XRInputRecorder.importJSON(JSON.stringify(calibratedRecording(1, frameData(2)))); + const recorder = new XRInputRecorder(); + + const firstFrame = makeFrame([source], undefined, undefined, 0, pose(11), firstSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(firstFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(firstFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(12); + recorder.stopReplay(); + + const movedViewerFrame = makeFrame([source], undefined, undefined, 1, pose(21), firstSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(movedViewerFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(movedViewerFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(12); + recorder.stopReplay(); + + const newSessionFrame = makeFrame([source], undefined, undefined, 2, pose(21), secondSession); + recorder.startReplay(loaded, true, 'frame'); + recorder.beginFrame(newSessionFrame, sceneSpace); + expect( + recorder.adaptTrackingFrame(newSessionFrame).getPose(grip, sceneSpace)?.transform.position.x + ).toBeCloseTo(22); + }); + + test('keeps in-memory recording and replay in the same reference space unchanged', () => { + const grip = {} as XRSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + } as XRInputSource; + const session = makeSession([source]); + const recordingFrame = makeFrame( + [source], + space => (space === grip ? pose(2) : null), + undefined, + 0, + pose(1), + session + ); + const recorder = new XRInputRecorder(); + recorder.startRecording(); + recorder.beginFrame(recordingFrame, sceneSpace); + recorder.stopRecording(); + const saved = recorder.getRecording(); + + const replayFrame = makeFrame([source], undefined, undefined, 1, pose(11), session); + recorder.startReplay(saved, true, 'frame'); + recorder.beginFrame(replayFrame, sceneSpace); + + const replayed = recorder.adaptTrackingFrame(replayFrame).getPose(grip, sceneSpace); + expect(replayed?.transform.position.x).toBeCloseTo(2); }); }); @@ -388,7 +548,9 @@ describe('serialization', () => { recorder.startRecording(); recorder.beginFrame(makeFrame(), sceneSpace); recorder.stopRecording(); - expect(XRInputRecorder.importJSON(recorder.exportJSON()).frames).toHaveLength(1); + const imported = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(imported.frames).toHaveLength(1); + expect(imported.calibration?.mode).toBe('viewer-start-yaw'); expect(typeof recorder.getRecording().recordedAt).toBe('number'); }); @@ -423,6 +585,8 @@ describe('serialization', () => { { version: 1, frames: null }, { version: 1, frames: [{ ...frameData(), timeMs: undefined }] }, { version: 1, frames: [frameData(2), frameData(1)] }, + { version: 1, calibration: { mode: 'viewer-start-yaw', pose: {} }, frames: [] }, + { version: 1, calibration: { mode: 'unknown', pose: {} }, frames: [] }, null, 42, [], diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 8838d38ba5..07863cca08 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -61,6 +61,12 @@ export type RecordedFrame = { export type Recording = { version: 1; recordedAt?: number; + /** Optional cross-session origin calibration. Absent in legacy recordings. */ + calibration?: { + mode: 'viewer-start-yaw'; + /** Gravity-aligned viewer pose in the recording's scene reference space. */ + pose: PoseData; + }; frames: RecordedFrame[]; }; @@ -83,6 +89,14 @@ function serializePose(pose: XRPose | null | undefined): SerializedPose { return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; } +function isPoseData(value: unknown): value is PoseData { + if (typeof value !== 'object' || value === null) return false; + const pose = value as Partial; + return [pose.px, pose.py, pose.pz, pose.ox, pose.oy, pose.oz, pose.ow].every(component => + Number.isFinite(component) + ); +} + function serializeJoint(pose: XRJointPose | null | undefined): SerializedJoint { const serialized = serializePose(pose); return serialized ? { ...serialized, radius: pose?.radius ?? 0.005 } : null; @@ -134,7 +148,7 @@ function makePose(pose: PoseData): XRPose { emulatedPosition: true, linearVelocity: null, angularVelocity: null, - } as XRPose; + } as unknown as XRPose; } function makeJointPose(joint: Exclude): XRJointPose { @@ -334,10 +348,15 @@ function replayDuration(frames: RecordedFrame[]): number { return 0; } -/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ -function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - const p = baseFromScene.position; - const q = baseFromScene.orientation; +/** Apply a baseSpace <- sceneSpace transform to a pose expressed in sceneSpace. */ +function transformByPose(pose: T, baseFromScene: PoseData): T { + const p = { x: baseFromScene.px, y: baseFromScene.py, z: baseFromScene.pz }; + const q = { + x: baseFromScene.ox, + y: baseFromScene.oy, + z: baseFromScene.oz, + w: baseFromScene.ow, + }; // Rotate the recorded translation by q using v' = v + qw*t + cross(q.xyz, t). const tx = 2 * (q.y * pose.pz - q.z * pose.py); @@ -356,6 +375,87 @@ function transformFromScene(pose: T, baseFromScene: XRRigidT }; } +/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ +function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { + const { position: p, orientation: q } = baseFromScene; + return transformByPose(pose, { + px: p.x, + py: p.y, + pz: p.z, + ox: q.x, + oy: q.y, + oz: q.z, + ow: q.w, + }); +} + +function inversePose(pose: PoseData): PoseData { + const magnitudeSquared = + pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; + const scale = magnitudeSquared > 0 ? 1 / magnitudeSquared : 1; + const inverseOrientation = { + ox: -pose.ox * scale, + oy: -pose.oy * scale, + oz: -pose.oz * scale, + ow: pose.ow * scale, + }; + const inverseTranslation = transformByPose( + { px: -pose.px, py: -pose.py, pz: -pose.pz, ox: 0, oy: 0, oz: 0, ow: 1 }, + { px: 0, py: 0, pz: 0, ...inverseOrientation } + ); + return { + px: inverseTranslation.px, + py: inverseTranslation.py, + pz: inverseTranslation.pz, + ...inverseOrientation, + }; +} + +/** Keep translation and heading while removing viewer pitch and roll. */ +function gravityAlignedViewerPose(pose: PoseData): PoseData { + const yaw = Math.atan2( + 2 * (pose.ox * pose.oz + pose.ow * pose.oy), + 1 - 2 * (pose.ox * pose.ox + pose.oy * pose.oy) + ); + const halfYaw = yaw / 2; + return { + px: pose.px, + py: pose.py, + pz: pose.pz, + ox: 0, + oy: Math.sin(halfYaw), + oz: 0, + ow: Math.cos(halfYaw), + }; +} + +function captureViewerCalibration( + frame: XRFrame, + referenceSpace: XRReferenceSpace +): Recording['calibration'] | undefined { + const viewerPose = serializePose(frame.getViewerPose(referenceSpace)); + return viewerPose + ? { mode: 'viewer-start-yaw', pose: gravityAlignedViewerPose(viewerPose) } + : undefined; +} + +function sceneAlignment(recordedViewer: PoseData, currentViewer: PoseData): PoseData { + // currentScene <- recordedScene = currentScene <- viewer <- recordedScene. + return transformByPose(inversePose(recordedViewer), currentViewer); +} + +type RecordingContext = { + session: XRSession; + referenceSpace: XRReferenceSpace; + referenceSpaceEpoch: number; +}; + +type CachedAlignment = { + referenceSpace: XRReferenceSpace; + referenceSpaceEpoch: number; + transform: PoseData | null; +}; + /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -381,6 +481,17 @@ export class XRInputRecorder { private _sceneReferenceSpace: XRReferenceSpace | null = null; private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; + private _recordingCalibration: Recording['calibration']; + private _recordingContext: RecordingContext | null = null; + private _recordingContexts = new WeakMap(); + private _alignmentCache = new WeakMap>(); + private _replayRecording: Recording | null = null; + private _replaySourceContext: RecordingContext | null = null; + private _replaySceneAlignment: PoseData | null = null; + private _replayAlignmentReady = false; + private _observedSession: XRSession | null = null; + private _observedReferenceSpace: XRReferenceSpace | null = null; + private _referenceSpaceEpochs = new WeakMap(); get mode() { return this._mode; @@ -404,6 +515,8 @@ export class XRInputRecorder { this._currentFrame = null; this._recordingStartTime = null; this._recordedAt = Date.now(); + this._recordingCalibration = undefined; + this._recordingContext = null; this._mode = 'recording'; } @@ -422,6 +535,10 @@ export class XRInputRecorder { this._replayElapsedMs = 0; this._lastReplayDisplayTime = null; this._currentFrame = null; + this._replayRecording = recording; + this._replaySourceContext = this._recordingContexts.get(recording) ?? null; + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; this._mode = 'replaying'; } @@ -429,6 +546,10 @@ export class XRInputRecorder { if (this._mode !== 'replaying') return; this._currentFrame = null; this._lastReplayDisplayTime = null; + this._replayRecording = null; + this._replaySourceContext = null; + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; this._mode = 'idle'; } @@ -443,7 +564,7 @@ export class XRInputRecorder { connected = true, captureLive = false ): void { - this._sceneReferenceSpace = sceneReferenceSpace; + this._observeReferenceSpace(frame.session, sceneReferenceSpace); if (this._mode === 'idle') { this._currentFrame = @@ -458,6 +579,15 @@ export class XRInputRecorder { if (this._mode === 'recording') { if (!sceneReferenceSpace) return; + if (!this._recordingCalibration) { + this._recordingCalibration = captureViewerCalibration(frame, sceneReferenceSpace); + if (!this._recordingCalibration) return; + this._recordingContext = { + session: frame.session, + referenceSpace: sceneReferenceSpace, + referenceSpaceEpoch: this._referenceSpaceEpoch(frame.session), + }; + } const now = frameTimestampMs(frame); this._recordingStartTime ??= now; const timeMs = Math.max(0, now - this._recordingStartTime); @@ -471,6 +601,11 @@ export class XRInputRecorder { return; } + if (!sceneReferenceSpace || !this._prepareReplayAlignment(frame, sceneReferenceSpace)) { + this._currentFrame = null; + return; + } + if (this._replayPacing === 'time') { this._advanceTimedReplay(frameTimestampMs(frame)); return; @@ -547,6 +682,15 @@ export class XRInputRecorder { if (!Array.isArray(recording.frames)) { throw new Error('Malformed recording: frames is not an array'); } + if ( + recording.calibration !== undefined && + (typeof recording.calibration !== 'object' || + recording.calibration === null || + recording.calibration.mode !== 'viewer-start-yaw' || + !isPoseData(recording.calibration.pose)) + ) { + throw new Error('Malformed recording: calibration is invalid'); + } let previousTime = -1; for (const frame of recording.frames) { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { @@ -558,11 +702,16 @@ export class XRInputRecorder { } getRecording(): Recording { - return { + const recording: Recording = { version: 1, recordedAt: this._recordedAt, + calibration: this._recordingCalibration, frames: [...this._frames], }; + if (this._recordingContext) { + this._recordingContexts.set(recording, this._recordingContext); + } + return recording; } private _assertIdle(): void { @@ -571,6 +720,81 @@ export class XRInputRecorder { } } + private _referenceSpaceEpoch(session: XRSession): number { + return this._referenceSpaceEpochs.get(session) ?? 0; + } + + private _observeReferenceSpace( + session: XRSession, + referenceSpace: XRReferenceSpace | null + ): void { + if (session === this._observedSession && referenceSpace === this._observedReferenceSpace) { + return; + } + + this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + if (session === this._observedSession && this._observedReferenceSpace !== null) { + this._referenceSpaceEpochs.set(session, this._referenceSpaceEpoch(session) + 1); + } else if (!this._referenceSpaceEpochs.has(session)) { + this._referenceSpaceEpochs.set(session, 0); + } + + this._observedSession = session; + this._observedReferenceSpace = referenceSpace; + this._sceneReferenceSpace = referenceSpace; + referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); + this._replayAlignmentReady = false; + } + + private _onReferenceSpaceReset = (): void => { + if (!this._observedSession) return; + this._referenceSpaceEpochs.set( + this._observedSession, + this._referenceSpaceEpoch(this._observedSession) + 1 + ); + this._replaySceneAlignment = null; + this._replayAlignmentReady = false; + this._currentFrame = null; + }; + + private _prepareReplayAlignment(frame: XRFrame, referenceSpace: XRReferenceSpace): boolean { + if (this._replayAlignmentReady) return true; + const recording = this._replayRecording; + if (!recording) return false; + + const referenceSpaceEpoch = this._referenceSpaceEpoch(frame.session); + let sessionCache = this._alignmentCache.get(recording); + const cached = sessionCache?.get(frame.session); + if ( + cached && + cached.referenceSpace === referenceSpace && + cached.referenceSpaceEpoch === referenceSpaceEpoch + ) { + this._replaySceneAlignment = cached.transform; + this._replayAlignmentReady = true; + return true; + } + + let transform: PoseData | null = null; + const source = this._replaySourceContext; + const sameReferenceSpace = + source?.session === frame.session && + source.referenceSpace === referenceSpace && + source.referenceSpaceEpoch === referenceSpaceEpoch; + if (!sameReferenceSpace && recording.calibration) { + const currentCalibration = captureViewerCalibration(frame, referenceSpace); + if (!currentCalibration) return false; + transform = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + } + + sessionCache ??= new WeakMap(); + sessionCache.set(frame.session, { referenceSpace, referenceSpaceEpoch, transform }); + this._alignmentCache.set(recording, sessionCache); + this._replaySceneAlignment = transform; + this._replayAlignmentReady = true; + return true; + } + private _proxySession(session: XRSession, replay: RecordedFrame): XRSession { const inputSources = Array.from(session.inputSources, source => { const hand = source.handedness; @@ -622,7 +846,7 @@ export class XRInputRecorder { return transformed ? makeJointPose(transformed) : undefined; } } - return frame.getJointPose(joint, baseSpace) ?? undefined; + return frame.getJointPose?.(joint, baseSpace) ?? undefined; } private _poseInBase( @@ -640,8 +864,12 @@ export class XRInputRecorder { baseSpace: XRSpace ): T | null { if (!pose || !this._sceneReferenceSpace) return null; - if (baseSpace === this._sceneReferenceSpace) return pose; + if (!this._replayAlignmentReady) return null; + const currentScenePose = this._replaySceneAlignment + ? transformByPose(pose, this._replaySceneAlignment) + : pose; + if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); - return relation ? transformFromScene(pose, relation.transform) : null; + return relation ? transformFromScene(currentScenePose, relation.transform) : null; } } From 89d29b78a308b55b8c14b16f6655bc1df8813767 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Fri, 11 Sep 2026 15:49:55 -0700 Subject: [PATCH 2/4] fix(webxr): replay hands without live tracking and preserve alignment Signed-off-by: Yanzi Zhu --- deps/cloudxr/webxr_client/src/CloudXRUI.tsx | 29 +- .../webxr_client/src/RecorderComponent.tsx | 4 +- .../webxr_client/src/RecorderContext.test.tsx | 95 ++++++ .../webxr_client/src/RecorderContext.tsx | 36 +- .../webxr_client/src/xrInputRecorder.test.ts | 317 +++++++++++++++++- .../webxr_client/src/xrInputRecorder.ts | 300 ++++++++++------- .../webxr_client/src/xrReplaySession.ts | 68 ++++ 7 files changed, 720 insertions(+), 129 deletions(-) create mode 100644 deps/cloudxr/webxr_client/src/RecorderContext.test.tsx create mode 100644 deps/cloudxr/webxr_client/src/xrReplaySession.ts diff --git a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx index 8355e0e32c..988640cb4b 100644 --- a/deps/cloudxr/webxr_client/src/CloudXRUI.tsx +++ b/deps/cloudxr/webxr_client/src/CloudXRUI.tsx @@ -603,10 +603,37 @@ export default function CloudXR3DUI({ {recorder.mode === 'recording' ? `REC ${recorder.recordedFrameCount} frames` : recorder.mode === 'replaying' - ? 'Replaying' + ? recorder.replayNeedsCalibration + ? 'Replay paused' + : 'Replaying' : 'Recording'} + {recorder.replayNeedsCalibration && recorder.mode === 'replaying' && ( + + Return your headset to its recording-start position and heading, then + calibrate. + + )} + {recorder.mode === 'idle' && recorder.recordingInterrupted && ( + + Recording stopped because the tracking origin changed. Save it or start a + new recording. + + )} + {recorder.mode === 'idle' && !recorder.recordingInterrupted && ( + + Note your headset position and heading when starting a recording. Use the + same pose to calibrate replay in a new session. + + )} + {recorder.mode === 'replaying' && recorder.replayNeedsCalibration && ( + + )} {recorder.mode !== 'replaying' && ( { @@ -55,6 +55,8 @@ export function RecorderComponent({ isConnected, showTrace }: RecorderComponentP showTrace && isVisible ); + onFrameState(); + if (recorder.mode === 'recording') { tickRef.current++; if (tickRef.current % 30 === 0) { diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx new file mode 100644 index 0000000000..8ec67646f7 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/RecorderContext.test.tsx @@ -0,0 +1,95 @@ +/** @jest-environment jsdom */ + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { act } from 'react'; +import { createRoot, type Root } from 'react-dom/client'; + +import { type RecorderContextValue, RecorderProvider, useRecorder } from './RecorderContext'; + +let current: RecorderContextValue; +let root: Root; + +function Probe() { + current = useRecorder(); + return null; +} + +function frame(session: XRSession): XRFrame { + return { + session, + predictedDisplayTime: 0, + getViewerPose: () => ({ + transform: { position: { x: 0, y: 0, z: 0 }, orientation: { x: 0, y: 0, z: 0, w: 1 } }, + }), + } as unknown as XRFrame; +} + +beforeEach(() => { + Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true }); + root = createRoot(document.createElement('div')); + act(() => + root.render( + + + + ) + ); +}); + +afterEach(() => act(() => root.unmount())); + +test('exposes cross-session calibration and resumes after the operator calibrates', () => { + const space = new EventTarget() as XRReferenceSpace; + const original = { inputSources: [] } as unknown as XRSession; + act(() => current.startRecord()); + act(() => current.recorder.beginFrame(frame(original), space)); + act(() => current.stopRecord()); + expect(current.savedRecording?.frames).toHaveLength(1); + act(() => current.startReplay()); + const next = { inputSources: [] } as unknown as XRSession; + act(() => { + current.recorder.beginFrame(frame(next), space); + current.onFrameState(); + }); + expect(current.mode).toBe('replaying'); + expect(current.replayNeedsCalibration).toBe(true); + act(() => current.calibrateReplay()); + act(() => { + current.recorder.beginFrame(frame(next), space); + current.onFrameState(); + }); + expect(current.replayNeedsCalibration).toBe(false); + expect(current.recorder.currentFrame).not.toBeNull(); +}); + +test('saves interrupted recording and updates the UI when the tracking origin changes', () => { + const session = { inputSources: [] } as unknown as XRSession; + const space = new EventTarget() as XRReferenceSpace; + act(() => current.startRecord()); + act(() => current.recorder.beginFrame(frame(session), space)); + act(() => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform: null })); + current.onFrameState(); + }); + expect(current.mode).toBe('idle'); + expect(current.recordingInterrupted).toBe(true); + expect(current.savedRecording?.frames).toHaveLength(1); + act(() => current.startRecord()); + expect(current.recordingInterrupted).toBe(false); +}); diff --git a/deps/cloudxr/webxr_client/src/RecorderContext.tsx b/deps/cloudxr/webxr_client/src/RecorderContext.tsx index 8db8fae7b9..4381a72192 100644 --- a/deps/cloudxr/webxr_client/src/RecorderContext.tsx +++ b/deps/cloudxr/webxr_client/src/RecorderContext.tsx @@ -23,7 +23,15 @@ * to descendant components. */ -import React, { createContext, useCallback, useContext, useMemo, useRef, useState } from 'react'; +import React, { + createContext, + useCallback, + useContext, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; import { type Recording, type ReplayPacing, XRInputRecorder } from './xrInputRecorder'; @@ -37,6 +45,10 @@ export interface RecorderContextValue { stopRecord: () => void; startReplay: () => void; stopReplay: () => void; + calibrateReplay: () => void; + replayNeedsCalibration: boolean; + recordingInterrupted: boolean; + onFrameState: () => void; setReplayPacing: (pacing: ReplayPacing) => void; onSaveRecording: () => void; onLoadRecording: () => void; @@ -59,9 +71,11 @@ function setLoadStatus(message: string, type: 'success' | 'error'): void { export function RecorderProvider({ children }: { children: React.ReactNode }) { const recorder = useMemo(() => new XRInputRecorder(), []); + useEffect(() => () => recorder.dispose(), [recorder]); const [mode, setMode] = useState<'idle' | 'recording' | 'replaying'>('idle'); const [savedRecording, setSavedRecordingState] = useState(null); const [recordedFrameCount, setRecordedFrameCount] = useState(0); + const [replayNeedsCalibration, setReplayNeedsCalibration] = useState(false); const [replayPacing, setReplayPacingState] = useState('time'); const fileInputRef = useRef(null); @@ -85,6 +99,7 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { if (recorder.mode !== 'idle' || !savedRecording) return; recorder.startReplay(savedRecording, true, replayPacing); setMode('replaying'); + setReplayNeedsCalibration(recorder.replayNeedsCalibration); }, [recorder, replayPacing, savedRecording]); const stopReplay = useCallback(() => { @@ -93,6 +108,21 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { setMode('idle'); }, [recorder]); + const calibrateReplay = useCallback(() => recorder.calibrateReplay(), [recorder]); + + const onFrameState = useCallback(() => { + if (mode !== recorder.mode) { + if (mode === 'recording') { + setSavedRecordingState(recorder.getRecording()); + setRecordedFrameCount(recorder.recordedFrameCount); + } + setMode(recorder.mode); + } + if (replayNeedsCalibration !== recorder.replayNeedsCalibration) { + setReplayNeedsCalibration(recorder.replayNeedsCalibration); + } + }, [mode, recorder, replayNeedsCalibration]); + const setReplayPacing = useCallback((pacing: ReplayPacing) => { setReplayPacingState(pacing); }, []); @@ -155,6 +185,10 @@ export function RecorderProvider({ children }: { children: React.ReactNode }) { stopRecord, startReplay, stopReplay, + calibrateReplay, + replayNeedsCalibration, + recordingInterrupted: recorder.recordingInterrupted, + onFrameState, setReplayPacing, onSaveRecording, onLoadRecording, diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index ea50ce8651..551d32c506 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -60,9 +60,15 @@ function gamepad(axis: number): Gamepad { } function makeSession(inputSources: XRInputSource[] = []): XRSession { - return { inputSources } as unknown as XRSession; + return Object.assign(new EventTarget(), { inputSources }) as unknown as XRSession; } +// Frames in a test share a session unless the test explicitly starts another. +let defaultSession: XRSession; +beforeEach(() => { + defaultSession = makeSession(); +}); + type PoseResolver = (space: XRSpace, baseSpace: XRSpace) => XRPose | null; type JointResolver = (joint: XRJointSpace, baseSpace: XRSpace) => XRJointPose | null; @@ -72,8 +78,9 @@ function makeFrame( getJointPose: JointResolver = () => null, predictedDisplayTime = 0, viewerPose: XRPose | null = pose(0), - session: XRSession = makeSession(inputSources) + session: XRSession = defaultSession ): XRFrame { + Object.assign(session, { inputSources }); return { session, predictedDisplayTime, @@ -336,12 +343,14 @@ describe('canonical scene-space capture', () => { }); describe('scoped CloudXR replay frame', () => { - test('does not alter global prototypes and returns real frames outside replay', () => { + test('preserves live tracking outside replay without changing browser frames', () => { const recorder = new XRInputRecorder(); const frame = makeFrame(); const originalGetPose = frame.getPose; recorder.startRecording(); - expect(recorder.adaptTrackingFrame(frame)).toBe(frame); + const adapted = recorder.adaptTrackingFrame(frame); + expect(adapted.session.inputSources).toBe(frame.session.inputSources); + expect(adapted.getViewerPose(sceneSpace)).toEqual(frame.getViewerPose(sceneSpace)); expect(frame.getPose).toBe(originalGetPose); recorder.stopRecording(); }); @@ -438,6 +447,9 @@ describe('scoped CloudXR replay frame', () => { const recorder = new XRInputRecorder(); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(frame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); expect(replayed?.transform.position.x).toBeCloseTo(12); @@ -467,6 +479,9 @@ describe('scoped CloudXR replay frame', () => { const recorder = new XRInputRecorder(); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(frame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); const replayed = recorder.adaptTrackingFrame(frame).getPose(grip, sceneSpace); expect(replayed?.transform.position.x).toBeCloseTo(-1); @@ -490,6 +505,9 @@ describe('scoped CloudXR replay frame', () => { const firstFrame = makeFrame([source], undefined, undefined, 0, pose(11), firstSession); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(firstFrame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(firstFrame, sceneSpace); expect( recorder.adaptTrackingFrame(firstFrame).getPose(grip, sceneSpace)?.transform.position.x ).toBeCloseTo(12); @@ -506,6 +524,9 @@ describe('scoped CloudXR replay frame', () => { const newSessionFrame = makeFrame([source], undefined, undefined, 2, pose(21), secondSession); recorder.startReplay(loaded, true, 'frame'); recorder.beginFrame(newSessionFrame, sceneSpace); + expect(recorder.replayNeedsCalibration).toBe(true); + recorder.calibrateReplay(); + recorder.beginFrame(newSessionFrame, sceneSpace); expect( recorder.adaptTrackingFrame(newSessionFrame).getPose(grip, sceneSpace)?.transform.position.x ).toBeCloseTo(22); @@ -542,6 +563,294 @@ describe('scoped CloudXR replay frame', () => { }); }); +describe('explicit replay calibration and reference-space resets', () => { + function setup(pacing: 'frame' | 'time' = 'frame') { + const grip = {} as XRSpace; + const wrist = {} as XRJointSpace; + const source = { + handedness: 'left', + gripSpace: grip, + targetRaySpace: {} as XRSpace, + gamepad: gamepad(1), + hand: new Map([['wrist', wrist]]), + } as unknown as XRInputSource; + const session = makeSession([source]); + const space = new EventTarget() as XRReferenceSpace; + const recorder = new XRInputRecorder(); + const saved = calibratedRecording(0, timedFrame(0, 1), timedFrame(100, 3)); + const frame = (viewerX: number, time = 0, viewer: XRPose | null = pose(viewerX)) => + makeFrame( + [source], + () => pose(999), + () => jointPose(999), + time, + viewer, + session + ); + const advance = (viewerX: number, time = 0) => { + const current = frame(viewerX, time); + recorder.beginFrame(current, space); + return recorder.adaptTrackingFrame(current); + }; + const reset = (transform: XRRigidTransform | null) => { + space.dispatchEvent(Object.assign(new Event('reset'), { transform })); + }; + recorder.startReplay(saved, false, pacing); + return { recorder, grip, wrist, source, session, space, saved, frame, advance, reset }; + } + + test('suppresses live input and waits for a deliberate calibration with a valid viewer pose', () => { + const { recorder, grip, wrist, space, frame, advance } = setup(); + const waiting = advance(10); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.currentFrame).toBeNull(); + expect(waiting.getPose(grip, space)).toBeUndefined(); + expect(waiting.getJointPose?.(wrist, space)).toBeUndefined(); + expect(waiting.session.inputSources).toHaveLength(0); + + recorder.calibrateReplay(); + recorder.beginFrame(frame(10, 0, null), space); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.replayFrameIndex).toBe(0); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); + expect(recorder.replayNeedsCalibration).toBe(false); + }); + + test.each(['frame', 'time'] as const)( + '%s replay preserves world placement after headset motion and repeated translation resets', + pacing => { + const { recorder, grip, wrist, space, advance, reset } = setup(pacing); + advance(10); + recorder.calibrateReplay(); + expect(advance(10).getPose(grip, space)?.transform.position.x).toBeCloseTo(11); + reset(new XRRigidTransform({ x: -100 })); + const firstReset = advance(112, pacing === 'time' ? 50 : 1); + expect(recorder.replayNeedsCalibration).toBe(false); + expect(firstReset.getPose(grip, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 112 : 113 + ); + expect(firstReset.getJointPose?.(wrist, space)?.transform.position.x).toBeCloseTo( + pacing === 'time' ? 114 : 115 + ); + reset(new XRRigidTransform({ x: -20 })); + expect(advance(132, 100).getPose(grip, space)?.transform.position.x).toBeCloseTo(133); + } + ); + + test('uses the inverse reset rotation for position and orientation', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + saved.frames = [saved.frames[0]]; + recorder.stopReplay(); + recorder.startReplay(saved, false, 'frame'); + advance(10); + recorder.calibrateReplay(); + advance(10); + const q = Math.sqrt(0.5); + reset(new XRRigidTransform({}, { x: 0, y: q, z: 0, w: q })); + const result = advance(999).getPose(grip, space)!; + expect(result.transform.position.x).toBeCloseTo(0); + expect(result.transform.position.z).toBeCloseTo(11); + expect(result.transform.orientation.y).toBeCloseTo(-q); + expect(result.transform.orientation.w).toBeCloseTo(q); + }); + + test('unknown reset pauses the replay clock and requires another explicit calibration', () => { + const { recorder, grip, space, advance, reset } = setup('time'); + advance(10, 1000); + recorder.calibrateReplay(); + advance(10, 1000); + expect(advance(12, 1050).getPose(grip, space)?.transform.position.x).toBeCloseTo(12); + reset(null); + expect(advance(112, 2000).getPose(grip, space)).toBeUndefined(); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.currentFrame).toBeNull(); + recorder.calibrateReplay(); + expect(advance(110, 5000).getPose(grip, space)?.transform.position.x).toBeCloseTo(112); + expect(recorder.currentFrame?.timeMs).toBe(50); + }); + + test('preserves cached placement when a known reset occurs while replay is stopped', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + advance(10); + recorder.calibrateReplay(); + advance(10); + recorder.stopReplay(); + reset(new XRRigidTransform({ x: -100 })); + recorder.startReplay(saved, false, 'frame'); + expect(advance(112).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + expect(recorder.replayNeedsCalibration).toBe(false); + }); + + test('calibration after a reset is stored in the original reference frame', () => { + const { recorder, grip, space, saved, advance, reset } = setup(); + advance(10); + reset(new XRRigidTransform({ x: -100 })); + recorder.calibrateReplay(); + expect(advance(110).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + recorder.stopReplay(); + recorder.startReplay(saved, false, 'frame'); + expect(advance(115).getPose(grip, space)?.transform.position.x).toBeCloseTo(111); + }); + + test('reference-space replacement invalidates placement and pending calibration', () => { + const { recorder, grip, frame, advance } = setup(); + advance(10); + recorder.calibrateReplay(); + const replacement = new EventTarget() as XRReferenceSpace; + const current = frame(20); + recorder.beginFrame(current, replacement); + expect(recorder.replayNeedsCalibration).toBe(true); + expect(recorder.adaptTrackingFrame(current).getPose(grip, replacement)).toBeUndefined(); + recorder.calibrateReplay(); + recorder.beginFrame(current, replacement); + expect( + recorder.adaptTrackingFrame(current).getPose(grip, replacement)?.transform.position.x + ).toBeCloseTo(21); + }); + + test.each([true, false])('stops recording on reset (known transform: %s)', known => { + const { recorder, grip, space, frame, reset } = setup(); + recorder.stopReplay(); + recorder.startRecording(); + recorder.beginFrame(frame(0), space); + reset(known ? new XRRigidTransform({ x: -100 }) : null); + expect(recorder.mode).toBe('idle'); + expect(recorder.recordingInterrupted).toBe(true); + recorder.beginFrame(frame(100), space); + expect(recorder.getRecording().frames).toHaveLength(1); + const saved = recorder.getRecording(); + recorder.startReplay(saved, false, 'frame'); + recorder.beginFrame(frame(100), space); + expect(recorder.replayNeedsCalibration).toBe(!known); + if (known) { + expect( + recorder.adaptTrackingFrame(frame(100)).getPose(grip, space)?.transform.position.x + ).toBeCloseTo(1099); + } + }); +}); + +describe('recorded hands without live tracking', () => { + function benchmark(pacing: 'frame' | 'time') { + const session = makeSession(); + const recorder = new XRInputRecorder(); + const frame = (sources: XRInputSource[] = [], time = 0) => + makeFrame( + sources, + () => null, + () => null, + time, + pose(0), + session + ); + // CloudXR initializes its active-hand set on the first tracking frame and + // subsequently refreshes it from inputsourceschange, not from joint poses. + const active = new Set(); + const refresh = (current: XRSession) => { + active.clear(); + for (const source of current.inputSources) { + if (source.hand) active.add(source.handedness); + } + }; + const idle = recorder.adaptTrackingFrame(frame()); + refresh(idle.session); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => refresh(event.session)); + idle.session.addEventListener('inputsourceschange', changed); + const read = (current: XRFrame) => { + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = Array.from(adapted.session.inputSources).find(s => s.handedness === 'left'); + if (!source?.hand || !active.has(source.handedness)) return null; + return adapted.getJointPose?.(source.hand.get('wrist')!, sceneSpace)?.transform.position.x; + }; + recorder.startReplay(recording(timedFrame(0, 1), timedFrame(100, 3)), false, pacing); + return { recorder, session, frame, active, changed, read, idle }; + } + + test.each(['frame', 'time'] as const)( + '%s replay advances when hands were never detected', + pacing => { + const { read, frame, active, changed, session } = benchmark(pacing); + expect(read(frame())).toBeCloseTo(3); + expect(active.has('left')).toBe(true); + expect(read(frame([], pacing === 'time' ? 50 : 1))).toBeCloseTo(pacing === 'time' ? 4 : 5); + expect(changed).toHaveBeenCalledTimes(1); + expect(session.inputSources).toHaveLength(0); + } + ); + + test('live hands appearing and disappearing do not gate or replace recorded hands', () => { + const { read, frame, changed, session, active, recorder } = benchmark('time'); + expect(read(frame())).toBeCloseTo(3); + const live = { + handedness: 'left', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const visible = frame([live], 25); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [live], + removed: [], + }) + ); + expect(read(visible)).toBeCloseTo(3.5); + const absent = frame([], 50); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [live], + }) + ); + expect(active.has('left')).toBe(true); + expect(read(absent)).toBeCloseTo(4); + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(absent); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + }); + + test('recorded tracking loss removes the source, and recorded recovery restores it', () => { + const { recorder, read, frame, active } = benchmark('frame'); + recorder.stopReplay(); + const lost = timedFrame(50, 2); + lost.handJoints.left = { wrist: null }; + recorder.startReplay(recording(timedFrame(0, 1), lost, timedFrame(100, 3)), false, 'frame'); + expect(read(frame())).toBeCloseTo(3); + expect(read(frame())).toBeNull(); + expect(active.size).toBe(0); + expect(read(frame())).toBeCloseTo(5); + }); + + test('forwards native events outside replay and supports listener cleanup', () => { + const { recorder, session, frame, changed, idle } = benchmark('frame'); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame()); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + expect(changed.mock.calls[0][0].session).toBe(idle.session); + idle.session.removeEventListener('inputsourceschange', changed); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { + session, + added: [], + removed: [], + }) + ); + expect(changed).toHaveBeenCalledTimes(1); + recorder.dispose(); + }); +}); + describe('serialization', () => { test('round-trips version 1 recordings', () => { const recorder = new XRInputRecorder(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 07863cca08..8bf4072485 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -22,6 +22,8 @@ * UI pointers, and CloudXR rendering/reprojection. */ +import { XRReplaySession } from './xrReplaySession'; + type PoseData = { px: number; py: number; @@ -72,7 +74,7 @@ export type Recording = { export type ReplayPacing = 'frame' | 'time'; -type Handedness = 'left' | 'right'; +const IDENTITY_POSE: PoseData = { px: 0, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }; function emptyFrame(timeMs = 0): RecordedFrame { return { @@ -85,7 +87,11 @@ function emptyFrame(timeMs = 0): RecordedFrame { function serializePose(pose: XRPose | null | undefined): SerializedPose { if (!pose) return null; - const { position: p, orientation: o } = pose.transform; + return serializeTransform(pose.transform); +} + +function serializeTransform(transform: XRRigidTransform): PoseData { + const { position: p, orientation: o } = transform; return { px: p.x, py: p.y, pz: p.z, ox: o.x, oy: o.y, oz: o.z, ow: o.w }; } @@ -140,6 +146,7 @@ function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = } function makePose(pose: PoseData): XRPose { + // WebXR permits null velocities; @types/webxr currently excludes null. return { transform: new XRRigidTransform( { x: pose.px, y: pose.py, z: pose.pz, w: 1 }, @@ -178,16 +185,6 @@ function bindWebXRMember(target: object, property: PropertyKey): unknown { return typeof value === 'function' ? value.bind(target) : value; } -function proxyInputSource(source: XRInputSource, gamepad: SerializedGamepad | null): XRInputSource { - const replayGamepad = gamepad ? makeGamepad(gamepad) : null; - return new Proxy(source, { - get(target, property) { - if (property === 'gamepad') return replayGamepad; - return bindWebXRMember(target, property); - }, - }); -} - function lerp(from: number, to: number, alpha: number): number { return from + (to - from) * alpha; } @@ -377,16 +374,7 @@ function transformByPose(pose: T, baseFromScene: PoseData): /** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - const { position: p, orientation: q } = baseFromScene; - return transformByPose(pose, { - px: p.x, - py: p.y, - pz: p.z, - ox: q.x, - oy: q.y, - oz: q.z, - ow: q.w, - }); + return transformByPose(pose, serializeTransform(baseFromScene)); } function inversePose(pose: PoseData): PoseData { @@ -444,18 +432,6 @@ function sceneAlignment(recordedViewer: PoseData, currentViewer: PoseData): Pose return transformByPose(inversePose(recordedViewer), currentViewer); } -type RecordingContext = { - session: XRSession; - referenceSpace: XRReferenceSpace; - referenceSpaceEpoch: number; -}; - -type CachedAlignment = { - referenceSpace: XRReferenceSpace; - referenceSpaceEpoch: number; - transform: PoseData | null; -}; - /** * Monotonic frame clock in ms. Some runtimes (e.g. PICO) leave * XRFrame.predictedDisplayTime undefined; without a fallback that yields NaN, @@ -482,16 +458,19 @@ export class XRInputRecorder { private _recordingStartTime: number | null = null; private _recordedAt: number | undefined; private _recordingCalibration: Recording['calibration']; - private _recordingContext: RecordingContext | null = null; - private _recordingContexts = new WeakMap(); - private _alignmentCache = new WeakMap>(); + private _recordingAlignment: PoseData | null = null; + private _alignmentCache = new WeakMap(); private _replayRecording: Recording | null = null; - private _replaySourceContext: RecordingContext | null = null; private _replaySceneAlignment: PoseData | null = null; - private _replayAlignmentReady = false; + private _calibrationRequested = false; private _observedSession: XRSession | null = null; private _observedReferenceSpace: XRReferenceSpace | null = null; - private _referenceSpaceEpochs = new WeakMap(); + // Cached alignments use the origin observed at session/reference-space entry. + private _currentFromInitial: PoseData = IDENTITY_POSE; + private _recordingInterrupted = false; + private _trackingSession: XRReplaySession | null = null; + private _recordedHandSources: XRInputSource[] = []; + private _sourceProxies = new WeakMap(); get mode() { return this._mode; @@ -509,6 +488,25 @@ export class XRInputRecorder { return this._currentFrame; } + get replayNeedsCalibration(): boolean { + return ( + this._mode === 'replaying' && + !!this._replayRecording?.calibration && + !this._alignmentCache.has(this._replayRecording) + ); + } + + get recordingInterrupted(): boolean { + return this._recordingInterrupted; + } + + /** Call after the operator returns to the recording's physical starting pose. */ + calibrateReplay(): void { + if (this.replayNeedsCalibration && this._observedReferenceSpace) { + this._calibrationRequested = true; + } + } + startRecording(): void { this._assertIdle(); this._frames = []; @@ -516,7 +514,8 @@ export class XRInputRecorder { this._recordingStartTime = null; this._recordedAt = Date.now(); this._recordingCalibration = undefined; - this._recordingContext = null; + this._recordingAlignment = null; + this._recordingInterrupted = false; this._mode = 'recording'; } @@ -529,6 +528,31 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); this._replayFrames = recording.frames; + this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { + const names = new Set(); + for (const frame of recording.frames) { + for (const name of Object.keys(frame.handJoints[handedness])) names.add(name); + } + if (!names.size) return []; + const hand = new Map(); + for (const name of names) { + hand.set( + name as XRHandJoint, + Object.assign(new EventTarget(), { jointName: name as XRHandJoint }) + ); + } + return [ + { + handedness, + // @types/webxr still requires obsolete numeric constants on XRHand. + hand: hand as XRHand, + targetRayMode: 'tracked-pointer' as const, + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: ['generic-hand-select'], + }, + ]; + }); this._replayIndex = 0; this._loopReplay = loop; this._replayPacing = pacing; @@ -536,9 +560,8 @@ export class XRInputRecorder { this._lastReplayDisplayTime = null; this._currentFrame = null; this._replayRecording = recording; - this._replaySourceContext = this._recordingContexts.get(recording) ?? null; this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._mode = 'replaying'; } @@ -547,9 +570,9 @@ export class XRInputRecorder { this._currentFrame = null; this._lastReplayDisplayTime = null; this._replayRecording = null; - this._replaySourceContext = null; + this._recordedHandSources = []; this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._mode = 'idle'; } @@ -582,11 +605,7 @@ export class XRInputRecorder { if (!this._recordingCalibration) { this._recordingCalibration = captureViewerCalibration(frame, sceneReferenceSpace); if (!this._recordingCalibration) return; - this._recordingContext = { - session: frame.session, - referenceSpace: sceneReferenceSpace, - referenceSpaceEpoch: this._referenceSpaceEpoch(frame.session), - }; + this._recordingAlignment = inversePose(this._currentFromInitial); } const now = frameTimestampMs(frame); this._recordingStartTime ??= now; @@ -602,6 +621,7 @@ export class XRInputRecorder { } if (!sceneReferenceSpace || !this._prepareReplayAlignment(frame, sceneReferenceSpace)) { + this._lastReplayDisplayTime = null; this._currentFrame = null; return; } @@ -641,18 +661,26 @@ export class XRInputRecorder { * objects are modified, and callers retain the original frame for rendering. */ adaptTrackingFrame = (frame: XRFrame): XRFrame => { - const replay = this._currentFrame; - if (this._mode !== 'replaying' || !replay) return frame; + if (this._trackingSession?.nativeSession !== frame.session) { + this._trackingSession?.dispose(); + this._trackingSession = new XRReplaySession(frame.session); + } + const replaying = this._mode === 'replaying'; + this._trackingSession.setInputSources( + replaying ? this._replayInputSources(frame.session) : null + ); + const session = this._trackingSession.session; + // Waiting for calibration must not silently substitute live robot inputs. + const replay = this._currentFrame ?? emptyFrame(0); - const session = this._proxySession(frame.session, replay); return new Proxy(frame, { get: (target, property) => { if (property === 'session') return session; - if (property === 'getPose') { + if (replaying && property === 'getPose') { return (space: XRSpace, baseSpace: XRSpace) => this._replayPose(target, replay, space, baseSpace); } - if (property === 'getJointPose') { + if (replaying && property === 'getJointPose') { return (joint: XRJointSpace, baseSpace: XRSpace) => this._replayJoint(target, replay, joint, baseSpace); } @@ -708,8 +736,8 @@ export class XRInputRecorder { calibration: this._recordingCalibration, frames: [...this._frames], }; - if (this._recordingContext) { - this._recordingContexts.set(recording, this._recordingContext); + if (this._recordingAlignment) { + this._alignmentCache.set(recording, this._recordingAlignment); } return recording; } @@ -720,10 +748,6 @@ export class XRInputRecorder { } } - private _referenceSpaceEpoch(session: XRSession): number { - return this._referenceSpaceEpochs.get(session) ?? 0; - } - private _observeReferenceSpace( session: XRSession, referenceSpace: XRReferenceSpace | null @@ -733,81 +757,115 @@ export class XRInputRecorder { } this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); - if (session === this._observedSession && this._observedReferenceSpace !== null) { - this._referenceSpaceEpochs.set(session, this._referenceSpaceEpoch(session) + 1); - } else if (!this._referenceSpaceEpochs.has(session)) { - this._referenceSpaceEpochs.set(session, 0); - } - + this._interruptRecording(); + this._invalidateAlignments(); this._observedSession = session; this._observedReferenceSpace = referenceSpace; this._sceneReferenceSpace = referenceSpace; referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); - this._replayAlignmentReady = false; } - private _onReferenceSpaceReset = (): void => { - if (!this._observedSession) return; - this._referenceSpaceEpochs.set( - this._observedSession, - this._referenceSpaceEpoch(this._observedSession) + 1 - ); + private _interruptRecording(): void { + if (this._mode === 'recording' && this._frames.length > 0) { + // One recording must not mix coordinates from different reference-space origins. + this.stopRecording(); + this._recordingInterrupted = true; + } + } + + private _invalidateAlignments(): void { + this._alignmentCache = new WeakMap(); + this._recordingAlignment = null; + this._currentFromInitial = IDENTITY_POSE; + this._replaySceneAlignment = null; + this._calibrationRequested = false; + this._lastReplayDisplayTime = null; + this._currentFrame = null; + } + + private _onReferenceSpaceReset = (event: XRReferenceSpaceEvent): void => { + this._interruptRecording(); + if (!event.transform) { + this._invalidateAlignments(); + return; + } + + // The event gives oldScene <- newScene. Preserve world placement using its + // inverse; the current viewer pose includes operator motion, not just drift. + const oldFromNew = serializeTransform(event.transform); + this._currentFromInitial = transformByPose(this._currentFromInitial, inversePose(oldFromNew)); this._replaySceneAlignment = null; - this._replayAlignmentReady = false; + this._calibrationRequested = false; this._currentFrame = null; }; private _prepareReplayAlignment(frame: XRFrame, referenceSpace: XRReferenceSpace): boolean { - if (this._replayAlignmentReady) return true; + if (this._replaySceneAlignment) return true; const recording = this._replayRecording; if (!recording) return false; - const referenceSpaceEpoch = this._referenceSpaceEpoch(frame.session); - let sessionCache = this._alignmentCache.get(recording); - const cached = sessionCache?.get(frame.session); - if ( - cached && - cached.referenceSpace === referenceSpace && - cached.referenceSpaceEpoch === referenceSpaceEpoch - ) { - this._replaySceneAlignment = cached.transform; - this._replayAlignmentReady = true; - return true; - } - - let transform: PoseData | null = null; - const source = this._replaySourceContext; - const sameReferenceSpace = - source?.session === frame.session && - source.referenceSpace === referenceSpace && - source.referenceSpaceEpoch === referenceSpaceEpoch; - if (!sameReferenceSpace && recording.calibration) { - const currentCalibration = captureViewerCalibration(frame, referenceSpace); - if (!currentCalibration) return false; - transform = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + let initialFromRecorded = this._alignmentCache.get(recording); + if (!initialFromRecorded) { + let currentFromRecorded = IDENTITY_POSE; + if (recording.calibration) { + if (!this._calibrationRequested) return false; + const currentCalibration = captureViewerCalibration(frame, referenceSpace); + if (!currentCalibration) return false; + currentFromRecorded = sceneAlignment(recording.calibration.pose, currentCalibration.pose); + } + initialFromRecorded = transformByPose( + currentFromRecorded, + inversePose(this._currentFromInitial) + ); + this._alignmentCache.set(recording, initialFromRecorded); + this._calibrationRequested = false; } - sessionCache ??= new WeakMap(); - sessionCache.set(frame.session, { referenceSpace, referenceSpaceEpoch, transform }); - this._alignmentCache.set(recording, sessionCache); - this._replaySceneAlignment = transform; - this._replayAlignmentReady = true; + this._replaySceneAlignment = transformByPose(initialFromRecorded, this._currentFromInitial); return true; } - private _proxySession(session: XRSession, replay: RecordedFrame): XRSession { - const inputSources = Array.from(session.inputSources, source => { - const hand = source.handedness; - return hand === 'left' || hand === 'right' - ? proxyInputSource(source, replay.gamepads[hand]) - : source; - }); + dispose(): void { + this._trackingSession?.dispose(); + this._trackingSession = null; + this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._observedReferenceSpace = null; + this._observedSession = null; + this._invalidateAlignments(); + } - return new Proxy(session, { - get(target, property) { - if (property === 'inputSources') return inputSources; - return bindWebXRMember(target, property); - }, + private _replayInputSources(session: XRSession): XRInputSource[] { + const recordedHands = new Set(this._recordedHandSources.map(source => source.handedness)); + const sources = [ + ...this._recordedHandSources.filter(source => { + const hand = source.handedness as 'left' | 'right'; + return Object.values(this._currentFrame?.handJoints[hand] ?? {}).some(Boolean); + }), + ...Array.from(session.inputSources).filter(source => !recordedHands.has(source.handedness)), + ]; + return sources.map(source => { + let proxy = this._sourceProxies.get(source); + if (!proxy) { + let lastGamepad: SerializedGamepad | null | undefined; + let replayGamepad: Gamepad | null = null; + proxy = new Proxy(source, { + get: (target, property) => { + const hand = target.handedness; + if (property === 'gamepad' && (hand === 'left' || hand === 'right')) { + const gamepad = this._currentFrame?.gamepads[hand]; + // WebXR gamepad identity is stable for repeated reads of one frame. + if (gamepad !== lastGamepad) { + replayGamepad = gamepad ? makeGamepad(gamepad) : null; + lastGamepad = gamepad; + } + return replayGamepad; + } + return bindWebXRMember(target, property); + }, + }); + this._sourceProxies.set(source, proxy); + } + return proxy; }); } @@ -817,7 +875,7 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { @@ -836,7 +894,7 @@ export class XRInputRecorder { joint: XRJointSpace, baseSpace: XRSpace ): XRJointPose | undefined { - for (const source of frame.session.inputSources) { + for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { const hand = source.handedness; if ((hand !== 'left' && hand !== 'right') || !source.hand) continue; for (const [name, candidate] of source.hand.entries()) { @@ -864,10 +922,8 @@ export class XRInputRecorder { baseSpace: XRSpace ): T | null { if (!pose || !this._sceneReferenceSpace) return null; - if (!this._replayAlignmentReady) return null; - const currentScenePose = this._replaySceneAlignment - ? transformByPose(pose, this._replaySceneAlignment) - : pose; + if (!this._replaySceneAlignment) return null; + const currentScenePose = transformByPose(pose, this._replaySceneAlignment); if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); return relation ? transformFromScene(currentScenePose, relation.transform) : null; diff --git a/deps/cloudxr/webxr_client/src/xrReplaySession.ts b/deps/cloudxr/webxr_client/src/xrReplaySession.ts new file mode 100644 index 0000000000..b375c45b96 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/xrReplaySession.ts @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** CloudXR's input-source events must describe the same sources as its tracking frame. */ +export class XRReplaySession { + private _sources: XRInputSource[] | null = null; + private readonly _events = new EventTarget(); + readonly session: XRSession; + + constructor(readonly nativeSession: XRSession) { + this.session = new Proxy(nativeSession, { + get: (target, property) => { + if (property === 'inputSources') return this._sources ?? target.inputSources; + if (property === 'addEventListener' || property === 'removeEventListener') { + return ( + type: string, + listener: EventListenerOrEventListenerObject, + options?: boolean | AddEventListenerOptions + ) => { + const owner = type === 'inputsourceschange' ? this._events : target; + owner[property](type, listener, options); + }; + } + const value = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + nativeSession.addEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + setInputSources(sources: XRInputSource[] | null): void { + const previous = Array.from(this._sources ?? this.nativeSession.inputSources); + this._sources = sources; + const next = Array.from(sources ?? this.nativeSession.inputSources); + const added = next.filter(source => !previous.includes(source)); + const removed = previous.filter(source => !next.includes(source)); + if (added.length || removed.length) this._dispatch(added, removed); + } + + dispose(): void { + this.nativeSession.removeEventListener('inputsourceschange', this._onNativeSourcesChange); + } + + private _onNativeSourcesChange = (event: XRInputSourcesChangeEvent): void => { + // Recorded sources do not disappear when the browser loses sight of a hand. + if (this._sources === null) this._dispatch(Array.from(event.added), Array.from(event.removed)); + }; + + private _dispatch(added: XRInputSource[], removed: XRInputSource[]): void { + this._events.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session: this.session, added, removed }) + ); + } +} From 343358ef78ea53e4633e4f265d8d95071ec5d587 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Fri, 11 Sep 2026 20:26:58 -0700 Subject: [PATCH 3/4] fix(webxr): replay controllers without live hardware Signed-off-by: Yanzi Zhu --- .../webxr_client/src/xrInputRecorder.test.ts | 195 ++++++++++++++++++ .../webxr_client/src/xrInputRecorder.ts | 104 ++++++++-- .../webxr_client/src/xrReplaySession.ts | 2 +- 3 files changed, 288 insertions(+), 13 deletions(-) diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index 551d32c506..f43fed9871 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -273,6 +273,7 @@ describe('lifecycle and frame advancement', () => { recorder.beginFrame(makeFrame(), sceneSpace, true, true); expect(recorder.currentFrame).toEqual({ timeMs: 0, + controllerProfiles: { left: null, right: null }, poses: { leftGrip: null, leftAim: null, rightGrip: null, rightAim: null }, gamepads: { left: null, right: null }, handJoints: { left: {}, right: {} }, @@ -851,6 +852,200 @@ describe('recorded hands without live tracking', () => { }); }); +describe('controller replay without live hardware', () => { + function controllerFrame(timeMs: number, x: number, profiles?: string[]): RecordedFrame { + const sample = timedFrame(timeMs, x); + sample.handJoints.left = {}; + if (profiles) sample.controllerProfiles = { left: profiles, right: null }; + return sample; + } + + test.each(['time', 'frame'] as const)( + 'replays poses and controls without live sources (%s)', + pacing => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const frame = (time: number) => makeFrame([], undefined, undefined, time, pose(0), session); + const scoped = recorder.adaptTrackingFrame(frame(0)).session; + const active = new Set(); + const changed = jest.fn((event: XRInputSourcesChangeEvent) => { + active.clear(); + for (const source of event.session.inputSources) { + if (source.gamepad && !source.hand) active.add(source.handedness); + } + }); + scoped.addEventListener('inputsourceschange', changed); + recorder.startReplay( + recording(controllerFrame(0, 0), controllerFrame(100, 1)), + false, + pacing + ); + let firstSource: XRInputSource | undefined; + for (const [time, expected] of [ + [0, 0], + [50, pacing === 'time' ? 0.5 : 1], + [100, 1], + ]) { + const current = frame(time); + recorder.beginFrame(current, sceneSpace); + const adapted = recorder.adaptTrackingFrame(current); + const source = adapted.session.inputSources[0]; + firstSource ??= source; + expect(source).toBe(firstSource); + expect(active.has('left')).toBe(true); + expect(source.hand).toBeUndefined(); + expect(source.profiles).toEqual(['generic-trigger-squeeze-thumbstick']); + expect(adapted.getPose(source.gripSpace!, sceneSpace)?.transform.position.x).toBe(expected); + expect(adapted.getPose(source.targetRaySpace, sceneSpace)?.transform.position.x).toBe( + expected + 1 + ); + expect(source.gamepad?.axes).toEqual([expected]); + expect(source.gamepad?.buttons[0]).toEqual({ + value: expected, + pressed: true, + touched: true, + }); + expect(source.gamepad).toBe(source.gamepad); + expect(session.inputSources).toHaveLength(0); + } + expect(changed).toHaveBeenCalledTimes(1); + recorder.stopReplay(); + recorder.adaptTrackingFrame(frame(100)); + expect(active.size).toBe(0); + expect(changed).toHaveBeenCalledTimes(2); + recorder.dispose(); + } + ); + + test('keeps recorded profiles and controls through live controller connection changes', () => { + const recorder = new XRInputRecorder(); + const session = makeSession(); + const changed = jest.fn(); + const initial = makeFrame([], undefined, undefined, 0, pose(0), session); + recorder.adaptTrackingFrame(initial).session.addEventListener('inputsourceschange', changed); + const profiles = ['meta-quest-touch-plus', 'oculus-touch-v3']; + recorder.startReplay( + recording(controllerFrame(0, 1, profiles), controllerFrame(100, 3, profiles)), + false + ); + recorder.beginFrame(initial, sceneSpace); + const source = recorder.adaptTrackingFrame(initial).session.inputSources[0]; + const live = { + handedness: 'left', + profiles: ['pico-4u'], + gamepad: gamepad(99), + } as XRInputSource; + for (const [time, inputs] of [ + [50, [live]], + [100, []], + ] as const) { + const frame = makeFrame([...inputs], undefined, undefined, time, pose(0), session); + session.dispatchEvent( + Object.assign(new Event('inputsourceschange'), { session, added: inputs, removed: [] }) + ); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources).toEqual([source]); + expect(source.profiles).toEqual(profiles); + expect(source.gamepad?.axes[0]).toBe(time === 50 ? 2 : 3); + } + expect(changed).toHaveBeenCalledTimes(1); + }); + + test('round-trips profiles and replays a controller alongside a recorded hand', () => { + const recorder = new XRInputRecorder(); + const profiles = ['pico-4u', 'oculus-touch-v2']; + const controller = { + handedness: 'left', + profiles, + gamepad: gamepad(0.75), + gripSpace: {}, + targetRaySpace: {}, + } as XRInputSource; + const hand = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + recorder.startRecording(); + recorder.beginFrame( + makeFrame( + [controller, hand], + () => pose(2), + () => jointPose(3) + ), + sceneSpace + ); + recorder.stopRecording(); + const saved = XRInputRecorder.importJSON(recorder.exportJSON()); + expect(saved.frames[0].controllerProfiles).toEqual({ left: profiles, right: null }); + delete saved.calibration; + recorder.startReplay(saved); + const frame = makeFrame(); + recorder.beginFrame(frame, sceneSpace); + const sources = Array.from(recorder.adaptTrackingFrame(frame).session.inputSources); + expect(sources.find(source => source.handedness === 'left')?.profiles).toEqual(profiles); + expect(sources.find(source => source.handedness === 'right')?.hand).toBeDefined(); + }); + + test('preserves gaps and device switches at recorded timestamps', () => { + const recorder = new XRInputRecorder(); + const first = controllerFrame(0, 1, ['meta-quest-touch-plus']); + const other = controllerFrame(100, 9, ['pico-4u']); + const lost = controllerFrame(200, 10); + lost.gamepads.left = null; + lost.poses.leftGrip = lost.poses.leftAim = null; + recorder.startReplay( + recording(first, other, lost, timedFrame(300, 20), controllerFrame(400, 30)), + false + ); + const sources: XRInputSource[] = []; + for (const time of [0, 50, 100, 200, 300, 400]) { + const frame = makeFrame([], undefined, undefined, time); + recorder.beginFrame(frame, sceneSpace); + const adapted = recorder.adaptTrackingFrame(frame); + const source = adapted.session.inputSources[0]; + if (time === 200) { + expect(adapted.session.inputSources).toHaveLength(0); + } else { + sources.push(source); + expect(adapted.session.inputSources).toHaveLength(1); + expect(!!source.hand).toBe(time === 300); + expect(source.gamepad?.axes[0]).toBe( + time < 100 ? 1 : time === 100 ? 9 : time === 300 ? 20 : 30 + ); + } + } + expect(sources[0]).toBe(sources[1]); + expect(sources[2]).not.toBe(sources[0]); + expect(sources[2].profiles).toEqual(['pico-4u']); + }); + + test('waits for calibration and restores the live controller on stop', () => { + const recorder = new XRInputRecorder(); + const live = { handedness: 'left', gamepad: gamepad(99) } as XRInputSource; + const frame = makeFrame([live]); + recorder.startReplay(calibratedRecording(0, controllerFrame(0, 1))); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources).toHaveLength(0); + recorder.calibrateReplay(); + recorder.beginFrame(frame, sceneSpace); + expect(recorder.adaptTrackingFrame(frame).session.inputSources[0].gamepad?.axes).toEqual([1]); + recorder.stopReplay(); + expect(recorder.adaptTrackingFrame(frame).session.inputSources[0]).toBe(live); + }); + + test.each([null, {}, { left: 'quest', right: null }, { left: [42], right: null }])( + 'rejects malformed controller profiles (%j)', + controllerProfiles => { + expect(() => + XRInputRecorder.importJSON( + JSON.stringify({ version: 1, frames: [{ ...frameData(), controllerProfiles }] }) + ) + ).toThrow('controllerProfiles is invalid'); + } + ); +}); + describe('serialization', () => { test('round-trips version 1 recordings', () => { const recorder = new XRInputRecorder(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 8bf4072485..03dea38ed8 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -56,6 +56,8 @@ export type RecordedFrame = { rightAim: SerializedPose; }; gamepads: Hands; + /** Controller profiles select SDK button mappings; absent in legacy recordings. */ + controllerProfiles?: Hands; /** Joint poses keyed by XRHandJoint name, also in scene space. */ handJoints: Hands; }; @@ -122,12 +124,16 @@ function serializeGamepad(gamepad: Gamepad | null | undefined): SerializedGamepa function captureFrame(frame: XRFrame, referenceSpace: XRReferenceSpace, timeMs = 0): RecordedFrame { const captured = emptyFrame(timeMs); + captured.controllerProfiles = { left: null, right: null }; for (const source of frame.session.inputSources) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; captured.gamepads[hand] = serializeGamepad(source.gamepad); + if (!source.hand && source.gamepad) { + captured.controllerProfiles[hand] = Array.from(source.profiles ?? []); + } captured.poses[`${hand}Grip`] = source.gripSpace ? serializePose(frame.getPose(source.gripSpace, referenceSpace)) : null; @@ -288,24 +294,45 @@ function interpolateJoints(from: JointPoses, to: JointPoses, alpha: number): Joi return result; } +function recordedControllerProfiles(frame: RecordedFrame, hand: 'left' | 'right'): string[] | null { + if (!frame.gamepads[hand]) return null; + if (frame.controllerProfiles !== undefined) return frame.controllerProfiles[hand]; + // Hand sources can expose gamepads too. Even null joint samples identify a hand. + // Legacy controller recordings have no profile, so use the SDK's generic mapping. + return Object.keys(frame.handJoints[hand]).length ? null : ['generic-trigger-squeeze-thumbstick']; +} + +function controllerKey(hand: string, profiles: string[]): string { + return JSON.stringify([hand, profiles]); +} + function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number): RecordedFrame { const interval = to.timeMs - from.timeMs; const alpha = interval > 0 ? (timeMs - from.timeMs) / interval : 0; + // Device switches are discrete even when two controllers have the same layout. + const sameDevice = (hand: 'left' | 'right') => + JSON.stringify(recordedControllerProfiles(from, hand)) === + JSON.stringify(recordedControllerProfiles(to, hand)); + const leftTo = sameDevice('left') ? to : from; + const rightTo = sameDevice('right') ? to : from; return { timeMs, + ...(from.controllerProfiles !== undefined + ? { controllerProfiles: from.controllerProfiles } + : {}), poses: { - leftGrip: interpolateOptionalPose(from.poses.leftGrip, to.poses.leftGrip, alpha), - leftAim: interpolateOptionalPose(from.poses.leftAim, to.poses.leftAim, alpha), - rightGrip: interpolateOptionalPose(from.poses.rightGrip, to.poses.rightGrip, alpha), - rightAim: interpolateOptionalPose(from.poses.rightAim, to.poses.rightAim, alpha), + leftGrip: interpolateOptionalPose(from.poses.leftGrip, leftTo.poses.leftGrip, alpha), + leftAim: interpolateOptionalPose(from.poses.leftAim, leftTo.poses.leftAim, alpha), + rightGrip: interpolateOptionalPose(from.poses.rightGrip, rightTo.poses.rightGrip, alpha), + rightAim: interpolateOptionalPose(from.poses.rightAim, rightTo.poses.rightAim, alpha), }, gamepads: { - left: interpolateGamepad(from.gamepads.left, to.gamepads.left, alpha), - right: interpolateGamepad(from.gamepads.right, to.gamepads.right, alpha), + left: interpolateGamepad(from.gamepads.left, leftTo.gamepads.left, alpha), + right: interpolateGamepad(from.gamepads.right, rightTo.gamepads.right, alpha), }, handJoints: { - left: interpolateJoints(from.handJoints.left, to.handJoints.left, alpha), - right: interpolateJoints(from.handJoints.right, to.handJoints.right, alpha), + left: interpolateJoints(from.handJoints.left, leftTo.handJoints.left, alpha), + right: interpolateJoints(from.handJoints.right, rightTo.handJoints.right, alpha), }, }; } @@ -470,6 +497,7 @@ export class XRInputRecorder { private _recordingInterrupted = false; private _trackingSession: XRReplaySession | null = null; private _recordedHandSources: XRInputSource[] = []; + private _recordedControllerSources = new Map(); private _sourceProxies = new WeakMap(); get mode() { @@ -528,6 +556,24 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); this._replayFrames = recording.frames; + this._recordedControllerSources.clear(); + // Replay owns device presence as well as poses, for both hands and controllers. + for (const frame of recording.frames) { + for (const handedness of ['left', 'right'] as const) { + const profiles = recordedControllerProfiles(frame, handedness); + if (!profiles) continue; + const key = controllerKey(handedness, profiles); + if (!this._recordedControllerSources.has(key)) { + this._recordedControllerSources.set(key, { + handedness, + targetRayMode: 'tracked-pointer', + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: [...profiles], + }); + } + } + } this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { const names = new Set(); for (const frame of recording.frames) { @@ -571,6 +617,7 @@ export class XRInputRecorder { this._lastReplayDisplayTime = null; this._replayRecording = null; this._recordedHandSources = []; + this._recordedControllerSources.clear(); this._replaySceneAlignment = null; this._calibrationRequested = false; this._mode = 'idle'; @@ -724,6 +771,20 @@ export class XRInputRecorder { if (!Number.isFinite(frame?.timeMs) || frame.timeMs < 0 || frame.timeMs < previousTime) { throw new Error('Malformed recording: frame timeMs must be finite and monotonic'); } + if (frame.controllerProfiles !== undefined) { + const profiles = frame.controllerProfiles; + if ( + !profiles || + !(['left', 'right'] as const).every( + hand => + profiles[hand] === null || + (Array.isArray(profiles[hand]) && + profiles[hand]!.every(profile => typeof profile === 'string')) + ) + ) { + throw new Error('Malformed recording: controllerProfiles is invalid'); + } + } previousTime = frame.timeMs; } return recording; @@ -835,13 +896,28 @@ export class XRInputRecorder { } private _replayInputSources(session: XRSession): XRInputSource[] { - const recordedHands = new Set(this._recordedHandSources.map(source => source.handedness)); + const recordedSources = [ + ...this._recordedHandSources, + ...this._recordedControllerSources.values(), + ]; + const recordedSides = new Set(recordedSources.map(source => source.handedness)); + const replay = this._currentFrame; const sources = [ ...this._recordedHandSources.filter(source => { const hand = source.handedness as 'left' | 'right'; - return Object.values(this._currentFrame?.handJoints[hand] ?? {}).some(Boolean); + return ( + replay && + !recordedControllerProfiles(replay, hand) && + Object.values(replay.handJoints[hand]).some(Boolean) + ); + }), + ...(['left', 'right'] as const).flatMap(hand => { + const profiles = replay && recordedControllerProfiles(replay, hand); + const source = + profiles && this._recordedControllerSources.get(controllerKey(hand, profiles)); + return source ? [source] : []; }), - ...Array.from(session.inputSources).filter(source => !recordedHands.has(source.handedness)), + ...Array.from(session.inputSources).filter(source => !recordedSides.has(source.handedness)), ]; return sources.map(source => { let proxy = this._sourceProxies.get(source); @@ -875,7 +951,11 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { + for (const source of [ + ...this._recordedHandSources, + ...this._recordedControllerSources.values(), + ...frame.session.inputSources, + ]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { diff --git a/deps/cloudxr/webxr_client/src/xrReplaySession.ts b/deps/cloudxr/webxr_client/src/xrReplaySession.ts index b375c45b96..7fd94e8cd6 100644 --- a/deps/cloudxr/webxr_client/src/xrReplaySession.ts +++ b/deps/cloudxr/webxr_client/src/xrReplaySession.ts @@ -56,7 +56,7 @@ export class XRReplaySession { } private _onNativeSourcesChange = (event: XRInputSourcesChangeEvent): void => { - // Recorded sources do not disappear when the browser loses sight of a hand. + // Recorded device presence is independent of live tracking and controller disconnects. if (this._sources === null) this._dispatch(Array.from(event.added), Array.from(event.removed)); }; From 2a8dd23eb946659ef5e148acbfc9360410520de3 Mon Sep 17 00:00:00 2001 From: Yanzi Zhu Date: Sat, 12 Sep 2026 09:08:49 -0700 Subject: [PATCH 4/4] fix(webxr): simplify replay sources and align traces Signed-off-by: Yanzi Zhu --- .../src/TraceVisualization.test.ts | 88 ++++++++ .../webxr_client/src/TraceVisualization.tsx | 16 +- .../webxr_client/src/xrInputRecorder.test.ts | 14 ++ .../webxr_client/src/xrInputRecorder.ts | 188 ++++++------------ 4 files changed, 175 insertions(+), 131 deletions(-) create mode 100644 deps/cloudxr/webxr_client/src/TraceVisualization.test.ts diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts new file mode 100644 index 0000000000..9f84dcaa09 --- /dev/null +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.test.ts @@ -0,0 +1,88 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { ReactElement } from 'react'; +import { type Points, Vector3 } from 'three'; + +import { TraceVisualization } from './TraceVisualization'; +import type { RecordedFrame } from './xrInputRecorder'; + +let mockFrame: () => void; +const sample: RecordedFrame = { + timeMs: 0, + poses: { + leftGrip: { px: 1, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, + leftAim: null, + rightGrip: null, + rightAim: null, + }, + gamepads: { left: null, right: null }, + handJoints: { left: {}, right: {} }, +}; +const mockRecorder = { + currentFrame: sample as RecordedFrame | null, + replaySceneAlignment: { px: 10, py: 0, pz: 0, ox: 0, oy: 0, oz: 0, ow: 1 }, +}; + +jest.mock('@react-three/fiber', () => ({ + useFrame: (callback: () => void) => { + mockFrame = callback; + }, +})); +jest.mock('./RecorderContext', () => ({ + useRecorder: () => ({ recorder: mockRecorder, mode: 'replaying' }), +})); +jest.mock('react', () => ({ + ...jest.requireActual('react'), + useRef: () => ({ current: null }), + useEffect: jest.fn(), +})); + +test('places the whole replay trail in calibrated scene space and hides it while paused', () => { + const tree = TraceVisualization({ showTrace: true }); + const points = (tree.props.children as ReactElement<{ object: Points }>[])[0].props.object; + const worldPoint = (index: number) => { + points.updateMatrixWorld(); + return new Vector3() + .fromBufferAttribute(points.geometry.getAttribute('position'), index) + .applyMatrix4(points.matrixWorld); + }; + mockFrame(); + expect(worldPoint(0).x).toBeCloseTo(11); + + // A reset changes the placement of earlier samples as well as the newest one. + mockRecorder.replaySceneAlignment = { + px: 100, + py: 0, + pz: 0, + ox: 0, + oy: Math.SQRT1_2, + oz: 0, + ow: Math.SQRT1_2, + }; + mockFrame(); + for (const index of [0, 1]) { + expect(worldPoint(index).x).toBeCloseTo(100); + expect(worldPoint(index).z).toBeCloseTo(-1); + } + mockRecorder.currentFrame = null; + mockFrame(); + expect(points.visible).toBe(false); + for (const { props } of tree.props.children as ReactElement<{ object: Points }>[]) { + props.object.geometry.dispose(); + } +}); diff --git a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx index 9d5d7f38f7..1df584ce25 100644 --- a/deps/cloudxr/webxr_client/src/TraceVisualization.tsx +++ b/deps/cloudxr/webxr_client/src/TraceVisualization.tsx @@ -128,15 +128,25 @@ export function TraceVisualization({ showTrace }: { showTrace: boolean }) { ); useFrame(() => { - if (!showTrace) { + const frame = recorder.currentFrame; + if (!showTrace || !frame) { channels.forEach(channel => { channel.points.visible = false; }); return; } - const frame = recorder.currentFrame; - if (!frame) return; + // Transform the whole trail so origin resets also move its earlier samples. + const alignment = recorder.replaySceneAlignment; + for (const { points } of channels) { + points.position.set(alignment?.px ?? 0, alignment?.py ?? 0, alignment?.pz ?? 0); + points.quaternion.set( + alignment?.ox ?? 0, + alignment?.oy ?? 0, + alignment?.oz ?? 0, + alignment?.ow ?? 1 + ); + } updateChannel(channels[0], frame.poses.leftGrip); updateChannel(channels[1], frame.poses.rightGrip); updateChannel(channels[2], frame.handJoints.left.wrist); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts index f43fed9871..fe85337f77 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.test.ts @@ -814,6 +814,20 @@ describe('recorded hands without live tracking', () => { expect(changed).toHaveBeenCalledTimes(2); }); + test('does not expose an unrecorded live hand to the SDK during replay', () => { + const { recorder, read, frame, active } = benchmark('frame'); + const liveRight = { + handedness: 'right', + hand: new Map([['wrist', {}]]), + targetRaySpace: {}, + } as unknown as XRInputSource; + const current = frame([liveRight]); + expect(read(current)).toBeCloseTo(3); + expect([...active]).toEqual(['left']); + expect(recorder.adaptTrackingFrame(current).session.inputSources).toHaveLength(1); + expect(current.session.inputSources).toEqual([liveRight]); + }); + test('recorded tracking loss removes the source, and recorded recovery restores it', () => { const { recorder, read, frame, active } = benchmark('frame'); recorder.stopReplay(); diff --git a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts index 03dea38ed8..c06c1b15f6 100644 --- a/deps/cloudxr/webxr_client/src/xrInputRecorder.ts +++ b/deps/cloudxr/webxr_client/src/xrInputRecorder.ts @@ -302,10 +302,6 @@ function recordedControllerProfiles(frame: RecordedFrame, hand: 'left' | 'right' return Object.keys(frame.handJoints[hand]).length ? null : ['generic-trigger-squeeze-thumbstick']; } -function controllerKey(hand: string, profiles: string[]): string { - return JSON.stringify([hand, profiles]); -} - function interpolateFrame(from: RecordedFrame, to: RecordedFrame, timeMs: number): RecordedFrame { const interval = to.timeMs - from.timeMs; const alpha = interval > 0 ? (timeMs - from.timeMs) / interval : 0; @@ -399,11 +395,6 @@ function transformByPose(pose: T, baseFromScene: PoseData): }; } -/** Apply the real baseSpace <- sceneSpace transform to a recorded pose. */ -function transformFromScene(pose: T, baseFromScene: XRRigidTransform): T { - return transformByPose(pose, serializeTransform(baseFromScene)); -} - function inversePose(pose: PoseData): PoseData { const magnitudeSquared = pose.ox * pose.ox + pose.oy * pose.oy + pose.oz * pose.oz + pose.ow * pose.ow; @@ -474,7 +465,6 @@ function frameTimestampMs(frame: XRFrame): number { export class XRInputRecorder { private _mode: 'idle' | 'recording' | 'replaying' = 'idle'; private _frames: RecordedFrame[] = []; - private _replayFrames: RecordedFrame[] = []; private _replayIndex = 0; private _loopReplay = true; private _replayPacing: ReplayPacing = 'time'; @@ -491,14 +481,11 @@ export class XRInputRecorder { private _replaySceneAlignment: PoseData | null = null; private _calibrationRequested = false; private _observedSession: XRSession | null = null; - private _observedReferenceSpace: XRReferenceSpace | null = null; // Cached alignments use the origin observed at session/reference-space entry. private _currentFromInitial: PoseData = IDENTITY_POSE; private _recordingInterrupted = false; private _trackingSession: XRReplaySession | null = null; - private _recordedHandSources: XRInputSource[] = []; - private _recordedControllerSources = new Map(); - private _sourceProxies = new WeakMap(); + private _replaySources = new Map(); get mode() { return this._mode; @@ -516,6 +503,11 @@ export class XRInputRecorder { return this._currentFrame; } + /** Places recording-space traces in the same scene coordinates as SDK replay. */ + get replaySceneAlignment(): Readonly | null { + return this._replaySceneAlignment; + } + get replayNeedsCalibration(): boolean { return ( this._mode === 'replaying' && @@ -530,7 +522,7 @@ export class XRInputRecorder { /** Call after the operator returns to the recording's physical starting pose. */ calibrateReplay(): void { - if (this.replayNeedsCalibration && this._observedReferenceSpace) { + if (this.replayNeedsCalibration && this._sceneReferenceSpace) { this._calibrationRequested = true; } } @@ -555,50 +547,7 @@ export class XRInputRecorder { startReplay(recording: Recording, loop = true, pacing: ReplayPacing = 'time'): void { this._assertIdle(); - this._replayFrames = recording.frames; - this._recordedControllerSources.clear(); - // Replay owns device presence as well as poses, for both hands and controllers. - for (const frame of recording.frames) { - for (const handedness of ['left', 'right'] as const) { - const profiles = recordedControllerProfiles(frame, handedness); - if (!profiles) continue; - const key = controllerKey(handedness, profiles); - if (!this._recordedControllerSources.has(key)) { - this._recordedControllerSources.set(key, { - handedness, - targetRayMode: 'tracked-pointer', - targetRaySpace: new EventTarget(), - gripSpace: new EventTarget(), - profiles: [...profiles], - }); - } - } - } - this._recordedHandSources = (['left', 'right'] as const).flatMap(handedness => { - const names = new Set(); - for (const frame of recording.frames) { - for (const name of Object.keys(frame.handJoints[handedness])) names.add(name); - } - if (!names.size) return []; - const hand = new Map(); - for (const name of names) { - hand.set( - name as XRHandJoint, - Object.assign(new EventTarget(), { jointName: name as XRHandJoint }) - ); - } - return [ - { - handedness, - // @types/webxr still requires obsolete numeric constants on XRHand. - hand: hand as XRHand, - targetRayMode: 'tracked-pointer' as const, - targetRaySpace: new EventTarget(), - gripSpace: new EventTarget(), - profiles: ['generic-hand-select'], - }, - ]; - }); + this._replaySources.clear(); this._replayIndex = 0; this._loopReplay = loop; this._replayPacing = pacing; @@ -616,8 +565,7 @@ export class XRInputRecorder { this._currentFrame = null; this._lastReplayDisplayTime = null; this._replayRecording = null; - this._recordedHandSources = []; - this._recordedControllerSources.clear(); + this._replaySources.clear(); this._replaySceneAlignment = null; this._calibrationRequested = false; this._mode = 'idle'; @@ -662,7 +610,8 @@ export class XRInputRecorder { return; } - if (this._replayFrames.length === 0) { + const frames = this._replayRecording!.frames; + if (frames.length === 0) { this._currentFrame = null; return; } @@ -678,8 +627,8 @@ export class XRInputRecorder { return; } - this._currentFrame = this._replayFrames[this._replayIndex]; - if (this._replayIndex < this._replayFrames.length - 1) { + this._currentFrame = frames[this._replayIndex]; + if (this._replayIndex < frames.length - 1) { this._replayIndex++; } else if (this._loopReplay) { this._replayIndex = 0; @@ -692,13 +641,14 @@ export class XRInputRecorder { } this._lastReplayDisplayTime = displayTime; - const firstTime = this._replayFrames[0].timeMs; - const lastTime = this._replayFrames[this._replayFrames.length - 1].timeMs; - const duration = replayDuration(this._replayFrames); + const frames = this._replayRecording!.frames; + const firstTime = frames[0].timeMs; + const lastTime = frames[frames.length - 1].timeMs; + const duration = replayDuration(frames); const elapsed = this._loopReplay && duration > 0 ? this._replayElapsedMs % duration : this._replayElapsedMs; const sampleTime = Math.min(firstTime + elapsed, lastTime); - const sample = sampleAtTime(this._replayFrames, sampleTime); + const sample = sampleAtTime(frames, sampleTime); this._currentFrame = sample.frame; this._replayIndex = sample.index; } @@ -713,9 +663,7 @@ export class XRInputRecorder { this._trackingSession = new XRReplaySession(frame.session); } const replaying = this._mode === 'replaying'; - this._trackingSession.setInputSources( - replaying ? this._replayInputSources(frame.session) : null - ); + this._trackingSession.setInputSources(replaying ? this._replayInputSources() : null); const session = this._trackingSession.session; // Waiting for calibration must not silently substitute live robot inputs. const replay = this._currentFrame ?? emptyFrame(0); @@ -813,15 +761,14 @@ export class XRInputRecorder { session: XRSession, referenceSpace: XRReferenceSpace | null ): void { - if (session === this._observedSession && referenceSpace === this._observedReferenceSpace) { + if (session === this._observedSession && referenceSpace === this._sceneReferenceSpace) { return; } - this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); this._interruptRecording(); this._invalidateAlignments(); this._observedSession = session; - this._observedReferenceSpace = referenceSpace; this._sceneReferenceSpace = referenceSpace; referenceSpace?.addEventListener?.('reset', this._onReferenceSpaceReset); } @@ -889,59 +836,46 @@ export class XRInputRecorder { dispose(): void { this._trackingSession?.dispose(); this._trackingSession = null; - this._observedReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); - this._observedReferenceSpace = null; + this._sceneReferenceSpace?.removeEventListener?.('reset', this._onReferenceSpaceReset); + this._sceneReferenceSpace = null; this._observedSession = null; this._invalidateAlignments(); } - private _replayInputSources(session: XRSession): XRInputSource[] { - const recordedSources = [ - ...this._recordedHandSources, - ...this._recordedControllerSources.values(), - ]; - const recordedSides = new Set(recordedSources.map(source => source.handedness)); + private _replayInputSources(): XRInputSource[] { const replay = this._currentFrame; - const sources = [ - ...this._recordedHandSources.filter(source => { - const hand = source.handedness as 'left' | 'right'; - return ( - replay && - !recordedControllerProfiles(replay, hand) && - Object.values(replay.handJoints[hand]).some(Boolean) - ); - }), - ...(['left', 'right'] as const).flatMap(hand => { - const profiles = replay && recordedControllerProfiles(replay, hand); - const source = - profiles && this._recordedControllerSources.get(controllerKey(hand, profiles)); - return source ? [source] : []; - }), - ...Array.from(session.inputSources).filter(source => !recordedSides.has(source.handedness)), - ]; - return sources.map(source => { - let proxy = this._sourceProxies.get(source); - if (!proxy) { - let lastGamepad: SerializedGamepad | null | undefined; - let replayGamepad: Gamepad | null = null; - proxy = new Proxy(source, { - get: (target, property) => { - const hand = target.handedness; - if (property === 'gamepad' && (hand === 'left' || hand === 'right')) { - const gamepad = this._currentFrame?.gamepads[hand]; - // WebXR gamepad identity is stable for repeated reads of one frame. - if (gamepad !== lastGamepad) { - replayGamepad = gamepad ? makeGamepad(gamepad) : null; - lastGamepad = gamepad; - } - return replayGamepad; - } - return bindWebXRMember(target, property); - }, - }); - this._sourceProxies.set(source, proxy); + if (!replay) return []; + // Only recorded devices belong in the SDK view; live sources can otherwise + // activate a hand/controller that is absent from the recording. + return (['left', 'right'] as const).flatMap(handedness => { + const joints = replay.handJoints[handedness]; + const profiles = recordedControllerProfiles(replay, handedness); + if (!profiles && !Object.values(joints).some(Boolean)) return []; + + const key = JSON.stringify([handedness, profiles]); + let source = this._replaySources.get(key); + if (!source) { + source = { + handedness, + targetRayMode: 'tracked-pointer', + targetRaySpace: new EventTarget(), + gripSpace: new EventTarget(), + profiles: profiles ? [...profiles] : ['generic-hand-select'], + // @types/webxr still requires obsolete numeric constants on XRHand. + hand: profiles ? undefined : (new Map() as XRHand), + }; + this._replaySources.set(key, source); } - return proxy; + if (source.hand) { + const hand = source.hand as Map; + for (const name of Object.keys(joints) as XRHandJoint[]) { + if (!hand.has(name)) + hand.set(name, Object.assign(new EventTarget(), { jointName: name })); + } + } + const gamepad = replay.gamepads[handedness]; + Object.assign(source, { gamepad: gamepad ? makeGamepad(gamepad) : null }); + return [source]; }); } @@ -951,11 +885,7 @@ export class XRInputRecorder { space: XRSpace, baseSpace: XRSpace ): XRPose | undefined { - for (const source of [ - ...this._recordedHandSources, - ...this._recordedControllerSources.values(), - ...frame.session.inputSources, - ]) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if (hand !== 'left' && hand !== 'right') continue; if (space === source.gripSpace) { @@ -974,7 +904,7 @@ export class XRInputRecorder { joint: XRJointSpace, baseSpace: XRSpace ): XRJointPose | undefined { - for (const source of [...this._recordedHandSources, ...frame.session.inputSources]) { + for (const source of [...this._replaySources.values(), ...frame.session.inputSources]) { const hand = source.handedness; if ((hand !== 'left' && hand !== 'right') || !source.hand) continue; for (const [name, candidate] of source.hand.entries()) { @@ -1006,6 +936,8 @@ export class XRInputRecorder { const currentScenePose = transformByPose(pose, this._replaySceneAlignment); if (baseSpace === this._sceneReferenceSpace) return currentScenePose; const relation = frame.getPose(this._sceneReferenceSpace, baseSpace); - return relation ? transformFromScene(currentScenePose, relation.transform) : null; + return relation + ? transformByPose(currentScenePose, serializeTransform(relation.transform)) + : null; } }