Add per-frame up vectors to camera tracks

README.md

@@ -287,9 +287,10 @@ Apply when writing `.webp` (lossless WebP rendered via GPU rasterizer).
                                         where the motion between instants exceeds a couple of pixels. Default: 16.
     --camera-track     <path>           Render a camera animation as a frame sequence: a supersplat editor project
                                         (.ssproj directory or its document.json), a viewer settings.json with
-                                        animTracks, or a JSON { frameRate, frames: [{ position, target, fov }] }.
+                                        animTracks, or a JSON { frameRate, frames: [{ position, target, fov, up }] }.
                                         Frames are written as <name>.NNNN.webp. Replaces --camera-pos/--camera-target;
-                                        the track's target is the defocus focus point. With --shutter, each frame is
+                                        the track's target is the defocus focus point. A frame's up vector tilts the
+                                        camera; frames without one use --camera-up. With --shutter, each frame is
                                         motion-blurred over that fraction of the frame interval.
     --frames           <a[-b]>          Inclusive frame range of the track to render. Default: all frames.
 ```

src/cli/index.ts

@@ -519,7 +519,8 @@ const parseArguments = async () => {
         throw new Error(`Invalid --webp-effort value: ${v['webp-effort']}. Must be in [0, 9].`);
     }
     // Camera animation: an editor project (.ssproj directory or its document.json),
-    // viewer settings.json, or a plain frames list. Poses without a fov use --camera-fov.
+    // viewer settings.json, or a plain frames list. Poses without a fov use --camera-fov,
+    // poses without an up vector use --camera-up.
     let renderCameraTrack: CameraTrack | undefined;
     if (v['camera-track'] !== undefined) {
         let trackPath = v['camera-track'];
@@ -532,7 +533,7 @@ const parseArguments = async () => {
         } catch (e) {
             throw new Error(`Failed to read camera track JSON: ${trackPath} (${(e as Error).message})`);
         }
-        renderCameraTrack = loadCameraTrack(trackJson, renderFov ?? 60);
+        renderCameraTrack = loadCameraTrack(trackJson, renderFov ?? 60, renderUp);
     }
     let renderFrames: [number, number] | undefined;
     if (v.frames !== undefined) {
@@ -952,9 +953,10 @@ IMAGE OUTPUT (.webp) — lossless WebP rendered via GPU rasterizer
                                             where the motion between instants exceeds a couple of pixels. Default: 16.
         --camera-track     <path>           Render a camera animation as a frame sequence: a supersplat editor project
                                             (.ssproj directory or its document.json), a viewer settings.json with
-                                            animTracks, or a JSON { frameRate, frames: [{ position, target, fov }] }.
+                                            animTracks, or a JSON { frameRate, frames: [{ position, target, fov, up }] }.
                                             Frames are written as <name>.NNNN.webp. Replaces --camera-pos/--camera-target;
-                                            the track's target is the defocus focus point. With --shutter, each frame is
+                                            the track's target is the defocus focus point. A frame's up vector tilts the
+                                            camera; frames without one use --camera-up. With --shutter, each frame is
                                             motion-blurred over that fraction of the frame interval.
         --frames           <a[-b]>          Inclusive frame range of the track to render. Default: all frames.
 

src/lib/render/camera-track.ts

@@ -13,22 +13,34 @@
  *   timeline does, so rendered frames match what the editor shows.
  * - The supersplat viewer's `settings.json`: the first of `animTracks`
  *   (keyframe times in frames, `spline` or `step` interpolation, loop mode).
- * - A plain per-frame list: `{ frameRate?, frames: [{ position, target, fov? }] }`,
+ * - A plain per-frame list: `{ frameRate?, frames: [{ position, target, fov?, up? }] }`,
  *   linearly interpolated between entries for fractional times.
  *
  * All poses are in the PlayCanvas default (viewer/editor) space, like the
  * writer's camera options; the target doubles as the defocus focus point.
+ * Only the frame list can tilt the camera: a frame's `up` sets its up
+ * vector (roll about the view direction); poses without one, and the editor
+ * and viewer formats, which carry none, use the caller's default up.
  */
 
+import { logger } from '../utils';
+
 type Vec3Like = { x: number; y: number; z: number };
 
-/** A camera pose on a track: position, look-at target and vertical fov in degrees. */
+/**
+ * A camera pose on a track: position, look-at target, vertical fov in
+ * degrees and, optionally, a unit up vector (the renderer's `up` option
+ * applies when absent, so tracks predating `up` keep working).
+ */
 type TrackPose = {
     position: Vec3Like;
     target: Vec3Like;
+    up?: Vec3Like;
     fov: number;
 };
 
+const DEFAULT_UP: Vec3Like = { x: 0, y: 1, z: 0 };
+
 /** A camera animation track evaluated in frame time. */
 interface CameraTrack {
     /** Frames per second. */
@@ -175,18 +187,20 @@ const finiteNumber = (v: unknown, what: string, dflt?: number): number => {
  * @param smoothness - Spline tangent scale (0 linear, 1 smooth).
  * @param loopLength - Period in frames for a looping spline, or null to hold the end poses.
  * @param step - Hold each key until the next instead of interpolating.
+ * @param up - Up vector shared by every pose (the formats have no per-key up).
  * @returns The track.
  */
-const splineTrack = (keys: Keyframes, frameRate: number, frameCount: number, smoothness: number, loopLength: number | null, step: boolean): CameraTrack => {
+const splineTrack = (keys: Keyframes, frameRate: number, frameCount: number, smoothness: number, loopLength: number | null, step: boolean, up: Vec3Like): CameraTrack => {
     const { times, points } = keys;
     const result = new Array<number>(7);
     const toPose = (): TrackPose => ({
         position: { x: result[0], y: result[1], z: result[2] },
         target: { x: result[3], y: result[4], z: result[5] },
+        up,
         fov: result[6]
     });
     if (times.length === 1) {
-        const p: TrackPose = { position: { x: points[0], y: points[1], z: points[2] }, target: { x: points[3], y: points[4], z: points[5] }, fov: points[6] };
+        const p: TrackPose = { position: { x: points[0], y: points[1], z: points[2] }, target: { x: points[3], y: points[4], z: points[5] }, up, fov: points[6] };
         return { frameRate, frameCount, poseAt: () => p };
     }
     if (step) {
@@ -222,9 +236,10 @@ const splineTrack = (keys: Keyframes, frameRate: number, frameCount: number, smo
  *
  * @param doc - Parsed `document.json`.
  * @param defaultFov - Fallback vertical fov in degrees when neither a pose nor the document carries one.
+ * @param up - Up vector for every pose.
  * @returns The track.
  */
-const fromEditorDocument = (doc: any, defaultFov: number): CameraTrack => {
+const fromEditorDocument = (doc: any, defaultFov: number, up: Vec3Like): CameraTrack => {
     const timeline = doc.timeline ?? {};
     const frameCount = Math.floor(finiteNumber(timeline.frames, 'timeline.frames'));
     const frameRate = finiteNumber(timeline.frameRate, 'timeline.frameRate', 30);
@@ -250,7 +265,7 @@ const fromEditorDocument = (doc: any, defaultFov: number): CameraTrack => {
         times.push(frame);
         points.push(position.x, position.y, position.z, target.x, target.y, target.z, finiteNumber(p.fov, 'pose fov', docFov));
     }
-    return splineTrack({ times, points }, frameRate, frameCount, smoothness, loop ? frameCount : null, false);
+    return splineTrack({ times, points }, frameRate, frameCount, smoothness, loop ? frameCount : null, false, up);
 };
 
 /**
@@ -260,9 +275,10 @@ const fromEditorDocument = (doc: any, defaultFov: number): CameraTrack => {
  * at the end; `none` and `pingpong` hold the end poses).
  *
  * @param settings - Parsed viewer `settings.json`.
+ * @param up - Up vector for every pose.
  * @returns The track.
  */
-const fromViewerSettings = (settings: any): CameraTrack => {
+const fromViewerSettings = (settings: any, up: Vec3Like): CameraTrack => {
     const track = settings.animTracks?.[0];
     if (!track) {
         throw new Error('camera track: the settings have no animation tracks');
@@ -284,29 +300,68 @@ const fromViewerSettings = (settings: any): CameraTrack => {
     const extra = duration === times[times.length - 1] / frameRate ? 1 : 0;
     const loopLength = track.loopMode === 'repeat' ? (duration + extra) * frameRate : null;
     const frameCount = Math.round(duration * frameRate);
-    return splineTrack({ times, points }, frameRate, frameCount, smoothness, loopLength, track.interpolation === 'step');
+    return splineTrack({ times, points }, frameRate, frameCount, smoothness, loopLength, track.interpolation === 'step', up);
+};
+
+const unitVec3 = (v: Vec3Like, what: string): Vec3Like => {
+    const len = Math.hypot(v.x, v.y, v.z);
+    if (len === 0) {
+        throw new Error(`camera track: ${what} must not be a zero vector`);
+    }
+    return { x: v.x / len, y: v.y / len, z: v.z / len };
 };
 
 /**
  * Plain per-frame list, linearly interpolated between entries so shutter
- * slices can fall between frames.
+ * slices can fall between frames. Up vectors are normalized on load and
+ * interpolated by normalized lerp. Adjacent frames whose ups oppose each
+ * other are a 180° roll whose direction the lerp cannot pick (it passes
+ * through zero), so the camera rolls through its right side: the segment
+ * interpolates via the earlier frame's right vector, with a warning.
  *
  * @param json - Parsed `{ frameRate?, frames[] }` object.
  * @param defaultFov - Fallback vertical fov in degrees for frames without one.
+ * @param defaultUp - Fallback up vector for frames without one.
  * @returns The track.
  */
-const fromFrameList = (json: any, defaultFov: number): CameraTrack => {
+const fromFrameList = (json: any, defaultFov: number, defaultUp: Vec3Like): CameraTrack => {
     const frames: any[] = json.frames;
     if (!Array.isArray(frames) || frames.length === 0) {
         throw new Error('camera track: `frames` must be a non-empty array');
     }
     const frameRate = finiteNumber(json.frameRate, 'frameRate', 30);
-    const poses: TrackPose[] = frames.map((f, i) => ({
+    const unitDefaultUp = unitVec3(defaultUp, 'default up');
+    const poses = frames.map((f, i) => ({
         position: vec3Of(f.position, `frames[${i}].position`),
         target: vec3Of(f.target, `frames[${i}].target`),
+        up: f.up === undefined ? unitDefaultUp : unitVec3(vec3Of(f.up, `frames[${i}].up`), `frames[${i}].up`),
         fov: finiteNumber(f.fov, `frames[${i}].fov`, defaultFov)
     }));
+    // Midpoint up per segment, set only where the endpoint ups oppose.
+    const mids: (Vec3Like | null)[] = [];
+    for (let i = 1; i < poses.length; i++) {
+        const { position, target, up: a } = poses[i - 1];
+        const b = poses[i].up;
+        let mid: Vec3Like | null = null;
+        if (a.x * b.x + a.y * b.y + a.z * b.z < -1 + 1e-6) {
+            const fx = target.x - position.x, fy = target.y - position.y, fz = target.z - position.z;
+            const rx = fy * a.z - fz * a.y, ry = fz * a.x - fx * a.z, rz = fx * a.y - fy * a.x;
+            const rlen = Math.hypot(rx, ry, rz);
+            // A degenerate frame (target at the position, or up along the view) has no
+            // right vector; leave it for the renderer's pose checks to report.
+            if (rlen > 0) {
+                mid = { x: rx / rlen, y: ry / rlen, z: rz / rlen };
+                logger.warn(`camera track: frames[${i - 1}].up and frames[${i}].up point in opposite directions; rolling through the camera's right side`);
+            }
+        }
+        mids.push(mid);
+    }
     const lerp = (a: number, b: number, t: number) => a + (b - a) * t;
+    const nlerpUp = (a: Vec3Like, b: Vec3Like, t: number): Vec3Like => {
+        const ux = lerp(a.x, b.x, t), uy = lerp(a.y, b.y, t), uz = lerp(a.z, b.z, t);
+        const ulen = Math.hypot(ux, uy, uz) || 1;
+        return { x: ux / ulen, y: uy / ulen, z: uz / ulen };
+    };
     return {
         frameRate,
         frameCount: poses.length,
@@ -316,9 +371,13 @@ const fromFrameList = (json: any, defaultFov: number): CameraTrack => {
             const i1 = Math.min(i0 + 1, poses.length - 1);
             const t = f - i0;
             const a = poses[i0], b = poses[i1];
+            const mid = i0 < i1 ? mids[i0] : null;
+            const up = !mid ? nlerpUp(a.up, b.up, t) :
+                t < 0.5 ? nlerpUp(a.up, mid, t * 2) : nlerpUp(mid, b.up, t * 2 - 1);
             return {
                 position: { x: lerp(a.position.x, b.position.x, t), y: lerp(a.position.y, b.position.y, t), z: lerp(a.position.z, b.position.z, t) },
                 target: { x: lerp(a.target.x, b.target.x, t), y: lerp(a.target.y, b.target.y, t), z: lerp(a.target.z, b.target.z, t) },
+                up,
                 fov: lerp(a.fov, b.fov, t)
             };
         }
@@ -331,16 +390,18 @@ const fromFrameList = (json: any, defaultFov: number): CameraTrack => {
  * @param json - Parsed contents of an editor `document.json`, a viewer
  * `settings.json`, or a plain `{ frameRate?, frames[] }` list.
  * @param defaultFov - Vertical fov in degrees for poses that carry none.
+ * @param defaultUp - Up vector for poses that carry none (only frame-list
+ * entries can carry their own). Default: world +Y.
  * @returns The track.
  */
-const loadCameraTrack = (json: unknown, defaultFov: number): CameraTrack => {
+const loadCameraTrack = (json: unknown, defaultFov: number, defaultUp: Vec3Like = DEFAULT_UP): CameraTrack => {
     if (!json || typeof json !== 'object') {
         throw new Error('camera track: expected a JSON object');
     }
     const j = json as any;
-    if (Array.isArray(j.poseSets) && j.timeline) return fromEditorDocument(j, defaultFov);
-    if (Array.isArray(j.animTracks)) return fromViewerSettings(j);
-    if (Array.isArray(j.frames)) return fromFrameList(j, defaultFov);
+    if (Array.isArray(j.poseSets) && j.timeline) return fromEditorDocument(j, defaultFov, defaultUp);
+    if (Array.isArray(j.animTracks)) return fromViewerSettings(j, defaultUp);
+    if (Array.isArray(j.frames)) return fromFrameList(j, defaultFov, defaultUp);
     throw new Error('camera track: unrecognised format (expected an editor document with poseSets/timeline, viewer settings with animTracks, or a frames list)');
 };
 

src/lib/types.ts

@@ -161,7 +161,7 @@ type Options = {
 
     /**
      * Camera animation to render as a frame sequence (see `loadCameraTrack`).
-     * Replaces `renderCameraPosition` / `renderLookAt` / `renderFov`; the
+     * Replaces `renderCameraPosition` / `renderLookAt` / `renderUp` / `renderFov`; the
      * output filename gains a zero-padded frame index before its extension.
      * With `renderShutter` set, each frame is motion-blurred over that
      * fraction of the frame interval.

src/lib/writers/write-image.ts

@@ -357,7 +357,7 @@ const writeImage = async (options: WriteImageOptions, fs: FileSystem): Promise<v
     const poseAt: (t: number) => Pose = cameraTrack ?
         (t) => {
             const p = cameraTrack.poseAt(t);
-            return { pos: toDataPoint(p.position), tgt: toDataPoint(p.target), up: upStart, fov: projection === 'equirect' ? 0 : p.fov };
+            return { pos: toDataPoint(p.position), tgt: toDataPoint(p.target), up: p.up ? toDataDir(p.up) : upStart, fov: projection === 'equirect' ? 0 : p.fov };
         } :
         (t) => {
             const pos = {

test/camera-track.test.mjs

@@ -0,0 +1,79 @@
+import assert from 'node:assert/strict';
+import { describe, it } from 'node:test';
+
+import { loadCameraTrack } from '../src/lib/render/camera-track.js';
+
+const close = (a, b, eps = 1e-9) => {
+    assert.ok(Math.abs(a.x - b.x) < eps && Math.abs(a.y - b.y) < eps && Math.abs(a.z - b.z) < eps, `${JSON.stringify(a)} != ${JSON.stringify(b)}`);
+};
+
+describe('camera track up vectors', () => {
+    it('frame list: per-frame up, normalized lerp between frames, default for frames without one', () => {
+        const track = loadCameraTrack({
+            frames: [
+                { position: [0, 0, 5], target: [0, 0, 0], up: [0, 1, 0] },
+                { position: [0, 0, 5], target: [0, 0, 0], up: [1, 0, 0] },
+                { position: [0, 0, 5], target: [0, 0, 0] }
+            ]
+        }, 60, { x: 0, y: 0, z: 1 });
+        close(track.poseAt(0).up, { x: 0, y: 1, z: 0 });
+        close(track.poseAt(1).up, { x: 1, y: 0, z: 0 });
+        const s = Math.SQRT1_2;
+        close(track.poseAt(0.5).up, { x: s, y: s, z: 0 });
+        close(track.poseAt(2).up, { x: 0, y: 0, z: 1 });
+    });
+
+    it('frame list: normalizes ups on load so magnitude does not bias the lerp', () => {
+        const track = loadCameraTrack({
+            frames: [
+                { position: [0, 0, 5], target: [0, 0, 0], up: [0, 3, 0] },
+                { position: [0, 0, 5], target: [0, 0, 0], up: [0.5, 0, 0] }
+            ]
+        }, 60);
+        const s = Math.SQRT1_2;
+        close(track.poseAt(0).up, { x: 0, y: 1, z: 0 });
+        close(track.poseAt(0.5).up, { x: s, y: s, z: 0 });
+    });
+
+    it('frame list: rejects malformed and zero ups', () => {
+        const pose = up => ({ position: [0, 0, 5], target: [0, 0, 0], up });
+        assert.throws(() => loadCameraTrack({ frames: [pose([0, 1])] }, 60), /frames\[0\]\.up must be an array of three numbers/);
+        assert.throws(() => loadCameraTrack({ frames: [pose([0, 0, 0])] }, 60), /frames\[0\]\.up must not be a zero vector/);
+        assert.throws(() => loadCameraTrack({ frames: [pose(undefined)] }, 60, { x: 0, y: 0, z: 0 }), /default up must not be a zero vector/);
+    });
+
+    it('frame list: opposing ups roll through the right vector instead of collapsing', () => {
+        // Camera at +Z looking at the origin: forward is -Z, right = forward × up = +X for up +Y.
+        const pose = up => ({ position: [0, 0, 5], target: [0, 0, 0], up });
+        const track = loadCameraTrack({ frames: [pose([0, 1, 0]), pose([0, -3, 0])] }, 60);
+        close(track.poseAt(0.5).up, { x: 1, y: 0, z: 0 });
+        const s = Math.SQRT1_2;
+        close(track.poseAt(0.25).up, { x: s, y: s, z: 0 });
+        close(track.poseAt(0.75).up, { x: s, y: -s, z: 0 });
+        for (let t = 0; t <= 1; t += 1 / 16) {
+            const u = track.poseAt(t).up;
+            assert.ok(Math.abs(Math.hypot(u.x, u.y, u.z) - 1) < 1e-9, `up at ${t} is not unit length`);
+        }
+    });
+
+    it('editor document and viewer settings use the default up', () => {
+        const up = { x: 0, y: 0, z: -1 };
+        const editor = loadCameraTrack({
+            timeline: { frames: 10, frameRate: 30 },
+            poseSets: [{ poses: [
+                { frame: 0, position: [0, 0, 5], target: [0, 0, 0], fov: 60 },
+                { frame: 5, position: [5, 0, 0], target: [0, 0, 0], fov: 60 }
+            ] }]
+        }, 60, up);
+        close(editor.poseAt(2.5).up, up);
+        const viewer = loadCameraTrack({
+            animTracks: [{
+                duration: 1,
+                frameRate: 30,
+                keyframes: { times: [0, 30], values: { position: [0, 0, 5, 5, 0, 0], target: [0, 0, 0, 0, 0, 0], fov: [60, 60] } }
+            }]
+        }, 60, up);
+        close(viewer.poseAt(15).up, up);
+        close(loadCameraTrack({ frames: [{ position: [0, 0, 5], target: [0, 0, 0] }] }, 60).poseAt(0).up, { x: 0, y: 1, z: 0 });
+    });
+});