diff --git a/docs/H264_PERFORMANCE.md b/docs/H264_PERFORMANCE.md index 202d2e5..9fb8e29 100644 --- a/docs/H264_PERFORMANCE.md +++ b/docs/H264_PERFORMANCE.md @@ -13,12 +13,16 @@ npm run benchmark:h264 -- \ --file /absolute/path/representative.mcap \ --base-url http://127.0.0.1:4173 \ --duration 60 \ + --speed 8 \ + --no-seeks \ --output benchmark-h264.json ``` The benchmark exposes the original file through a temporary read-only Range/CORS server. It records -progress updates, random seeks, Image panel H.264 metrics, decode/page/console errors, and Chromium -JS heap when available. Run `npm run benchmark:h264 -- --help` for all options. +progress updates, optional random seeks, aggregate metrics for every Image panel, decode/page/console +errors, and Chromium JS heap when available. `--speed` is a load input only: adaptive behavior is +driven by measured media lag and queue pressure, not by fixed playback-rate branches. Run +`npm run benchmark:h264 -- --help` for all options. ## Manual browser pass @@ -35,6 +39,8 @@ Suggested acceptance targets: movement in about one second. - No decode, page, or console errors; Image metrics are non-negative and pressure is `normal`, `degraded`, or `recovery`. +- Decoder input remains bounded, media lag repeatedly recovers after pressure, rendered frames keep + increasing, and resync count does not grow continuously under a steady workload. - The H.264 pending queue is hard-bounded at 120 frames and a 1,000 ms media-time span. Soft pressure keeps a sole complete GOP intact; if either hard bound is exceeded without a newer IDR suffix that fits, the worker keeps the current picture, drops the complete backlog, and waits for diff --git a/package-lock.json b/package-lock.json index 170b1fd..ec898cf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@ioai/rosview", - "version": "1.7.4", + "version": "1.7.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ioai/rosview", - "version": "1.7.4", + "version": "1.7.5", "license": "MIT", "devDependencies": { "@eslint/js": "^9.39.4", diff --git a/package.json b/package.json index d90d7a3..f9f9304 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ioai/rosview", - "version": "1.7.4", + "version": "1.7.5", "description": "High-performance robotics data visualization for MCAP, ROS bag, ROS2 db3, HDF5 and BVH — embeddable React component and standalone SPA", "keywords": [ "ros", diff --git a/scripts/benchmark-h264.mjs b/scripts/benchmark-h264.mjs index c6d2c7e..936e738 100644 --- a/scripts/benchmark-h264.mjs +++ b/scripts/benchmark-h264.mjs @@ -18,6 +18,8 @@ Options: --file Absolute path to a local MCAP file (required) --base-url Running rosview URL (default: ${DEFAULT_BASE_URL}) --duration Benchmark duration in seconds (default: ${DEFAULT_DURATION_SECONDS}) + --speed Playback speed used as load input (default: 1) + --no-seeks Disable random seek stress during sampling --output Also write the JSON result to this path --help Show this help `; @@ -27,6 +29,8 @@ function parseArgs(argv) { const options = { baseUrl: DEFAULT_BASE_URL, durationSeconds: DEFAULT_DURATION_SECONDS, + speed: 1, + seeks: true, file: undefined, output: undefined, help: false, @@ -35,6 +39,7 @@ function parseArgs(argv) { ['--file', 'file'], ['--base-url', 'baseUrl'], ['--duration', 'durationSeconds'], + ['--speed', 'speed'], ['--output', 'output'], ]); @@ -44,6 +49,10 @@ function parseArgs(argv) { options.help = true; continue; } + if (argument === '--no-seeks') { + options.seeks = false; + continue; + } const separator = argument.indexOf('='); const name = separator >= 0 ? argument.slice(0, separator) : argument; const key = valueOptions.get(name); @@ -54,7 +63,7 @@ function parseArgs(argv) { if (!value || value.startsWith('--')) { throw new Error(`${name} requires a value`); } - options[key] = key === 'durationSeconds' ? Number(value) : value; + options[key] = key === 'durationSeconds' || key === 'speed' ? Number(value) : value; } return options; } @@ -85,6 +94,9 @@ async function validateOptions(options) { if (!Number.isFinite(options.durationSeconds) || options.durationSeconds < 2) { throw new Error('--duration must be a number of at least 2 seconds'); } + if (!Number.isFinite(options.speed) || options.speed < 0.1 || options.speed > 8) { + throw new Error('--speed must be between 0.1 and 8'); + } if (options.output) { options.output = path.resolve(options.output); } @@ -206,12 +218,9 @@ async function readProgress(fill) { }); } -async function readImagePanel(page) { - const panel = page.getByTestId('image-panel').first(); - if (!(await panel.isVisible().catch(() => false))) { - return null; - } - return panel.evaluate((element) => { +async function readImagePanels(page) { + const panels = page.getByTestId('image-panel'); + return panels.evaluateAll((elements) => elements.map((element) => { const numberAttribute = (name) => { const value = element.getAttribute(name); if (value === null) return null; @@ -222,8 +231,12 @@ async function readImagePanel(page) { pressure: element.getAttribute('data-h264-pressure'), queueFrames: numberAttribute('data-h264-queue-frames'), droppedFrames: numberAttribute('data-h264-dropped-frames'), + decodeQueueSize: numberAttribute('data-h264-decode-queue'), + mediaLagMs: numberAttribute('data-h264-media-lag-ms'), + resyncCount: numberAttribute('data-h264-resync-count'), + renderedFrames: numberAttribute('data-h264-rendered-frames'), }; - }); + })); } function summarizeProgress(samples) { @@ -276,6 +289,10 @@ async function runBenchmark(options, fixtureUrl) { .waitFor({ state: 'visible', timeout: 90_000 }); const play = page.getByRole('button', { name: 'Play playback' }); await play.waitFor({ state: 'visible', timeout: 30_000 }); + if (options.speed !== 1) { + await page.getByTestId('playback-speed-trigger').click(); + await page.getByRole('menuitem', { name: `${options.speed}x`, exact: true }).click(); + } await play.click(); const fill = page.getByTestId('playback-progress-fill'); @@ -286,7 +303,9 @@ async function runBenchmark(options, fixtureUrl) { const heapSamples = []; const seeks = []; const durationMs = options.durationSeconds * 1_000; - const seekTimes = [0.35, 0.6, 0.82].map((ratio) => durationMs * ratio); + const seekTimes = options.seeks + ? [0.35, 0.6, 0.82].map((ratio) => durationMs * ratio) + : []; let nextSeek = 0; const samplingStartedAt = Date.now(); @@ -312,8 +331,8 @@ async function runBenchmark(options, fixtureUrl) { await resume.click(); } samples.push({ elapsedMs, width: await readProgress(fill) }); - const image = await readImagePanel(page); - if (image) imageSamples.push({ elapsedMs, ...image }); + const panels = await readImagePanels(page); + if (panels.length > 0) imageSamples.push({ elapsedMs, panels }); const heapBytes = await readHeap(page); if (heapBytes !== null) heapSamples.push({ elapsedMs, bytes: heapBytes }); await page.waitForTimeout(SAMPLE_INTERVAL_MS); @@ -326,13 +345,23 @@ async function runBenchmark(options, fixtureUrl) { const statusTexts = await page.getByTestId('image-panel-status').allTextContents().catch(() => []); const heapValues = heapSamples.map(({ bytes }) => bytes); const latestImage = imageSamples.at(-1) ?? null; + const latestPanels = latestImage?.panels ?? []; const metricsReasonable = - latestImage === null || - (['normal', 'degraded', 'recovery'].includes(latestImage.pressure) && - Number.isInteger(latestImage.queueFrames) && - latestImage.queueFrames >= 0 && - Number.isInteger(latestImage.droppedFrames) && - latestImage.droppedFrames >= 0); + latestPanels.every((panel) => + ['normal', 'degraded', 'recovery'].includes(panel.pressure) && + Number.isInteger(panel.queueFrames) && + panel.queueFrames >= 0 && + Number.isInteger(panel.droppedFrames) && + panel.droppedFrames >= 0 && + Number.isInteger(panel.decodeQueueSize) && + panel.decodeQueueSize >= 0 && + Number.isFinite(panel.mediaLagMs) && + panel.mediaLagMs >= 0 && + Number.isInteger(panel.resyncCount) && + panel.resyncCount >= 0 && + Number.isInteger(panel.renderedFrames) && + panel.renderedFrames >= 0 + ); return { schemaVersion: 1, @@ -342,6 +371,7 @@ async function runBenchmark(options, fixtureUrl) { fileBytes: options.fileSize, baseUrl: options.baseUrl, durationSeconds: options.durationSeconds, + speed: options.speed, }, environment: { platform: process.platform, diff --git a/src/core/players/IterablePlayer.test.ts b/src/core/players/IterablePlayer.test.ts index 2490e6e..c7874e5 100644 --- a/src/core/players/IterablePlayer.test.ts +++ b/src/core/players/IterablePlayer.test.ts @@ -120,6 +120,25 @@ afterEach(() => { messageBus.reset(); }); +describe('IterablePlayer playback speed', () => { + it('uses a deterministic 10x maximum instead of a best-effort sentinel', async () => { + const player = new IterablePlayer(makeSource([])); + let latestState: PlayerState | undefined; + player.setListener((state) => { + latestState = state; + }); + await player.initialize({}); + + player.setSpeed(10); + expect(latestState?.activeData?.speed).toBe(10); + + player.setSpeed(64); + expect(latestState?.activeData?.speed).toBe(10); + + player.close(); + }); +}); + describe('IterablePlayer high-frequency lane', () => { it('routes video-only topics outside the generic message bus', async () => { const source = makeSource([makeImageMessage()]); diff --git a/src/core/players/IterablePlayer.ts b/src/core/players/IterablePlayer.ts index 5a914f5..5711ddb 100644 --- a/src/core/players/IterablePlayer.ts +++ b/src/core/players/IterablePlayer.ts @@ -6,7 +6,7 @@ import type { StreamMessagesInTimeRangeArgs, Subscription, } from '@/core/types/player'; -import { PLAYBACK_SPEED_MAX } from '@/core/types/player'; +import { MAX_PLAYBACK_SPEED } from '@/core/types/player'; import type { DataQualityReport, Time, Initialization, MessageEvent, TimeRange } from '@/core/types/ros'; import type { ISourceHandle } from '@/infra/workers/ISourceHandle'; import type { IMessageCursor } from '@/infra/workers/types'; @@ -469,7 +469,7 @@ export class IterablePlayer implements Player { this._advancePlaybackEpoch(); this._isPlaying = true; const now = performance.now(); - this._clock.play(this._currentTime, this._speedFactor(), now); + this._clock.play(this._currentTime, this._speed, now); this._lastTickWallMs = now; this._pageSuspended = typeof document !== "undefined" && document.hidden; if (this._pageSuspended) { @@ -600,19 +600,11 @@ export class IterablePlayer implements Player { } setSpeed(speed: number): void { - if (speed === PLAYBACK_SPEED_MAX) { - this._speed = PLAYBACK_SPEED_MAX; - } else { - this._speed = Math.min(8, Math.max(0.1, speed)); - } - this._clock.setSpeed(this._speedFactor(), performance.now()); + this._speed = Math.min(MAX_PLAYBACK_SPEED, Math.max(0.1, speed)); + this._clock.setSpeed(this._speed, performance.now()); this._emitState(); } - private _speedFactor(): number { - return this._speed === PLAYBACK_SPEED_MAX ? 64 : this._speed; - } - setSamplingFps(fps: number): void { const clamped = Math.max(1, Math.min(MAX_SAMPLING_FPS, Math.round(fps))); this._samplingFps = clamped; diff --git a/src/core/types/player.ts b/src/core/types/player.ts index 8d0daae..fb57653 100644 --- a/src/core/types/player.ts +++ b/src/core/types/player.ts @@ -97,8 +97,8 @@ export interface HighFrequencyConsumer { onMessageBatch?: (messages: MessageEvent[]) => void; } -/** Sentinel playback speed for “as fast as possible” (see IterablePlayer / PlaybackBar). */ -export const PLAYBACK_SPEED_MAX = -1; +/** Highest deterministic playback rate exposed by the visualization player. */ +export const MAX_PLAYBACK_SPEED = 10; export interface Player { setListener(listener: (state: PlayerState) => void): void; diff --git a/src/features/panels/Audio/AudioPanel.tsx b/src/features/panels/Audio/AudioPanel.tsx index 94e57da..767da75 100644 --- a/src/features/panels/Audio/AudioPanel.tsx +++ b/src/features/panels/Audio/AudioPanel.tsx @@ -3,7 +3,6 @@ import { useIntl } from 'react-intl'; import { useShallow } from 'zustand/react/shallow'; import { timeToNs } from '@/core/analysis/timeSeries'; import type { Player } from '@/core/types/player'; -import { PLAYBACK_SPEED_MAX } from '@/core/types/player'; import { messageBus } from '@/core/pipeline/messageBus'; import { useSubscriberSeq } from '@/core/pipeline/useMessageBus'; import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline'; @@ -100,7 +99,6 @@ export const AudioPanel: React.FC = (props) => { const allowPlayback = useMemo(() => { if (!isPlaying || config.mute) return false; - if (speed === PLAYBACK_SPEED_MAX) return false; return Math.abs(speed - 1) < 1e-4; }, [isPlaying, config.mute, speed]); @@ -259,7 +257,7 @@ export const AudioPanel: React.FC = (props) => { const statusLabel = useMemo(() => { if (!config.topic) return formatMessage({ id: 'panels.audio.status.waitingTopic' }); if (!allowPlayback && isPlaying && !config.mute) { - if (speed === PLAYBACK_SPEED_MAX || Math.abs(speed - 1) >= 1e-4) { + if (Math.abs(speed - 1) >= 1e-4) { return formatMessage({ id: 'panels.audio.status.mutedNon1x' }); } } diff --git a/src/features/panels/Image/ImagePanel.tsx b/src/features/panels/Image/ImagePanel.tsx index 1dd1e61..1a4510e 100644 --- a/src/features/panels/Image/ImagePanel.tsx +++ b/src/features/panels/Image/ImagePanel.tsx @@ -1,6 +1,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { useIntl } from 'react-intl'; import type { Player } from '@/core/types/player'; +import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline'; import type { MessageEvent as RosMessageEvent } from '@/core/types/ros'; import { scheduleFrame } from '@/shared/utils/rafScheduler'; import { toNano } from '@/shared/utils/time'; @@ -45,6 +46,9 @@ export type ImagePanelProps = ImageConfig & { export const ImagePanel: React.FC = (props) => { const { formatMessage } = useIntl(); + const isPlaying = useMessagePipeline( + (state) => state.playerState.activeData?.isPlaying ?? false, + ); const { player, panelId, @@ -250,15 +254,24 @@ export const ImagePanel: React.FC = (props) => { }; }, [player, mainConsumerId, h264ConsumerId, topic]); - // Reset on playback rewind; for H264, rebuild decoder state from the nearest keyframe. + // Keep the worker's media deadline current. On rewind, rebuild H.264 state + // from the nearest complete random-access point. useEffect(() => { return player.subscribeCurrentTime((time) => { + workerRef.current?.postMessage({ + type: 'playback', + currentTime: time, + isPlaying, + } satisfies ImageRenderWorkerRequest); const nowNs = toNano(time); const previousNs = lastPlaybackTimeNsRef.current; if (previousNs != null && nowNs + 5_000_000n < previousNs) { const worker = workerRef.current; if (worker && topic && h264ModeRef.current) { - worker.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + worker.postMessage({ + type: 'reset', + preserveFrame: true, + } satisfies ImageRenderWorkerRequest); void repairH264Seek(player, worker, topic, time); } else { workerRef.current?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); @@ -266,7 +279,7 @@ export const ImagePanel: React.FC = (props) => { } lastPlaybackTimeNsRef.current = nowNs; }); - }, [player, topic]); + }, [isPlaying, player, topic]); // Send color/depth decode options when they change — triggers immediate redraw in worker useEffect(() => { @@ -315,6 +328,10 @@ export const ImagePanel: React.FC = (props) => { data-h264-pressure={metrics?.pressureMode} data-h264-queue-frames={metrics?.queueFrames} data-h264-dropped-frames={metrics?.droppedFrames} + data-h264-decode-queue={metrics?.decodeQueueSize} + data-h264-media-lag-ms={metrics?.mediaLagMs} + data-h264-resync-count={metrics?.resyncCount} + data-h264-rendered-frames={metrics?.renderedFrames} > }[] = []; - #lastDeltaTimeKey = 0n; #lastTimestampUs = -1; #configuredCodec: string | null = null; #streamCodec: string | null = null; - #mutex = Promise.resolve(undefined); - #resolveFrame: ((frame: VideoFrame) => void) | null = null; - #rejectFrame: ((err: Error) => void) | null = null; - #outputTimeout: ReturnType | null = null; + #generation = 0; + #submitted = new Map(); + #callbacks: { + output: (output: { + videoFrame: VideoFrame; + sourceFrame: ImageWorkerFrameEnvelope; + decodeMs: number; + }) => void; + error: (error: Error) => void; + dequeue: () => void; + }; + + public constructor( + callbacks: { + output: (output: { + videoFrame: VideoFrame; + sourceFrame: ImageWorkerFrameEnvelope; + decodeMs: number; + }) => void; + error: (error: Error) => void; + dequeue: () => void; + }, + ) { + this.#callbacks = callbacks; + } public dispose(): void { this.reset(); - this.#lastKeyChunks = []; - this.#lastDeltaTimeKey = 0n; this.#lastTimestampUs = -1; - this.#mutex = Promise.resolve(); } public reset(): void { - this.#cancelOutputWait(new Error('H.264 decode was reset')); + this.#generation += 1; if (this.#decoder && this.#decoder.state !== 'closed') { this.#decoder.close(); } this.#decoder = null; this.#configuredCodec = null; this.#streamCodec = null; - this.#lastKeyChunks = []; - this.#lastDeltaTimeKey = 0n; + this.#submitted.clear(); } public get codec(): string | undefined { return this.#configuredCodec ?? undefined; } - public decodeFrame(data: Uint8Array, sortTimeKey: bigint): Promise { + public get decodeQueueSize(): number { + return this.#decoder?.state === 'configured' ? this.#decoder.decodeQueueSize : 0; + } + + public async submitFrame( + frame: ImageWorkerFrameEnvelope, + data: Uint8Array, + sortTimeKey: bigint, + ): Promise { if (typeof VideoDecoder === 'undefined') { - return Promise.reject(new Error('WebCodecs VideoDecoder is not supported')); + throw new Error('WebCodecs VideoDecoder is not supported'); } - const run = async (): Promise => { - await this.#ensureDecoder(data); - const decoder = this.#decoder!; - const type = getH264ChunkType(data); - const nalTypes = scanH264NalTypes(data); - - if (containsH264IdrNal(data)) { - // Worker messages already own their ArrayBuffer; retaining the view is safe. - this.#lastKeyChunks.push({ timeKey: sortTimeKey, data }); - if (this.#lastKeyChunks.length > MAX_KEY_CHUNKS) { - this.#lastKeyChunks.shift(); - } - } - - if (!nalTypes.some((nalType) => nalType === 1 || nalType === 5)) { - decoder.decode( - new EncodedVideoChunk({ - type, - timestamp: this.#monotonicTimestampUs(sortTimeKey), - data, - }), - ); - return null; - } - - let lastFrame: VideoFrame | null = null; - if (type === 'delta' && sortTimeKey < this.#lastDeltaTimeKey) { - for (const { timeKey, data: keyData } of this.#lastKeyChunks) { - if (timeKey < sortTimeKey) { - try { - const recovered = await this.#decodeOneChunk(decoder, keyData, 'key', timeKey); - lastFrame?.close(); - lastFrame = recovered; - } catch { - // Ignore GOP repair failures. - } - } - } - } - this.#lastDeltaTimeKey = sortTimeKey; - const frame = await this.#decodeOneChunk(decoder, data, type, sortTimeKey); - lastFrame?.close(); - return frame; - }; - - const promise = this.#mutex.then(() => run()); - this.#mutex = promise.then( - () => undefined, - () => undefined, - ); - return promise; + const generation = this.#generation; + await this.#ensureDecoder(data); + if (generation !== this.#generation) { + return; + } + const decoder = this.#decoder!; + const timestamp = this.#monotonicTimestampUs(sortTimeKey); + const hasVcl = scanH264NalTypes(data).some((nalType) => nalType === 1 || nalType === 5); + if (hasVcl) { + this.#submitted.set(timestamp, { + frame, + startedAt: performance.now(), + generation, + }); + } + try { + decoder.decode( + new EncodedVideoChunk({ + type: getH264ChunkType(data), + timestamp, + data, + }), + ); + } catch (error) { + this.#submitted.delete(timestamp); + throw error; + } } async #ensureDecoder(data: Uint8Array): Promise { @@ -215,12 +222,23 @@ class WorkerH264Decoder { this.#decoder = new VideoDecoder({ output: (frame) => { - this.#finishOk(frame); + const submitted = this.#submitted.get(frame.timestamp); + this.#submitted.delete(frame.timestamp); + if (!submitted || submitted.generation !== this.#generation) { + frame.close(); + return; + } + this.#callbacks.output({ + videoFrame: frame, + sourceFrame: submitted.frame, + decodeMs: performance.now() - submitted.startedAt, + }); }, error: (error) => { - this.#finishErr(new Error(String(error))); + this.#callbacks.error(new Error(String(error))); }, }); + this.#decoder.addEventListener('dequeue', this.#callbacks.dequeue); try { this.#decoder.configure(supportedConfig); this.#configuredCodec = supportedConfig.codec; @@ -233,91 +251,11 @@ class WorkerH264Decoder { } } - async #decodeOneChunk( - decoder: VideoDecoder, - data: Uint8Array, - type: 'key' | 'delta', - timeKey: bigint, - ): Promise { - const wait = this.#beginWait(); - try { - decoder.decode( - new EncodedVideoChunk({ - type, - timestamp: this.#monotonicTimestampUs(timeKey), - data, - }), - ); - } catch (error) { - this.#clearOutputWait(); - throw error instanceof Error ? error : new Error(String(error)); - } - return await wait; - } - - #beginWait(): Promise { - return new Promise((resolve, reject) => { - this.#resolveFrame = resolve; - this.#rejectFrame = reject; - this.#outputTimeout = setTimeout(() => { - if (this.#rejectFrame) { - const rejectFrame = this.#rejectFrame; - this.#clearOutputWait(); - rejectFrame(new Error('H.264 decode timed out')); - } - }, OUTPUT_TIMEOUT_MS); - }); - } - - #clearOutputWait(): void { - if (this.#outputTimeout != null) { - clearTimeout(this.#outputTimeout); - this.#outputTimeout = null; - } - this.#resolveFrame = null; - this.#rejectFrame = null; - } - - #cancelOutputWait(error: Error): void { - const rejectFrame = this.#rejectFrame; - this.#clearOutputWait(); - rejectFrame?.(error); - } - #monotonicTimestampUs(timeKey: bigint): number { const timestamp = monotonicH264TimestampUs(timeKey, this.#lastTimestampUs); this.#lastTimestampUs = timestamp; return timestamp; } - - #finishOk(frame: VideoFrame): void { - if (this.#outputTimeout != null) { - clearTimeout(this.#outputTimeout); - this.#outputTimeout = null; - } - if (!this.#resolveFrame) { - frame.close(); - return; - } - const resolveFrame = this.#resolveFrame; - this.#resolveFrame = null; - this.#rejectFrame = null; - resolveFrame(frame); - } - - #finishErr(error: Error): void { - if (this.#outputTimeout != null) { - clearTimeout(this.#outputTimeout); - this.#outputTimeout = null; - } - if (!this.#rejectFrame) { - return; - } - const rejectFrame = this.#rejectFrame; - this.#resolveFrame = null; - this.#rejectFrame = null; - rejectFrame(error); - } } // ---------- Cached frame state ---------- @@ -362,7 +300,12 @@ class ImageRenderWorkerRuntime { #pendingFrame: ImageWorkerFrameEnvelope | null = null; #pendingH264Frames: ImageWorkerFrameEnvelope[] = []; #isProcessing = false; - #decoder = new WorkerH264Decoder(); + #decoder: WorkerH264Decoder; + #pendingDecodedH264: { + videoFrame: VideoFrame; + sourceFrame: ImageWorkerFrameEnvelope; + } | null = null; + #h264RenderTimer: ReturnType | null = null; #lastPostedUiPhase: ImageSurfaceStatus['phase'] = 'idle'; #haltUntilReset = false; /** Reused RGBA buffer for raw frames; resized as needed. */ @@ -382,6 +325,11 @@ class ImageRenderWorkerRuntime { #lastH264BitmapAt = -Infinity; #droppedH264Frames = 0; #renderedH264Frames = 0; + #h264ResyncCount = 0; + #lastH264ResyncAt = -Infinity; + #playbackTimeNs: bigint | null = null; + #lastDecodedH264TimeNs: bigint | null = null; + #isPlaying = false; #lastMetricsAt = -Infinity; #epoch = 0; @@ -389,6 +337,14 @@ class ImageRenderWorkerRuntime { if (!this.#bufferCtx) { throw new Error('Buffer canvas context is unavailable in worker'); } + this.#decoder = new WorkerH264Decoder({ + output: (output) => this.#handleH264Output(output), + error: (error) => this.#handleH264DecoderError(error), + dequeue: () => { + this.#updateH264Pressure(); + void this.#drainLatestFrame(); + }, + }); } public handle(message: ImageRenderWorkerRequest): void { @@ -424,6 +380,14 @@ class ImageRenderWorkerRuntime { this.#redrawRawCached(); return; + case 'playback': + this.#playbackTimeNs = timeToKey(message.currentTime); + this.#isPlaying = message.isPlaying; + this.#updateH264Pressure(); + this.#trimPendingH264FramesIfNeeded(); + this.#emitMetricsIfDue(); + return; + case 'frame': if (this.#haltUntilReset) { return; @@ -438,20 +402,24 @@ class ImageRenderWorkerRuntime { this.#epoch += 1; this.#pendingFrame = null; this.#pendingH264Frames = []; + this.#disposePendingH264Output(); this.#haltUntilReset = false; this.#resetH264RuntimeState(); this.#decoder.reset(); this.#disposeAuxiliaryDecodeState(); - this.#disposeCachedBitmap(); - this.#cachedFrame = null; - this.#clearCanvas(); - this.#emitStatus({ phase: 'idle' }); + if (!message.preserveFrame) { + this.#disposeCachedBitmap(); + this.#cachedFrame = null; + this.#clearCanvas(); + this.#emitStatus({ phase: 'idle' }); + } return; case 'dispose': this.#epoch += 1; this.#pendingFrame = null; this.#pendingH264Frames = []; + this.#disposePendingH264Output(); this.#haltUntilReset = false; this.#decoder.dispose(); this.#disposeAuxiliaryDecodeState(); @@ -510,9 +478,12 @@ class ImageRenderWorkerRuntime { this.#pendingH264Frames, this.#h264RecentConfig, ); - if (selection.resync) { + const resyncAllowed = + hardLimitExceeded || + performance.now() - this.#lastH264ResyncAt >= H264_RESYNC_COOLDOWN_MS; + if (selection.resync && resyncAllowed) { this.#pendingH264Frames = selection.frames; - this.#h264NeedsResync = true; + this.#resyncH264Decoder(); this.#droppedH264Frames += selection.droppedFrames; } @@ -540,7 +511,7 @@ class ImageRenderWorkerRuntime { this.#pendingH264Frames = plan.frames; this.#h264WaitingForIdr = true; this.#h264ConfigBeforeIdr = [...this.#h264RecentConfig]; - this.#h264NeedsResync = true; + this.#resyncH264Decoder(); this.#updateH264Pressure(); this.#emitMetricsIfDue(true); } @@ -563,12 +534,22 @@ class ImageRenderWorkerRuntime { const epoch = this.#epoch; try { let frame: ImageWorkerFrameEnvelope | null; - while ((frame = this.#takeNextFrame())) { + while (true) { + if ( + this.#pendingH264Frames.length > 0 && + this.#decoder.decodeQueueSize >= H264_DECODE_QUEUE_HIGH_WATER + ) { + break; + } + frame = this.#takeNextFrame(); + if (!frame) { + break; + } if (epoch !== this.#epoch) { break; } if (isH264Frame(frame) && this.#h264NeedsResync) { - this.#decoder.reset(); + this.#resyncH264Decoder(); this.#h264NeedsResync = false; } await this.#decodeAndRender(frame, epoch); @@ -580,7 +561,11 @@ class ImageRenderWorkerRuntime { } } finally { this.#isProcessing = false; - if (this.#pendingFrame || this.#pendingH264Frames.length > 0) { + if ( + this.#pendingFrame || + (this.#pendingH264Frames.length > 0 && + this.#decoder.decodeQueueSize < H264_DECODE_QUEUE_HIGH_WATER) + ) { void this.#drainLatestFrame(); } } @@ -614,66 +599,9 @@ class ImageRenderWorkerRuntime { const sortKey = timeToKey(frame.receiveTime); if (kind === 'h264') { - const decodeStartedAt = performance.now(); - const videoFrame = await this.#decoder.decodeFrame(bytes, sortKey); - if (!videoFrame) { - this.#h264DecodeMs = updateDecodeDurationEwma( - this.#h264DecodeMs, - performance.now() - decodeStartedAt, - ); - this.#emitMetricsIfDue(); - return; - } - try { - if (epoch !== this.#epoch) { - return; - } - this.#h264DecodeMs = updateDecodeDurationEwma( - this.#h264DecodeMs, - performance.now() - decodeStartedAt, - ); - this.#updateH264Pressure(); - const now = performance.now(); - const shouldRender = - this.#h264Pressure.mode !== 'degraded' || - now - this.#lastH264RenderAt >= H264_DEGRADED_RENDER_INTERVAL_MS; - if (!shouldRender) { - this.#droppedH264Frames += 1; - this.#emitMetricsIfDue(); - return; - } - - const width = videoFrame.displayWidth || videoFrame.codedWidth; - const height = videoFrame.displayHeight || videoFrame.codedHeight; - this.#drawCanvasImageSource(videoFrame, width, height); - this.#lastH264RenderAt = now; - this.#renderedH264Frames += 1; - this.#emitStatus({ - phase: 'ready', - width, - height, - encoding: frame.format, - receiveTime: frame.receiveTime, - }); - // Bitmap caching is only for resize/option redraws. Sampling it at - // low frequency avoids a GPU copy for every decoded video frame. - if (this.#h264Pressure.mode === 'normal' && now - this.#lastH264BitmapAt >= 500) { - try { - const bitmap = await createImageBitmap(videoFrame); - if (epoch === this.#epoch) { - this.#storeBitmap(bitmap, width, height, frame.format, frame.receiveTime); - this.#lastH264BitmapAt = now; - } else { - bitmap.close(); - } - } catch { - // The frame is already on screen; cache creation is optional. - } - } - this.#emitMetricsIfDue(); - } finally { - videoFrame.close(); - } + await this.#decoder.submitFrame(frame, bytes, sortKey); + this.#updateH264Pressure(); + this.#emitMetricsIfDue(); return; } @@ -732,36 +660,10 @@ class ImageRenderWorkerRuntime { return; } if (isH264Frame(frame)) { - // Decoder failures usually invalidate the current GOP. Preserve the - // current canvas and resume only when a real IDR arrives. - this.#decoder.reset(); - const hasPendingIdr = this.#pendingH264Frames.some( - (pending) => isH264Frame(pending) && containsH264IdrNal(pending.data), + this.#droppedH264Frames += 1; + this.#handleH264DecoderError( + error instanceof Error ? error : new Error(String(error)), ); - if (hasPendingIdr) { - const recovery = selectLatestCompleteH264Gop( - this.#pendingH264Frames, - this.#h264RecentConfig, - true, - ); - this.#pendingH264Frames = recovery.frames; - this.#h264WaitingForIdr = false; - this.#h264NeedsResync = true; - this.#droppedH264Frames += recovery.droppedFrames + 1; - } else { - this.#droppedH264Frames += this.#pendingH264Frames.length + 1; - this.#pendingH264Frames = []; - this.#h264WaitingForIdr = true; - this.#h264ConfigBeforeIdr = [...this.#h264RecentConfig]; - this.#h264NeedsResync = false; - } - if (this.#renderedH264Frames === 0 && !this.#cachedFrame) { - this.#emitStatus({ - phase: 'error', - message: error instanceof Error ? error.message : String(error), - }); - } - this.#emitMetricsIfDue(true); return; } this.#haltUntilReset = true; @@ -772,16 +674,171 @@ class ImageRenderWorkerRuntime { } } + #handleH264Output(output: { + videoFrame: VideoFrame; + sourceFrame: ImageWorkerFrameEnvelope; + decodeMs: number; + }): void { + const frameTimeNs = timeToKey(output.sourceFrame.receiveTime); + this.#lastDecodedH264TimeNs = frameTimeNs; + this.#h264DecodeMs = updateDecodeDurationEwma(this.#h264DecodeMs, output.decodeMs); + + if ( + this.#isPlaying && + shouldDropDecodedH264Frame(this.#playbackTimeNs, frameTimeNs) + ) { + output.videoFrame.close(); + this.#droppedH264Frames += 1; + this.#updateH264Pressure(); + this.#emitMetricsIfDue(); + return; + } + + if (this.#pendingDecodedH264) { + this.#pendingDecodedH264.videoFrame.close(); + this.#droppedH264Frames += 1; + } + this.#pendingDecodedH264 = { + videoFrame: output.videoFrame, + sourceFrame: output.sourceFrame, + }; + this.#scheduleH264Render(); + this.#updateH264Pressure(); + this.#emitMetricsIfDue(); + } + + #scheduleH264Render(): void { + if (this.#h264RenderTimer != null || !this.#pendingDecodedH264) { + return; + } + const renderIntervalMs = + this.#h264Pressure.mode === 'normal' + ? H264_RENDER_INTERVAL_MS + : H264_PRESSURED_RENDER_INTERVAL_MS; + const delayMs = Math.max( + 0, + renderIntervalMs - (performance.now() - this.#lastH264RenderAt), + ); + if (delayMs <= 0) { + void this.#renderPendingH264Output(); + return; + } + this.#h264RenderTimer = setTimeout(() => { + this.#h264RenderTimer = null; + void this.#renderPendingH264Output(); + }, delayMs); + } + + async #renderPendingH264Output(): Promise { + const pending = this.#pendingDecodedH264; + this.#pendingDecodedH264 = null; + if (!pending) { + return; + } + const { videoFrame, sourceFrame } = pending; + const now = performance.now(); + try { + const frameTimeNs = timeToKey(sourceFrame.receiveTime); + if ( + this.#isPlaying && + shouldDropDecodedH264Frame(this.#playbackTimeNs, frameTimeNs) + ) { + this.#droppedH264Frames += 1; + return; + } + + const width = videoFrame.displayWidth || videoFrame.codedWidth; + const height = videoFrame.displayHeight || videoFrame.codedHeight; + this.#drawCanvasImageSource(videoFrame, width, height); + this.#lastH264RenderAt = now; + this.#renderedH264Frames += 1; + this.#emitStatus({ + phase: 'ready', + width, + height, + encoding: sourceFrame.kind === 'compressed' ? sourceFrame.format : 'h264', + receiveTime: sourceFrame.receiveTime, + }); + + if (this.#h264Pressure.mode === 'normal' && now - this.#lastH264BitmapAt >= 500) { + try { + const bitmap = await createImageBitmap(videoFrame); + this.#storeBitmap( + bitmap, + width, + height, + sourceFrame.kind === 'compressed' ? sourceFrame.format : 'h264', + sourceFrame.receiveTime, + ); + this.#lastH264BitmapAt = now; + } catch { + // The frame is already visible; resize caching is optional. + } + } + } finally { + videoFrame.close(); + this.#emitMetricsIfDue(); + if (this.#pendingDecodedH264) { + this.#scheduleH264Render(); + } + } + } + + #handleH264DecoderError(error: Error): void { + this.#resyncH264Decoder(); + const recovery = selectLatestCompleteH264Gop( + this.#pendingH264Frames, + this.#h264RecentConfig, + true, + ); + if (recovery.resync) { + this.#pendingH264Frames = recovery.frames; + this.#h264WaitingForIdr = false; + this.#droppedH264Frames += recovery.droppedFrames; + void this.#drainLatestFrame(); + } else { + this.#droppedH264Frames += this.#pendingH264Frames.length; + this.#pendingH264Frames = []; + this.#h264WaitingForIdr = true; + this.#h264ConfigBeforeIdr = [...this.#h264RecentConfig]; + } + if (this.#renderedH264Frames === 0 && !this.#cachedFrame) { + this.#emitStatus({ phase: 'error', message: error.message }); + } + this.#emitMetricsIfDue(true); + } + + #resyncH264Decoder(): void { + this.#decoder.reset(); + this.#disposePendingH264Output(); + this.#h264NeedsResync = false; + this.#h264ResyncCount += 1; + this.#lastH264ResyncAt = performance.now(); + } + + #disposePendingH264Output(): void { + if (this.#h264RenderTimer != null) { + clearTimeout(this.#h264RenderTimer); + this.#h264RenderTimer = null; + } + this.#pendingDecodedH264?.videoFrame.close(); + this.#pendingDecodedH264 = null; + } + #updateH264Pressure(): void { const previousMode = this.#h264Pressure.mode; + const mediaLagMs = + !this.#isPlaying || this.#lastDecodedH264TimeNs == null + ? 0 + : decodedFrameLatenessMs(this.#playbackTimeNs, this.#lastDecodedH264TimeNs); this.#h264Pressure = updateH264Pressure(this.#h264Pressure, { queueFrames: this.#pendingH264Frames.length, queueSpanMs: h264QueueSpanMs(this.#pendingH264Frames), decodeMs: this.#h264DecodeMs, + decodeQueueSize: this.#decoder.decodeQueueSize, + mediaLagMs, }); if (previousMode !== this.#h264Pressure.mode) { - this.#applyViewport(); - this.#redrawCachedFrame(); this.#emitMetricsIfDue(true); } } @@ -798,6 +855,9 @@ class ImageRenderWorkerRuntime { this.#lastH264BitmapAt = -Infinity; this.#droppedH264Frames = 0; this.#renderedH264Frames = 0; + this.#h264ResyncCount = 0; + this.#lastH264ResyncAt = -Infinity; + this.#lastDecodedH264TimeNs = null; this.#lastMetricsAt = -Infinity; } @@ -807,6 +867,10 @@ class ImageRenderWorkerRuntime { return; } this.#lastMetricsAt = now; + const mediaLagMs = + this.#lastDecodedH264TimeNs == null + ? 0 + : decodedFrameLatenessMs(this.#playbackTimeNs, this.#lastDecodedH264TimeNs); const metrics: ImageRenderMetrics = { pressureMode: this.#h264Pressure.mode, queueFrames: this.#pendingH264Frames.length, @@ -814,6 +878,9 @@ class ImageRenderWorkerRuntime { decodeMs: this.#h264DecodeMs, droppedFrames: this.#droppedH264Frames, renderedFrames: this.#renderedH264Frames, + decodeQueueSize: this.#decoder.decodeQueueSize, + mediaLagMs, + resyncCount: this.#h264ResyncCount, codec: this.#decoder.codec, }; workerScope.postMessage({ type: 'metrics', metrics } satisfies ImageRenderWorkerEvent); diff --git a/src/features/panels/Image/core/h264Backpressure.test.ts b/src/features/panels/Image/core/h264Backpressure.test.ts index 400ca43..6c81da1 100644 --- a/src/features/panels/Image/core/h264Backpressure.test.ts +++ b/src/features/panels/Image/core/h264Backpressure.test.ts @@ -2,12 +2,22 @@ import { describe, expect, it } from 'vitest'; import { H264_MAX_PENDING_FRAMES, H264_MAX_PENDING_SPAN_MS, + decodedFrameLatenessMs, initialH264PressureState, isH264HardLimitExceeded, + shouldDropDecodedH264Frame, updateDecodeDurationEwma, updateH264Pressure, } from './h264Backpressure'; +const healthy = { + queueFrames: 2, + queueSpanMs: 20, + decodeMs: 10, + decodeQueueSize: 1, + mediaLagMs: 20, +}; + describe('H.264 adaptive backpressure', () => { it('treats frame count and queue span as strict hard bounds', () => { expect(isH264HardLimitExceeded(H264_MAX_PENDING_FRAMES, H264_MAX_PENDING_SPAN_MS)).toBe(false); @@ -20,6 +30,8 @@ describe('H.264 adaptive backpressure', () => { queueFrames: 20, queueSpanMs: 400, decodeMs: 10, + decodeQueueSize: 1, + mediaLagMs: 20, }); expect(next.mode).toBe('degraded'); }); @@ -29,23 +41,37 @@ describe('H.264 adaptive backpressure', () => { queueFrames: 80, queueSpanMs: 500, decodeMs: 60, + decodeQueueSize: 8, + mediaLagMs: 500, }); - state = updateH264Pressure(state, { queueFrames: 2, queueSpanMs: 20, decodeMs: 10 }); + state = updateH264Pressure(state, healthy); expect(state.mode).toBe('recovery'); for (let i = 0; i < 10; i++) { - state = updateH264Pressure(state, { queueFrames: 2, queueSpanMs: 20, decodeMs: 10 }); + state = updateH264Pressure(state, healthy); } expect(state.mode).toBe('recovery'); - state = updateH264Pressure(state, { queueFrames: 2, queueSpanMs: 20, decodeMs: 10 }); + state = updateH264Pressure(state, healthy); expect(state.mode).toBe('normal'); }); it('relapses quickly when recovery pressure rises again', () => { let state = { mode: 'degraded' as const, healthySamples: 0 }; - state = updateH264Pressure(state, { queueFrames: 0, queueSpanMs: 0, decodeMs: 5 }); + state = updateH264Pressure(state, { + queueFrames: 0, + queueSpanMs: 0, + decodeMs: 5, + decodeQueueSize: 0, + mediaLagMs: 0, + }); expect(state.mode).toBe('recovery'); - state = updateH264Pressure(state, { queueFrames: 45, queueSpanMs: 300, decodeMs: 20 }); + state = updateH264Pressure(state, { + queueFrames: 45, + queueSpanMs: 300, + decodeMs: 20, + decodeQueueSize: 6, + mediaLagMs: 300, + }); expect(state.mode).toBe('degraded'); }); @@ -53,4 +79,23 @@ describe('H.264 adaptive backpressure', () => { expect(updateDecodeDurationEwma(20, 40)).toBe(24); expect(updateDecodeDurationEwma(0, 15)).toBe(15); }); + + it('uses actual media lag instead of playback speed', () => { + const overloaded = updateH264Pressure(initialH264PressureState(), { + ...healthy, + mediaLagMs: 400, + }); + const capable = updateH264Pressure(initialH264PressureState(), healthy); + + expect(overloaded.mode).toBe('degraded'); + expect(capable.mode).toBe('normal'); + }); + + it('drops decoded output only after it misses the media deadline', () => { + const playback = 1_000_000_000n; + expect(decodedFrameLatenessMs(playback, 950_000_000n)).toBe(50); + expect(shouldDropDecodedH264Frame(playback, 900_000_000n)).toBe(false); + expect(shouldDropDecodedH264Frame(playback, 850_000_000n)).toBe(true); + expect(shouldDropDecodedH264Frame(null, 0n)).toBe(false); + }); }); diff --git a/src/features/panels/Image/core/h264Backpressure.ts b/src/features/panels/Image/core/h264Backpressure.ts index b00000b..3cdce2a 100644 --- a/src/features/panels/Image/core/h264Backpressure.ts +++ b/src/features/panels/Image/core/h264Backpressure.ts @@ -9,6 +9,8 @@ export interface H264PressureObservation { queueFrames: number; queueSpanMs: number; decodeMs: number; + decodeQueueSize: number; + mediaLagMs: number; } /** @@ -17,11 +19,34 @@ export interface H264PressureObservation { */ export const H264_MAX_PENDING_FRAMES = 120; export const H264_MAX_PENDING_SPAN_MS = 1_000; -export const H264_DEGRADED_RENDER_INTERVAL_MS = 80; +export const H264_DECODE_QUEUE_HIGH_WATER = 4; +export const H264_RENDER_INTERVAL_MS = 1000 / 60; +export const H264_PRESSURED_RENDER_INTERVAL_MS = 1000 / 30; +export const H264_OUTPUT_DEADLINE_MS = 120; -const ENTER_DEGRADED = { frames: 72, spanMs: 350, decodeMs: 55 }; -const ENTER_RECOVERY = { frames: 18, spanMs: 120, decodeMs: 32 }; -const RELAPSE = { frames: 40, spanMs: 250, decodeMs: 45 }; +const ENTER_DEGRADED = { + frames: 72, + spanMs: 350, + decodeMs: 55, + // A full bounded decode pipeline is healthy by itself. Only treat the + // decoder queue as overload when it exceeds the configured feeder bound. + decodeQueueSize: H264_DECODE_QUEUE_HIGH_WATER * 2, + mediaLagMs: 350, +}; +const ENTER_RECOVERY = { + frames: 18, + spanMs: 120, + decodeMs: 32, + decodeQueueSize: 1, + mediaLagMs: 120, +}; +const RELAPSE = { + frames: 40, + spanMs: 250, + decodeMs: 45, + decodeQueueSize: H264_DECODE_QUEUE_HIGH_WATER + 2, + mediaLagMs: 250, +}; const RECOVERY_SAMPLES = 12; export function initialH264PressureState(): H264PressureState { @@ -43,15 +68,21 @@ export function updateH264Pressure( const overloaded = observation.queueFrames >= ENTER_DEGRADED.frames || observation.queueSpanMs >= ENTER_DEGRADED.spanMs || - observation.decodeMs >= ENTER_DEGRADED.decodeMs; + observation.decodeMs >= ENTER_DEGRADED.decodeMs || + observation.decodeQueueSize >= ENTER_DEGRADED.decodeQueueSize || + observation.mediaLagMs >= ENTER_DEGRADED.mediaLagMs; const healthy = observation.queueFrames <= ENTER_RECOVERY.frames && observation.queueSpanMs <= ENTER_RECOVERY.spanMs && - observation.decodeMs <= ENTER_RECOVERY.decodeMs; + observation.decodeMs <= ENTER_RECOVERY.decodeMs && + observation.decodeQueueSize <= ENTER_RECOVERY.decodeQueueSize && + observation.mediaLagMs <= ENTER_RECOVERY.mediaLagMs; const relapsed = observation.queueFrames >= RELAPSE.frames || observation.queueSpanMs >= RELAPSE.spanMs || - observation.decodeMs >= RELAPSE.decodeMs; + observation.decodeMs >= RELAPSE.decodeMs || + observation.decodeQueueSize >= RELAPSE.decodeQueueSize || + observation.mediaLagMs >= RELAPSE.mediaLagMs; if (state.mode === 'normal') { return overloaded ? { mode: 'degraded', healthySamples: 0 } : state; @@ -77,3 +108,18 @@ export function updateDecodeDurationEwma(previousMs: number, sampleMs: number): } return previousMs === 0 ? sampleMs : previousMs * 0.8 + sampleMs * 0.2; } + +export function decodedFrameLatenessMs(playbackTimeNs: bigint | null, frameTimeNs: bigint): number { + if (playbackTimeNs == null) { + return 0; + } + return Math.max(0, Number(playbackTimeNs - frameTimeNs) / 1_000_000); +} + +export function shouldDropDecodedH264Frame( + playbackTimeNs: bigint | null, + frameTimeNs: bigint, + deadlineMs = H264_OUTPUT_DEADLINE_MS, +): boolean { + return decodedFrameLatenessMs(playbackTimeNs, frameTimeNs) > deadlineMs; +} diff --git a/src/features/panels/Image/core/h264SeekRepair.ts b/src/features/panels/Image/core/h264SeekRepair.ts index c825ed5..97943c3 100644 --- a/src/features/panels/Image/core/h264SeekRepair.ts +++ b/src/features/panels/Image/core/h264SeekRepair.ts @@ -94,7 +94,10 @@ export async function repairH264Seek( continue; } - worker.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + worker.postMessage({ + type: 'reset', + preserveFrame: true, + } satisfies ImageRenderWorkerRequest); for (const event of repairFrames) { const next = toWorkerFrame(event); if (!next) { diff --git a/src/features/panels/Image/core/imageWorkerProtocol.ts b/src/features/panels/Image/core/imageWorkerProtocol.ts index 84fcb16..2db2ebe 100644 --- a/src/features/panels/Image/core/imageWorkerProtocol.ts +++ b/src/features/panels/Image/core/imageWorkerProtocol.ts @@ -54,12 +54,18 @@ export type ImageRenderWorkerRequest = type: 'rawDecodeOptions'; options: Partial; } + | { + type: 'playback'; + currentTime: Time; + isPlaying: boolean; + } | { type: 'frame'; frame: ImageWorkerFrameEnvelope; } | { type: 'reset'; + preserveFrame?: boolean; } | { type: 'dispose'; @@ -72,6 +78,9 @@ export interface ImageRenderMetrics { decodeMs: number; droppedFrames: number; renderedFrames: number; + decodeQueueSize: number; + mediaLagMs: number; + resyncCount: number; codec?: string; } diff --git a/src/features/workspace/playback/PlaybackBar.tsx b/src/features/workspace/playback/PlaybackBar.tsx index 6b73722..6f484bc 100644 --- a/src/features/workspace/playback/PlaybackBar.tsx +++ b/src/features/workspace/playback/PlaybackBar.tsx @@ -1,5 +1,4 @@ import React, { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'; -import { PLAYBACK_SPEED_MAX } from '@/core/types/player'; import { CalendarClock, Pause, Play, SkipBack, SkipForward, Timer } from 'lucide-react'; import { useShallow } from 'zustand/react/shallow'; import { useIntl } from 'react-intl'; @@ -53,7 +52,7 @@ function percentToTime(percent: number, start: Time, end: Time): Time { return { sec, nsec }; } -const PRESET_SPEEDS = [0.1, 0.25, 0.5, 1, 2, 4, 8] as const; +const PRESET_SPEEDS = [0.1, 0.25, 0.5, 1, 2, 4, 8, 10] as const; const PRESET_SAMPLING_FPS = [15, 30, 45] as const; const MENUBAR_PLAYBACK_SPEED = 'playback-menubar-speed'; @@ -599,9 +598,7 @@ export const PlaybackBar: React.FC = ({ player, extensionConte setPlaybackSettingsMenubarValue(MENUBAR_PLAYBACK_SPEED); }} > - {speed === PLAYBACK_SPEED_MAX - ? formatMessage({ id: 'playback.speedMax' }) - : `${speed}x`} + {speed}x @@ -616,14 +613,6 @@ export const PlaybackBar: React.FC = ({ player, extensionConte {item}x ))} - { - player.setSpeed(PLAYBACK_SPEED_MAX); - }} - > - {formatMessage({ id: 'playback.speedMax' })} - diff --git a/src/shared/hooks/useKeyboardShortcuts.ts b/src/shared/hooks/useKeyboardShortcuts.ts index 7160ae0..a3dd53d 100644 --- a/src/shared/hooks/useKeyboardShortcuts.ts +++ b/src/shared/hooks/useKeyboardShortcuts.ts @@ -1,6 +1,6 @@ import { useEffect, useRef } from 'react'; import type { Player } from '@/core/types/player'; -import { PLAYBACK_SPEED_MAX } from '@/core/types/player'; +import { MAX_PLAYBACK_SPEED } from '@/core/types/player'; import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline'; import type { MessagePipelineState } from '@/core/pipeline/store'; import { resolveStepMsFromModifiers } from '@/shared/utils/playbackStep'; @@ -56,8 +56,8 @@ export function useKeyboardShortcuts(player: Player) { case 'BracketLeft': { e.preventDefault(); const cur = speedRef.current; - if (cur === PLAYBACK_SPEED_MAX) { - player.setSpeed(4); + if (cur > 8) { + player.setSpeed(8); } else { player.setSpeed(Math.max(0.1, cur / 2)); } @@ -66,11 +66,8 @@ export function useKeyboardShortcuts(player: Player) { case 'BracketRight': { e.preventDefault(); const cur = speedRef.current; - if (cur === PLAYBACK_SPEED_MAX) { - break; - } if (cur >= 8) { - player.setSpeed(PLAYBACK_SPEED_MAX); + player.setSpeed(MAX_PLAYBACK_SPEED); } else { player.setSpeed(Math.min(8, cur * 2)); } diff --git a/src/shared/intl/messages/en/playback.json b/src/shared/intl/messages/en/playback.json index 4f15644..e626d10 100644 --- a/src/shared/intl/messages/en/playback.json +++ b/src/shared/intl/messages/en/playback.json @@ -1,6 +1,5 @@ { "playback.annotationsEmpty": "No annotation ranges", - "playback.speedMax": "Max (best effort)", "playback.play": "Play playback", "playback.pause": "Pause playback", "playback.timeMode.relative.aria": "Relative time from log start; click to switch to absolute local time", diff --git a/src/shared/intl/messages/ja/playback.json b/src/shared/intl/messages/ja/playback.json index d43b13f..ed8ebd2 100644 --- a/src/shared/intl/messages/ja/playback.json +++ b/src/shared/intl/messages/ja/playback.json @@ -1,6 +1,5 @@ { "playback.annotationsEmpty": "注釈範囲がありません", - "playback.speedMax": "最大（ベストエフォート）", "playback.play": "再生", "playback.pause": "一時停止", "playback.timeMode.relative.aria": "ログ先頭からの相対時刻です。クリックで絶対時刻に切り替え", diff --git a/src/shared/intl/messages/zh/playback.json b/src/shared/intl/messages/zh/playback.json index 30b6e22..8ac8235 100644 --- a/src/shared/intl/messages/zh/playback.json +++ b/src/shared/intl/messages/zh/playback.json @@ -1,6 +1,5 @@ { "playback.annotationsEmpty": "暂无标注区间", - "playback.speedMax": "最大（尽力加速）", "playback.play": "播放", "playback.pause": "暂停", "playback.timeMode.relative.aria": "当前为相对时间，点击切换为绝对时间", diff --git a/tests/image-h264.spec.ts b/tests/image-h264.spec.ts index 79276ee..a8e2c90 100644 --- a/tests/image-h264.spec.ts +++ b/tests/image-h264.spec.ts @@ -49,10 +49,33 @@ test('H.264 CompressedImage decodes without error', async ({ page }) => { const metrics = await imagePanel.evaluate((element) => ({ queueFrames: Number(element.getAttribute('data-h264-queue-frames')), droppedFrames: Number(element.getAttribute('data-h264-dropped-frames')), + decodeQueueSize: Number(element.getAttribute('data-h264-decode-queue')), + mediaLagMs: Number(element.getAttribute('data-h264-media-lag-ms')), + resyncCount: Number(element.getAttribute('data-h264-resync-count')), + renderedFrames: Number(element.getAttribute('data-h264-rendered-frames')), })); expect(Number.isInteger(metrics.queueFrames)).toBe(true); expect(metrics.queueFrames).toBeGreaterThanOrEqual(0); expect(Number.isInteger(metrics.droppedFrames)).toBe(true); expect(metrics.droppedFrames).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(metrics.decodeQueueSize)).toBe(true); + expect(metrics.decodeQueueSize).toBeGreaterThanOrEqual(0); + expect(Number.isFinite(metrics.mediaLagMs)).toBe(true); + expect(metrics.mediaLagMs).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(metrics.resyncCount)).toBe(true); + expect(metrics.resyncCount).toBeGreaterThanOrEqual(0); + expect(Number.isInteger(metrics.renderedFrames)).toBe(true); + expect(metrics.renderedFrames).toBeGreaterThanOrEqual(0); + + await page.getByTestId('playback-speed-trigger').click(); + await page.getByRole('menuitem', { name: '8x', exact: true }).click(); + const resume = page.getByRole('button', { name: 'Play playback' }); + if (await resume.isVisible().catch(() => false)) { + await resume.click(); + } + await page.waitForTimeout(1_000); + await expect(imageStatus).toBeVisible(); + await expect(imagePanel).toHaveAttribute('data-h264-pressure', /^(normal|degraded|recovery)$/); + expect(await page.getByText(/decode failed|could not be decoded/i).count()).toBe(0); } });