Fill stream pauses with black video to keep chat/chapters in sync

client-vue/src/components/vod/VodItem.vue

@@ -451,13 +451,22 @@ function openPlayer(): void {
         hash += `&video_path=${props.vod.webpath}/${props.vod.basename}_vod.mp4`;
     }
 
+    // stream pauses are cut out of the captured video, chat and chapters need to be synced to it
+    const syncToVideo = playerSettings.value.vodSource == "captured" && props.vod.needs_video_sync;
+
     if (playerSettings.value.chatSource == "captured") {
-        hash += `&chatfile=${props.vod.webpath}/${props.vod.basename}.chatdump`;
+        if (syncToVideo) {
+            hash += `&chatfile=${store.cfg<string>("basepath", "")}/api/v0/vod/${props.vod.uuid}/synced.chatdump`;
+        } else {
+            hash += `&chatfile=${props.vod.webpath}/${props.vod.basename}.chatdump`;
+        }
     } else {
         hash += `&chatfile=${props.vod.webpath}/${props.vod.basename}_chat.json`;
     }
 
-    hash += `&chapters=${props.vod.chapters?.map((c) => `${c.offset}:${encodeURIComponent(c.title.replaceAll(":", "").replaceAll(";", ""))}`).join(";")}`;
+    hash += `&chapters=${props.vod.chapters
+        ?.map((c) => `${syncToVideo ? c.video_offset : c.offset}:${encodeURIComponent(c.title.replaceAll(":", "").replaceAll(";", ""))}`)
+        .join(";")}`;
 
     // url.searchParams.set("offset", this.playerSettings.offset.toString());
     // window.open(url.toString(), "_blank");

client-vue/src/components/vod/VodItemChapters.vue

@@ -42,8 +42,8 @@
                             }"
                         >
                             <!-- start timestamp -->
-                            <td data-contents="offset" :title="formatDate(chapter.started_at)">
-                                {{ chapter.offset !== undefined ? humanDuration(chapter.offset) : "Unknown" }}
+                            <td data-contents="offset" :title="chapterOffsetTitle(chapter)">
+                                {{ chapter.video_offset !== undefined ? humanDuration(chapter.video_offset) : "Unknown" }}
                             </td>
 
                             <!-- start time -->
@@ -58,15 +58,15 @@
 
                             <!-- end time -->
                             <td v-if="showAdvanced" data-contents="ended_at">
-                                <template v-if="chapter.offset !== undefined && chapter.duration !== undefined">
-                                    {{ humanDuration(chapter.offset + chapter.duration) }}
+                                <template v-if="chapter.video_offset !== undefined && chapter.video_duration !== undefined">
+                                    {{ humanDuration(chapter.video_offset + chapter.video_duration) }}
                                 </template>
                             </td>
 
                             <!-- duration -->
                             <td data-contents="duration">
-                                <template v-if="chapter.duration">
-                                    {{ niceDuration(chapter.duration) }}
+                                <template v-if="chapter.video_duration">
+                                    {{ niceDuration(chapter.video_duration) }}
                                 </template>
                                 <template v-else>
                                     <duration-display :start-date="chapter.started_at.toISOString()" output-style="human" />
@@ -89,7 +89,7 @@
                                         <a
                                             class="px-1"
                                             target="_blank"
-                                            :href="playerLink(chapter.offset)"
+                                            :href="playerLink(chapter.video_offset)"
                                             title="Open in player"
                                             :class="{ 'is-spoiler': store.clientCfg('hideChapterTitlesAndGames') }"
                                         >
@@ -104,7 +104,11 @@
                                             :to="{
                                                 name: 'Editor',
                                                 params: { uuid: vod.uuid },
-                                                query: { start: chapter.offset, end: (chapter.offset || 0) + (chapter.duration || 0), chapter: chapterIndex },
+                                                query: {
+                                                    start: chapter.video_offset,
+                                                    end: (chapter.video_offset || 0) + (chapter.video_duration || 0),
+                                                    chapter: chapterIndex,
+                                                },
                                             }"
                                         >
                                             <span class="icon"><font-awesome-icon icon="cut" /></span>
@@ -233,8 +237,20 @@ const hasViewerCount = computed(() => {
 function playerLink(offset = 0, chatdownload = false): string {
     if (!props.vod || !store.config) return "#";
     const video_path = `${props.vod.webpath}/${props.vod.basename}.mp4`;
-    const chat_path = `${props.vod.webpath}/${props.vod.basename}.${chatdownload ? "chat" : "chatdump"}`;
-    return `${store.cfg<string>("basepath", "")}/vodplayer#source=file_http&video_path=${video_path}&chatfile=${chat_path}&offset=${offset}`;
+    let chat_path = `${props.vod.webpath}/${props.vod.basename}.${chatdownload ? "chat" : "chatdump"}`;
+    if (!chatdownload && props.vod.needs_video_sync) {
+        // stream pauses were cut out of the video, use the chat dump synced to it
+        chat_path = `${store.cfg<string>("basepath", "")}/api/v0/vod/${props.vod.uuid}/synced.chatdump`;
+    }
+    return `${store.cfg<string>("basepath", "")}/vodplayer#source=file_http&video_path=${video_path}&chatfile=${chat_path}&t=${Math.floor(offset)}`;
+}
+
+function chapterOffsetTitle(chapter: ChapterTypes): string {
+    let title = formatDate(chapter.started_at);
+    if (chapter.offset !== undefined && chapter.video_offset !== undefined && Math.round(chapter.offset) !== Math.round(chapter.video_offset)) {
+        title += ` (${humanDuration(chapter.offset)} into the stream, ad breaks are cut out of the video)`;
+    }
+    return title;
 }
 
 function twitchVideoLink(video_id: string): string {

client-vue/src/core/Providers/Base/BaseVOD.ts

@@ -69,6 +69,10 @@ export default class BaseVOD {
 
     viewers: VodViewerEntry[] = [];
     stream_pauses: StreamPause[] = [];
+    /** The stream pauses were filled with black video during conversion */
+    stream_pauses_padded = false;
+    /** Chat and chapter times need to be mapped to the captured video because of stream pauses */
+    needs_video_sync = false;
 
     get current_chapter(): BaseVODChapter | undefined {
         if (this.chapters.length > 0) {
@@ -145,6 +149,8 @@ export default class BaseVOD {
         vod.stream_pauses = apiResponse.stream_pauses
             ? apiResponse.stream_pauses.map((entry) => ({ start: new Date(entry.start), end: new Date(entry.end) }))
             : [];
+        vod.stream_pauses_padded = apiResponse.stream_pauses_padded ?? false;
+        vod.needs_video_sync = apiResponse.needs_video_sync ?? false;
 
         return vod;
 

client-vue/src/core/Providers/Base/BaseVODChapter.ts

@@ -8,9 +8,15 @@ export class BaseVODChapter {
      */
     public started_at!: Date;
 
+    /** Seconds since the stream started */
     public offset?: number;
     public duration?: number;
 
+    /** Position in the captured video, corrected for stream pauses (ad breaks) */
+    public video_offset?: number;
+    /** Duration in the captured video, corrected for stream pauses (ad breaks) */
+    public video_duration?: number;
+
     // public game_id?: string;
     // public game?: TwitchGame;
 
@@ -34,6 +40,8 @@ export class BaseVODChapter {
         chapter.started_at = new Date(apiResponse.started_at);
         chapter.offset = apiResponse.offset;
         chapter.duration = apiResponse.duration;
+        chapter.video_offset = apiResponse.video_offset ?? apiResponse.offset;
+        chapter.video_duration = apiResponse.video_duration ?? apiResponse.duration;
         // chapter.strings = apiResponse.strings;
         // chapter.game_id = apiResponse.game_id;
         chapter.title = apiResponse.title;

client-vue/src/core/Providers/Twitch/TwitchVODChapter.ts

@@ -19,6 +19,8 @@ export class TwitchVODChapter extends BaseVODChapter {
         chapter.started_at = new Date(apiResponse.started_at);
         chapter.offset = apiResponse.offset;
         chapter.duration = apiResponse.duration;
+        chapter.video_offset = apiResponse.video_offset ?? apiResponse.offset;
+        chapter.video_duration = apiResponse.video_duration ?? apiResponse.duration;
         // chapter.strings = apiResponse.strings;
         chapter.game_id = apiResponse.game_id;
         chapter.title = apiResponse.title;

client-vue/src/views/EditorView.vue

@@ -156,12 +156,12 @@
                 <ul class="list">
                     <li v-for="(chapter, i) in vodData.chapters" :key="i">
                         <a
-                            v-if="chapter.offset !== undefined && chapter.duration !== undefined"
-                            @click="setCutpoints(chapter.offset || 0, chapter.duration || 0)"
+                            v-if="chapter.video_offset !== undefined && chapter.video_duration !== undefined"
+                            @click="setCutpoints(chapter.video_offset || 0, chapter.video_duration || 0)"
                         >
                             <div class="chapter-time">
-                                {{ humanDuration(chapter.offset) }} - {{ humanDuration(chapter.offset + chapter.duration) }} ({{
-                                    humanDuration(chapter.duration)
+                                {{ humanDuration(chapter.video_offset) }} - {{ humanDuration(chapter.video_offset + chapter.video_duration) }} ({{
+                                    humanDuration(chapter.video_duration)
                                 }})
                             </div>
                             <div class="chapter-game">
@@ -479,8 +479,8 @@ function timelineMouseMove(event: MouseEvent): void {
 
     if (vodData.value) {
         const idx = vodData.value.chapters.findIndex((chapter, i) => {
-            const o = i == 0 ? 0 : chapter.offset || 0;
-            return chapter.offset && chapter.duration && seconds >= o && seconds < o + chapter.duration;
+            const o = i == 0 ? 0 : chapter.video_offset || 0;
+            return chapter.video_offset && chapter.video_duration && seconds >= o && seconds < o + chapter.video_duration;
         });
         if (idx !== -1) {
             currentChapterHoverIndex.value = idx;
@@ -619,7 +619,7 @@ function submitForm(event: Event): void {
 
 function chapterWidth(chapter: BaseVODChapter): number {
     // const chapterOffset = chapter.offset || 0;
-    const chapterDuration = chapter.duration || 0;
+    const chapterDuration = chapter.video_duration || 0;
     const width = (chapterDuration / videoDuration.value) * 100;
     return width;
 }
@@ -697,8 +697,8 @@ watch(
 );
 
 function chapterStyle(index: number, chapter: BaseVODChapter): HTMLAttributes["style"] {
-    const chapterOffset = index == 0 ? 0 : chapter.offset || 0;
-    const chapterDuration = chapter.duration || 0;
+    const chapterOffset = index == 0 ? 0 : chapter.video_offset || 0;
+    const chapterDuration = chapter.video_duration || 0;
     const left = (chapterOffset / videoDuration.value) * 100;
     // cap width at duration
     const width = Math.min((chapterDuration / videoDuration.value) * 100, 100 - left);

client-vue/test/mockdata.ts

@@ -138,6 +138,8 @@ export const MockApiVODData: ApiTwitchVod = {
     channel_uuid: "",
     viewers: [],
     stream_pauses: [],
+    stream_pauses_padded: false,
+    needs_video_sync: false,
 };
 
 export const MockApiChapterData: ApiVodTwitchChapter = {
@@ -146,6 +148,8 @@ export const MockApiChapterData: ApiVodTwitchChapter = {
     duration: 0,
     started_at: "",
     offset: 0,
+    video_offset: 0,
+    video_duration: 0,
     is_mature: false,
     game_id: undefined,
     game_name: undefined,

common/Api/Client.ts

@@ -23,7 +23,12 @@ export interface ApiVodBaseChapter {
     duration: number;
     started_at: string;
     // datetime: PHPDateTimeJSON;
+    /** Seconds since the stream started */
     offset: number;
+    /** Position in the captured video, corrected for stream pauses (ad breaks) */
+    video_offset: number;
+    /** Duration in the captured video, corrected for stream pauses (ad breaks) */
+    video_duration: number;
 }
 
 export interface ApiVodTwitchChapter extends ApiVodBaseChapter {
@@ -120,6 +125,10 @@ export interface ApiBaseVod {
 
     viewers: { amount: number; timestamp: string }[];
     stream_pauses: { start: string; end: string }[];
+    /** The stream pauses were filled with black video during conversion */
+    stream_pauses_padded: boolean;
+    /** Chat and chapter times need to be mapped to the captured video because of stream pauses */
+    needs_video_sync: boolean;
 
     bookmarks: VODBookmark[];
 }

common/ServerConfig.ts

@@ -704,6 +704,13 @@ export const settingsFields = createSettingsFields({
         type: "boolean",
         default: false,
     },
+    "video.pad_stream_pauses": {
+        group: "Video",
+        text: "Fill ad breaks with black video",
+        type: "boolean",
+        default: false,
+        help: "When converting a capture, insert black, silent video where streamlink filtered out ad breaks, so the video is as long as the stream and chat and chapters stay in sync. Only a short black clip is encoded, the capture itself is still copied. Falls back to a normal conversion if it fails.",
+    },
     create_video_chapters: {
         group: "Video",
         text: "Create video chapters",

server/src/Controllers/Vod.ts

@@ -392,7 +392,8 @@ export async function RenderWizard(
                 chat_font,
                 chat_font_size,
                 chat_source == "downloaded",
-                true
+                true,
+                vod_source != "downloaded"
             );
         } catch (error) {
             res.api(400, {
@@ -700,9 +701,10 @@ export async function CutVod(
     }
 
     if (vod.is_chat_downloaded || vod.is_chatdump_captured) {
+        // the clip is cut from the captured video, so use the chat dump synced to it
         const chat_file_in = vod.is_chat_downloaded
             ? vod.path_chat
-            : vod.path_chatdump;
+            : await vod.getChatdumpForVideo();
         const chat_file_out = path.join(
             BaseConfigDataFolder.saved_clips,
             `${vod.basename}_${time_in}-${time_out}_${segment_name}_chat.json`
@@ -896,6 +898,40 @@ export async function GetSync(
     */
 }
 
+/**
+ * Send the captured chat dump timed to the captured video.
+ * Stream pauses (ad breaks) are cut out of the video, so the chat is synced to match.
+ * Sends the original chat dump if no sync is needed.
+ */
+export async function GetVideoChatdump(
+    req: express.Request,
+    res: express.Response
+): Promise<void> {
+    const vod = LiveStreamDVR.getInstance().getVodByUUID(req.params.uuid);
+
+    if (!vod) {
+        res.api(400, {
+            status: "ERROR",
+            message: "Vod not found",
+        } as ApiErrorResponse);
+        return;
+    }
+
+    if (!vod.is_chatdump_captured) {
+        res.api(404, {
+            status: "ERROR",
+            message: "No chat dump captured for this vod",
+        } as ApiErrorResponse);
+        return;
+    }
+
+    const chatdump = await vod.getChatdumpForVideo();
+
+    res.sendFile(chatdump, {
+        headers: { "Content-Type": "application/json" },
+    });
+}
+
 export async function RenameVod(
     req: express.Request,
     res: express.Response

server/src/Core/Helper.ts

@@ -276,7 +276,11 @@ export class Helper {
     public static path_ffprobe(): string | false {
         const f = this.path_ffmpeg();
         if (!f) return false;
-        return f.replace("ffmpeg.exe", "ffprobe.exe");
+        // ffprobe is next to ffmpeg, replace only the file name (ffmpeg or ffmpeg.exe)
+        return path.join(
+            path.dirname(f),
+            path.basename(f).replace(/^ffmpeg/i, "ffprobe")
+        );
     }
 
     public static python_scripts_dir_name(): string {