From 67271474c0d7e18cf2da84021e667c9aa0cf9d5b Mon Sep 17 00:00:00 2001 From: KrX3D Date: Sat, 29 Aug 2026 22:32:13 +0200 Subject: [PATCH 1/2] perf: resume collection from YouTube's own batches instead of refetching them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The accumulator added over the last two PRs still never completed, and the log finally shows why it never could: playlist.continuation.detected itemCount:15 hasContinuation:true playlist.continuation.detected itemCount:15 hasContinuation:true playlist.continuation.detected itemCount:15 hasContinuation:true That is 15 initial + 45 = 60 of 68 items, and every one says there is more to come. YouTube stops there of its own accord — it never fetches the last 8-item batch unless the user scrolls — so the "no continuation token" completion signal the accumulator waits for simply never arrives. Hooking the array-root path was not wrong, there was just no final native batch to catch. Meanwhile the collector, started at page load, worked from the INITIAL token and refetched batches 2-5 from scratch: 53 items, 45 of which were already on screen, at 2.5s per request. Roughly ten seconds of helper tiles for eight missing items. So the two halves are now joined, which is what was actually being asked for: hold the collector until the native burst goes quiet (NATIVE_SETTLE_MS, reset by each arriving batch), then start it with the items already accumulated as its seed and the NEWEST continuation token YouTube reached. _collectAll already treats plc.contents as its starting set, so it simply carries on and fetches only what is genuinely missing — one batch here instead of four. Also drops BATCH_FETCH_DELAY_MS from 2500ms to 400ms. That figure was picked while timing was wrongly blamed for the {"error":...} rejections; the real cause was missing request headers, and nothing has failed since those were added. With seeding it is usually one fetch, so most of that 2.5s was pure latency. Expected: native settles ~0.75s, collector fills the gap, cache and reload land in roughly 2-3s instead of ~10.5s. --- mods/features/adblock.js | 8 ++-- mods/features/playlistBatchCollect.js | 58 ++++++++++++++++++++++++--- 2 files changed, 57 insertions(+), 9 deletions(-) diff --git a/mods/features/adblock.js b/mods/features/adblock.js index 7ca59452..b603bfa6 100644 --- a/mods/features/adblock.js +++ b/mods/features/adblock.js @@ -5,7 +5,7 @@ import { timelyAction, longPressData, MenuServiceItemRenderer, ShelfRenderer, Ti import { PatchSettings } from '../ui/customYTSettings.js'; import { t } from 'i18next'; import './logServer.js'; -import { autoStartCollect, getCachedFullPlaylist, noteInitialPlaylistContents, noteContinuationBatch } from './playlistBatchCollect.js'; +import { autoStartCollect, scheduleCollectAfterNativeSettles, getCachedFullPlaylist, noteInitialPlaylistContents, noteContinuationBatch } from './playlistBatchCollect.js'; import { appendFileOnlyLog, detectAndStorePage, @@ -680,7 +680,7 @@ function processResponsePayload(payload, detectedPage) { // never fires, so the slow re-download stays on the critical path. // Must run BEFORE filterContinuationItems, which reduces an all-watched // batch to the single kept helper. - noteContinuationBatch(String(window.location?.hash || ''), plc.contents, !!plc?.continuations); + noteContinuationBatch(String(window.location?.hash || ''), plc.contents, plc.continuations); plc.contents = filterContinuationItems(plc.contents, detectedPage, !!plc?.continuations, 'arrayPayload.playlist.continuation'); } const arrayTopPlaylistRenderer = payload?.contents?.tvBrowseRenderer?.content?.tvSurfaceContentRenderer?.content?.twoColumnRenderer?.rightColumn?.playlistVideoListRenderer; @@ -956,7 +956,7 @@ JSON.parse = function () { // it — see playlistBatchCollect.js's autoStartCollect() doc comment. // No-ops when serving from cache: continuations is null, so there is no // token to collect with. - autoStartCollect(topPlaylistRenderer.continuations); + scheduleCollectAfterNativeSettles(topPlaylistRenderer.continuations, 'page_load'); filterPlaylistRendererContents(topPlaylistRenderer, detectedPage, 'playlist.renderer'); } @@ -1005,7 +1005,7 @@ JSON.parse = function () { // so it collects real items rather than the single kept helper. This is // the same data the background collector would re-download, only it is // already here and roughly ten seconds sooner. - noteContinuationBatch(String(window.location?.hash || ''), plc.contents, hasContinuation); + noteContinuationBatch(String(window.location?.hash || ''), plc.contents, plc.continuations); plc.contents = filterContinuationItems(plc.contents, detectedPage, hasContinuation, 'playlist.continuation'); } diff --git a/mods/features/playlistBatchCollect.js b/mods/features/playlistBatchCollect.js index 0c6ac422..d8a42334 100644 --- a/mods/features/playlistBatchCollect.js +++ b/mods/features/playlistBatchCollect.js @@ -138,6 +138,9 @@ function _clearState() { // collection on every later one. window.__ttCollectCancel = false; window.__ttContinuationAcc = null; + clearTimeout(window.__ttNativeSettleTimer); + window.__ttNativeSettleTimer = null; + window.__ttLatestContinuations = null; } window.addEventListener('hashchange', _clearState); window.addEventListener('popstate', _clearState); @@ -206,7 +209,11 @@ function _makeAbort() { // anti-automation throttling on the endpoint — a real scroll physically // can't happen that fast. So instead of firing every background fetch back // to back, wait a human-scroll-like interval before each one. -const BATCH_FETCH_DELAY_MS = 2500; +// Lowered from 2500ms. That figure was chosen while timing was wrongly blamed +// for the {"error":...} rejections; the real cause was missing request headers, +// and no fetch has failed since those were added. It also costs real time now +// that seeding means usually a single fetch — 2.5s of it was pure latency. +const BATCH_FETCH_DELAY_MS = 400; function _delay(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } @@ -485,7 +492,7 @@ function maybeReloadForFullPlaylist(key) { // so it can hand the raw items straight here. When a response arrives with no // continuation token the playlist is complete, and the reload can happen // immediately instead of waiting for a redundant re-download. -export function noteContinuationBatch(key, contents, hasMore) { +export function noteContinuationBatch(key, contents, continuations) { if (!configRead('enablePlaylistBatchCollect')) return; if (!Array.isArray(contents) || !contents.length) return; if (getCachedFullPlaylist(key)) return; // already complete; this is the reloaded pass @@ -506,7 +513,11 @@ export function noteContinuationBatch(key, contents, hasMore) { acc.contents.push(item); } - if (hasMore) return; + // Each native batch pushes the settle timer out and supersedes the token, so + // the collector resumes from the furthest point YouTube reached. + scheduleCollectAfterNativeSettles(continuations, 'native_batch'); + + if (continuations) return; // No continuation token: this was the last batch, so initial + everything // accumulated is the whole playlist. @@ -532,7 +543,43 @@ export function noteContinuationBatch(key, contents, hasMore) { // or not) — the InnerTube context is client/session info, not tied to any // one request, so reusing it here is the same assumption _collectAll already // makes when reusing one captured context across every batch it fetches. -export function autoStartCollect(continuations) { +// Wait for YouTube to stop loading on its own, then resume from where it got +// to — rather than re-downloading everything it already delivered. +// +// Measured on a 68-video all-watched playlist: YouTube fetches the initial 15 +// plus three continuations of 15 (60 of 68) in about 0.75s and then stops. All +// three carry hasContinuation:true, so there is never a completion signal, and +// the last 8 items are never fetched unless the user scrolls. Starting the +// collector at page load meant it re-fetched batches 2-5 from scratch — 53 +// items, 45 of which were already on screen — taking ~10s, with helper tiles +// visible the whole time. +// +// So: hold off until the native burst goes quiet, then hand the collector the +// items already accumulated plus the newest continuation token. It fetches +// only what is genuinely missing (one batch here instead of four). +const NATIVE_SETTLE_MS = 1200; + +export function scheduleCollectAfterNativeSettles(continuations, reason) { + if (!configRead('enablePlaylistBatchCollect')) return; + if (window.__ttPrefetchStarted || window.__ttPrefetchedBatch) return; + + const key = playlistKeyFromHash(); + if (getCachedFullPlaylist(key)) return; // reloaded pass: already complete + + // Newest token wins — each native continuation supersedes the last. + if (_getToken(continuations)) window.__ttLatestContinuations = continuations; + + clearTimeout(window.__ttNativeSettleTimer); + window.__ttNativeSettleTimer = setTimeout(() => { + if (playlistKeyFromHash() !== key) return; + const acc = window.__ttContinuationAcc; + const seed = (acc && acc.key === key && Array.isArray(acc.contents)) ? acc.contents : []; + _log('playlist.batch_collect.native_settled', { reason, seeded: seed.length }); + autoStartCollect(window.__ttLatestContinuations, seed); + }, NATIVE_SETTLE_MS); +} + +export function autoStartCollect(continuations, seedContents) { if (!configRead('enablePlaylistBatchCollect')) return; if (window.__ttPrefetchStarted || window.__ttPrefetchedBatch) return; @@ -548,6 +595,7 @@ export function autoStartCollect(continuations) { window.__ttCollectCancel = false; _log('playlist.batch_collect.auto_triggered', { headerKeys: _lastBrowseHeaders ? Object.keys(_lastBrowseHeaders) : null, + seeded: Array.isArray(seedContents) ? seedContents.length : 0, }); window.__ttPrefetchStarted = true; @@ -560,7 +608,7 @@ export function autoStartCollect(continuations) { ;(async () => { let collected = null; try { - collected = await _collectAll(url, { contents: [], continuations }, context, headers); + collected = await _collectAll(url, { contents: Array.isArray(seedContents) ? seedContents : [], continuations }, context, headers); } catch (err) { _log('playlist.batch_collect.auto_error', { err: String(err?.message || err) }); } From 926ac32a0063e9fc3a3991ac41d87030efd058d5 Mon Sep 17 00:00:00 2001 From: KrX3D Date: Sat, 29 Aug 2026 22:37:36 +0200 Subject: [PATCH 2/2] chore: drop now-unused autoStartCollect import flagged by CodeQL Valid finding. This PR replaced adblock.js's direct autoStartCollect() call with scheduleCollectAfterNativeSettles(), so the import was left behind referenced only by a comment. - Removed it from adblock.js's import list. - Unexported autoStartCollect: nothing outside the module calls it any more, it is reached only through the scheduler. - Corrected two doc comments that still described adblock.js invoking it directly on page load, which is no longer how the flow works. No behaviour change. --- mods/features/adblock.js | 9 +++++---- mods/features/playlistBatchCollect.js | 10 ++++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mods/features/adblock.js b/mods/features/adblock.js index b603bfa6..9ce30b03 100644 --- a/mods/features/adblock.js +++ b/mods/features/adblock.js @@ -5,7 +5,7 @@ import { timelyAction, longPressData, MenuServiceItemRenderer, ShelfRenderer, Ti import { PatchSettings } from '../ui/customYTSettings.js'; import { t } from 'i18next'; import './logServer.js'; -import { autoStartCollect, scheduleCollectAfterNativeSettles, getCachedFullPlaylist, noteInitialPlaylistContents, noteContinuationBatch } from './playlistBatchCollect.js'; +import { scheduleCollectAfterNativeSettles, getCachedFullPlaylist, noteInitialPlaylistContents, noteContinuationBatch } from './playlistBatchCollect.js'; import { appendFileOnlyLog, detectAndStorePage, @@ -951,9 +951,10 @@ JSON.parse = function () { noteInitialPlaylistContents(playlistKey, topPlaylistRenderer.contents); } storePlaylistContinuationToken(topPlaylistRenderer.continuations, 'topPlaylist'); - // Start collecting the rest of the playlist in the background right - // now, instead of waiting for the user to scroll down once to trigger - // it — see playlistBatchCollect.js's autoStartCollect() doc comment. + // Queue background collection of whatever YouTube does not fetch + // itself — deferred until its own loading goes quiet, so the collector + // resumes from there instead of refetching. See + // scheduleCollectAfterNativeSettles() in playlistBatchCollect.js. // No-ops when serving from cache: continuations is null, so there is no // token to collect with. scheduleCollectAfterNativeSettles(topPlaylistRenderer.continuations, 'page_load'); diff --git a/mods/features/playlistBatchCollect.js b/mods/features/playlistBatchCollect.js index d8a42334..60bcf46c 100644 --- a/mods/features/playlistBatchCollect.js +++ b/mods/features/playlistBatchCollect.js @@ -7,9 +7,11 @@ * no 614KB re-parse. * * Two trigger paths feed the same background collector: - * - autoStartCollect(): called by adblock.js right after the initial - * playlist page's own continuation token is parsed, so collection starts - * on page load without needing any scroll at all. + * - scheduleCollectAfterNativeSettles(): called by adblock.js on playlist + * page load and again for each continuation YouTube delivers itself. It + * waits for that native loading to go quiet, then hands autoStartCollect() + * the items already accumulated plus the newest token, so collection + * resumes from where YouTube stopped rather than refetching from batch 2. * - The XHR send() seed path below: a fallback for when the auto-trigger's * captured context/url isn't available yet (e.g. very first browse of a * session) — starts from the first scroll-triggered continuation request. @@ -579,7 +581,7 @@ export function scheduleCollectAfterNativeSettles(continuations, reason) { }, NATIVE_SETTLE_MS); } -export function autoStartCollect(continuations, seedContents) { +function autoStartCollect(continuations, seedContents) { if (!configRead('enablePlaylistBatchCollect')) return; if (window.__ttPrefetchStarted || window.__ttPrefetchedBatch) return;