diff --git a/src/components/webhook-loop-mount.tsx b/src/components/webhook-loop-mount.tsx index 5ac1436af..5e96dc3d1 100644 --- a/src/components/webhook-loop-mount.tsx +++ b/src/components/webhook-loop-mount.tsx @@ -1,7 +1,6 @@ import { useEffect, useRef } from "react"; import { useAuth } from "@/lib/auth"; import { useSettings } from "@/lib/settings"; -import { proactiveStorageCleanup } from "@/lib/storage-recovery"; import { runWebhookTick } from "@/lib/webhook-engine"; const TICK_INTERVAL_MS = 30 * 60 * 1000; @@ -17,10 +16,6 @@ export function WebhookLoopMount() { authRef.current = authKey; const runningRef = useRef(false); - useEffect(() => { - proactiveStorageCleanup(); - }, []); - useEffect(() => { const hasDestination = !!settings.webhooks.discordUrl || diff --git a/src/lib/onboarding.tsx b/src/lib/onboarding.tsx index b6c822bae..3258d7fbe 100644 --- a/src/lib/onboarding.tsx +++ b/src/lib/onboarding.tsx @@ -7,6 +7,7 @@ import { useState, type ReactNode, } from "react"; +import { setItemWithRecovery } from "@/lib/storage-recovery"; type Flags = { onboarded: boolean; nudges: Record }; @@ -31,7 +32,7 @@ export function parseOnboardingFlags(raw: string | null): Flags { return { ...DEFAULT, ...parsed, - nudges: { [CONTENT_ADVISORY_NUDGE]: true, ...(parsed.nudges ?? {}) }, + nudges: { [CONTENT_ADVISORY_NUDGE]: true, ...parsed.nudges }, }; } catch { return DEFAULT; @@ -46,7 +47,10 @@ export function OnboardingProvider({ children }: { children: ReactNode }) { ); useEffect(() => { - localStorage.setItem(STORAGE_KEY, JSON.stringify(flags)); + // This provider wraps the whole app, so a bare setItem throwing here (a full store + // throws on every write) takes the tree down during commit, before anything can + // signal the boot splash to clear. Prune and keep going instead. + setItemWithRecovery(STORAGE_KEY, JSON.stringify(flags)); }, [flags]); const finishOnboarding = useCallback(() => setFlags((f) => ({ ...f, onboarded: true })), []); diff --git a/src/lib/storage-recovery.ts b/src/lib/storage-recovery.ts index 755722689..feab7d071 100644 --- a/src/lib/storage-recovery.ts +++ b/src/lib/storage-recovery.ts @@ -1,67 +1,63 @@ -const PRUNABLE_EXACT = new Set([ - "harbor.omdb.v1", +// What may be evicted when the store is full. Prunable means the network or a recomputation can +// give it back; anything the user made or did stays, however big it is: playback history, last +// season, live-TV watch counts and manual EPG matches, Discover's learned affinity, uploaded +// stream badges and their rules, the legacy parental pin, the build rating, completed library +// repairs, webhook delivery history, addon velocity history, and OMDb's daily request +// budget. Open the module that +// owns a key before adding it here - backup.ts groups by prefix, so its sections mix real records +// with caches and cannot answer this on their own. Keys of a few KB are not worth the risk either +// way; this list is for what actually fills the store. +// +// These are prefixes, and stop before any version stamp on purpose. Pinning a version means the +// live cache silently stops being prunable the day its owner bumps it, which is how seven of +// these were unreachable until now. +const PRUNABLE_PREFIXES = [ + // Only OMDb's rating and negative-result caches are rebuildable. The daily + // request budget and key-validity state must survive recovery. + "harbor.omdb.v", "harbor.omdb.misses", - "harbor.omdb.budget", - "harbor.iptv.hydration.v2", - "harbor.iptv.epgmap.v1", - "harbor.iptv.stats.v1", - "harbor.dead-streams.v1", - "harbor.discover.v1", - "harbor.discover.events.v1", - "harbor.discover.awards.v1", - "harbor.discover.rows.v1", - "harbor.scroll.v1", + "harbor.iptv.hydration.v", + "harbor.dead-streams.v", + "harbor.discover.events.v", + "harbor.discover.awards.v", + "harbor.discover.rows.v", + "harbor.scroll.v", "harbor.webhook.lastTick", - "harbor.calendar.webhook.last", "harbor.anidbtvdbcache", "harbor.armcache", "harbor.armkitsucache", "harbor.armsrcmalcache", "harbor.extkitsucache", - "harbor.imdb.tmdb.v1", - "harbor.tmdb.imdb.v1", - "harbor.tmdb.personName.v1", - "harbor.awards.wikidata.v8", - "harbor.awards.wikidata.v9", - "harbor.awards.wikidata.v10", - "harbor.anime_awards.metas.v2", - "harbor.animefillercache.v2", "harbor.jikancatalog", + "harbor.imdb.tmdb.v", + "harbor.tmdb.imdb.v", + "harbor.tmdb.personName.v", + "harbor.awards.wikidata.v", + "harbor.graph.wikidata.", + "harbor.anime_awards.metas.", + "harbor.animefillercache.v", "harbor.malscorecache", - "harbor.anime.recs_by_mal.v1", - "harbor.anime.toppicks.cache.v1", - "harbor.anime.toppicks.shown.v1", - "harbor.anime.toppicks.visit.v1", - "harbor.anime.mal_id_by_franchise.v1", - "harbor.picker-cache.v4", - "harbor.picker-cache.v5", - "harbor.shows.hero.pool.v2", + "harbor.anime.recs_by_mal.v", + "harbor.anime.toppicks.cache.", + "harbor.anime.mal_id_by_franchise.v", + "harbor.anime.art.v", + "harbor.anime.hero.hosted.v", + "harbor.anilist.collection.v", + "harbor.shows.hero.pool.v", + "harbor.heroPool.v", + "harbor.mal.hero.v", "harbor.mdblist.cards", - "harbor.build.rating.v1", - "harbor.parental", - "harbor.lastseason.v1", - "harbor.surprise.recent.v1", - "harbor.stremio-addons.velocity.v1", - "harbor.playback-history.v1", - "harbor.streamBadges.v1", - "harbor.manga.popular.v1", - "harbor.ebook.metadata.v1", - "harbor.heroPool.v1", -]); - -const PRUNABLE_PREFIXES = [ - "harbor.libraryNameRepair.v1.", - "harbor.anilist.collection.v1.", - "harbor.manga.cache.v1.", - "harbor.manga.cache.v2.", + "harbor.manga.popular.v", + "harbor.manga.cache.v", "harbor.manga.art.", - "harbor.ebook.openlibrary.v1.", + "harbor.ebook.metadata.v", + "harbor.ebook.nyt.v", + "harbor.ebook.openlibrary.v", + "harbor.sports.board.v", "harbor.tvdbo.", - "harbor.playback-history.v1.", ]; function isPrunable(key: string): boolean { - if (PRUNABLE_EXACT.has(key)) return true; return PRUNABLE_PREFIXES.some((p) => key.startsWith(p)); } @@ -256,8 +252,51 @@ function purgeOldCacheVersions(): void { } } +const PROBE_KEY = "harbor.storage.probe"; +/** Room for one more character proves nothing: the next boot write is a settings or profile + * blob, so ask for enough space to land one of those. */ +const PROBE_CHARS = 32 * 1024; + +/** + * The size thresholds below only measure what is prunable, so a store filled mostly by + * keys we never prune stays under them while every write already throws. Ask the store + * itself instead. + */ +function storageRejectsWrites(): boolean { + try { + localStorage.setItem(PROBE_KEY, "x".repeat(PROBE_CHARS)); + localStorage.removeItem(PROBE_KEY); + return false; + } catch (e) { + localStorage.removeItem(PROBE_KEY); + return isQuotaError(e); + } +} + +/** + * Boot only: the store cannot take a realistic write, so free until it can and stop there. + * Dropping every cache instead would cost the user a full refetch of the home feed to buy + * headroom nothing is asking for. + */ +function freeUntilWritable(): void { + for (const { key } of prunableEntries()) { + pruneOne(key); + if (!storageRejectsWrites()) return; + } +} + export function proactiveStorageCleanup(): void { if (typeof localStorage === "undefined") return; + // Runs before createRoot, so a storage that is blocked or corrupt rather than merely full + // must not reject mount() - that leaves the boot splash up with nothing behind it. + try { + pruneUntilUsable(); + } catch (e) { + console.warn("[storage] proactive cleanup skipped", e); + } +} + +function pruneUntilUsable(): void { purgeDeadCaches(); purgeOldCacheVersions(); capLruPrefixes(); @@ -272,5 +311,7 @@ export function proactiveStorageCleanup(): void { console.info( `[storage] proactive total cleanup: ${r.pruned.length} caches, ${r.freedBytes} bytes`, ); + } else if (storageRejectsWrites()) { + freeUntilWritable(); } } diff --git a/src/main.tsx b/src/main.tsx index 9090ece50..78a2b1925 100644 --- a/src/main.tsx +++ b/src/main.tsx @@ -20,6 +20,7 @@ import "@/index.css"; import "flag-icons/css/flag-icons.min.css"; import { startTaskbarProgress } from "@/lib/download/taskbar-progress"; import { completeBetaReturnPreferences } from "@/lib/updater/beta-return"; +import { proactiveStorageCleanup } from "@/lib/storage-recovery"; // Loaded only when a DJ window is actually opened; not part of the core entry graph. const DjDeckApp = lazy(() => @@ -220,7 +221,16 @@ function MainRoot() { return ; } +/** PIP, modal, captions, HDR overlay, DJ deck and the detached player all load this same + * index.html and share its localStorage. Opening one must not evict caches the main window + * is in the middle of using. */ +const isAuxWindow = isPip || isVideoPip || isModal || isHdrOverlay || isCaptions || isDjDeck; + async function mount() { + // Must run before anything writes: on a full localStorage every setItem throws, and the + // first unguarded one during the initial render kills the tree before onReady fires, so + // the boot splash never clears. Cleaning up from inside a component is too late. + if (!isAuxWindow) proactiveStorageCleanup(); await loadInstallFeatures(); const musicEnabled = getInstallFeatures().music; if ( diff --git a/tests/storage-quota-boot.test.ts b/tests/storage-quota-boot.test.ts new file mode 100644 index 000000000..84cb4b0e9 --- /dev/null +++ b/tests/storage-quota-boot.test.ts @@ -0,0 +1,242 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +/** + * A full localStorage used to brick the boot: every setItem threw, the first unguarded one + * ran during the initial render, and the tree died before anything cleared the splash. + * proactiveStorageCleanup() is what has to make the store writable again, so it has to fire + * on a store that actually rejects writes — not only on one whose prunable half is large. + */ +function installQuotaStorage(seed: Array<[string, string]>) { + const store = new Map(); + const used = () => [...store].reduce((n, [k, v]) => n + k.length + v.length, 0); + // Seeded right up to the limit, so the store starts out rejecting every write. + let budget = Number.POSITIVE_INFINITY; + const stub = { + getItem: (k: string) => store.get(k) ?? null, + removeItem: (k: string) => void store.delete(k), + key: (i: number) => [...store.keys()][i] ?? null, + get length() { + return store.size; + }, + setItem(k: string, v: string) { + const next = + used() - (store.get(k)?.length ?? 0) - (store.has(k) ? k.length : 0) + k.length + v.length; + if (next > budget) throw new DOMException("quota", "QuotaExceededError"); + store.set(k, v); + }, + }; + for (const [k, v] of seed) store.set(k, v); + budget = used(); + (globalThis as { localStorage?: unknown }).localStorage = stub; + return store; +} + +const filler = (n: number) => "x".repeat(n); + +test("a localStorage that rejects every write is pruned back into a writable state", async () => { + // Full, but the prunable share stays under PROACTIVE_TOTAL_THRESHOLD, which is exactly the + // shape that used to slip past the size heuristics and leave the store unwritable. + const store = installQuotaStorage([ + ["harbor.playback-history.v1.p_demo", filler(400_000)], + ["harbor.streamBadges.v1", filler(250_000)], + ["harbor.libraryNameRepair.v1.p_demo", "1699999999999"], + [ + "harbor.omdb.budget", + JSON.stringify({ + used: 920, + limit: 1000, + resetAt: 2000000000000, + exhausted: false, + keyInvalid: false, + }), + ], + ["harbor.settings", filler(200_000)], + ["harbor.mal.hero.v1", filler(100_000)], + ]); + + assert.throws(() => localStorage.setItem("harbor.onboarding", "{}")); + + const { proactiveStorageCleanup } = await import("@/lib/storage-recovery"); + proactiveStorageCleanup(); + + localStorage.setItem("harbor.onboarding", "{}"); + assert.equal(store.get("harbor.onboarding"), "{}"); + + // The cache goes; the 400k of continue watching next to it does not, however full we are. + assert.equal(store.has("harbor.mal.hero.v1"), false); + assert.ok(store.has("harbor.playback-history.v1.p_demo")); + assert.ok(store.has("harbor.streamBadges.v1")); + assert.equal(store.get("harbor.libraryNameRepair.v1.p_demo"), "1699999999999"); + assert.equal(JSON.parse(store.get("harbor.omdb.budget")!).used, 920); + assert.ok(store.has("harbor.settings")); +}); + +test("boot cleanup preserves large user histories instead of treating them as caches", async () => { + // A >256 KB snapshot record used to be pruned before checking whether storage was full. + // Its historical stars cannot be reconstructed from today's community-addon index. + const velocity = JSON.stringify({ + snapshots: [{ fetchedAt: 1, stars: { addon: filler(280_000) } }], + }); + const ledger = JSON.stringify({ + "__baseline__:all:discord": "2026-11-05", + "all:discord:tt123": "2026-11-05", + }); + const store = installQuotaStorage([ + ["harbor.settings", filler(300_000)], + ["harbor.stremio-addons.velocity.v1", velocity], + ["harbor.calendar.webhook.last", ledger], + ["harbor.mal.hero.v1", filler(100_000)], + ]); + + const { proactiveStorageCleanup } = await import("@/lib/storage-recovery"); + proactiveStorageCleanup(); + + assert.equal(store.get("harbor.stremio-addons.velocity.v1"), velocity); + assert.equal(store.get("harbor.calendar.webhook.last"), ledger); + assert.equal(store.has("harbor.mal.hero.v1"), false); + localStorage.setItem("harbor.onboarding", "{}"); +}); + +test("boot cleanup stops as soon as the store takes writes again", async () => { + // Freeing the one biggest cache is enough here, so the rest must still be there: buying + // headroom nobody asked for costs the user a refetch of everything it dropped. + const store = installQuotaStorage([ + ["harbor.settings", filler(300_000)], + ["harbor.mal.hero.v1", filler(200_000)], + ["harbor.ebook.openlibrary.v2.anonymous.abc", filler(150_000)], + ["harbor.sports.board.v3:SOCCER@20261031@day", filler(140_000)], + ]); + + const { proactiveStorageCleanup } = await import("@/lib/storage-recovery"); + proactiveStorageCleanup(); + + localStorage.setItem("harbor.onboarding", "{}"); + assert.equal(store.has("harbor.mal.hero.v1"), false); + assert.ok(store.has("harbor.ebook.openlibrary.v2.anonymous.abc")); + assert.ok(store.has("harbor.sports.board.v3:SOCCER@20261031@day")); +}); + +test("setItemWithRecovery reports failure instead of throwing when nothing is left to prune", async () => { + installQuotaStorage([["harbor.settings", filler(900)]]); + const { setItemWithRecovery } = await import("@/lib/storage-recovery"); + assert.equal(setItemWithRecovery("harbor.onboarding", filler(500)), false); +}); + +/** + * The ordering in the tests above means a wrongly listed key may simply never be reached, so + * guard the list itself with the sweep that spares nothing prunable. + */ +test("the full sweep evicts caches and nothing the user did", async () => { + const store = installQuotaStorage([ + ["harbor.playback-history.v1.p_demo", filler(1_000)], + ["harbor.iptv.stats.v1", filler(1_000)], + ["harbor.iptv.epgmap.v1", filler(1_000)], + ["harbor.discover.v1", filler(1_000)], + ["harbor.lastseason.v1", filler(1_000)], + ["harbor.parental", filler(1_000)], + ["harbor.build.rating.v1", filler(1_000)], + ["harbor.streamBadges.v1", filler(1_000)], + [ + "harbor.calendar.webhook.last", + JSON.stringify({ + "__baseline__:all:discord": "2026-11-05", + "all:discord:tt123": "2026-11-05", + }), + ], + [ + "harbor.stremio-addons.velocity.v1", + JSON.stringify({ + snapshots: [ + { fetchedAt: 1, stars: { addon: 3 } }, + { fetchedAt: 2, stars: { addon: 20 } }, + ], + }), + ], + ["harbor.libraryNameRepair.v1.p_demo", "1699999999999"], + ["harbor.libraryNameRepair.v99.p_demo", "1699999999999"], + [ + "harbor.omdb.budget", + JSON.stringify({ + used: 920, + limit: 1000, + resetAt: 2000000000000, + exhausted: false, + keyInvalid: false, + }), + ], + ["harbor.settings", filler(1_000)], + ["harbor.omdb.v1", filler(1_000)], + ["harbor.omdb.misses", filler(1_000)], + ["harbor.mal.hero.v1", filler(1_000)], + ["harbor.tvdbo.tt0903747", filler(1_000)], + ]); + + const { freeStorageSpace } = await import("@/lib/storage-recovery"); + freeStorageSpace(); + + assert.deepEqual([...store.keys()].sort(), [ + "harbor.build.rating.v1", + "harbor.calendar.webhook.last", + "harbor.discover.v1", + "harbor.iptv.epgmap.v1", + "harbor.iptv.stats.v1", + "harbor.lastseason.v1", + "harbor.libraryNameRepair.v1.p_demo", + "harbor.libraryNameRepair.v99.p_demo", + "harbor.omdb.budget", + "harbor.parental", + "harbor.playback-history.v1.p_demo", + "harbor.settings", + "harbor.streamBadges.v1", + "harbor.stremio-addons.velocity.v1", + ]); +}); + +test("a cache stays prunable after its owner bumps the version", async () => { + // Pinning versions in the list is how harbor.jikancatalog2, harbor.armkitsucache.v3 and five + // others quietly became unreachable: the list still named the version before. + const store = installQuotaStorage([ + ["harbor.jikancatalog2", filler(1_000)], + ["harbor.armkitsucache.v3", filler(1_000)], + ["harbor.anidbtvdbcache.v2", filler(1_000)], + ["harbor.armsrcmalcache.v2", filler(1_000)], + ["harbor.extkitsucache.v2", filler(1_000)], + ["harbor.anime_awards.metas.v6", filler(1_000)], + ["harbor.anime.toppicks.cache.v2", filler(1_000)], + ["harbor.awards.wikidata.v10", filler(1_000)], + ["harbor.graph.wikidata.v2", filler(1_000)], + ["harbor.mal.hero.v1", filler(1_000)], + ["harbor.ebook.nyt.v1", filler(1_000)], + ["harbor.anime.art.v2", filler(1_000)], + ["harbor.anime.hero.hosted.v5", filler(1_000)], + ["harbor.sports.board.v3:SOCCER@20261031@day", filler(1_000)], + ["harbor.ebook.openlibrary.v2.anonymous.abc", filler(1_000)], + ["harbor.manga.cache.v2.one-piece", filler(1_000)], + // Versions nobody has shipped yet: the next bump must not need this list touched again. + ["harbor.armkitsucache.v99", filler(1_000)], + ["harbor.mal.hero.v9", filler(1_000)], + ["harbor.sports.board.v9:SOCCER@20261031@day", filler(1_000)], + ["harbor.manga.cache.v9.one-piece", filler(1_000)], + ["harbor.settings", filler(1_000)], + ]); + + const { freeStorageSpace } = await import("@/lib/storage-recovery"); + freeStorageSpace(); + + assert.deepEqual([...store.keys()], ["harbor.settings"]); +}); + +test("boot recovery frees room for a real write, not for one character", async () => { + // Many small caches behind a big record: a one-character probe is satisfied by the first + // prune, and the settings or profile blob that follows still throws. + const seed: Array<[string, string]> = [["harbor.settings", filler(500_000)]]; + for (let i = 0; i < 10; i++) seed.push([`harbor.tvdbo.tt${i}`, filler(5_000)]); + installQuotaStorage(seed); + + const { proactiveStorageCleanup } = await import("@/lib/storage-recovery"); + proactiveStorageCleanup(); + + // Four times the largest prunable entry, so only a real headroom target can cover it. + localStorage.setItem("harbor.profiles.v1", filler(20_000)); +});