From 7e65d0d9ec8ca5c5a259ca5157e4d7420d503e16 Mon Sep 17 00:00:00 2001 From: ssavutu Date: Wed, 29 Jul 2026 23:46:03 -0400 Subject: [PATCH] fix(media): run the filesystem index in the background MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The index could never complete over HTTP. It walks ~145k filesystem entries on the real corpus and takes minutes, but it ran inside the request on r.Context(), so the first proxy to give up killed it: Nginx cuts an idle upstream read at 60s (observed: status=500 duration_ms=60045, surfacing to the client as Cloudflare's 504), and Cloudflare would cut in at ~100s regardless. The walk aborted cleanly and committed what it had — 6485 of ~24k originals — but no amount of retrying could finish it, and the Media -> Reindex button was unusable by construction. POST /v1/media/index now starts the walk on a background context that outlives the request and returns 202 immediately; GET /v1/media/index reports {running, started_at, finished_at, progress, error}. A second POST while a run is in flight returns 409 rather than starting a duplicate. Runs are capped at two hours so a wedged filesystem cannot leave the job permanently "running" and block every later attempt. Progress counts entries walked, not files indexed: the corpus is mostly WordPress derivatives skipped before any stat, so an indexed-file counter appears frozen for long stretches. MediaIndexResponse gains Walked for this. The frontend polls every 2s and shows live counts in the button. A 409 is treated as "join the run in progress" rather than an error, so clicking Reindex while one is already going attaches to it. Also fixes a real bug this surfaced: refresh() was setSearch(current => current), which React bails out of, so the list never actually reloaded after an upload or index. It now bumps a token the load effect depends on. Co-Authored-By: Claude Opus 5 --- deploy/README.md | 16 +- frontend/src/pages/mediaView.tsx | 58 +++++++- server/docs/docs.go | 55 ++++++- server/docs/swagger.json | 55 ++++++- server/docs/swagger.yaml | 38 ++++- server/internal/database/media.go | 34 +++++ .../database/media_integration_test.go | 37 +++++ server/internal/handlers/media.go | 140 ++++++++++++++++-- .../handlers/media_integration_test.go | 80 ++++++++++ server/internal/handlers/media_test.go | 100 +++++++++++++ server/internal/models/api_responses.go | 20 ++- server/internal/routes/routes.go | 3 + 12 files changed, 596 insertions(+), 40 deletions(-) diff --git a/deploy/README.md b/deploy/README.md index 1f452b7..5d29a0a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -150,7 +150,8 @@ the database. After the media rsync completes, populate it once from the CMS (Media -> Reindex) or directly: ```bash -curl -X POST https://localhost/v1/media/index # admin session required +curl -X POST https://localhost/v1/media/index # admin session required; returns 202 +curl https://localhost/v1/media/index # poll progress ``` It walks `MEDIA_ROOT/wp-content/uploads`, skips WordPress's generated `-WxH` @@ -159,8 +160,17 @@ already-indexed files are skipped and any alt text set in the CMS is preserved so re-run it after any later out-of-band rsync. Uploads through the CMS index themselves and need no reindex. -Note this walks the whole tree, so on a large corpus over CephFS the first run -takes a while; run it once at cutover rather than on a schedule. +**The index runs in the background.** `POST` returns `202` immediately and `GET` +reports `{running, progress:{walked, scanned, added, skipped}, error}`; a second +`POST` while one is in flight returns `409`. This is not cosmetic: the real corpus +is ~145k filesystem entries and the walk takes minutes, while Nginx cuts an idle +upstream read at 60s and Cloudflare at ~100s. A synchronous version was cancelled +by those proxies every time and could never finish. A run is capped at two hours +so a wedged filesystem cannot leave the job stuck "running" forever. + +Progress is reported per entry *walked* rather than per file indexed, because the +corpus is mostly derivatives that are skipped without a stat — counting only +indexed files would look frozen for long stretches. ### Disk diff --git a/frontend/src/pages/mediaView.tsx b/frontend/src/pages/mediaView.tsx index b72f4c4..a6b909a 100644 --- a/frontend/src/pages/mediaView.tsx +++ b/frontend/src/pages/mediaView.tsx @@ -27,11 +27,20 @@ type MediaResponse = { } type IndexReport = { + walked?: number scanned?: number added?: number skipped?: number } +type IndexStatus = { + running: boolean + started_at?: string + finished_at?: string + progress?: IndexReport + error?: string +} + const PAGE_SIZE = 60 function formatBytes(bytes?: number) { @@ -73,6 +82,8 @@ function MediaView() { const [selected, setSelected] = useState(null) const [uploadStatus, setUploadStatus] = useState(null) const [isIndexing, setIsIndexing] = useState(false) + const [indexProgress, setIndexProgress] = useState(null) + const [reloadToken, setReloadToken] = useState(0) // Debounce so typing doesn't fire a request per keystroke; the filter itself // is applied server-side because the library is far too large to filter here. @@ -117,7 +128,7 @@ function MediaView() { void load() return () => controller.abort() - }, [fetchPage]) + }, [fetchPage, reloadToken]) const loadMore = async () => { setIsLoadingMore(true) @@ -133,7 +144,10 @@ function MediaView() { } } - const refresh = () => setSearch((current) => current) + // Bumping a token is what actually re-runs the load effect. Re-setting `search` + // to its current value does not: React bails out of a same-value setState, so + // the effect never re-fires and the list silently stays stale. + const refresh = useCallback(() => setReloadToken((n) => n + 1), []) const handleUpload = async (files: FileList | null) => { if (!files || files.length === 0) return @@ -223,6 +237,32 @@ function MediaView() { } } + // The index walks ~145k filesystem entries and takes minutes, so the server + // runs it in the background and we poll. A synchronous request could never + // finish: Nginx cuts an upstream read at 60s and Cloudflare at ~100s. + const pollIndexStatus = useCallback(async (): Promise => { + const response = await apiFetch("/v1/media/index") + if (!response.ok) throw new Error(await errorMessage(response, `Status check failed (${response.status})`)) + const status = (await response.json()) as IndexStatus + + const p = status.progress + if (status.running) { + setIndexProgress( + `Indexing… ${String(p?.walked ?? 0)} files scanned, ${String(p?.added ?? 0)} added`, + ) + return true + } + + setIndexProgress(null) + if (status.error) { + setError(`Reindex failed: ${status.error}`) + } else if (status.finished_at) { + setNotice(`Indexed ${String(p?.scanned ?? 0)} assets — ${String(p?.added ?? 0)} new, ${String(p?.skipped ?? 0)} already known.`) + if ((p?.added ?? 0) > 0) refresh() + } + return false + }, [apiFetch, refresh]) + const handleReindex = async () => { setIsIndexing(true) setError(null) @@ -230,15 +270,19 @@ function MediaView() { try { const response = await apiFetch("/v1/media/index", { method: "POST" }) - if (!response.ok) { + // 409 means one is already running — join its progress rather than error. + if (!response.ok && response.status !== 409) { setError(await errorMessage(response, `Reindex failed (${response.status})`)) + setIsIndexing(false) return } - const report = (await response.json()) as IndexReport - setNotice(`Indexed ${String(report.scanned ?? 0)} files — ${String(report.added ?? 0)} new.`) - if ((report.added ?? 0) > 0) refresh() + + while (await pollIndexStatus()) { + await new Promise((resolve) => setTimeout(resolve, 2000)) + } } catch (err) { setError(err instanceof Error ? err.message : "Unable to reindex media.") + setIndexProgress(null) } finally { setIsIndexing(false) } @@ -268,7 +312,7 @@ function MediaView() { type="button" >