From d31cf5b6ee611294a88ff39d004664afe4113985 Mon Sep 17 00:00:00 2001 From: Ronny Trommer Date: Thu, 8 Oct 2026 13:07:36 +0200 Subject: [PATCH 1/2] fix(index): stop blocking page render on the modarchive RSS feed getServerSideProps awaited modarchive.org/rss.php on every request to find the newest module id. The feed takes 2.5 to 4 s to answer, so the start page TTFB was about 3 s for every visitor. Move the lookup into lib/modarchive/latest-id.ts. getLatestId() returns the cached id, or RANDOM_MAX before the first fetch lands, and starts one background refresh when the value is older than an hour. A failed or timed-out (10 s) fetch keeps the previous value. Local production build: TTFB 45 ms on the first request, 4 ms after. Closes #208 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GwowRkcBNzHsDGBXywedzR Signed-off-by: Ronny Trommer --- lib/modarchive/latest-id.test.ts | 73 ++++++++++++++++++++++++++++++++ lib/modarchive/latest-id.ts | 40 +++++++++++++++++ pages/index.tsx | 17 +++----- 3 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 lib/modarchive/latest-id.test.ts create mode 100644 lib/modarchive/latest-id.ts diff --git a/lib/modarchive/latest-id.test.ts b/lib/modarchive/latest-id.test.ts new file mode 100644 index 0000000..0541620 --- /dev/null +++ b/lib/modarchive/latest-id.test.ts @@ -0,0 +1,73 @@ +/* + * Copyright 2026 Ronny Trommer + * SPDX-License-Identifier: GPL-3.0-or-later + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +const rss = (id: number) => + `https://modarchive.org/index.php?request=view_by_moduleid&query=${id}` + + ``; + +async function load() { + vi.resetModules(); + return (await import("./latest-id")).getLatestId; +} + +describe("getLatestId", () => { + let fetchMock: ReturnType; + + beforeEach(() => { + fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + it("returns the fallback without waiting for the feed", async () => { + fetchMock.mockReturnValue(new Promise(() => {})); + const getLatestId = await load(); + expect(getLatestId(42)).toBe(42); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("serves the fetched id once the refresh lands, without refetching", async () => { + fetchMock.mockResolvedValue(new Response(rss(200123))); + const getLatestId = await load(); + getLatestId(42); + await vi.waitFor(() => expect(getLatestId(42)).toBe(200123)); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("starts only one refresh while one is in flight", async () => { + fetchMock.mockReturnValue(new Promise(() => {})); + const getLatestId = await load(); + getLatestId(42); + getLatestId(42); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("keeps the fallback when the feed fails", async () => { + fetchMock.mockRejectedValue(new Error("down")); + const getLatestId = await load(); + getLatestId(42); + await new Promise((r) => setTimeout(r, 0)); + expect(getLatestId(42)).toBe(42); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("refreshes again after the TTL expires", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + fetchMock.mockResolvedValueOnce(new Response(rss(100))); + fetchMock.mockResolvedValueOnce(new Response(rss(101))); + const getLatestId = await load(); + getLatestId(42); + await vi.waitFor(() => expect(getLatestId(42)).toBe(100)); + vi.setSystemTime(Date.now() + 61 * 60 * 1000); + getLatestId(42); + await vi.waitFor(() => expect(getLatestId(42)).toBe(101)); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); +}); diff --git a/lib/modarchive/latest-id.ts b/lib/modarchive/latest-id.ts new file mode 100644 index 0000000..f18105a --- /dev/null +++ b/lib/modarchive/latest-id.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2026 Ronny Trommer + * SPDX-License-Identifier: GPL-3.0-or-later + */ + +// Newest modarchive.org module id, used as the upper bound for random +// picks. The RSS feed takes several seconds to answer, so the page never +// awaits it: callers get the cached value (or the fallback before the +// first fetch lands) and a stale value triggers a background refresh. + +const RSS_URL = "https://modarchive.org/rss.php?request=uploads"; +const TTL_MS = 60 * 60 * 1000; +const TIMEOUT_MS = 10_000; + +let cached: number | undefined; +let checkedAt = 0; +let refreshing: Promise | undefined; + +async function refresh(): Promise { + try { + const res = await fetch(RSS_URL, { + signal: AbortSignal.timeout(TIMEOUT_MS), + }); + const rss = await res.text(); + const id = Number(rss.split("downloads.php?moduleid=")[1].split("#")[0]); + if (Number.isInteger(id) && id > 0) cached = id; + } catch { + // Keep the previous value; retry after the next TTL window. + } finally { + checkedAt = Date.now(); + refreshing = undefined; + } +} + +export function getLatestId(fallback: number): number { + if (!refreshing && Date.now() - checkedAt > TTL_MS) { + refreshing = refresh(); + } + return cached ?? fallback; +} diff --git a/pages/index.tsx b/pages/index.tsx index a9eb9f0..0497140 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -1,3 +1,7 @@ +/* + * Copyright 2026 Ronny Trommer + * SPDX-License-Identifier: GPL-3.0-or-later + */ import React from "react"; import Head from "next/head"; import type { GetServerSideProps } from "next"; @@ -5,6 +9,7 @@ import type { GetServerSideProps } from "next"; import Player from "../components/Player"; import Footer from "../components/Footer"; import { modArchive, library, type Source } from "../components/sources"; +import { getLatestId } from "../lib/modarchive/latest-id"; import { getRandomInt, getRandomFromArray, @@ -111,17 +116,7 @@ export const getServerSideProps: GetServerSideProps = async ({ backSideContent = ""; } - let latestId: number; - try { - const rss_req = await fetch( - "https://modarchive.org/rss.php?request=uploads", - { method: "GET" } - ); - const rss = await rss_req.text(); - latestId = Number(rss.split("downloads.php?moduleid=")[1].split("#")[0]); - } catch { - latestId = RANDOM_MAX; - } + const latestId = getLatestId(RANDOM_MAX); let initialSource: Source | null = null; if (query.source === "modarchive" && query.id) { From e671d37b2aaa49e411b813c1a191f419d55bdb70 Mon Sep 17 00:00:00 2001 From: Ronny Trommer Date: Thu, 8 Oct 2026 22:30:08 +0200 Subject: [PATCH 2/2] fix(index): harden latest-id refresh after review - Clear the in-flight promise via .finally() in getLatestId, so a synchronous throw inside fetch can no longer leave a settled promise in `refreshing` and block every later refresh. - Retry a failed fetch after one minute instead of a full hour, so a cold-start failure does not pin visitors to RANDOM_MAX. - Reuse fetchHtml (project User-Agent, throws on non-2xx) with a new optional AbortSignal parameter instead of a bare fetch. - Document the long-lived server process assumption. - Tests for non-2xx, retry after failure, and synchronous throw. Refs #208 Assisted-by: ClaudeCode:claude-opus-5-5 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GwowRkcBNzHsDGBXywedzR Signed-off-by: Ronny Trommer --- lib/modarchive/fetch.ts | 9 +++++--- lib/modarchive/latest-id.test.ts | 35 +++++++++++++++++++++++++++++--- lib/modarchive/latest-id.ts | 28 ++++++++++++++----------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/lib/modarchive/fetch.ts b/lib/modarchive/fetch.ts index e07f664..6130c3d 100644 --- a/lib/modarchive/fetch.ts +++ b/lib/modarchive/fetch.ts @@ -8,12 +8,15 @@ // without poisoning the cache. No retries — failures are surfaced // immediately so the chart pane can offer a Retry button. -const USER_AGENT = - "CoolModFiles (+https://github.com/no42-org/CoolModFiles)"; +const USER_AGENT = "CoolModFiles (+https://github.com/no42-org/CoolModFiles)"; -export async function fetchHtml(url: string): Promise { +export async function fetchHtml( + url: string, + signal?: AbortSignal +): Promise { const res = await fetch(url, { method: "GET", + signal, redirect: "follow", headers: { "User-Agent": USER_AGENT, diff --git a/lib/modarchive/latest-id.test.ts b/lib/modarchive/latest-id.test.ts index 0541620..e2e4760 100644 --- a/lib/modarchive/latest-id.test.ts +++ b/lib/modarchive/latest-id.test.ts @@ -49,13 +49,42 @@ describe("getLatestId", () => { expect(fetchMock).toHaveBeenCalledTimes(1); }); - it("keeps the fallback when the feed fails", async () => { - fetchMock.mockRejectedValue(new Error("down")); + it("keeps the fallback when the feed fails and retries after a minute", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + fetchMock.mockRejectedValueOnce(new Error("down")); + fetchMock.mockResolvedValueOnce(new Response(rss(300))); const getLatestId = await load(); getLatestId(42); - await new Promise((r) => setTimeout(r, 0)); + await new Promise((r) => setTimeout(r, 10)); expect(getLatestId(42)).toBe(42); expect(fetchMock).toHaveBeenCalledTimes(1); + vi.setSystemTime(Date.now() + 61 * 1000); + getLatestId(42); + await vi.waitFor(() => expect(getLatestId(42)).toBe(300)); + expect(fetchMock).toHaveBeenCalledTimes(2); + }); + + it("keeps the fallback on a non-2xx response", async () => { + fetchMock.mockResolvedValue(new Response(rss(500), { status: 503 })); + const getLatestId = await load(); + getLatestId(42); + await new Promise((r) => setTimeout(r, 10)); + expect(getLatestId(42)).toBe(42); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + it("recovers when fetch throws synchronously", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + fetchMock.mockImplementationOnce(() => { + throw new Error("sync"); + }); + fetchMock.mockResolvedValueOnce(new Response(rss(400))); + const getLatestId = await load(); + getLatestId(42); + await new Promise((r) => setTimeout(r, 10)); + vi.setSystemTime(Date.now() + 61 * 1000); + getLatestId(42); + await vi.waitFor(() => expect(getLatestId(42)).toBe(400)); }); it("refreshes again after the TTL expires", async () => { diff --git a/lib/modarchive/latest-id.ts b/lib/modarchive/latest-id.ts index f18105a..ff67d09 100644 --- a/lib/modarchive/latest-id.ts +++ b/lib/modarchive/latest-id.ts @@ -7,34 +7,38 @@ // picks. The RSS feed takes several seconds to answer, so the page never // awaits it: callers get the cached value (or the fallback before the // first fetch lands) and a stale value triggers a background refresh. +// Assumes a long-lived server process (the Docker standalone deploy); +// a serverless runtime may freeze before the refresh completes. + +import { fetchHtml } from "./fetch"; const RSS_URL = "https://modarchive.org/rss.php?request=uploads"; const TTL_MS = 60 * 60 * 1000; +const RETRY_MS = 60 * 1000; const TIMEOUT_MS = 10_000; let cached: number | undefined; -let checkedAt = 0; +let nextCheckAt = 0; let refreshing: Promise | undefined; async function refresh(): Promise { try { - const res = await fetch(RSS_URL, { - signal: AbortSignal.timeout(TIMEOUT_MS), - }); - const rss = await res.text(); + const rss = await fetchHtml(RSS_URL, AbortSignal.timeout(TIMEOUT_MS)); const id = Number(rss.split("downloads.php?moduleid=")[1].split("#")[0]); - if (Number.isInteger(id) && id > 0) cached = id; + if (!Number.isInteger(id) || id <= 0) throw new Error(`bad id ${id}`); + cached = id; + nextCheckAt = Date.now() + TTL_MS; } catch { - // Keep the previous value; retry after the next TTL window. - } finally { - checkedAt = Date.now(); - refreshing = undefined; + // Keep the previous value; retry sooner than a full TTL. + nextCheckAt = Date.now() + RETRY_MS; } } export function getLatestId(fallback: number): number { - if (!refreshing && Date.now() - checkedAt > TTL_MS) { - refreshing = refresh(); + if (!refreshing && Date.now() >= nextCheckAt) { + refreshing = refresh().finally(() => { + refreshing = undefined; + }); } return cached ?? fallback; }