Fix list loading through Ultraviolet and Scramjet proxies

AGENTS.md

@@ -0,0 +1,34 @@
+# AGENTS.md
+
+## Cursor Cloud specific instructions
+
+### What this repo is
+`proxy-list` is a **static website** (all UI assets live in `docs/`, main page is `docs/index.html`)
+served in development by a thin **Cloudflare Worker** (`workers/site.js`, config `wrangler.jsonc`).
+The Worker serves the `docs/` assets plus a few JSON APIs: `POST /api/link-click`,
+`POST /api/link-clicks/get`, `POST /api/presence-ping`, `GET /api/presence-active`.
+There is **no build/compile step** for the site — `docs/data.json` (the ~29k-link dataset the UI
+loads) is committed, so the site renders without running any tooling.
+
+### Run / build / test (the dev environment already has deps installed)
+- Run the app (dev): `npm run dev` → `wrangler dev`, serves on `http://localhost:8787`.
+  It serves the static `docs/` site and the `/api/*` Worker endpoints together.
+- Build: `npm run build` is a no-op message (static site, nothing to compile).
+- Tests: there is no test runner configured. The one standalone check is
+  `node scripts/test_data_loader_urls.js` (asserts URL-resolution logic in `docs/data-loader.js`;
+  prints `ok` on success).
+- Lint: no linter is configured.
+
+### Non-obvious gotchas
+- **`python` is not on PATH — only `python3`.** The npm script `build:filter-stats` and the
+  README/CONTRIBUTING maintenance commands invoke bare `python`, which fails here. Run the optional
+  Python tooling in `scripts/` with `python3 scripts/<name>.py` instead. (`requests` is available.)
+  These Python scripts are optional maintenance tooling (link checker, `list.md`→JSON converters,
+  filter-stats) and are **not** needed to run or demo the site.
+- **Firebase is optional and degrades gracefully.** Without the Worker secrets
+  `FIREBASE_PROJECT_ID` / `FIREBASE_CLIENT_EMAIL` / `FIREBASE_PRIVATE_KEY`, `/api/link-click` and
+  presence endpoints fall back to in-edge Cache counters and return a `warning` field
+  (`via: "edge"`). Sign-in, saved links, admin pages, and global/shared stats require real Firebase
+  config; local dev works fine without it.
+- **Port 8787 is shared** by both `wrangler dev` and the optional self-hosted webhook server
+  (`scripts/github_webhook_server.py`). Do not run both at once.

docs/data-loader.js

@@ -244,6 +244,69 @@
     return out;
   }
 
+  function yieldToMainThread() {
+    return new Promise(function (resolve) {
+      if (typeof requestAnimationFrame === "function") {
+        requestAnimationFrame(function () {
+          setTimeout(resolve, 0);
+        });
+      } else {
+        setTimeout(resolve, 0);
+      }
+    });
+  }
+
+  /** Expand compact rows in chunks so web proxies and low-end devices stay responsive. */
+  async function expandAllLinksAsync(payload, options) {
+    if (!isCompactPayload(payload)) {
+      return Array.isArray(payload) ? payload : payload.links || [];
+    }
+    var opts = Object.assign({ shareProviderArrays: true, yieldEvery: 500 }, options || {});
+    var total = payload.links.length;
+    var out = new Array(total);
+    var step = Math.max(50, Number(opts.yieldEvery) || 500);
+    for (var i = 0; i < total; i++) {
+      out[i] = expandRow(payload, payload.links[i], opts);
+      if (i > 0 && i % step === 0) {
+        if (typeof opts.onProgress === "function") {
+          try {
+            opts.onProgress(i / total);
+          } catch (_) {}
+        }
+        await yieldToMainThread();
+      }
+    }
+    if (typeof opts.onProgress === "function") {
+      try {
+        opts.onProgress(1);
+      } catch (_) {}
+    }
+    return out;
+  }
+
+  async function fetchJsonAsset(name, options) {
+    var opts = options || {};
+    var base = opts.baseUrl != null ? opts.baseUrl : listAssetBaseUrl();
+    var timeoutMs = opts.timeoutMs != null ? opts.timeoutMs : 20000;
+    var errors = [];
+    var candidates = listAssetUrlCandidates(name, base);
+    for (var i = 0; i < candidates.length; i++) {
+      var url = candidates[i];
+      try {
+        var res = await fetchWithTimeout(url, opts.fetchInit || {}, timeoutMs);
+        if (!res.ok) {
+          errors.push(url + " HTTP " + res.status);
+          continue;
+        }
+        return readJsonResponse(res);
+      } catch (err) {
+        errors.push(url + ": " + (err && err.message ? err.message : String(err)));
+      }
+    }
+    var detail = errors.length ? errors.join("; ") : "no URLs attempted";
+    throw new Error("Could not load " + name + " (" + detail + ")");
+  }
+
   function linkCount(normalized) {
     if (normalized.compact) return normalized.compact.links.length;
     return (normalized.links || []).length;
@@ -330,6 +393,8 @@
     normalizePayload: normalizePayload,
     resolveExpandedLinks: resolveExpandedLinks,
     expandAllLinks: expandAllLinks,
+    expandAllLinksAsync: expandAllLinksAsync,
+    fetchJsonAsset: fetchJsonAsset,
     linkCount: linkCount,
     isRelativeUrl: isRelativeUrl,
     directoryOfSrc: directoryOfSrc,