Move watchlist caching to R2 and add Stremio E2E coverage

.github/workflows/ci.yml

@@ -30,6 +30,60 @@ jobs:
       - name: Lint, build, and test
         run: pnpm turbo run lint build test
 
+  e2e:
+    name: E2E
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.30.1
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: pnpm
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: 2.98.2
+
+      - name: Start local Supabase (db + REST only)
+        run: supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta
+
+      - name: Start local R2-compatible store
+        run: |
+          docker run --rm -d --name stremlist-e2e-r2 \
+            -p 127.0.0.1:7431:9000 \
+            -e MINIO_ROOT_USER=stremlist-e2e \
+            -e MINIO_ROOT_PASSWORD=stremlist-e2e-secret \
+            quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z server /data
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+
+      - name: Install Playwright Chromium
+        run: pnpm --filter @stremlist/e2e exec playwright install --with-deps chromium
+
+      - name: Run E2E tests
+        run: pnpm --filter @stremlist/e2e test:e2e
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: apps/e2e/playwright-report
+          retention-days: 7
+
   version-bump:
     name: Auto version bump
     needs: ci

.gitignore

@@ -45,3 +45,7 @@ lerna-debug.log*
 !robots.txt
 # Supabase CLI local state
 supabase/.temp/
+
+# Playwright
+test-results/
+playwright-report/

README.md

@@ -12,7 +12,7 @@ Stremlist is a Stremio addon that turns your IMDb watchlist into a Stremio catal
 - Optional Rating Poster Database (RPDB) poster support via API key
 - Simple install flow through a hosted configuration UI
 - Cache-first watchlist serving with periodic auto-refresh and a manual "Refresh now" control
-- Lightweight backend with Supabase for user management and watchlist caching
+- Lightweight backend with Supabase for user configuration and Cloudflare R2 for watchlist caching
 - Monorepo architecture with Turborepo (`apps` + `packages`)
 
 ## Monorepo Structure
@@ -34,7 +34,8 @@ This repository follows the Turborepo recommended structure:
 
 - Frontend and backend are deployed on [Vercel](https://vercel.com)
 - Backend serves Stremio addon endpoints and configuration flow
-- Supabase stores user configuration and cached watchlist data
+- Supabase stores user configuration
+- Cloudflare R2 stores gzip-compressed watchlist cache objects
 
 ## Getting Started
 
@@ -106,16 +107,20 @@ http://localhost:7001/manifest.json
 
 Set backend env vars in `apps/backend/.env`.
 
-| Variable | Required | Description | Default |
-| --- | --- | --- | --- |
-| `PORT` | No | Backend HTTP port | `7001` |
-| `FRONTEND_URL` | No | URL used for `/:userId/configure` redirect | `https://stremlist.com` |
-| `SUPABASE_URL` | Yes | Supabase project URL | - |
-| `SUPABASE_SERVICE_ROLE_KEY` | Yes | Supabase service role key | - |
-| `CACHE_TTL_MINUTES` | No | How long a cached watchlist is served before it is refreshed on the next request | `30` |
-| `REFRESH_COOLDOWN_SECONDS` | No | Minimum time between manual "Refresh now" requests per user | `60` |
-| `RESEND_API_KEY` | No | Resend API key for newsletter subscription endpoint | - |
-| `RESEND_AUDIENCE_ID` | No | Resend audience ID for newsletter subscription endpoint | - |
+| Variable                    | Required | Description                                                                      | Default                 |
+| --------------------------- | -------- | -------------------------------------------------------------------------------- | ----------------------- |
+| `PORT`                      | No       | Backend HTTP port                                                                | `7001`                  |
+| `FRONTEND_URL`              | No       | URL used for `/:userId/configure` redirect                                       | `https://stremlist.com` |
+| `SUPABASE_URL`              | Yes      | Supabase project URL                                                             | -                       |
+| `SUPABASE_SERVICE_ROLE_KEY` | Yes      | Supabase service role key                                                        | -                       |
+| `R2_ACCOUNT_ID`             | Yes      | Cloudflare account ID used by the R2 S3 endpoint                                 | -                       |
+| `R2_ACCESS_KEY_ID`          | Yes      | Bucket-scoped R2 API token access key                                            | -                       |
+| `R2_SECRET_ACCESS_KEY`      | Yes      | Bucket-scoped R2 API token secret                                                | -                       |
+| `R2_BUCKET`                 | Yes      | Private R2 cache bucket name                                                     | -                       |
+| `CACHE_TTL_MINUTES`         | No       | How long a cached watchlist is served before it is refreshed on the next request | `30`                    |
+| `REFRESH_COOLDOWN_SECONDS`  | No       | Minimum time between manual "Refresh now" requests per user                      | `60`                    |
+| `RESEND_API_KEY`            | No       | Resend API key for newsletter subscription endpoint                              | -                       |
+| `RESEND_AUDIENCE_ID`        | No       | Resend audience ID for newsletter subscription endpoint                          | -                       |
 
 ## Type Generation
 
@@ -127,10 +132,13 @@ pnpm generate:types
 
 This updates `packages/shared/src/database.types.ts`.
 
+The production R2 rollout and cleanup procedure is documented in
+[`docs/r2-cache-migration.md`](docs/r2-cache-migration.md).
+
 ## License
 
 ISC
 
 ## Disclaimer
 
-This project is not affiliated with IMDb or Stremio.
+This project is not affiliated with IMDb or Stremio.

apps/backend/package.json

@@ -24,6 +24,7 @@
     "cleanup:invalid-users": "tsx src/scripts/cleanup-invalid-users.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1118.0",
     "@hono/zod-validator": "^0.4.3",
     "@stremlist/shared": "workspace:*",
     "@supabase/supabase-js": "^2.95.3",

apps/backend/src/__tests__/catalog-fallback.test.ts

@@ -1,16 +1,22 @@
+import type { StremioMeta } from "@stremlist/shared";
 import { describe, it, expect, beforeEach, vi } from "vitest";
 
 vi.mock("../lib/supabase", async () => {
   return await import("./helpers/mock-supabase.js");
 });
 
+vi.mock("../services/watchlist-cache", async () => {
+  return await import("./helpers/mock-watchlist-cache.js");
+});
+
 vi.mock("../lib/resend", () => ({
   resend: { contacts: { create: vi.fn() } },
 }));
 
 import app from "../index.js";
 import * as scraper from "../services/imdb-scraper";
 import { db } from "./helpers/mock-supabase.js";
+import { cache } from "./helpers/mock-watchlist-cache.js";
 
 const OWNER = "ur216216210";
 const UUID_1 = "6bde5e3d-617f-4912-950a-2f9acf815b7e";
@@ -26,13 +32,13 @@ function seedUser(imdbUserId: string) {
   });
 }
 
-function seedWatchlist(id: string) {
+function seedWatchlist(id: string, sortOption = "added_at-asc") {
   db.getTable("user_watchlists").push({
     id,
     owner_user_id: OWNER,
     imdb_user_id: OWNER,
     catalog_title: "",
-    sort_option: "added_at-asc",
+    sort_option: sortOption,
     position: 0,
     created_at: new Date().toISOString(),
     updated_at: new Date().toISOString(),
@@ -41,25 +47,14 @@ function seedWatchlist(id: string) {
 
 function seedCache(
   watchlistId: string,
-  metas: { id: string; type: string }[],
+  metas: StremioMeta[],
   cachedAt?: string,
 ) {
-  // An empty `metas` seeds zero rows — the normalised equivalent of an empty
-  // blob: the next read sees no rows and treats it as a cache miss.
-  const at = cachedAt ?? new Date().toISOString();
-  metas.forEach((meta, i) => {
-    db.getTable("watchlist_cache_items").push({
-      watchlist_id: watchlistId,
-      item_id: meta.id,
-      type: meta.type,
-      position: i,
-      data: meta,
-      cached_at: at,
-    });
-  });
+  if (metas.length === 0) return;
+  cache.seed(watchlistId, metas, cachedAt ? new Date(cachedAt) : new Date());
 }
 
-const CACHED_MOVIE = {
+const CACHED_MOVIE: StremioMeta = {
   id: "tt0111161",
   type: "movie",
   name: "The Shawshank Redemption",
@@ -86,6 +81,7 @@ function requestMovieCatalog() {
 
 beforeEach(() => {
   db.reset();
+  cache.reset();
   vi.restoreAllMocks();
 });
 
@@ -173,3 +169,89 @@ describe("catalog route degrades gracefully on fetch failure", () => {
     expect((await res.json()) as CatalogResponse).toEqual({ metas: [] });
   });
 });
+
+describe("catalog pagination", () => {
+  it("serves Stremio pages of at most 100 items using the skip extra", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1);
+    seedCache(
+      UUID_1,
+      Array.from(
+        { length: 205 },
+        (_, index): StremioMeta => ({
+          ...CACHED_MOVIE,
+          id: `tt${String(index).padStart(7, "0")}`,
+          name: `Movie ${index}`,
+        }),
+      ),
+    );
+
+    const first = await requestMovieCatalog();
+    const second = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`,
+    );
+    const last = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=200.json`,
+    );
+
+    const firstBody = (await first.json()) as CatalogResponse;
+    const secondBody = (await second.json()) as CatalogResponse;
+    const lastBody = (await last.json()) as CatalogResponse;
+    expect(firstBody.metas).toHaveLength(100);
+    expect(secondBody.metas).toHaveLength(100);
+    expect(lastBody.metas).toHaveLength(5);
+    expect(firstBody.metas[0].id).toBe("tt0000000");
+    expect(secondBody.metas[0].id).toBe("tt0000100");
+    expect(lastBody.metas[0].id).toBe("tt0000200");
+    expect(first.headers.get("Cache-Control")).toBe("no-store");
+    expect(first.headers.get("Vercel-CDN-Cache-Control")).toBeNull();
+  });
+
+  it("keeps random pages stable and non-overlapping within a cache generation", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1, "random");
+    seedCache(
+      UUID_1,
+      Array.from(
+        { length: 205 },
+        (_, index): StremioMeta => ({
+          ...CACHED_MOVIE,
+          id: `tt${String(index).padStart(7, "0")}`,
+          name: `Movie ${index}`,
+        }),
+      ),
+    );
+
+    const first = await requestMovieCatalog();
+    const second = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`,
+    );
+    const repeatedFirst = await requestMovieCatalog();
+
+    const firstIds = ((await first.json()) as CatalogResponse).metas.map(
+      (meta) => meta.id,
+    );
+    const secondIds = ((await second.json()) as CatalogResponse).metas.map(
+      (meta) => meta.id,
+    );
+    const repeatedFirstIds = (
+      (await repeatedFirst.json()) as CatalogResponse
+    ).metas.map((meta) => meta.id);
+
+    expect(repeatedFirstIds).toEqual(firstIds);
+    expect(new Set([...firstIds, ...secondIds]).size).toBe(200);
+  });
+
+  it("rejects an invalid skip value", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1);
+    seedCache(UUID_1, [CACHED_MOVIE]);
+
+    const res = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=wat.json`,
+    );
+
+    expect(res.status).toBe(400);
+    expect((await res.json()) as CatalogResponse).toEqual({ metas: [] });
+  });
+});

apps/backend/src/__tests__/helpers/mock-watchlist-cache.ts

@@ -0,0 +1,73 @@
+import type { StremioMeta, WatchlistData } from "@stremlist/shared";
+
+interface Entry {
+  data: WatchlistData;
+  cachedAt: Date;
+  generation: string;
+}
+
+class InMemoryWatchlistCache {
+  private entries = new Map<string, Entry>();
+
+  reset(): void {
+    this.entries.clear();
+  }
+
+  seed(watchlistId: string, metas: StremioMeta[], cachedAt = new Date()): void {
+    this.entries.set(watchlistId, {
+      data: { metas: structuredClone(metas) },
+      cachedAt,
+      generation: `${watchlistId}:${cachedAt.toISOString()}`,
+    });
+  }
+
+  get(watchlistId: string): Entry | null {
+    return this.entries.get(watchlistId) ?? null;
+  }
+
+  delete(watchlistId: string): void {
+    this.entries.delete(watchlistId);
+  }
+}
+
+export const cache = new InMemoryWatchlistCache();
+
+export function getCachedWatchlist(watchlistId: string): Promise<Entry | null> {
+  return Promise.resolve(cache.get(watchlistId));
+}
+
+export function writeCachedWatchlist(
+  watchlistId: string,
+  watchlistData: WatchlistData,
+  cachedAt = new Date(),
+): Promise<string> {
+  const seen = new Set<string>();
+  const metas = watchlistData.metas.filter((meta) => {
+    const key = `${meta.type}:${meta.id}`;
+    if (seen.has(key)) return false;
+    seen.add(key);
+    return true;
+  });
+  if (metas.length === 0) cache.delete(watchlistId);
+  else cache.seed(watchlistId, metas, cachedAt);
+  return Promise.resolve(`${watchlistId}:${cachedAt.toISOString()}`);
+}
+
+export function findCachedMeta(
+  watchlistIds: string[],
+  type: string,
+  id: string,
+): Promise<StremioMeta | null> {
+  for (const watchlistId of watchlistIds) {
+    const found = cache
+      .get(watchlistId)
+      ?.data.metas.find((meta) => meta.type === type && meta.id === id);
+    if (found) return Promise.resolve(found);
+  }
+  return Promise.resolve(null);
+}
+
+export function deleteCachedWatchlist(watchlistId: string): Promise<void> {
+  cache.delete(watchlistId);
+  return Promise.resolve();
+}