release: v1.8.0 — R2 watchlist cache + E2E coverage

.github/workflows/ci.yml

@@ -1,8 +1,9 @@
 name: CI
 
 on:
+  workflow_dispatch:
   push:
-    branches: [main]
+    branches: [main, staging]
   pull_request:
     branches: [main, staging]
 
@@ -30,6 +31,68 @@ jobs:
       - name: Lint, build, and test
         run: pnpm turbo run lint build test
 
+      - name: Smoke-test backend bundle
+        env:
+          SUPABASE_URL: https://example.invalid
+          SUPABASE_SERVICE_ROLE_KEY: test
+          RESEND_API_KEY: re_test
+          RESEND_AUDIENCE_ID: test
+        run: node --input-type=module --eval 'await import("./apps/backend/dist/index.js")'
+
+  e2e:
+    name: E2E
+    if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.30.1
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: pnpm
+
+      - uses: supabase/setup-cli@v1
+        with:
+          version: 2.98.2
+
+      - name: Start local Supabase (db + REST only)
+        run: supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta
+
+      - name: Start local R2-compatible store
+        run: |
+          docker run --rm -d --name stremlist-e2e-r2 \
+            -p 127.0.0.1:7431:9000 \
+            -e MINIO_ROOT_USER=stremlist-e2e \
+            -e MINIO_ROOT_PASSWORD=stremlist-e2e-secret \
+            quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z server /data
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }}
+
+      - name: Install Playwright Chromium
+        run: pnpm --filter @stremlist/e2e exec playwright install --with-deps chromium
+
+      - name: Run E2E tests
+        run: pnpm --filter @stremlist/e2e test:e2e
+
+      - name: Upload Playwright report
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: playwright-report
+          path: apps/e2e/playwright-report
+          retention-days: 7
+
   version-bump:
     name: Auto version bump
     needs: ci

.gitignore

@@ -45,3 +45,7 @@ lerna-debug.log*
 !robots.txt
 # Supabase CLI local state
 supabase/.temp/
+
+# Playwright
+test-results/
+playwright-report/

.sops.yaml

@@ -0,0 +1,5 @@
+creation_rules:
+  - path_regex: '(^|/)\.env(\.[A-Za-z0-9_-]+)?\.enc$'
+    age:
+      - age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f # mac
+      - age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf # dev-tower

README.md

@@ -12,7 +12,7 @@ Stremlist is a Stremio addon that turns your IMDb watchlist into a Stremio catal
 - Optional Rating Poster Database (RPDB) poster support via API key
 - Simple install flow through a hosted configuration UI
 - Cache-first watchlist serving with periodic auto-refresh and a manual "Refresh now" control
-- Lightweight backend with Supabase for user management and watchlist caching
+- Lightweight backend with Supabase for user configuration and Cloudflare R2 for watchlist caching
 - Monorepo architecture with Turborepo (`apps` + `packages`)
 
 ## Monorepo Structure
@@ -34,7 +34,8 @@ This repository follows the Turborepo recommended structure:
 
 - Frontend and backend are deployed on [Vercel](https://vercel.com)
 - Backend serves Stremio addon endpoints and configuration flow
-- Supabase stores user configuration and cached watchlist data
+- Supabase stores user configuration
+- Cloudflare R2 stores gzip-compressed watchlist cache objects
 
 ## Getting Started
 
@@ -106,16 +107,20 @@ http://localhost:7001/manifest.json
 
 Set backend env vars in `apps/backend/.env`.
 
-| Variable | Required | Description | Default |
-| --- | --- | --- | --- |
-| `PORT` | No | Backend HTTP port | `7001` |
-| `FRONTEND_URL` | No | URL used for `/:userId/configure` redirect | `https://stremlist.com` |
-| `SUPABASE_URL` | Yes | Supabase project URL | - |
-| `SUPABASE_SERVICE_ROLE_KEY` | Yes | Supabase service role key | - |
-| `CACHE_TTL_MINUTES` | No | How long a cached watchlist is served before it is refreshed on the next request | `30` |
-| `REFRESH_COOLDOWN_SECONDS` | No | Minimum time between manual "Refresh now" requests per user | `60` |
-| `RESEND_API_KEY` | No | Resend API key for newsletter subscription endpoint | - |
-| `RESEND_AUDIENCE_ID` | No | Resend audience ID for newsletter subscription endpoint | - |
+| Variable                    | Required | Description                                                                      | Default                 |
+| --------------------------- | -------- | -------------------------------------------------------------------------------- | ----------------------- |
+| `PORT`                      | No       | Backend HTTP port                                                                | `7001`                  |
+| `FRONTEND_URL`              | No       | URL used for `/:userId/configure` redirect                                       | `https://stremlist.com` |
+| `SUPABASE_URL`              | Yes      | Supabase project URL                                                             | -                       |
+| `SUPABASE_SERVICE_ROLE_KEY` | Yes      | Supabase service role key                                                        | -                       |
+| `R2_ACCOUNT_ID`             | Yes      | Cloudflare account ID used by the R2 S3 endpoint                                 | -                       |
+| `R2_ACCESS_KEY_ID`          | Yes      | Bucket-scoped R2 API token access key                                            | -                       |
+| `R2_SECRET_ACCESS_KEY`      | Yes      | Bucket-scoped R2 API token secret                                                | -                       |
+| `R2_BUCKET`                 | Yes      | Private R2 cache bucket name                                                     | -                       |
+| `CACHE_TTL_MINUTES`         | No       | How long a cached watchlist is served before it is refreshed on the next request | `30`                    |
+| `REFRESH_COOLDOWN_SECONDS`  | No       | Minimum time between manual "Refresh now" requests per user                      | `60`                    |
+| `RESEND_API_KEY`            | No       | Resend API key for newsletter subscription endpoint                              | -                       |
+| `RESEND_AUDIENCE_ID`        | No       | Resend audience ID for newsletter subscription endpoint                          | -                       |
 
 ## Type Generation
 
@@ -127,10 +132,13 @@ pnpm generate:types
 
 This updates `packages/shared/src/database.types.ts`.
 
+The production R2 rollout and cleanup procedure is documented in
+[`docs/r2-cache-migration.md`](docs/r2-cache-migration.md).
+
 ## License
 
 ISC
 
 ## Disclaimer
 
-This project is not affiliated with IMDb or Stremio.
+This project is not affiliated with IMDb or Stremio.

apps/backend/.env.enc

@@ -0,0 +1,18 @@
+{
+	"data": "ENC[AES256_GCM,data:WLea9WPQipmxDWU+N30X9R5cg7gXXwOr4YbqMQTa1ObhjPHzFo3z+MBXvQiJghw1x+qLgVfF8X/QQ7kQIIwooiRRdhTinQUEE2PFxoO9VRrjuOa3+meiC0GyamCOXojdandwK0QhAKuBIVnUC6ckLWnDxAsjMJUSAia4jyfZ3IAfRPZxocwQce38HS3R8odKU6+EU+69nrQeT5bF9WdvfAeEgiY9HzTBsx+W7vKLow9HS4Ae50eHnDlTwMgYvrSZ65/HbxmtS2tFR2Kbtjlp4Pv+OPkIb1WROCTPvI+kElrvBRzmLdutoYSh1TnmfWNoKUSrUArS/xjCLSc98EhZWLvLS7q0Nu4XQFdoCT/yv8gwdCmoExryxYwSg7q2B0wSumBIBVwOaekbI7SQPipE1LiXm4plIox+MdMN6/ZZzVvn7xCR/qtaCfn8Yqvtu7H+ROBx8dQyzW9pEjCCtd9oDFgPKEX+sGvmizviFsKn0PvNnOvw2ZPG/Nzb2niMd4jmJgvcfLZI3Y5Kx+zcRt2Mn1vtmsMseffk6Pq/1hMrUFVXuNd5DnKEnGyDfrHwPhrA2R/zN2LEwyTgeeNT95JPT1STkvLbBqC/Fg2OBX78ak/qK6SmAVxu15qW7OjoQ5J78mcILeHevck+yvfztQ4XOGgv5hU9BpXI7wT//ErD1+yx6oy5zPSi+LhdeKtWHmzuHpTf5hOoj52mHaYh5gP8pHkdPv5Snl1jsr/WuvRVq2mAJmBhkX3lORGHpvBrmo+7TmiskOu7dCJZpA3IcV6ncjkXr/qoXtxv+Mldt3GqQCsbbbdxWrYW/V4=,iv:Ujc90jNL6IVFa4rKVRPQWH3gVpXOfxKwkxbmpCy9F8Y=,tag:RvM8CCBR2ZVa9657yqsIFw==,type:str]",
+	"sops": {
+		"age": [
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvM2t3MHo1NmY0cDMrUTFw\nSDJFeE9YaGVWZUhkUDZtMVVjNUhIdVRkUkdRCmVSM3poR2xrQmw0Ky9ZVlY5dC9r\nREk4dG5DMUtVZUE0NlFTME80czMxTDQKLS0tIDRua3kwQW94K2JhU1JxRkRnUGN1\nSDJ0aGVqM2pvOGxIUDNHOHhzcFRIZmsKVwtUVOHka3wx5NW87CU3UktLCwsnTYqq\nOiJY+5/8tQVUJk6CoO3M/3MRhVmXEjGnfa7v8T0H4Xa3ySg1OGhzNg==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f"
+			},
+			{
+				"enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwWmF5NS81TERzcUNVY2NJ\nb0o4RndZREIrMDBES3FtcmtzRk5QSVZDK0dzCnV1NDJlNGtPUFFOcTl5d0hzLzJi\nSVlOT2p6SUZQazJHZldPbTM3VHk0UWcKLS0tIFkvZmpKOHRIOWhMY1c2UWRPRkhC\nL3FtUkNFZkpsTXJnMzQzc2NGWVhBRWsKBZg4S0qVEVZfKvqt2aDttfxw7N7Y7rpy\nGOBA6D1F8qXl2YWrwdhCgnoKryuIqHixpeViR9YCPrqlUWINp12FAQ==\n-----END AGE ENCRYPTED FILE-----\n",
+				"recipient": "age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf"
+			}
+		],
+		"lastmodified": "2026-08-26T11:21:17Z",
+		"mac": "ENC[AES256_GCM,data:r+ilB7Knq7ohVRGataZRsjJsPREUeO8L2SDG6pOI7Njy44wwvl+wS4eSicDxzGrEMi9IIVJJWr72nkib0QU0ea1RwWpK19ACPXp9mY61YJjisq/EX7AO1CB+w/xqEOwzmDrLSK4jMchFegK0PjGThfjevi4G5ptqG0BEkxrEUao=,iv:iKzqaagRrW6RXR7QiIlnNpHPSMbYfYBrca9cme/5x0c=,tag:dd5/Sun8gYdTvo5uxr5SNg==,type:str]",
+		"version": "3.13.3"
+	}
+}

apps/backend/build.ts

@@ -7,6 +7,7 @@ await esbuild.build({
   platform: "node",
   format: "esm",
   bundle: true,
+  external: ["@aws-sdk/client-s3"],
 });
 
 // Build the client entry (used by frontend for RPC types) without bundling

apps/backend/package.json

@@ -24,6 +24,7 @@
     "cleanup:invalid-users": "tsx src/scripts/cleanup-invalid-users.ts"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1118.0",
     "@hono/zod-validator": "^0.4.3",
     "@stremlist/shared": "workspace:*",
     "@supabase/supabase-js": "^2.95.3",

apps/backend/src/__tests__/catalog-fallback.test.ts

@@ -1,16 +1,22 @@
+import type { StremioMeta } from "@stremlist/shared";
 import { describe, it, expect, beforeEach, vi } from "vitest";
 
 vi.mock("../lib/supabase", async () => {
   return await import("./helpers/mock-supabase.js");
 });
 
+vi.mock("../services/watchlist-cache", async () => {
+  return await import("./helpers/mock-watchlist-cache.js");
+});
+
 vi.mock("../lib/resend", () => ({
   resend: { contacts: { create: vi.fn() } },
 }));
 
 import app from "../index.js";
 import * as scraper from "../services/imdb-scraper";
 import { db } from "./helpers/mock-supabase.js";
+import { cache } from "./helpers/mock-watchlist-cache.js";
 
 const OWNER = "ur216216210";
 const UUID_1 = "6bde5e3d-617f-4912-950a-2f9acf815b7e";
@@ -26,13 +32,13 @@ function seedUser(imdbUserId: string) {
   });
 }
 
-function seedWatchlist(id: string) {
+function seedWatchlist(id: string, sortOption = "added_at-asc") {
   db.getTable("user_watchlists").push({
     id,
     owner_user_id: OWNER,
     imdb_user_id: OWNER,
     catalog_title: "",
-    sort_option: "added_at-asc",
+    sort_option: sortOption,
     position: 0,
     created_at: new Date().toISOString(),
     updated_at: new Date().toISOString(),
@@ -41,25 +47,14 @@ function seedWatchlist(id: string) {
 
 function seedCache(
   watchlistId: string,
-  metas: { id: string; type: string }[],
+  metas: StremioMeta[],
   cachedAt?: string,
 ) {
-  // An empty `metas` seeds zero rows — the normalised equivalent of an empty
-  // blob: the next read sees no rows and treats it as a cache miss.
-  const at = cachedAt ?? new Date().toISOString();
-  metas.forEach((meta, i) => {
-    db.getTable("watchlist_cache_items").push({
-      watchlist_id: watchlistId,
-      item_id: meta.id,
-      type: meta.type,
-      position: i,
-      data: meta,
-      cached_at: at,
-    });
-  });
+  if (metas.length === 0) return;
+  cache.seed(watchlistId, metas, cachedAt ? new Date(cachedAt) : new Date());
 }
 
-const CACHED_MOVIE = {
+const CACHED_MOVIE: StremioMeta = {
   id: "tt0111161",
   type: "movie",
   name: "The Shawshank Redemption",
@@ -86,6 +81,7 @@ function requestMovieCatalog() {
 
 beforeEach(() => {
   db.reset();
+  cache.reset();
   vi.restoreAllMocks();
 });
 
@@ -173,3 +169,89 @@ describe("catalog route degrades gracefully on fetch failure", () => {
     expect((await res.json()) as CatalogResponse).toEqual({ metas: [] });
   });
 });
+
+describe("catalog pagination", () => {
+  it("serves Stremio pages of at most 100 items using the skip extra", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1);
+    seedCache(
+      UUID_1,
+      Array.from(
+        { length: 205 },
+        (_, index): StremioMeta => ({
+          ...CACHED_MOVIE,
+          id: `tt${String(index).padStart(7, "0")}`,
+          name: `Movie ${index}`,
+        }),
+      ),
+    );
+
+    const first = await requestMovieCatalog();
+    const second = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`,
+    );
+    const last = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=200.json`,
+    );
+
+    const firstBody = (await first.json()) as CatalogResponse;
+    const secondBody = (await second.json()) as CatalogResponse;
+    const lastBody = (await last.json()) as CatalogResponse;
+    expect(firstBody.metas).toHaveLength(100);
+    expect(secondBody.metas).toHaveLength(100);
+    expect(lastBody.metas).toHaveLength(5);
+    expect(firstBody.metas[0].id).toBe("tt0000000");
+    expect(secondBody.metas[0].id).toBe("tt0000100");
+    expect(lastBody.metas[0].id).toBe("tt0000200");
+    expect(first.headers.get("Cache-Control")).toBe("no-store");
+    expect(first.headers.get("Vercel-CDN-Cache-Control")).toBeNull();
+  });
+
+  it("keeps random pages stable and non-overlapping within a cache generation", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1, "random");
+    seedCache(
+      UUID_1,
+      Array.from(
+        { length: 205 },
+        (_, index): StremioMeta => ({
+          ...CACHED_MOVIE,
+          id: `tt${String(index).padStart(7, "0")}`,
+          name: `Movie ${index}`,
+        }),
+      ),
+    );
+
+    const first = await requestMovieCatalog();
+    const second = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`,
+    );
+    const repeatedFirst = await requestMovieCatalog();
+
+    const firstIds = ((await first.json()) as CatalogResponse).metas.map(
+      (meta) => meta.id,
+    );
+    const secondIds = ((await second.json()) as CatalogResponse).metas.map(
+      (meta) => meta.id,
+    );
+    const repeatedFirstIds = (
+      (await repeatedFirst.json()) as CatalogResponse
+    ).metas.map((meta) => meta.id);
+
+    expect(repeatedFirstIds).toEqual(firstIds);
+    expect(new Set([...firstIds, ...secondIds]).size).toBe(200);
+  });
+
+  it("rejects an invalid skip value", async () => {
+    seedUser(OWNER);
+    seedWatchlist(UUID_1);
+    seedCache(UUID_1, [CACHED_MOVIE]);
+
+    const res = await app.request(
+      `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=wat.json`,
+    );
+
+    expect(res.status).toBe(400);
+    expect((await res.json()) as CatalogResponse).toEqual({ metas: [] });
+  });
+});