From 1a9aca07184caa8193ad66b67df3be398598af09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Sun, 9 Aug 2026 19:08:08 +0200 Subject: [PATCH 01/15] chore: remove .mcp.json MCP server config --- .mcp.json | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .mcp.json diff --git a/.mcp.json b/.mcp.json deleted file mode 100644 index b421c60..0000000 --- a/.mcp.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "mcpServers": { - "supabase": { - "type": "http", - "url": "https://mcp.supabase.com/mcp?project_ref=igbmxubzxjfdbgmawrtr" - } - } -} From a6f0313924c9045b97de8fe3cd21bb4f344cc1ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 13:23:19 +0200 Subject: [PATCH 02/15] chore(secrets): add SOPS-encrypted environment --- .sops.yaml | 5 +++++ apps/backend/.env.enc | 18 ++++++++++++++++++ apps/frontend/.env.enc | 18 ++++++++++++++++++ apps/frontend/.env.local.enc | 18 ++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 .sops.yaml create mode 100644 apps/backend/.env.enc create mode 100644 apps/frontend/.env.enc create mode 100644 apps/frontend/.env.local.enc diff --git a/.sops.yaml b/.sops.yaml new file mode 100644 index 0000000..c8962c7 --- /dev/null +++ b/.sops.yaml @@ -0,0 +1,5 @@ +creation_rules: + - path_regex: '(^|/)\.env(\.[A-Za-z0-9_-]+)?\.enc$' + age: + - age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f # mac + - age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf # dev-tower diff --git a/apps/backend/.env.enc b/apps/backend/.env.enc new file mode 100644 index 0000000..cc9f2d3 --- /dev/null +++ b/apps/backend/.env.enc @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:WLea9WPQipmxDWU+N30X9R5cg7gXXwOr4YbqMQTa1ObhjPHzFo3z+MBXvQiJghw1x+qLgVfF8X/QQ7kQIIwooiRRdhTinQUEE2PFxoO9VRrjuOa3+meiC0GyamCOXojdandwK0QhAKuBIVnUC6ckLWnDxAsjMJUSAia4jyfZ3IAfRPZxocwQce38HS3R8odKU6+EU+69nrQeT5bF9WdvfAeEgiY9HzTBsx+W7vKLow9HS4Ae50eHnDlTwMgYvrSZ65/HbxmtS2tFR2Kbtjlp4Pv+OPkIb1WROCTPvI+kElrvBRzmLdutoYSh1TnmfWNoKUSrUArS/xjCLSc98EhZWLvLS7q0Nu4XQFdoCT/yv8gwdCmoExryxYwSg7q2B0wSumBIBVwOaekbI7SQPipE1LiXm4plIox+MdMN6/ZZzVvn7xCR/qtaCfn8Yqvtu7H+ROBx8dQyzW9pEjCCtd9oDFgPKEX+sGvmizviFsKn0PvNnOvw2ZPG/Nzb2niMd4jmJgvcfLZI3Y5Kx+zcRt2Mn1vtmsMseffk6Pq/1hMrUFVXuNd5DnKEnGyDfrHwPhrA2R/zN2LEwyTgeeNT95JPT1STkvLbBqC/Fg2OBX78ak/qK6SmAVxu15qW7OjoQ5J78mcILeHevck+yvfztQ4XOGgv5hU9BpXI7wT//ErD1+yx6oy5zPSi+LhdeKtWHmzuHpTf5hOoj52mHaYh5gP8pHkdPv5Snl1jsr/WuvRVq2mAJmBhkX3lORGHpvBrmo+7TmiskOu7dCJZpA3IcV6ncjkXr/qoXtxv+Mldt3GqQCsbbbdxWrYW/V4=,iv:Ujc90jNL6IVFa4rKVRPQWH3gVpXOfxKwkxbmpCy9F8Y=,tag:RvM8CCBR2ZVa9657yqsIFw==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvM2t3MHo1NmY0cDMrUTFw\nSDJFeE9YaGVWZUhkUDZtMVVjNUhIdVRkUkdRCmVSM3poR2xrQmw0Ky9ZVlY5dC9r\nREk4dG5DMUtVZUE0NlFTME80czMxTDQKLS0tIDRua3kwQW94K2JhU1JxRkRnUGN1\nSDJ0aGVqM2pvOGxIUDNHOHhzcFRIZmsKVwtUVOHka3wx5NW87CU3UktLCwsnTYqq\nOiJY+5/8tQVUJk6CoO3M/3MRhVmXEjGnfa7v8T0H4Xa3ySg1OGhzNg==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBwWmF5NS81TERzcUNVY2NJ\nb0o4RndZREIrMDBES3FtcmtzRk5QSVZDK0dzCnV1NDJlNGtPUFFOcTl5d0hzLzJi\nSVlOT2p6SUZQazJHZldPbTM3VHk0UWcKLS0tIFkvZmpKOHRIOWhMY1c2UWRPRkhC\nL3FtUkNFZkpsTXJnMzQzc2NGWVhBRWsKBZg4S0qVEVZfKvqt2aDttfxw7N7Y7rpy\nGOBA6D1F8qXl2YWrwdhCgnoKryuIqHixpeViR9YCPrqlUWINp12FAQ==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf" + } + ], + "lastmodified": "2026-08-26T11:21:17Z", + "mac": "ENC[AES256_GCM,data:r+ilB7Knq7ohVRGataZRsjJsPREUeO8L2SDG6pOI7Njy44wwvl+wS4eSicDxzGrEMi9IIVJJWr72nkib0QU0ea1RwWpK19ACPXp9mY61YJjisq/EX7AO1CB+w/xqEOwzmDrLSK4jMchFegK0PjGThfjevi4G5ptqG0BEkxrEUao=,iv:iKzqaagRrW6RXR7QiIlnNpHPSMbYfYBrca9cme/5x0c=,tag:dd5/Sun8gYdTvo5uxr5SNg==,type:str]", + "version": "3.13.3" + } +} diff --git a/apps/frontend/.env.enc b/apps/frontend/.env.enc new file mode 100644 index 0000000..458df0c --- /dev/null +++ b/apps/frontend/.env.enc @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:WO43usXLFN/FGsd0CqIiqW0n9ew7V9OLSlY2GvGbJr/cgXu3TgSv,iv:nbdb5I8/4DGA0JKUtSVKoHsz8rcmhUfZ+NJ+zPJcAg0=,tag:AiknfuIr7N27ImeDN/rdAg==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGcXlmTThHKzhBbGFNVm10\nZm9SOE5uUm84M294cE9FYXNLWnJnZ1ViNERnCjgrRDlaZldqMGlESVVpYkR2Mkhn\nSFRNRmtZYW9sQU9ySGU4QmVsVFpSQ2sKLS0tIGsvR2xWSjdDRXFDR3Q5SHg3dzBj\nRFlidDVCMkRWdjhjSkpFOTd1bXIzTEUK9JaciyYggJ42VxUvQAYvPBzDErppP8ec\nuVITBRb8UlBShpcYiPlCRKAlSv4Bf2lBmvJOG2NNQxrMt3EX0YhPlA==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvM09YNjNtbW4wa2VsV2ww\nMmZyaDB6azc3dnJPdGZEZjJORUNpenZmeTJzCkFWV0xXTU5vSWthNVFPOTIvSURX\nZTFzOXpZQlVEb3ZHbkdVb28xcStycnMKLS0tIDVLU3lMSkkyWHh4OG1Va21iaVBi\nVnNzRmRva3kyRlQ2Nnc1VU1Qc3FHcVUK0n8/76+F8ZRRF7cvtpGvXtoVjbq9Hlsi\nz/YKKI5c2lNPmfEABWbnCbK7t0luAIxyS8qLVT5b17EUHCxZQS0Z5g==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf" + } + ], + "lastmodified": "2026-08-26T11:21:17Z", + "mac": "ENC[AES256_GCM,data:TYil2cEtH8PvJeA+K4ijrBF4gCqlQ6qs8KRuOhm4wJmj76IE8qhYkMFC7tPy/1PYfkwhKDMkcEu+xskUZgb2WTGA0pi6Oa01Ddrsx6KWlAsktsdQChbj8JfMMENcCl+ULKvSYYf7YKZdAZhct/p2ArrEWWDypFgxrHpHl1jRW88=,iv:GC78vqhkx8jHFlfT5rjUjNTJVu5yguu2e9m9Duca2zs=,tag:8Mrgb0g1yX9kNvmVwXW/Ug==,type:str]", + "version": "3.13.3" + } +} diff --git a/apps/frontend/.env.local.enc b/apps/frontend/.env.local.enc new file mode 100644 index 0000000..e59611b --- /dev/null +++ b/apps/frontend/.env.local.enc @@ -0,0 +1,18 @@ +{ + "data": "ENC[AES256_GCM,data:z4qlrKs4q8bRfuHSCouetyKBBDHOnNKIJqeTOLzGXZXgCA+HQllYsEa88CboF6RAHpAvhgTzt3TQIVxSOquSfe4sy7eCLI+AqOQs3OWH7nhTFF5ARMvXfjFZltakAi8erWZFS/QZXakHWIDEdbU+QL7iPjBjyKyMS6r1LsS4C7qmSJ0T9jd7sdRbyD4qLFvi,iv:KnMTrR7j1vudr+oDoOAg5KrsMH76RQHbkhNr68dH598=,tag:GL3bhGE5UgiwTNhS61GwAw==,type:str]", + "sops": { + "age": [ + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEWEhxN3dyWW9RcXJnZlU5\nMnZDVm8vdUNSRTFOQnJuSklLMjYyYkh1ekhvCm9tR2hzdTJ5djIrSXYwZTdyWkY5\nMzhGS1pwc3F4OWhkc2hMMFY5WnlvM0UKLS0tIDRuWndYb3BEM1ovR3R0b0dPMEhm\nMUJ2VFRTNVdKSTJ4cUo4NjhXcmd0WjgKWb8DrarHln8OhYx9BG4olYHZIyhMAh47\n1z/JGpEvHLVwU2SALWhJjLSC1KNS/FrlO17fcKSUTeqIRb9DjEKRSw==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1skqn0tvghvz8fvu6m5880w72z5p00822kcc3adlagl4fkyu2fu8sg3w83f" + }, + { + "enc": "-----BEGIN AGE ENCRYPTED FILE-----\nYWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA4b0o1bGZabSsvRm5iN2x5\nSE1kSXZReUtsb1g3b3lkOUJBTjBQb1VkbFVvCklyVEtGa2lNdEdJdkt6RjJYcDFm\nZitzb3VOMTcvR2Z5NEtUYjNmTXlmb00KLS0tIEk5YWFiVVVVTUNMemt2Ukt6WVJX\nYUJlYngxWjRBQkgvb2FHZmNnNWNXdlUK9qNFRy+fZa5SF6cXXS/lRZkeLVZtaV/A\nLe6kQdi/jWw8NVwCQWRz/+Tfh1QJplWZmLKa8efi03ewite4klYNjw==\n-----END AGE ENCRYPTED FILE-----\n", + "recipient": "age1cau5u3xfc0eat67zm5eqnasmrndewy5gyxjy26xflnvmva2m7egsn3pucf" + } + ], + "lastmodified": "2026-08-26T11:21:17Z", + "mac": "ENC[AES256_GCM,data:jdcxDXCMNwJgoR+/TRPLW6NaRnKHoJRAsKfJSCi1O45zDnI5g2OXAnr9tO7zexaLbTk21GZQ1RkFCcQ9jMcOYZVvlMzyxNNTxmNACkZh2Ki2paWii2B+/U/3FlcItgPXBlF+ck1KnDPDjwKOCr+kvGJ7HC6kRRM9PJ9G6bC6fp8=,iv:/0jvuXGAF3lCvsTIesKZW3TqWiqazHvH1baMnG+2FQ4=,tag:RQJtOFHTsWxGlq6c+QCvtA==,type:str]", + "version": "3.13.3" + } +} From 92fcc73bc3179257419025e0219ed9a2d543380a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 16:05:03 +0200 Subject: [PATCH 03/15] feat(e2e): add Stremio end-to-end test harness - Add Playwright API, UI, and live smoke coverage - Configure disposable Supabase CI jobs and local test setup - Update shared Stremio types, tooling, and database schema --- .github/workflows/ci.yml | 57 +++ .gitignore | 4 + apps/e2e/.prettierignore | 2 + apps/e2e/README.md | 68 +++ apps/e2e/env.ts | 52 ++ apps/e2e/eslint.config.mjs | 19 + apps/e2e/helpers/api.ts | 117 +++++ apps/e2e/helpers/db.ts | 39 ++ apps/e2e/helpers/stremio.ts | 76 +++ apps/e2e/helpers/test-data.ts | 32 ++ apps/e2e/package.json | 33 ++ apps/e2e/playwright.config.ts | 90 ++++ apps/e2e/preflight.ts | 27 + apps/e2e/tests/addon-api.spec.ts | 469 ++++++++++++++++++ apps/e2e/tests/configure-page.spec.ts | 170 +++++++ apps/e2e/tests/home-onboarding.spec.ts | 71 +++ apps/e2e/tests/stremio-catalogs.spec.ts | 178 +++++++ apps/e2e/tests/stremio-install.spec.ts | 65 +++ apps/e2e/tests/stremio-meta.spec.ts | 37 ++ apps/e2e/tsconfig.json | 14 + package.json | 1 + packages/shared/src/stremio.types.ts | 2 +- pnpm-lock.yaml | 74 +++ supabase/.gitignore | 8 + supabase/config.toml | 408 +++++++++++++++ .../20260219000000_baseline_schema.sql | 21 + turbo.json | 4 + 27 files changed, 2137 insertions(+), 1 deletion(-) create mode 100644 apps/e2e/.prettierignore create mode 100644 apps/e2e/README.md create mode 100644 apps/e2e/env.ts create mode 100644 apps/e2e/eslint.config.mjs create mode 100644 apps/e2e/helpers/api.ts create mode 100644 apps/e2e/helpers/db.ts create mode 100644 apps/e2e/helpers/stremio.ts create mode 100644 apps/e2e/helpers/test-data.ts create mode 100644 apps/e2e/package.json create mode 100644 apps/e2e/playwright.config.ts create mode 100644 apps/e2e/preflight.ts create mode 100644 apps/e2e/tests/addon-api.spec.ts create mode 100644 apps/e2e/tests/configure-page.spec.ts create mode 100644 apps/e2e/tests/home-onboarding.spec.ts create mode 100644 apps/e2e/tests/stremio-catalogs.spec.ts create mode 100644 apps/e2e/tests/stremio-install.spec.ts create mode 100644 apps/e2e/tests/stremio-meta.spec.ts create mode 100644 apps/e2e/tsconfig.json create mode 100644 supabase/.gitignore create mode 100644 supabase/config.toml create mode 100644 supabase/migrations/20260219000000_baseline_schema.sql diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef90f27..f13c3bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,63 @@ jobs: - name: Lint, build, and test run: pnpm turbo run lint build test + e2e: + name: E2E (${{ matrix.name }}) + # Local coverage and the small live smoke suite run in separate jobs. Each + # gets its own disposable Supabase stack, and an external outage cannot hide + # failures in the deterministic suite. + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - suite: local + name: Local + - suite: live + name: Live smoke + steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v4 + with: + version: 10.30.1 + + - uses: actions/setup-node@v4 + with: + node-version: 20 + cache: pnpm + + - uses: supabase/setup-cli@v1 + with: + version: 2.98.2 + + - name: Start local Supabase (db + REST only) + run: supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Cache Playwright browsers + uses: actions/cache@v4 + with: + path: ~/.cache/ms-playwright + key: playwright-${{ runner.os }}-${{ hashFiles('pnpm-lock.yaml') }} + + - name: Install Playwright Chromium + run: pnpm --filter @stremlist/e2e exec playwright install --with-deps chromium + + - name: Run E2E tests + run: pnpm --filter @stremlist/e2e test:e2e:${{ matrix.suite }} + + - name: Upload Playwright report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: playwright-report-${{ matrix.suite }} + path: apps/e2e/playwright-report + retention-days: 7 + version-bump: name: Auto version bump needs: ci diff --git a/.gitignore b/.gitignore index cac4628..1c0fdfa 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,7 @@ lerna-debug.log* !robots.txt # Supabase CLI local state supabase/.temp/ + +# Playwright +test-results/ +playwright-report/ diff --git a/apps/e2e/.prettierignore b/apps/e2e/.prettierignore new file mode 100644 index 0000000..f2459b4 --- /dev/null +++ b/apps/e2e/.prettierignore @@ -0,0 +1,2 @@ +playwright-report +test-results diff --git a/apps/e2e/README.md b/apps/e2e/README.md new file mode 100644 index 0000000..cd2e539 --- /dev/null +++ b/apps/e2e/README.md @@ -0,0 +1,68 @@ +# @stremlist/e2e + +End-to-end tests that exercise Stremlist the way a real user does: the addon +is installed into the **hosted Stremio Web app** (web.stremio.com) from a +backend running locally, with **live IMDb data** and a **local Supabase +stack**. The configure/onboarding pages of the frontend are covered too. + +## How it works + +- Playwright starts the backend (`:7301`) and the frontend (`:7302`) as web + servers with ports distinct from the dev ones, so tests can run next to a + normal dev session. +- The backend points at a local Supabase stack (`supabase start`), reset + between tests. Functional seeding goes through the backend's own HTTP API, + so tests exercise real code paths. +- Stremio Web runs in anonymous mode: each fresh browser context has its own + local addon collection. No Stremio account or shared state is involved. +- Chromium is launched with `--disable-features=LocalNetworkAccessChecks,...` + because Chrome otherwise blocks the HTTPS Stremio Web page from fetching the + addon on `127.0.0.1` (Local Network Access permission, never grantable in + headless runs). +- IMDb is live. Assertions are structural (ordering invariants, id shapes, + counts) or compare the Stremio UI against the addon's own catalog JSON from + the same run, so they do not depend on what is in the watchlist today. +- The default run contains deterministic local coverage plus four live smoke + tests. Broader live checks stay in the `live-regression` project and run only + when requested. + +## Running locally + +```sh +# One-time / per boot: start the local Supabase stack (needs Docker running) +supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta + +# From the repo root: local coverage plus the small live smoke suite +pnpm test:e2e + +# Or from this package, with more options +pnpm --filter @stremlist/e2e test:e2e:local # deterministic local suite +pnpm --filter @stremlist/e2e test:e2e:live # four live smoke tests +pnpm --filter @stremlist/e2e test:e2e:live:full # all live external checks +pnpm --filter @stremlist/e2e test:e2e:all # every project +pnpm --filter @stremlist/e2e test:e2e:headed # smoke tests with a browser +pnpm --filter @stremlist/e2e report # open the last HTML report +``` + +The suite deletes test users between cases. Foreign-key cascades clear their +watchlists and caches in the same database statement. The harness rejects any +non-loopback Supabase URL unless the caller provides the explicit destructive +confirmation described below. + +## Environment knobs + +| Variable | Purpose | +| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | +| `E2E_SUPABASE_URL` / `E2E_SUPABASE_SERVICE_ROLE_KEY` | Non-default local Supabase stack | +| `E2E_ALLOW_REMOTE_DATABASE=I_UNDERSTAND_THIS_WIPES_DATA` | Permit an isolated remote test project. Cleanup deletes every user and all dependent data | +| `E2E_IMDB_USER_ID` / `E2E_IMDB_USER_ID_2` | Override the public watchlists under test | +| `E2E_IMDB_LIST_ID` | Override the public `ls` list under test | +| `E2E_PRIVATE_IMDB_USER_ID` | Override the private watchlist under test (default: a maintainer-owned account kept private on purpose) | +| `E2E_PRIVATE_IMDB_LIST_ID` | Enable the private `ls` list test (skipped otherwise — no stable private list id is available) | + +## Known limitations + +- Drag-and-drop catalog reordering (pointer-based dnd-kit) is not covered. +- The newsletter endpoint is not covered (it would email real people). +- The live smoke and full regression suites depend on web.stremio.com and IMDb. + CI runs only the four smoke checks and retries them twice on failure. diff --git a/apps/e2e/env.ts b/apps/e2e/env.ts new file mode 100644 index 0000000..fd44750 --- /dev/null +++ b/apps/e2e/env.ts @@ -0,0 +1,52 @@ +// Shared constants for the E2E harness. Every port is distinct from the +// regular dev ports (7001/5173) so tests can run next to a dev session. + +export const BACKEND_PORT = 7301; +export const FRONTEND_PORT = 7302; + +export const BACKEND_URL = `http://127.0.0.1:${BACKEND_PORT}`; +export const FRONTEND_URL = `http://127.0.0.1:${FRONTEND_PORT}`; + +export const STREMIO_WEB_URL = "https://web.stremio.com"; + +// Local Supabase stack (supabase start). The service-role key below is the +// public, well-known key every local Supabase CLI stack ships with — it is not +// a secret. Both values can be overridden for non-default stacks. +export const SUPABASE_URL = + process.env.E2E_SUPABASE_URL ?? "http://127.0.0.1:54321"; +export const SUPABASE_SERVICE_ROLE_KEY = + process.env.E2E_SUPABASE_SERVICE_ROLE_KEY ?? + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU"; + +const REMOTE_DATABASE_CONFIRMATION = "I_UNDERSTAND_THIS_WIPES_DATA"; + +export function assertSafeSupabaseTarget(): void { + let hostname: string; + try { + hostname = new URL(SUPABASE_URL).hostname.toLowerCase(); + } catch { + throw new Error(`E2E_SUPABASE_URL is not a valid URL: ${SUPABASE_URL}`); + } + + const isLoopback = ["localhost", "127.0.0.1", "[::1]", "::1"].includes( + hostname, + ); + const remoteWipeConfirmed = + process.env.E2E_ALLOW_REMOTE_DATABASE === REMOTE_DATABASE_CONFIRMATION; + + if (!isLoopback && !remoteWipeConfirmed) { + throw new Error( + `Refusing to run destructive E2E cleanup against non-loopback Supabase host "${hostname}". ` + + `Use a disposable local stack, or set E2E_ALLOW_REMOTE_DATABASE=${REMOTE_DATABASE_CONFIRMATION} only for an isolated remote test project.`, + ); + } +} + +assertSafeSupabaseTarget(); + +// Short cooldown so refresh-throttle tests stay fast. +export const REFRESH_COOLDOWN_SECONDS = 2; + +export function addonManifestUrl(userId: string): string { + return `${BACKEND_URL}/${userId}/manifest.json`; +} diff --git a/apps/e2e/eslint.config.mjs b/apps/e2e/eslint.config.mjs new file mode 100644 index 0000000..b971edd --- /dev/null +++ b/apps/e2e/eslint.config.mjs @@ -0,0 +1,19 @@ +import baseConfig from "@stremlist/eslint-config/base"; +import prettier from "eslint-config-prettier/flat"; + +export default [ + { ignores: ["eslint.config.mjs", "playwright-report/**", "test-results/**"] }, + ...baseConfig, + { + languageOptions: { + parserOptions: { + project: "./tsconfig.json", + tsconfigRootDir: import.meta.dirname, + }, + }, + rules: { + curly: ["error", "multi-line"], + }, + }, + prettier, +]; diff --git a/apps/e2e/helpers/api.ts b/apps/e2e/helpers/api.ts new file mode 100644 index 0000000..08c13cb --- /dev/null +++ b/apps/e2e/helpers/api.ts @@ -0,0 +1,117 @@ +import type { + StremioManifest, + StremioMeta, + UserConfigResponse, + UserConfigUpdateWatchlist, +} from "@stremlist/shared"; +import { hcWithType } from "@stremlist/backend/client"; +import { BACKEND_URL } from "../env.js"; + +// Thin typed wrappers over the backend HTTP API. Tests use these both to +// arrange state (through real code paths) and to assert the addon protocol +// contract that Stremio clients consume. + +export type CatalogMeta = StremioMeta; +type Manifest = StremioManifest; +type UserConfig = UserConfigResponse; +const api = hcWithType(BACKEND_URL); + +async function getJson(path: string): Promise<{ status: number; body: T }> { + const response = await fetch(`${BACKEND_URL}${path}`); + return { status: response.status, body: (await response.json()) as T }; +} + +export async function getBaseManifest(): Promise { + return (await getJson("/manifest.json")).body; +} + +export async function getUserManifest(userId: string): Promise { + return (await getJson(`/${userId}/manifest.json`)).body; +} + +export async function getConfig( + userId: string, +): Promise<{ status: number; body: UserConfig }> { + const response = await api[":userId"].config.$get({ param: { userId } }); + return { + status: response.status, + body: (await response.json()) as UserConfig, + }; +} + +type ConfigWatchlistInput = UserConfigUpdateWatchlist; + +export async function postConfig( + userId: string, + watchlists: ConfigWatchlistInput[], + rpdbApiKey?: string, +): Promise<{ status: number; body: unknown }> { + const response = await api[":userId"].config.$post({ + param: { userId }, + json: { watchlists, rpdbApiKey }, + }); + return { status: response.status, body: await response.json() }; +} + +export async function getCatalog( + userId: string, + type: string, + catalogId: string, +): Promise<{ status: number; metas: CatalogMeta[] }> { + const { status, body } = await getJson<{ metas: CatalogMeta[] }>( + `/${userId}/catalog/${type}/${catalogId}.json`, + ); + return { status, metas: body.metas }; +} + +export async function getMeta( + userId: string, + type: string, + id: string, +): Promise<{ status: number; meta: CatalogMeta | null }> { + const { status, body } = await getJson<{ meta: CatalogMeta | null }>( + `/${userId}/meta/${type}/${id}.json`, + ); + return { status, meta: body.meta }; +} + +export async function refresh( + userId: string, +): Promise<{ status: number; body: Record }> { + const response = await api[":userId"].refresh.$post({ + param: { userId }, + }); + return { + status: response.status, + body: (await response.json()) as Record, + }; +} + +export async function validateUser( + userId: string, +): Promise> { + const response = await api.validate[":userId"].$get({ param: { userId } }); + return (await response.json()) as Record; +} + +export async function validateList( + listId: string, +): Promise> { + const response = await api["validate-list"][":listId"].$get({ + param: { listId }, + }); + return (await response.json()) as Record; +} + +/** + * Bootstrap a user exactly the way a real install does: the first manifest + * fetch upserts the user and seeds the default watchlist. Returns the config. + */ +export async function bootstrapUser(userId: string): Promise { + await getUserManifest(userId); + const { status, body } = await getConfig(userId); + if (status !== 200) { + throw new Error(`bootstrapUser(${userId}) got ${status}`); + } + return body; +} diff --git a/apps/e2e/helpers/db.ts b/apps/e2e/helpers/db.ts new file mode 100644 index 0000000..947ec47 --- /dev/null +++ b/apps/e2e/helpers/db.ts @@ -0,0 +1,39 @@ +import { createClient } from "@supabase/supabase-js"; +import type { Database } from "@stremlist/shared"; +import { SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL } from "../env.js"; + +// Service-role client: bypasses RLS, used only to reset and inspect state +// between tests. All functional seeding goes through the backend's own HTTP +// API so the tests exercise real code paths. +export const db = createClient( + SUPABASE_URL, + SUPABASE_SERVICE_ROLE_KEY, +); + +/** Delete test users. Foreign-key cascades reset every dependent table. */ +export async function resetDb(): Promise { + const { error } = await db + .from("users") + .delete() + .neq("imdb_user_id", "__e2e_no_matching_user__"); + if (error) throw new Error(`resetDb failed: ${error.message}`); +} + +/** Rewind a user's last_fetched_at so the refresh cooldown does not apply. */ +export async function clearRefreshCooldown(userId: string): Promise { + const past = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + const { error } = await db + .from("users") + .update({ last_fetched_at: past }) + .eq("imdb_user_id", userId); + if (error) throw new Error(`clearRefreshCooldown failed: ${error.message}`); +} + +export async function countCacheItems(watchlistId: string): Promise { + const { count, error } = await db + .from("watchlist_cache_items") + .select("*", { count: "exact", head: true }) + .eq("watchlist_id", watchlistId); + if (error) throw new Error(`countCacheItems failed: ${error.message}`); + return count ?? 0; +} diff --git a/apps/e2e/helpers/stremio.ts b/apps/e2e/helpers/stremio.ts new file mode 100644 index 0000000..50d0231 --- /dev/null +++ b/apps/e2e/helpers/stremio.ts @@ -0,0 +1,76 @@ +import type { Page } from "@playwright/test"; +import { expect } from "@playwright/test"; +import { STREMIO_WEB_URL } from "../env.js"; + +// Page helpers for the hosted Stremio Web app (web.stremio.com). The app runs +// in anonymous/local mode: a fresh browser context has no account and stores +// the addon collection in localStorage, so tests are fully isolated. + +export function addonsDeepLink(manifestUrl: string): string { + return `${STREMIO_WEB_URL}/#/addons?addon=${encodeURIComponent(manifestUrl)}`; +} + +export function discoverUrl( + manifestUrl: string, + type: "movie" | "series", + catalogId: string, +): string { + return `${STREMIO_WEB_URL}/#/discover/${encodeURIComponent(manifestUrl)}/${type}/${encodeURIComponent(catalogId)}`; +} + +export function detailUrl(type: "movie" | "series", metaId: string): string { + return `${STREMIO_WEB_URL}/#/detail/${type}/${metaId}/${metaId}`; +} + +/** + * Dismiss the "install the desktop app" prompt if it is showing. Its "Install" + * link would otherwise collide with the addon modal's Install button. + */ +export async function dismissDesktopAppPrompt(page: Page): Promise { + const dismiss = page.getByText("Don't show again", { exact: true }); + try { + await dismiss.click({ timeout: 3_000 }); + } catch { + // Prompt not shown — nothing to do. + } +} + +/** + * Open the addon deep link and complete the install through the modal. + * Resolves once the modal is gone and the addon shows as installed. + */ +export async function installAddon( + page: Page, + manifestUrl: string, +): Promise { + await page.goto(addonsDeepLink(manifestUrl)); + await dismissDesktopAppPrompt(page); + const installButton = page.getByText("Install", { exact: true }).last(); + await expect(installButton).toBeVisible(); + await installButton.click(); + // The modal closes on success; "Uninstall"/"Configure" appear on the card. + await expect( + page.getByText("Stremlist", { exact: true }).first(), + ).toBeVisible(); +} + +/** Uninstall the addon from the Addons page, confirming in the modal. */ +export async function uninstallAddon( + page: Page, + manifestUrl: string, +): Promise { + await page.goto(addonsDeepLink(manifestUrl)); + await dismissDesktopAppPrompt(page); + const uninstallButton = page.getByText("Uninstall", { exact: true }).last(); + await expect(uninstallButton).toBeVisible(); + await uninstallButton.click(); +} + +/** Titles of the meta items currently rendered on a Discover page, in order. */ +export async function discoverItemTitles(page: Page): Promise { + const links = page.locator('a[href^="#/detail/"][title]'); + await expect(links.first()).toBeVisible(); + return links.evaluateAll((elements) => + elements.map((el) => el.getAttribute("title") ?? ""), + ); +} diff --git a/apps/e2e/helpers/test-data.ts b/apps/e2e/helpers/test-data.ts new file mode 100644 index 0000000..297cf57 --- /dev/null +++ b/apps/e2e/helpers/test-data.ts @@ -0,0 +1,32 @@ +// Real IMDb ids used by the E2E suite. Tests hit live IMDb through the +// backend, so assertions stay structural (ordering invariants, counts, id +// shapes) instead of pinning exact titles that could change over time. + +// Small public watchlist (~19 items) — one of the ids the backend stress test +// already exercises, kept small so live fetches stay fast. +export const PUBLIC_USER = process.env.E2E_IMDB_USER_ID ?? "ur102135398"; + +// Second public watchlist for multi-catalog scenarios. +export const PUBLIC_USER_2 = process.env.E2E_IMDB_USER_ID_2 ?? "ur102551738"; + +// Long-standing public IMDb list ("Top 100 Greatest Movies of All Time"). +export const PUBLIC_LIST = process.env.E2E_IMDB_LIST_ID ?? "ls055592025"; + +// Syntactically valid ids that do not exist. IMDb user ids are ~9 digits; +// a 13-digit id is far outside the allocated range. +export const UNKNOWN_USER = "ur9999999999999"; +export const UNKNOWN_LIST = "ls9999999999999"; + +// Account whose watchlist is deliberately kept private for these tests +// (maintainer-owned). The p-handle resolves to the same account, covering the +// handle-resolution path for private sources too. +export const PRIVATE_USER = + process.env.E2E_PRIVATE_IMDB_USER_ID ?? "ur198342247"; +export const PRIVATE_P_HANDLE = "p.e4ialbfdp3rntdahbslk5yzovm"; + +// No stable private ls list is available; provide one via env to enable the +// private-list test. +export const PRIVATE_LIST = process.env.E2E_PRIVATE_IMDB_LIST_ID; + +// p-handle that must resolve to a canonical ur id (IMDb founder's profile). +export const P_HANDLE = "p.colneedham"; diff --git a/apps/e2e/package.json b/apps/e2e/package.json new file mode 100644 index 0000000..f80452a --- /dev/null +++ b/apps/e2e/package.json @@ -0,0 +1,33 @@ +{ + "name": "@stremlist/e2e", + "private": true, + "type": "module", + "scripts": { + "preflight": "tsx preflight.ts", + "test:e2e": "pnpm preflight && playwright test --project=local --project=live-smoke", + "test:e2e:local": "pnpm preflight && playwright test --project=local", + "test:e2e:live": "pnpm preflight && playwright test --project=live-smoke", + "test:e2e:live:full": "pnpm preflight && playwright test --project=live-smoke --project=live-regression", + "test:e2e:all": "pnpm preflight && playwright test", + "test:e2e:headed": "pnpm preflight && playwright test --project=live-smoke --headed", + "report": "playwright show-report", + "typecheck": "tsc --noEmit", + "lint": "eslint .", + "format": "prettier . --write", + "format:check": "prettier . --check" + }, + "devDependencies": { + "@playwright/test": "^1.62.1", + "@stremlist/backend": "workspace:*", + "@stremlist/eslint-config": "workspace:*", + "@stremlist/frontend": "workspace:*", + "@stremlist/shared": "workspace:*", + "@supabase/supabase-js": "^2.95.3", + "@types/node": "^20.11.17", + "eslint": "^9.39.2", + "eslint-config-prettier": "^10.1.8", + "prettier": "^3.8.1", + "tsx": "^4.7.1", + "typescript": "^5.8.3" + } +} diff --git a/apps/e2e/playwright.config.ts b/apps/e2e/playwright.config.ts new file mode 100644 index 0000000..04fe807 --- /dev/null +++ b/apps/e2e/playwright.config.ts @@ -0,0 +1,90 @@ +import { defineConfig, devices } from "@playwright/test"; +import { + BACKEND_PORT, + BACKEND_URL, + FRONTEND_PORT, + FRONTEND_URL, + REFRESH_COOLDOWN_SECONDS, + SUPABASE_SERVICE_ROLE_KEY, + SUPABASE_URL, +} from "./env.js"; + +export default defineConfig({ + testDir: "./tests", + // Tests within a project share one backend and one database. CI runs local + // and live-smoke projects as separate jobs with isolated Supabase stacks. + fullyParallel: false, + workers: 1, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + timeout: 120_000, + expect: { timeout: 20_000 }, + reporter: process.env.CI + ? [["list"], ["html", { open: "never" }], ["github"]] + : [["list"], ["html", { open: "never" }]], + use: { + trace: "retain-on-failure", + screenshot: "only-on-failure", + // stremio-web registers a service worker; blocking it keeps network + // behavior deterministic across fresh contexts. + serviceWorkers: "block", + launchOptions: { + args: [ + // Chrome blocks fetches from public HTTPS pages (web.stremio.com) to + // loopback addresses behind the Local Network Access permission, which + // headless runs can never grant. Older Chromium versions gate the same + // thing behind Private Network Access preflights. Disable both so the + // hosted Stremio Web app can talk to the local addon under test. + "--disable-features=LocalNetworkAccessChecks,BlockInsecurePrivateNetworkRequests,PrivateNetworkAccessSendPreflights,PrivateNetworkAccessRespectPreflightResults", + ], + }, + }, + projects: [ + { + name: "local", + grep: /@local/, + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "live-smoke", + grep: /@live-smoke/, + use: { ...devices["Desktop Chrome"] }, + }, + { + name: "live-regression", + grep: /@live-regression/, + use: { ...devices["Desktop Chrome"] }, + }, + ], + webServer: [ + { + command: "pnpm exec tsx src/dev.ts", + cwd: "../backend", + url: `${BACKEND_URL}/health`, + reuseExistingServer: !process.env.CI, + timeout: 60_000, + env: { + PORT: String(BACKEND_PORT), + SUPABASE_URL, + SUPABASE_SERVICE_ROLE_KEY, + FRONTEND_URL, + REFRESH_COOLDOWN_SECONDS: String(REFRESH_COOLDOWN_SECONDS), + CACHE_TTL_MINUTES: "30", + // The Resend SDK throws at import time without a key. Newsletter + // delivery is deliberately out of E2E scope (it would email real + // people), so a dummy key is enough to boot the app. + RESEND_API_KEY: "re_e2e_dummy_key", + }, + }, + { + command: `pnpm exec vite --port ${FRONTEND_PORT} --strictPort`, + cwd: "../frontend", + url: FRONTEND_URL, + reuseExistingServer: !process.env.CI, + timeout: 60_000, + env: { + VITE_BACKEND_URL: BACKEND_URL, + }, + }, + ], +}); diff --git a/apps/e2e/preflight.ts b/apps/e2e/preflight.ts new file mode 100644 index 0000000..ae9e447 --- /dev/null +++ b/apps/e2e/preflight.ts @@ -0,0 +1,27 @@ +import { SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL } from "./env.js"; + +// This script runs before Playwright starts its web servers, so database +// failures surface immediately instead of becoming a backend startup timeout. +try { + const response = await fetch( + `${SUPABASE_URL}/rest/v1/users?select=imdb_user_id&limit=1`, + { + headers: { + apikey: SUPABASE_SERVICE_ROLE_KEY, + Authorization: `Bearer ${SUPABASE_SERVICE_ROLE_KEY}`, + }, + }, + ); + if (!response.ok) { + throw new Error( + `Supabase responded ${response.status}: ${await response.text()}`, + ); + } +} catch (error) { + throw new Error( + `E2E Supabase stack is not reachable at ${SUPABASE_URL}.\n` + + `Start the local stack from the repository root with:\n` + + ` supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta\n` + + `Underlying error: ${error instanceof Error ? error.message : String(error)}`, + ); +} diff --git a/apps/e2e/tests/addon-api.spec.ts b/apps/e2e/tests/addon-api.spec.ts new file mode 100644 index 0000000..f0e0a9e --- /dev/null +++ b/apps/e2e/tests/addon-api.spec.ts @@ -0,0 +1,469 @@ +import { expect, test } from "@playwright/test"; +import { FRONTEND_URL, BACKEND_URL } from "../env.js"; +import { + bootstrapUser, + getBaseManifest, + getCatalog, + getConfig, + getMeta, + getUserManifest, + postConfig, + refresh, + validateList, + validateUser, + type CatalogMeta, +} from "../helpers/api.js"; +import { + clearRefreshCooldown, + countCacheItems, + resetDb, +} from "../helpers/db.js"; +import { + P_HANDLE, + PRIVATE_LIST, + PRIVATE_P_HANDLE, + PRIVATE_USER, + PUBLIC_LIST, + PUBLIC_USER, + UNKNOWN_LIST, + UNKNOWN_USER, +} from "../helpers/test-data.js"; + +// Addon protocol contract — the exact HTTP surface every Stremio client +// (web, desktop, mobile) consumes. Data comes from live IMDb, so assertions +// are structural: ordering invariants, id shapes, counts. + +const CATALOG_ID_PATTERN = + /^wl-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-(movie|series)$/; + +test.beforeEach(async () => { + await resetDb(); +}); + +test.describe("manifest", () => { + test("base manifest requires configuration", { tag: "@local" }, async () => { + const manifest = await getBaseManifest(); + expect(manifest.behaviorHints?.configurationRequired).toBe(true); + expect(manifest.behaviorHints?.configurable).toBe(true); + expect(manifest.name).toBe("Stremlist"); + }); + + test( + "first user manifest bootstraps the install", + { tag: "@local" }, + async () => { + const manifest = await getUserManifest(PUBLIC_USER); + expect(manifest.id).toBe(`com.stremlist.${PUBLIC_USER}`); + expect(manifest.behaviorHints?.configurationRequired).toBe(false); + // Default watchlist in split mode → one movie + one series catalog. + expect(manifest.catalogs).toHaveLength(2); + expect(manifest.catalogs.map((c) => c.type).sort()).toEqual([ + "movie", + "series", + ]); + for (const catalogRef of manifest.catalogs) { + expect(catalogRef.id).toMatch(CATALOG_ID_PATTERN); + expect(catalogRef.name).toContain("Stremlist"); + } + // The meta resource must stay declared — Stremio clients rely on it. + const metaResource = manifest.resources.find( + (r) => + typeof r === "object" && + r !== null && + (r as { name?: string }).name === "meta", + ) as { idPrefixes?: string[] } | undefined; + expect(metaResource?.idPrefixes).toEqual(["tt"]); + }, + ); + + test( + "display mode controls emitted catalogs", + { tag: "@local" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + await postConfig(PUBLIC_USER, [ + { + id: watchlist.id, + imdbUserId: watchlist.imdbUserId, + sortOption: "added_at-asc", + displayMode: "movie", + }, + ]); + const manifest = await getUserManifest(PUBLIC_USER); + expect(manifest.catalogs).toHaveLength(1); + expect(manifest.catalogs[0].type).toBe("movie"); + }, + ); +}); + +test.describe("catalogs", () => { + test( + "movie catalog serves the live IMDb watchlist", + { tag: "@live-smoke" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + const catalogId = `wl-${config.watchlists[0].id}-movie`; + const { status, metas } = await getCatalog( + PUBLIC_USER, + "movie", + catalogId, + ); + expect(status).toBe(200); + expect(metas.length).toBeGreaterThan(0); + for (const meta of metas) { + expect(meta.type).toBe("movie"); + expect(meta.id).toMatch(/^tt\d+$/); + expect(meta.name.length).toBeGreaterThan(0); + } + // The fetch must have populated the per-item cache. + expect(await countCacheItems(config.watchlists[0].id)).toBeGreaterThan(0); + }, + ); + + test( + "every sort option orders the catalog correctly", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + const catalogId = `wl-${watchlist.id}-movie`; + + const setSort = async (sortOption: string) => { + const { status } = await postConfig(PUBLIC_USER, [ + { id: watchlist.id, imdbUserId: watchlist.imdbUserId, sortOption }, + ]); + expect(status).toBe(200); + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + return metas; + }; + + const baseline = await setSort("added_at-asc"); + expect(baseline.length).toBeGreaterThan(1); + const ids = (metas: CatalogMeta[]) => metas.map((m) => m.id); + const years = (metas: CatalogMeta[]) => + metas.map((m) => parseInt(m.releaseInfo ?? "0", 10) || 0); + const ratings = (metas: CatalogMeta[]) => + metas.map((m) => parseFloat(m.imdbRating ?? "0") || 0); + const expectMonotonic = (values: number[], direction: "asc" | "desc") => { + for (let i = 1; i < values.length; i++) { + if (direction === "asc") + expect(values[i]).toBeGreaterThanOrEqual(values[i - 1]); + else expect(values[i]).toBeLessThanOrEqual(values[i - 1]); + } + }; + + const addedDesc = await setSort("added_at-desc"); + expect(ids(addedDesc)).toEqual([...ids(baseline)].reverse()); + + const titleAsc = await setSort("title-asc"); + const namesAsc = titleAsc.map((m) => m.name); + expect(namesAsc).toEqual( + [...namesAsc].sort((a, b) => a.localeCompare(b)), + ); + + const titleDesc = await setSort("title-desc"); + const namesDesc = titleDesc.map((m) => m.name); + expect(namesDesc).toEqual( + [...namesDesc].sort((a, b) => b.localeCompare(a)), + ); + + expectMonotonic(years(await setSort("year-asc")), "asc"); + expectMonotonic(years(await setSort("year-desc")), "desc"); + expectMonotonic(ratings(await setSort("rating-asc")), "asc"); + expectMonotonic(ratings(await setSort("rating-desc")), "desc"); + + const random = await setSort("random"); + expect(ids(random).sort()).toEqual(ids(baseline).sort()); + }, + ); + + test( + "ls list source serves a public IMDb list", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + await postConfig(PUBLIC_USER, [ + { imdbUserId: PUBLIC_LIST, sortOption: "added_at-asc" }, + ]); + const updated = await getConfig(PUBLIC_USER); + const watchlist = updated.body.watchlists[0]; + expect(watchlist.imdbUserId).toBe(PUBLIC_LIST); + const { metas } = await getCatalog( + PUBLIC_USER, + "movie", + `wl-${watchlist.id}-movie`, + ); + expect(metas.length).toBeGreaterThan(0); + expect(config.watchlists[0].id).not.toBe(watchlist.id); + }, + ); + + test( + "built-in chart catalog serves live chart data", + { tag: "@live-regression" }, + async () => { + await bootstrapUser(PUBLIC_USER); + await postConfig(PUBLIC_USER, [ + { + imdbUserId: "imdb:top-rated-movies", + sortOption: "added_at-asc", + displayMode: "movie", + }, + ]); + const { body } = await getConfig(PUBLIC_USER); + const chart = body.watchlists[0]; + const { metas } = await getCatalog( + PUBLIC_USER, + "movie", + `wl-${chart.id}-movie`, + ); + // IMDb Top 250 — allow slack for titles Stremio types can't represent. + expect(metas.length).toBeGreaterThan(200); + for (const meta of metas.slice(0, 10)) { + expect(meta.type).toBe("movie"); + expect(meta.id).toMatch(/^tt\d+$/); + } + }, + ); + + test("RPDB key rewrites posters", { tag: "@live-regression" }, async () => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + await postConfig( + PUBLIC_USER, + [ + { + id: watchlist.id, + imdbUserId: watchlist.imdbUserId, + sortOption: "added_at-asc", + }, + ], + "e2e-test-key", + ); + const { metas } = await getCatalog( + PUBLIC_USER, + "movie", + `wl-${watchlist.id}-movie`, + ); + expect(metas.length).toBeGreaterThan(0); + for (const meta of metas) { + expect(meta.poster).toContain( + "https://api.ratingposterdb.com/e2e-test-key/imdb/poster-default/", + ); + } + }); + + test( + "unknown watchlist degrades to an informational card, not a 500", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(UNKNOWN_USER); + const { status, metas } = await getCatalog( + UNKNOWN_USER, + "movie", + `wl-${config.watchlists[0].id}-movie`, + ); + expect(status).toBe(200); + expect(metas).toHaveLength(1); + expect(metas[0].id).toBe("stremlist:unavailable:not_found"); + expect(metas[0].name).toContain("not found"); + }, + ); + + test( + "private watchlist degrades to an informational card", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PRIVATE_USER); + const { status, metas } = await getCatalog( + PRIVATE_USER, + "movie", + `wl-${config.watchlists[0].id}-movie`, + ); + expect(status).toBe(200); + expect(metas).toHaveLength(1); + expect(metas[0].id).toBe("stremlist:unavailable:private"); + expect(metas[0].name).toContain("private"); + }, + ); + + test( + "malformed catalog requests return empty catalogs", + { tag: "@local" }, + async () => { + await bootstrapUser(PUBLIC_USER); + const unknownCatalog = await getCatalog( + PUBLIC_USER, + "movie", + "wl-00000000-0000-4000-8000-000000000000-movie", + ); + expect(unknownCatalog.status).toBe(200); + expect(unknownCatalog.metas).toEqual([]); + + const badType = await getCatalog( + PUBLIC_USER, + "channel", + "stremlist-movies", + ); + expect(badType.status).toBe(200); + expect(badType.metas).toEqual([]); + }, + ); +}); + +test.describe("meta", () => { + test( + "serves cached meta and falls back to null on misses", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + const catalogId = `wl-${config.watchlists[0].id}-movie`; + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + const first = metas[0]; + + const hit = await getMeta(PUBLIC_USER, "movie", first.id); + expect(hit.status).toBe(200); + expect(hit.meta?.name).toBe(first.name); + expect(hit.meta?.id).toBe(first.id); + + // Cache-only: unknown ids must return null (Stremio then asks Cinemeta), + // never 500. + const miss = await getMeta(PUBLIC_USER, "movie", "tt9999999999"); + expect(miss.status).toBe(200); + expect(miss.meta).toBeNull(); + }, + ); +}); + +test.describe("validation endpoints", () => { + test( + "validates public, unknown, and p-handle sources", + { tag: "@live-regression" }, + async () => { + expect(await validateUser(PUBLIC_USER)).toEqual({ + valid: true, + userId: PUBLIC_USER, + }); + expect(await validateUser(UNKNOWN_USER)).toEqual({ + valid: false, + reason: "not_found", + }); + expect(await validateList(PUBLIC_LIST)).toEqual({ valid: true }); + expect(await validateList(UNKNOWN_LIST)).toEqual({ + valid: false, + reason: "not_found", + }); + + // p-handles resolve to a canonical ur id first. The target account's + // watchlist visibility is not under our control, so only assert shape. + const handleResult = await validateUser(P_HANDLE); + if (handleResult.valid) { + expect(String(handleResult.userId)).toMatch(/^ur\d+$/); + } else { + expect(["private", "not_found"]).toContain(handleResult.reason); + } + }, + ); + + test("reports private sources", { tag: "@live-regression" }, async () => { + expect(await validateUser(PRIVATE_USER)).toEqual({ + valid: false, + reason: "private", + }); + // Same account via its p-handle: exercises handle resolution on a + // private source. + expect(await validateUser(PRIVATE_P_HANDLE)).toEqual({ + valid: false, + reason: "private", + }); + if (PRIVATE_LIST) { + expect(await validateList(PRIVATE_LIST)).toEqual({ + valid: false, + reason: "private", + }); + } + }); +}); + +test.describe("config API", () => { + test("rejects invalid configurations", { tag: "@local" }, async () => { + await bootstrapUser(PUBLIC_USER); + const valid = { imdbUserId: PUBLIC_USER, sortOption: "added_at-asc" }; + + expect((await postConfig(PUBLIC_USER, [])).status).toBe(400); + expect( + ( + await postConfig( + PUBLIC_USER, + Array.from({ length: 11 }, () => valid), + ) + ).status, + ).toBe(400); + expect( + ( + await postConfig(PUBLIC_USER, [ + { ...valid, catalogTitle: "x".repeat(31) }, + ]) + ).status, + ).toBe(400); + expect( + (await postConfig(PUBLIC_USER, [{ ...valid, sortOption: "bogus" }])) + .status, + ).toBe(400); + expect((await postConfig(PUBLIC_USER, [valid, valid])).status).toBe(400); + expect( + (await postConfig(PUBLIC_USER, [{ ...valid, imdbUserId: "banana" }])) + .status, + ).toBe(400); + }); + + test("404s for users that never installed", { tag: "@local" }, async () => { + const { status } = await getConfig(PUBLIC_USER); + expect(status).toBe(404); + }); +}); + +test.describe("refresh", () => { + test( + "refreshes from live IMDb and then throttles", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + await clearRefreshCooldown(PUBLIC_USER); + + const first = await refresh(PUBLIC_USER); + expect(first.status).toBe(200); + expect(first.body.ok).toBe(true); + expect(first.body.refreshed).toBe(1); + expect(first.body.failed).toBe(0); + expect(first.body.total).toBe(1); + expect(await countCacheItems(config.watchlists[0].id)).toBeGreaterThan(0); + + const second = await refresh(PUBLIC_USER); + expect(second.body.throttled).toBe(true); + }, + ); +}); + +test.describe("misc endpoints", () => { + test("health, stats, and configure redirect", { tag: "@local" }, async () => { + const health = await fetch(`${BACKEND_URL}/health`); + expect(health.status).toBe(200); + expect(((await health.json()) as { database: string }).database).toBe("up"); + + await bootstrapUser(PUBLIC_USER); + const stats = await fetch(`${BACKEND_URL}/stats`); + expect( + ((await stats.json()) as { activeUsers: number }).activeUsers, + ).toBeGreaterThanOrEqual(1); + + const configure = await fetch(`${BACKEND_URL}/${PUBLIC_USER}/configure`, { + redirect: "manual", + }); + expect(configure.status).toBe(302); + expect(configure.headers.get("location")).toBe( + `${FRONTEND_URL}/configure?userId=${PUBLIC_USER}`, + ); + }); +}); diff --git a/apps/e2e/tests/configure-page.spec.ts b/apps/e2e/tests/configure-page.spec.ts new file mode 100644 index 0000000..5f5e209 --- /dev/null +++ b/apps/e2e/tests/configure-page.spec.ts @@ -0,0 +1,170 @@ +import { expect, test } from "@playwright/test"; +import { FRONTEND_URL } from "../env.js"; +import { bootstrapUser, getConfig } from "../helpers/api.js"; +import { resetDb } from "../helpers/db.js"; +import { + PUBLIC_LIST, + PUBLIC_USER, + UNKNOWN_USER, +} from "../helpers/test-data.js"; + +// The /configure page: catalog management, options, refresh, install links. + +const configureUrl = (userId: string) => + `${FRONTEND_URL}/configure?userId=${userId}`; + +test.beforeEach(async () => { + await resetDb(); +}); + +test( + "loads the existing configuration", + { tag: "@local" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + + await expect(page.getByText("Catalog 1")).toBeVisible(); + await expect( + page.locator(`input[value="${PUBLIC_USER}"]`).first(), + ).toBeVisible(); + await expect( + page.getByRole("link", { name: "Open in Stremio Web" }), + ).toBeVisible(); + }, +); + +test( + "adds an ls list catalog and saves", + { tag: "@local" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + + await page.getByRole("button", { name: "Add Catalog" }).click(); + await expect(page.getByText("Catalog 2")).toBeVisible(); + + const idInputs = page.locator('input[placeholder*="ur12345678"]'); + await idInputs.last().fill(PUBLIC_LIST); + + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + + const { body } = await getConfig(PUBLIC_USER); + expect(body.watchlists).toHaveLength(2); + expect(body.watchlists.map((w) => w.imdbUserId)).toContain(PUBLIC_LIST); + }, +); + +test("adds a built-in chart catalog", { tag: "@local" }, async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + + await page.getByRole("button", { name: "Add Built-in Catalog" }).click(); + await page.getByRole("menuitem", { name: "Top 250 Movies" }).click(); + await expect(page.getByText("Built-in", { exact: true })).toBeVisible(); + + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + + const { body } = await getConfig(PUBLIC_USER); + expect(body.watchlists.map((w) => w.imdbUserId)).toContain( + "imdb:top-rated-movies", + ); +}); + +test( + "changes sort order and content filter", + { tag: "@local" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + + // Radix selects: one combobox for "Sort Order", one for "Show", in DOM order. + await page.getByRole("combobox").nth(0).click(); + await page.getByRole("option", { name: "Highest Rated" }).click(); + await page.getByRole("combobox").nth(1).click(); + await page.getByRole("option", { name: "Movies only" }).click(); + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + + const { body } = await getConfig(PUBLIC_USER); + expect(body.watchlists[0].sortOption).toBe("rating-desc"); + expect(body.watchlists[0].displayMode).toBe("movie"); + }, +); + +test("saves and clears the RPDB key", { tag: "@local" }, async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + + await page.locator("#rpdb-api-key").fill("e2e-rpdb-key"); + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + expect((await getConfig(PUBLIC_USER)).body.rpdbApiKey).toBe("e2e-rpdb-key"); + + await page.locator("#rpdb-api-key").fill(""); + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + expect((await getConfig(PUBLIC_USER)).body.rpdbApiKey).toBeNull(); +}); + +test("removes a catalog", { tag: "@local" }, async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await page.getByRole("button", { name: "Add Built-in Catalog" }).click(); + await page.getByRole("menuitem", { name: "Box Office (Weekend)" }).click(); + await expect(page.getByText("Catalog 2")).toBeVisible(); + + await page.getByLabel("Remove catalog").last().click(); + await expect(page.getByText("Catalog 2")).not.toBeVisible(); + + await page.getByRole("button", { name: "Save", exact: true }).click(); + await expect(page.getByText("Saved!", { exact: false })).toBeVisible(); + expect((await getConfig(PUBLIC_USER)).body.watchlists).toHaveLength(1); +}); + +test( + "manual refresh hits the backend and starts the cooldown", + { tag: "@live-regression" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(configureUrl(PUBLIC_USER)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + + const refreshResponse = page.waitForResponse( + (response) => + response.url().includes("/refresh") && + response.request().method() === "POST", + ); + await page.getByRole("button", { name: /Refresh now|Refresh in/ }).click(); + expect((await refreshResponse).status()).toBe(200); + }, +); + +test( + "unknown user is told to install first", + { tag: "@local" }, + async ({ page }) => { + await page.goto(configureUrl(UNKNOWN_USER)); + await expect( + page.getByText("User not found.", { exact: false }), + ).toBeVisible(); + }, +); + +test( + "without userId, entering an id loads its configuration", + { tag: "@local" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(`${FRONTEND_URL}/configure`); + await page.locator("#imdb-id").fill(PUBLIC_USER); + await expect(page).toHaveURL(new RegExp(`userId=${PUBLIC_USER}`)); + await expect(page.getByText("Catalog 1")).toBeVisible(); + }, +); diff --git a/apps/e2e/tests/home-onboarding.spec.ts b/apps/e2e/tests/home-onboarding.spec.ts new file mode 100644 index 0000000..8ccd944 --- /dev/null +++ b/apps/e2e/tests/home-onboarding.spec.ts @@ -0,0 +1,71 @@ +import { expect, test } from "@playwright/test"; +import { addonManifestUrl, FRONTEND_URL } from "../env.js"; +import { bootstrapUser } from "../helpers/api.js"; +import { resetDb } from "../helpers/db.js"; +import { + PUBLIC_USER, + PUBLIC_USER_2, + UNKNOWN_USER, +} from "../helpers/test-data.js"; + +// First-install flow on the Stremlist home page. + +test.beforeEach(async ({ page }) => { + await resetDb(); + await page.goto(FRONTEND_URL); +}); + +test( + "new user gets install actions after live validation", + { tag: "@live-smoke" }, + async ({ page }) => { + await page.locator("#imdb-id").fill(PUBLIC_USER_2); + + const webInstall = page.getByRole("link", { name: "Open in Stremio Web" }); + await expect(webInstall).toBeVisible(); + await expect(webInstall).toHaveAttribute( + "href", + `https://web.stremio.com/#/addons?addon=${encodeURIComponent(addonManifestUrl(PUBLIC_USER_2))}`, + ); + await expect( + page.getByRole("link", { name: "Open in Stremio Desktop" }), + ).toHaveAttribute( + "href", + `stremio://127.0.0.1:7301/${PUBLIC_USER_2}/manifest.json`, + ); + }, +); + +test( + "returning user is welcomed back", + { tag: "@live-regression" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + await page.locator("#imdb-id").fill(PUBLIC_USER); + await expect(page.getByText(`Welcome back, ${PUBLIC_USER}!`)).toBeVisible(); + }, +); + +test( + "unknown IMDb id shows the not-found error", + { tag: "@live-regression" }, + async ({ page }) => { + await page.locator("#imdb-id").fill(UNKNOWN_USER); + await expect( + page.getByText( + "This IMDb ID does not exist. Please check and try again.", + ), + ).toBeVisible(); + }, +); + +test( + "garbage input shows the format error", + { tag: "@local" }, + async ({ page }) => { + await page.locator("#imdb-id").fill("banana"); + await expect( + page.getByText("Could not find a valid IMDb ID", { exact: false }), + ).toBeVisible(); + }, +); diff --git a/apps/e2e/tests/stremio-catalogs.spec.ts b/apps/e2e/tests/stremio-catalogs.spec.ts new file mode 100644 index 0000000..a37799c --- /dev/null +++ b/apps/e2e/tests/stremio-catalogs.spec.ts @@ -0,0 +1,178 @@ +import { expect, test } from "@playwright/test"; +import { addonManifestUrl } from "../env.js"; +import { + bootstrapUser, + getCatalog, + getConfig, + postConfig, +} from "../helpers/api.js"; +import { resetDb } from "../helpers/db.js"; +import { + discoverItemTitles, + discoverUrl, + installAddon, + uninstallAddon, +} from "../helpers/stremio.js"; +import { + PRIVATE_USER, + PUBLIC_USER, + UNKNOWN_USER, +} from "../helpers/test-data.js"; + +// Catalog rendering inside the real Stremio Web app. Expected content is read +// from the addon's own catalog endpoint in the same run, so assertions stay +// deterministic even though the underlying IMDb data is live. + +test.beforeEach(async () => { + await resetDb(); +}); + +test( + "watchlist catalog renders in Discover, in catalog order", + { tag: "@live-smoke" }, + async ({ page }) => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + const catalogId = `wl-${watchlist.id}-movie`; + const manifestUrl = addonManifestUrl(PUBLIC_USER); + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + expect(metas.length).toBeGreaterThan(0); + + await installAddon(page, manifestUrl); + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + + const rendered = await discoverItemTitles(page); + expect(rendered.length).toBeGreaterThan(0); + const expected = metas.map((meta) => meta.name); + expect(rendered.slice(0, Math.min(5, expected.length))).toEqual( + expected.slice(0, Math.min(5, rendered.length)), + ); + }, +); + +test( + "sort option changes reorder the catalog without reinstalling", + { tag: "@live-regression" }, + async ({ page }) => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + const catalogId = `wl-${watchlist.id}-movie`; + const manifestUrl = addonManifestUrl(PUBLIC_USER); + await getCatalog(PUBLIC_USER, "movie", catalogId); + await installAddon(page, manifestUrl); + + await postConfig(PUBLIC_USER, [ + { + id: watchlist.id, + imdbUserId: watchlist.imdbUserId, + sortOption: "title-asc", + }, + ]); + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + const expected = metas.map((meta) => meta.name); + expect(expected).toEqual([...expected].sort((a, b) => a.localeCompare(b))); + + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + const rendered = await discoverItemTitles(page); + expect(rendered.slice(0, Math.min(5, expected.length))).toEqual( + expected.slice(0, Math.min(5, rendered.length)), + ); + }, +); + +test( + "built-in chart catalog renders after a reinstall", + { tag: "@live-regression" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + const manifestUrl = addonManifestUrl(PUBLIC_USER); + await installAddon(page, manifestUrl); + + // Adding a catalog changes the manifest, which Stremio only picks up on + // reinstall — exactly what the configure page tells the user to do. + await postConfig(PUBLIC_USER, [ + { imdbUserId: PUBLIC_USER, sortOption: "added_at-asc" }, + { + imdbUserId: "imdb:box-office", + sortOption: "added_at-asc", + displayMode: "movie", + }, + ]); + const { body } = await getConfig(PUBLIC_USER); + const chart = body.watchlists.find( + (w) => w.imdbUserId === "imdb:box-office", + ); + expect(chart).toBeDefined(); + const catalogId = `wl-${chart!.id}-movie`; + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + expect(metas.length).toBeGreaterThan(0); + + await uninstallAddon(page, manifestUrl); + await installAddon(page, manifestUrl); + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + + const rendered = await discoverItemTitles(page); + expect(rendered.length).toBeGreaterThan(0); + expect(rendered[0]).toBe(metas[0].name); + }, +); + +test( + "catalog rows appear on the Board", + { tag: "@live-regression" }, + async ({ page }) => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + // Distinctive title so the Board row is unambiguous. + await postConfig(PUBLIC_USER, [ + { + id: watchlist.id, + imdbUserId: watchlist.imdbUserId, + sortOption: "added_at-asc", + catalogTitle: "E2E QA", + }, + ]); + const manifestUrl = addonManifestUrl(PUBLIC_USER); + await getCatalog(PUBLIC_USER, "movie", `wl-${watchlist.id}-movie`); + + await installAddon(page, manifestUrl); + await page.goto("https://web.stremio.com/#/"); + await expect( + page.getByText("Stremlist E2E QA", { exact: false }).first(), + ).toBeAttached({ timeout: 30_000 }); + }, +); + +test( + "broken watchlist shows the informational card in Stremio", + { tag: "@live-regression" }, + async ({ page }) => { + const config = await bootstrapUser(UNKNOWN_USER); + const catalogId = `wl-${config.watchlists[0].id}-movie`; + const manifestUrl = addonManifestUrl(UNKNOWN_USER); + + await installAddon(page, manifestUrl); + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + await expect( + page.getByText("IMDb watchlist not found", { exact: false }).first(), + ).toBeVisible(); + }, +); + +test( + "private watchlist shows the private card in Stremio", + { tag: "@live-regression" }, + async ({ page }) => { + const config = await bootstrapUser(PRIVATE_USER); + const catalogId = `wl-${config.watchlists[0].id}-movie`; + const manifestUrl = addonManifestUrl(PRIVATE_USER); + + await installAddon(page, manifestUrl); + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + await expect( + page + .getByText("This IMDb watchlist is private", { exact: false }) + .first(), + ).toBeVisible(); + }, +); diff --git a/apps/e2e/tests/stremio-install.spec.ts b/apps/e2e/tests/stremio-install.spec.ts new file mode 100644 index 0000000..0151f5e --- /dev/null +++ b/apps/e2e/tests/stremio-install.spec.ts @@ -0,0 +1,65 @@ +import { expect, test } from "@playwright/test"; +import { addonManifestUrl, FRONTEND_URL } from "../env.js"; +import { bootstrapUser } from "../helpers/api.js"; +import { resetDb } from "../helpers/db.js"; +import { + addonsDeepLink, + dismissDesktopAppPrompt, + installAddon, +} from "../helpers/stremio.js"; +import { PUBLIC_USER } from "../helpers/test-data.js"; + +// Install lifecycle inside the real Stremio Web app (anonymous profile — +// a fresh browser context has its own local addon collection). + +test.beforeEach(async () => { + await resetDb(); +}); + +test( + "installs and uninstalls the addon through Stremio Web", + { tag: "@live-smoke" }, + async ({ page }) => { + await bootstrapUser(PUBLIC_USER); + const manifestUrl = addonManifestUrl(PUBLIC_USER); + + await installAddon(page, manifestUrl); + + // Re-opening the deep link on an installed addon offers Uninstall. + await page.goto(addonsDeepLink(manifestUrl)); + await page.reload(); + await dismissDesktopAppPrompt(page); + const uninstall = page.getByText("Uninstall", { exact: true }).last(); + await expect(uninstall).toBeVisible(); + await uninstall.click(); + + // And once uninstalled, the same deep link offers Install again. + await page.goto(addonsDeepLink(manifestUrl)); + await page.reload(); + await dismissDesktopAppPrompt(page); + await expect( + page.getByText("Install", { exact: true }).last(), + ).toBeVisible(); + }, +); + +test( + "configure page links straight into Stremio Web's install dialog", + { tag: "@live-regression" }, + async ({ page, context }) => { + await bootstrapUser(PUBLIC_USER); + await page.goto(`${FRONTEND_URL}/configure?userId=${PUBLIC_USER}`); + + const popupPromise = context.waitForEvent("page"); + await page.getByRole("link", { name: "Open in Stremio Web" }).click(); + const popup = await popupPromise; + await popup.waitForLoadState(); + expect(popup.url()).toBe( + `https://web.stremio.com/#/addons?addon=${encodeURIComponent(addonManifestUrl(PUBLIC_USER))}`, + ); + await dismissDesktopAppPrompt(popup); + await expect( + popup.getByText("Install", { exact: true }).last(), + ).toBeVisible(); + }, +); diff --git a/apps/e2e/tests/stremio-meta.spec.ts b/apps/e2e/tests/stremio-meta.spec.ts new file mode 100644 index 0000000..cff1960 --- /dev/null +++ b/apps/e2e/tests/stremio-meta.spec.ts @@ -0,0 +1,37 @@ +import { expect, test } from "@playwright/test"; +import { addonManifestUrl } from "../env.js"; +import { bootstrapUser, getCatalog } from "../helpers/api.js"; +import { resetDb } from "../helpers/db.js"; +import { discoverUrl, installAddon } from "../helpers/stremio.js"; +import { PUBLIC_USER } from "../helpers/test-data.js"; + +// Clicking through from a Stremlist catalog to a detail page in Stremio Web. + +test.beforeEach(async () => { + await resetDb(); +}); + +test( + "catalog items open their detail page", + { tag: "@live-regression" }, + async ({ page }) => { + const config = await bootstrapUser(PUBLIC_USER); + const watchlist = config.watchlists[0]; + const catalogId = `wl-${watchlist.id}-movie`; + const manifestUrl = addonManifestUrl(PUBLIC_USER); + const { metas } = await getCatalog(PUBLIC_USER, "movie", catalogId); + const first = metas[0]; + + await installAddon(page, manifestUrl); + await page.goto(discoverUrl(manifestUrl, "movie", catalogId)); + + const firstItem = page.locator('a[href^="#/detail/"]').first(); + await expect(firstItem).toBeVisible(); + await firstItem.click(); + + await expect(page).toHaveURL(new RegExp(`#/detail/movie/${first.id}`)); + // The detail page renders the title as a logo image, not text — assert on + // imagery for the exact tt id we clicked (logo/background src embed it). + await expect(page.locator(`img[src*="${first.id}"]`).first()).toBeVisible(); + }, +); diff --git a/apps/e2e/tsconfig.json b/apps/e2e/tsconfig.json new file mode 100644 index 0000000..a0d8688 --- /dev/null +++ b/apps/e2e/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "Bundler", + "strict": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true, + "types": ["node"], + "noEmit": true + }, + "include": ["tests", "fixtures", "helpers", "*.ts"], + "exclude": ["node_modules"] +} diff --git a/package.json b/package.json index 936674c..970b1ab 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "format": "turbo run format", "format:check": "turbo run format:check", "test": "turbo run test", + "test:e2e": "turbo run test:e2e", "typecheck": "turbo run typecheck", "generate:types": "pnpm supabase gen types typescript --project-id igbmxubzxjfdbgmawrtr --schema public > packages/shared/src/database.types.ts", "prepare": "husky" diff --git a/packages/shared/src/stremio.types.ts b/packages/shared/src/stremio.types.ts index 2ddef38..bd68b20 100644 --- a/packages/shared/src/stremio.types.ts +++ b/packages/shared/src/stremio.types.ts @@ -26,7 +26,7 @@ export interface UserConfigUpdateWatchlist { catalogTitle?: string; sortOption: string; displayMode?: DisplayMode; - position: number; + position?: number; } export interface UserConfigUpdatePayload { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a99ea8a..6a29e7a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,6 +82,45 @@ importers: specifier: ^4.0.18 version: 4.0.18(@types/node@20.19.33)(jiti@2.6.1)(lightningcss@1.30.2)(tsx@4.21.0)(yaml@2.8.2) + apps/e2e: + devDependencies: + '@playwright/test': + specifier: ^1.62.1 + version: 1.62.1 + '@stremlist/backend': + specifier: workspace:* + version: link:../backend + '@stremlist/eslint-config': + specifier: workspace:* + version: link:../../packages/eslint-config + '@stremlist/frontend': + specifier: workspace:* + version: link:../frontend + '@stremlist/shared': + specifier: workspace:* + version: link:../../packages/shared + '@supabase/supabase-js': + specifier: ^2.95.3 + version: 2.95.3 + '@types/node': + specifier: ^20.11.17 + version: 20.19.33 + eslint: + specifier: ^9.39.2 + version: 9.39.2(jiti@2.6.1) + eslint-config-prettier: + specifier: ^10.1.8 + version: 10.1.8(eslint@9.39.2(jiti@2.6.1)) + prettier: + specifier: ^3.8.1 + version: 3.8.1 + tsx: + specifier: ^4.7.1 + version: 4.21.0 + typescript: + specifier: ^5.8.3 + version: 5.9.3 + apps/frontend: dependencies: '@dnd-kit/helpers': @@ -782,6 +821,11 @@ packages: '@napi-rs/wasm-runtime@0.2.12': resolution: {integrity: sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==} + '@playwright/test@1.62.1': + resolution: {integrity: sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==} + engines: {node: '>=20'} + hasBin: true + '@preact/signals-core@1.14.0': resolution: {integrity: sha512-AowtCcCU/33lFlh1zRFf/u+12rfrhtNakj7UpaGEsmMwUKpKWMVvcktOGcwBBNiB4lWrZWc01LhiyyzVklJyaQ==} @@ -2054,6 +2098,11 @@ packages: react-dom: optional: true + fsevents@2.3.2: + resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} @@ -2425,6 +2474,16 @@ packages: engines: {node: '>=0.10'} hasBin: true + playwright-core@1.62.1: + resolution: {integrity: sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==} + engines: {node: '>=20'} + hasBin: true + + playwright@1.62.1: + resolution: {integrity: sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==} + engines: {node: '>=20'} + hasBin: true + postal-mime@2.7.3: resolution: {integrity: sha512-MjhXadAJaWgYzevi46+3kLak8y6gbg0ku14O1gO/LNOuay8dO+1PtcSGvAdgDR0DoIsSaiIA8y/Ddw6MnrO0Tw==} @@ -3315,6 +3374,10 @@ snapshots: '@tybys/wasm-util': 0.10.1 optional: true + '@playwright/test@1.62.1': + dependencies: + playwright: 1.62.1 + '@preact/signals-core@1.14.0': {} '@radix-ui/number@1.1.1': {} @@ -4484,6 +4547,9 @@ snapshots: react: 19.2.4 react-dom: 19.2.4(react@19.2.4) + fsevents@2.3.2: + optional: true + fsevents@2.3.3: optional: true @@ -4778,6 +4844,14 @@ snapshots: pidtree@0.6.0: {} + playwright-core@1.62.1: {} + + playwright@1.62.1: + dependencies: + playwright-core: 1.62.1 + optionalDependencies: + fsevents: 2.3.2 + postal-mime@2.7.3: {} postcss@8.5.6: diff --git a/supabase/.gitignore b/supabase/.gitignore new file mode 100644 index 0000000..ad9264f --- /dev/null +++ b/supabase/.gitignore @@ -0,0 +1,8 @@ +# Supabase +.branches +.temp + +# dotenvx +.env.keys +.env.local +.env.*.local diff --git a/supabase/config.toml b/supabase/config.toml new file mode 100644 index 0000000..2cdde7b --- /dev/null +++ b/supabase/config.toml @@ -0,0 +1,408 @@ +# For detailed configuration reference documentation, visit: +# https://supabase.com/docs/guides/local-development/cli/config +# A string used to distinguish different Supabase projects on the same host. Defaults to the +# working directory name when running `supabase init`. +project_id = "stremlist" + +[api] +enabled = true +# Port to use for the API URL. +port = 54321 +# Schemas to expose in your API. Tables, views and stored procedures in this schema will get API +# endpoints. `public` and `graphql_public` schemas are included by default. +schemas = ["public", "graphql_public"] +# Extra schemas to add to the search_path of every request. +extra_search_path = ["public", "extensions"] +# The maximum number of rows returns from a view, table, or stored procedure. Limits payload size +# for accidental or malicious requests. +max_rows = 1000 + +[api.tls] +# Enable HTTPS endpoints locally using a self-signed certificate. +enabled = false +# Paths to self-signed certificate pair. +# cert_path = "../certs/my-cert.pem" +# key_path = "../certs/my-key.pem" + +[db] +# Port to use for the local database URL. +port = 54322 +# Port used by db diff command to initialize the shadow database. +shadow_port = 54320 +# Maximum amount of time to wait for health check when starting the local database. +health_timeout = "2m" +# The database major version to use. This has to be the same as your remote database's. Run `SHOW +# server_version;` on the remote database to check. +major_version = 17 + +[db.pooler] +enabled = false +# Port to use for the local connection pooler. +port = 54329 +# Specifies when a server connection can be reused by other clients. +# Configure one of the supported pooler modes: `transaction`, `session`. +pool_mode = "transaction" +# How many server connections to allow per user/database pair. +default_pool_size = 20 +# Maximum number of client connections allowed. +max_client_conn = 100 + +# [db.vault] +# secret_key = "env(SECRET_VALUE)" + +[db.migrations] +# If disabled, migrations will be skipped during a db push or reset. +enabled = true +# Specifies an ordered list of schema files that describe your database. +# Supports glob patterns relative to supabase directory: "./schemas/*.sql" +schema_paths = [] + +[db.seed] +# If enabled, seeds the database after migrations during a db reset. +enabled = true +# Specifies an ordered list of seed files to load during db reset. +# Supports glob patterns relative to supabase directory: "./seeds/*.sql" +sql_paths = ["./seed.sql"] + +[db.network_restrictions] +# Enable management of network restrictions. +enabled = false +# List of IPv4 CIDR blocks allowed to connect to the database. +# Defaults to allow all IPv4 connections. Set empty array to block all IPs. +allowed_cidrs = ["0.0.0.0/0"] +# List of IPv6 CIDR blocks allowed to connect to the database. +# Defaults to allow all IPv6 connections. Set empty array to block all IPs. +allowed_cidrs_v6 = ["::/0"] + +# Uncomment to reject non-secure connections to the database. +# [db.ssl_enforcement] +# enabled = true + +[realtime] +enabled = true +# Bind realtime via either IPv4 or IPv6. (default: IPv4) +# ip_version = "IPv6" +# The maximum length in bytes of HTTP request headers. (default: 4096) +# max_header_length = 4096 + +[studio] +enabled = true +# Port to use for Supabase Studio. +port = 54323 +# External URL of the API server that frontend connects to. +api_url = "http://127.0.0.1" +# OpenAI API Key to use for Supabase AI in the Supabase Studio. +openai_api_key = "env(OPENAI_API_KEY)" + +# Email testing server. Emails sent with the local dev setup are not actually sent - rather, they +# are monitored, and you can view the emails that would have been sent from the web interface. +[inbucket] +enabled = true +# Port to use for the email testing server web interface. +port = 54324 +# Uncomment to expose additional ports for testing user applications that send emails. +# smtp_port = 54325 +# pop3_port = 54326 +# admin_email = "admin@email.com" +# sender_name = "Admin" + +[storage] +enabled = true +# The maximum file size allowed (e.g. "5MB", "500KB"). +file_size_limit = "50MiB" + +# Uncomment to configure local storage buckets +# [storage.buckets.images] +# public = false +# file_size_limit = "50MiB" +# allowed_mime_types = ["image/png", "image/jpeg"] +# objects_path = "./images" + +# Allow connections via S3 compatible clients +[storage.s3_protocol] +enabled = true + +# Image transformation API is available to Supabase Pro plan. +# [storage.image_transformation] +# enabled = true + +# Store analytical data in S3 for running ETL jobs over Iceberg Catalog +# This feature is only available on the hosted platform. +[storage.analytics] +enabled = false +max_namespaces = 5 +max_tables = 10 +max_catalogs = 2 + +# Analytics Buckets is available to Supabase Pro plan. +# [storage.analytics.buckets.my-warehouse] + +# Store vector embeddings in S3 for large and durable datasets +# This feature is only available on the hosted platform. +[storage.vector] +enabled = false +max_buckets = 10 +max_indexes = 5 + +# Vector Buckets is available to Supabase Pro plan. +# [storage.vector.buckets.documents-openai] + +[auth] +enabled = true +# The base URL of your website. Used as an allow-list for redirects and for constructing URLs used +# in emails. +site_url = "http://127.0.0.1:3000" +# The public URL that Auth serves on. Defaults to the API external URL with `/auth/v1` appended. +# external_url = "" +# A list of *exact* URLs that auth providers are permitted to redirect to post authentication. +additional_redirect_urls = ["https://127.0.0.1:3000"] +# How long tokens are valid for, in seconds. Defaults to 3600 (1 hour), maximum 604,800 (1 week). +jwt_expiry = 3600 +# JWT issuer URL. If not set, defaults to auth.external_url. +# jwt_issuer = "" +# Path to JWT signing key. DO NOT commit your signing keys file to git. +# signing_keys_path = "./signing_keys.json" +# If disabled, the refresh token will never expire. +enable_refresh_token_rotation = true +# Allows refresh tokens to be reused after expiry, up to the specified interval in seconds. +# Requires enable_refresh_token_rotation = true. +refresh_token_reuse_interval = 10 +# Allow/disallow new user signups to your project. +enable_signup = true +# Allow/disallow anonymous sign-ins to your project. +enable_anonymous_sign_ins = false +# Allow/disallow testing manual linking of accounts +enable_manual_linking = false +# Passwords shorter than this value will be rejected as weak. Minimum 6, recommended 8 or more. +minimum_password_length = 6 +# Passwords that do not meet the following requirements will be rejected as weak. Supported values +# are: `letters_digits`, `lower_upper_letters_digits`, `lower_upper_letters_digits_symbols` +password_requirements = "" + +# Configure passkey sign-ins. +# [auth.passkey] +# enabled = false + +# Configure WebAuthn relying party settings (required when passkey is enabled). +# [auth.webauthn] +# rp_display_name = "Supabase" +# rp_id = "localhost" +# rp_origins = ["http://127.0.0.1:3000"] + +[auth.rate_limit] +# Number of emails that can be sent per hour. Requires auth.email.smtp to be enabled. +email_sent = 2 +# Number of SMS messages that can be sent per hour. Requires auth.sms to be enabled. +sms_sent = 30 +# Number of anonymous sign-ins that can be made per hour per IP address. Requires enable_anonymous_sign_ins = true. +anonymous_users = 30 +# Number of sessions that can be refreshed in a 5 minute interval per IP address. +token_refresh = 150 +# Number of sign up and sign-in requests that can be made in a 5 minute interval per IP address (excludes anonymous users). +sign_in_sign_ups = 30 +# Number of OTP / Magic link verifications that can be made in a 5 minute interval per IP address. +token_verifications = 30 +# Number of Web3 logins that can be made in a 5 minute interval per IP address. +web3 = 30 + +# Configure one of the supported captcha providers: `hcaptcha`, `turnstile`. +# [auth.captcha] +# enabled = true +# provider = "hcaptcha" +# secret = "" + +[auth.email] +# Allow/disallow new user signups via email to your project. +enable_signup = true +# If enabled, a user will be required to confirm any email change on both the old, and new email +# addresses. If disabled, only the new email is required to confirm. +double_confirm_changes = true +# If enabled, users need to confirm their email address before signing in. +enable_confirmations = false +# If enabled, users will need to reauthenticate or have logged in recently to change their password. +secure_password_change = false +# Controls the minimum amount of time that must pass before sending another signup confirmation or password reset email. +max_frequency = "1s" +# Number of characters used in the email OTP. +otp_length = 6 +# Number of seconds before the email OTP expires (defaults to 1 hour). +otp_expiry = 3600 + +# Use a production-ready SMTP server +# [auth.email.smtp] +# enabled = true +# host = "smtp.sendgrid.net" +# port = 587 +# user = "apikey" +# pass = "env(SENDGRID_API_KEY)" +# admin_email = "admin@email.com" +# sender_name = "Admin" + +# Uncomment to customize email template +# [auth.email.template.invite] +# subject = "You have been invited" +# content_path = "./supabase/templates/invite.html" + +# Uncomment to customize notification email template +# [auth.email.notification.password_changed] +# enabled = true +# subject = "Your password has been changed" +# content_path = "./templates/password_changed_notification.html" + +[auth.sms] +# Allow/disallow new user signups via SMS to your project. +enable_signup = false +# If enabled, users need to confirm their phone number before signing in. +enable_confirmations = false +# Template for sending OTP to users +template = "Your code is {{ .Code }}" +# Controls the minimum amount of time that must pass before sending another sms otp. +max_frequency = "5s" + +# Use pre-defined map of phone number to OTP for testing. +# [auth.sms.test_otp] +# 4152127777 = "123456" + +# Configure logged in session timeouts. +# [auth.sessions] +# Force log out after the specified duration. +# timebox = "24h" +# Force log out if the user has been inactive longer than the specified duration. +# inactivity_timeout = "8h" + +# This hook runs before a new user is created and allows developers to reject the request based on the incoming user object. +# [auth.hook.before_user_created] +# enabled = true +# uri = "pg-functions://postgres/auth/before-user-created-hook" + +# This hook runs before a token is issued and allows you to add additional claims based on the authentication method used. +# [auth.hook.custom_access_token] +# enabled = true +# uri = "pg-functions:////" + +# Configure one of the supported SMS providers: `twilio`, `twilio_verify`, `messagebird`, `textlocal`, `vonage`. +[auth.sms.twilio] +enabled = false +account_sid = "" +message_service_sid = "" +# DO NOT commit your Twilio auth token to git. Use environment variable substitution instead: +auth_token = "env(SUPABASE_AUTH_SMS_TWILIO_AUTH_TOKEN)" + +# Multi-factor-authentication is available to Supabase Pro plan. +[auth.mfa] +# Control how many MFA factors can be enrolled at once per user. +max_enrolled_factors = 10 + +# Control MFA via App Authenticator (TOTP) +[auth.mfa.totp] +enroll_enabled = false +verify_enabled = false + +# Configure MFA via Phone Messaging +[auth.mfa.phone] +enroll_enabled = false +verify_enabled = false +otp_length = 6 +template = "Your code is {{ .Code }}" +max_frequency = "5s" + +# Configure MFA via WebAuthn +# [auth.mfa.web_authn] +# enroll_enabled = true +# verify_enabled = true + +# Use an external OAuth provider. The full list of providers are: `apple`, `azure`, `bitbucket`, +# `discord`, `facebook`, `github`, `gitlab`, `google`, `keycloak`, `linkedin_oidc`, `notion`, `twitch`, +# `twitter`, `x`, `slack`, `spotify`, `workos`, `zoom`. +[auth.external.apple] +enabled = false +client_id = "" +# DO NOT commit your OAuth provider secret to git. Use environment variable substitution instead: +secret = "env(SUPABASE_AUTH_EXTERNAL_APPLE_SECRET)" +# Overrides the default auth callback URL derived from auth.external_url. +redirect_uri = "" +# Overrides the default auth provider URL. Used to support self-hosted gitlab, single-tenant Azure, +# or any other third-party OIDC providers. +url = "" +# If enabled, the nonce check will be skipped. Required for local sign in with Google auth. +skip_nonce_check = false +# If enabled, it will allow the user to successfully authenticate when the provider does not return an email address. +email_optional = false + +# Allow Solana wallet holders to sign in to your project via the Sign in with Solana (SIWS, EIP-4361) standard. +# You can configure "web3" rate limit in the [auth.rate_limit] section and set up [auth.captcha] if self-hosting. +[auth.web3.solana] +enabled = false + +# Use Firebase Auth as a third-party provider alongside Supabase Auth. +[auth.third_party.firebase] +enabled = false +# project_id = "my-firebase-project" + +# Use Auth0 as a third-party provider alongside Supabase Auth. +[auth.third_party.auth0] +enabled = false +# tenant = "my-auth0-tenant" +# tenant_region = "us" + +# Use AWS Cognito (Amplify) as a third-party provider alongside Supabase Auth. +[auth.third_party.aws_cognito] +enabled = false +# user_pool_id = "my-user-pool-id" +# user_pool_region = "us-east-1" + +# Use Clerk as a third-party provider alongside Supabase Auth. +[auth.third_party.clerk] +enabled = false +# Obtain from https://clerk.com/setup/supabase +# domain = "example.clerk.accounts.dev" + +# OAuth server configuration +[auth.oauth_server] +# Enable OAuth server functionality +enabled = false +# Path for OAuth consent flow UI +authorization_url_path = "/oauth/consent" +# Allow dynamic client registration +allow_dynamic_registration = false + +[edge_runtime] +enabled = true +# Supported request policies: `oneshot`, `per_worker`. +# `per_worker` (default) — enables hot reload during local development. +# `oneshot` — fallback mode if hot reload causes issues (e.g. in large repos or with symlinks). +policy = "per_worker" +# Port to attach the Chrome inspector for debugging edge functions. +inspector_port = 8083 +# The Deno major version to use. +deno_version = 2 + +# [edge_runtime.secrets] +# secret_key = "env(SECRET_VALUE)" + +[analytics] +enabled = true +port = 54327 +# Configure one of the supported backends: `postgres`, `bigquery`. +backend = "postgres" + +# Experimental features may be deprecated any time +[experimental] +# Configures Postgres storage engine to use OrioleDB (S3) +orioledb_version = "" +# Configures S3 bucket URL, eg. .s3-.amazonaws.com +s3_host = "env(S3_HOST)" +# Configures S3 bucket region, eg. us-east-1 +s3_region = "env(S3_REGION)" +# Configures AWS_ACCESS_KEY_ID for S3 bucket +s3_access_key = "env(S3_ACCESS_KEY)" +# Configures AWS_SECRET_ACCESS_KEY for S3 bucket +s3_secret_key = "env(S3_SECRET_KEY)" + +# [experimental.pgdelta] +# When enabled, pg-delta becomes the active engine for supported schema flows. +# enabled = false +# Directory under `supabase/` where declarative files are written. +# declarative_schema_path = "./database" +# JSON string passed through to pg-delta SQL formatting. +# format_options = "{\"keywordCase\":\"upper\",\"indent\":2,\"maxWidth\":80,\"commaStyle\":\"trailing\"}" diff --git a/supabase/migrations/20260219000000_baseline_schema.sql b/supabase/migrations/20260219000000_baseline_schema.sql new file mode 100644 index 0000000..4011b8f --- /dev/null +++ b/supabase/migrations/20260219000000_baseline_schema.sql @@ -0,0 +1,21 @@ +-- Baseline for environments created from scratch (local dev, CI). +-- The production database predates migration tracking: `users` and the legacy +-- `watchlist_cache` blob table were created by hand, so the oldest committed +-- migration (20260220000000_enable_rls) assumes they already exist. Everything +-- here is IF NOT EXISTS so pushing this migration to production is a no-op. + +CREATE TABLE IF NOT EXISTS public.users ( + imdb_user_id text PRIMARY KEY, + is_active boolean NOT NULL DEFAULT true, + last_fetched_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + last_cache_served_at timestamptz +); + +-- Legacy blob cache, unused by the application since watchlist_cache_items. +-- Kept only because later migrations reference it. +CREATE TABLE IF NOT EXISTS public.watchlist_cache ( + imdb_user_id text PRIMARY KEY REFERENCES public.users(imdb_user_id) ON DELETE CASCADE, + cached_data jsonb, + cached_at timestamptz NOT NULL DEFAULT now() +); diff --git a/turbo.json b/turbo.json index d0071d3..03d027b 100644 --- a/turbo.json +++ b/turbo.json @@ -22,6 +22,10 @@ "test": { "dependsOn": ["^build"] }, + "test:e2e": { + "dependsOn": ["^build"], + "cache": false + }, "typecheck": { "dependsOn": ["^build"] } From 648917663dd39307be8032bd328a5673a9b864d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 16:23:49 +0200 Subject: [PATCH 04/15] fix(e2e): address Greptile review findings --- .github/workflows/ci.yml | 2 +- apps/e2e/helpers/db.ts | 5 +++-- apps/e2e/helpers/test-data.ts | 4 ++++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f13c3bf..6a483b0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: node-version: 20 cache: pnpm - - uses: supabase/setup-cli@v1 + - uses: supabase/setup-cli@ab058987d8d6c725971f6cf9d0b5c98467e30bd1 # v1 with: version: 2.98.2 diff --git a/apps/e2e/helpers/db.ts b/apps/e2e/helpers/db.ts index 947ec47..651aea5 100644 --- a/apps/e2e/helpers/db.ts +++ b/apps/e2e/helpers/db.ts @@ -1,6 +1,7 @@ import { createClient } from "@supabase/supabase-js"; import type { Database } from "@stremlist/shared"; import { SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL } from "../env.js"; +import { E2E_USER_IDS } from "./test-data.js"; // Service-role client: bypasses RLS, used only to reset and inspect state // between tests. All functional seeding goes through the backend's own HTTP @@ -10,12 +11,12 @@ export const db = createClient( SUPABASE_SERVICE_ROLE_KEY, ); -/** Delete test users. Foreign-key cascades reset every dependent table. */ +/** Delete only this run's test users. Foreign-key cascades reset their data. */ export async function resetDb(): Promise { const { error } = await db .from("users") .delete() - .neq("imdb_user_id", "__e2e_no_matching_user__"); + .in("imdb_user_id", [...E2E_USER_IDS]); if (error) throw new Error(`resetDb failed: ${error.message}`); } diff --git a/apps/e2e/helpers/test-data.ts b/apps/e2e/helpers/test-data.ts index 297cf57..2a523f3 100644 --- a/apps/e2e/helpers/test-data.ts +++ b/apps/e2e/helpers/test-data.ts @@ -9,6 +9,10 @@ export const PUBLIC_USER = process.env.E2E_IMDB_USER_ID ?? "ur102135398"; // Second public watchlist for multi-catalog scenarios. export const PUBLIC_USER_2 = process.env.E2E_IMDB_USER_ID_2 ?? "ur102551738"; +// Cleanup must stay scoped to identities explicitly owned by this E2E run. +// The local Supabase stack may share a persisted volume with development. +export const E2E_USER_IDS = [PUBLIC_USER, PUBLIC_USER_2] as const; + // Long-standing public IMDb list ("Top 100 Greatest Movies of All Time"). export const PUBLIC_LIST = process.env.E2E_IMDB_LIST_ID ?? "ls055592025"; From 5064df8b93a98ac408ec2a2312cce2286bb7b892 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 16:28:30 +0200 Subject: [PATCH 05/15] fix(e2e): include all persisted test users in cleanup --- apps/e2e/helpers/test-data.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/apps/e2e/helpers/test-data.ts b/apps/e2e/helpers/test-data.ts index 2a523f3..db6e9df 100644 --- a/apps/e2e/helpers/test-data.ts +++ b/apps/e2e/helpers/test-data.ts @@ -9,10 +9,6 @@ export const PUBLIC_USER = process.env.E2E_IMDB_USER_ID ?? "ur102135398"; // Second public watchlist for multi-catalog scenarios. export const PUBLIC_USER_2 = process.env.E2E_IMDB_USER_ID_2 ?? "ur102551738"; -// Cleanup must stay scoped to identities explicitly owned by this E2E run. -// The local Supabase stack may share a persisted volume with development. -export const E2E_USER_IDS = [PUBLIC_USER, PUBLIC_USER_2] as const; - // Long-standing public IMDb list ("Top 100 Greatest Movies of All Time"). export const PUBLIC_LIST = process.env.E2E_IMDB_LIST_ID ?? "ls055592025"; @@ -28,6 +24,15 @@ export const PRIVATE_USER = process.env.E2E_PRIVATE_IMDB_USER_ID ?? "ur198342247"; export const PRIVATE_P_HANDLE = "p.e4ialbfdp3rntdahbslk5yzovm"; +// Cleanup must stay scoped to identities explicitly owned by this E2E run. +// The local Supabase stack may share a persisted volume with development. +export const E2E_USER_IDS = [ + PUBLIC_USER, + PUBLIC_USER_2, + UNKNOWN_USER, + PRIVATE_USER, +] as const; + // No stable private ls list is available; provide one via env to enable the // private-list test. export const PRIVATE_LIST = process.env.E2E_PRIVATE_IMDB_LIST_ID; From 0843152d361c126d6c6252a53bd14faa05ec3148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 16:56:49 +0200 Subject: [PATCH 06/15] test(e2e): run full suite by default --- .github/workflows/ci.yml | 17 +++------------- apps/e2e/README.md | 38 ++++++++++++++++------------------- apps/e2e/env.ts | 2 +- apps/e2e/helpers/db.ts | 5 +---- apps/e2e/helpers/stremio.ts | 4 ---- apps/e2e/package.json | 9 +-------- apps/e2e/playwright.config.ts | 4 +--- apps/e2e/tsconfig.json | 2 +- turbo.json | 1 - 9 files changed, 25 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a483b0..866a45b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,20 +31,9 @@ jobs: run: pnpm turbo run lint build test e2e: - name: E2E (${{ matrix.name }}) - # Local coverage and the small live smoke suite run in separate jobs. Each - # gets its own disposable Supabase stack, and an external outage cannot hide - # failures in the deterministic suite. + name: E2E if: github.event_name == 'pull_request' runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - suite: local - name: Local - - suite: live - name: Live smoke steps: - uses: actions/checkout@v4 @@ -77,13 +66,13 @@ jobs: run: pnpm --filter @stremlist/e2e exec playwright install --with-deps chromium - name: Run E2E tests - run: pnpm --filter @stremlist/e2e test:e2e:${{ matrix.suite }} + run: pnpm --filter @stremlist/e2e test:e2e - name: Upload Playwright report if: failure() uses: actions/upload-artifact@v4 with: - name: playwright-report-${{ matrix.suite }} + name: playwright-report path: apps/e2e/playwright-report retention-days: 7 diff --git a/apps/e2e/README.md b/apps/e2e/README.md index cd2e539..430e99e 100644 --- a/apps/e2e/README.md +++ b/apps/e2e/README.md @@ -22,9 +22,8 @@ stack**. The configure/onboarding pages of the frontend are covered too. - IMDb is live. Assertions are structural (ordering invariants, id shapes, counts) or compare the Stremio UI against the addon's own catalog JSON from the same run, so they do not depend on what is in the watchlist today. -- The default run contains deterministic local coverage plus four live smoke - tests. Broader live checks stay in the `live-regression` project and run only - when requested. +- The default run and pull request CI execute all three projects: deterministic + local coverage, four live smoke tests, and the broader live regression suite. ## Running locally @@ -32,16 +31,13 @@ stack**. The configure/onboarding pages of the frontend are covered too. # One-time / per boot: start the local Supabase stack (needs Docker running) supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta -# From the repo root: local coverage plus the small live smoke suite +# From the repo root: run every E2E project pnpm test:e2e -# Or from this package, with more options -pnpm --filter @stremlist/e2e test:e2e:local # deterministic local suite -pnpm --filter @stremlist/e2e test:e2e:live # four live smoke tests -pnpm --filter @stremlist/e2e test:e2e:live:full # all live external checks -pnpm --filter @stremlist/e2e test:e2e:all # every project -pnpm --filter @stremlist/e2e test:e2e:headed # smoke tests with a browser -pnpm --filter @stremlist/e2e report # open the last HTML report +# Select one project while debugging +pnpm --filter @stremlist/e2e test:e2e --project=local +pnpm --filter @stremlist/e2e test:e2e --project=live-smoke +pnpm --filter @stremlist/e2e test:e2e --project=live-regression ``` The suite deletes test users between cases. Foreign-key cascades clear their @@ -51,18 +47,18 @@ confirmation described below. ## Environment knobs -| Variable | Purpose | -| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| `E2E_SUPABASE_URL` / `E2E_SUPABASE_SERVICE_ROLE_KEY` | Non-default local Supabase stack | -| `E2E_ALLOW_REMOTE_DATABASE=I_UNDERSTAND_THIS_WIPES_DATA` | Permit an isolated remote test project. Cleanup deletes every user and all dependent data | -| `E2E_IMDB_USER_ID` / `E2E_IMDB_USER_ID_2` | Override the public watchlists under test | -| `E2E_IMDB_LIST_ID` | Override the public `ls` list under test | -| `E2E_PRIVATE_IMDB_USER_ID` | Override the private watchlist under test (default: a maintainer-owned account kept private on purpose) | -| `E2E_PRIVATE_IMDB_LIST_ID` | Enable the private `ls` list test (skipped otherwise — no stable private list id is available) | +| Variable | Purpose | +| -------------------------------------------------------- | ----------------------------------------------------------------------------------------- | +| `E2E_SUPABASE_URL` / `E2E_SUPABASE_SERVICE_ROLE_KEY` | Non-default local Supabase stack | +| `E2E_ALLOW_REMOTE_DATABASE=I_UNDERSTAND_THIS_WIPES_DATA` | Permit an isolated remote test project. Cleanup deletes every user and all dependent data | +| `E2E_IMDB_USER_ID` / `E2E_IMDB_USER_ID_2` | Override the public watchlists under test | +| `E2E_IMDB_LIST_ID` | Override the public `ls` list under test | +| `E2E_PRIVATE_IMDB_USER_ID` | Override the private watchlist under test | +| `E2E_PRIVATE_IMDB_LIST_ID` | Enable the private `ls` list test | ## Known limitations - Drag-and-drop catalog reordering (pointer-based dnd-kit) is not covered. - The newsletter endpoint is not covered (it would email real people). -- The live smoke and full regression suites depend on web.stremio.com and IMDb. - CI runs only the four smoke checks and retries them twice on failure. +- The live smoke and regression suites depend on web.stremio.com and IMDb. CI + retries failures twice. diff --git a/apps/e2e/env.ts b/apps/e2e/env.ts index fd44750..fcbec1d 100644 --- a/apps/e2e/env.ts +++ b/apps/e2e/env.ts @@ -20,7 +20,7 @@ export const SUPABASE_SERVICE_ROLE_KEY = const REMOTE_DATABASE_CONFIRMATION = "I_UNDERSTAND_THIS_WIPES_DATA"; -export function assertSafeSupabaseTarget(): void { +function assertSafeSupabaseTarget(): void { let hostname: string; try { hostname = new URL(SUPABASE_URL).hostname.toLowerCase(); diff --git a/apps/e2e/helpers/db.ts b/apps/e2e/helpers/db.ts index 651aea5..54e6214 100644 --- a/apps/e2e/helpers/db.ts +++ b/apps/e2e/helpers/db.ts @@ -6,10 +6,7 @@ import { E2E_USER_IDS } from "./test-data.js"; // Service-role client: bypasses RLS, used only to reset and inspect state // between tests. All functional seeding goes through the backend's own HTTP // API so the tests exercise real code paths. -export const db = createClient( - SUPABASE_URL, - SUPABASE_SERVICE_ROLE_KEY, -); +const db = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY); /** Delete only this run's test users. Foreign-key cascades reset their data. */ export async function resetDb(): Promise { diff --git a/apps/e2e/helpers/stremio.ts b/apps/e2e/helpers/stremio.ts index 50d0231..dd100aa 100644 --- a/apps/e2e/helpers/stremio.ts +++ b/apps/e2e/helpers/stremio.ts @@ -18,10 +18,6 @@ export function discoverUrl( return `${STREMIO_WEB_URL}/#/discover/${encodeURIComponent(manifestUrl)}/${type}/${encodeURIComponent(catalogId)}`; } -export function detailUrl(type: "movie" | "series", metaId: string): string { - return `${STREMIO_WEB_URL}/#/detail/${type}/${metaId}/${metaId}`; -} - /** * Dismiss the "install the desktop app" prompt if it is showing. Its "Install" * link would otherwise collide with the addon modal's Install button. diff --git a/apps/e2e/package.json b/apps/e2e/package.json index f80452a..6ca7054 100644 --- a/apps/e2e/package.json +++ b/apps/e2e/package.json @@ -3,14 +3,7 @@ "private": true, "type": "module", "scripts": { - "preflight": "tsx preflight.ts", - "test:e2e": "pnpm preflight && playwright test --project=local --project=live-smoke", - "test:e2e:local": "pnpm preflight && playwright test --project=local", - "test:e2e:live": "pnpm preflight && playwright test --project=live-smoke", - "test:e2e:live:full": "pnpm preflight && playwright test --project=live-smoke --project=live-regression", - "test:e2e:all": "pnpm preflight && playwright test", - "test:e2e:headed": "pnpm preflight && playwright test --project=live-smoke --headed", - "report": "playwright show-report", + "test:e2e": "tsx preflight.ts && playwright test", "typecheck": "tsc --noEmit", "lint": "eslint .", "format": "prettier . --write", diff --git a/apps/e2e/playwright.config.ts b/apps/e2e/playwright.config.ts index 04fe807..36bccb6 100644 --- a/apps/e2e/playwright.config.ts +++ b/apps/e2e/playwright.config.ts @@ -11,8 +11,7 @@ import { export default defineConfig({ testDir: "./tests", - // Tests within a project share one backend and one database. CI runs local - // and live-smoke projects as separate jobs with isolated Supabase stacks. + // Every project shares one backend and database, so tests run serially. fullyParallel: false, workers: 1, forbidOnly: !!process.env.CI, @@ -69,7 +68,6 @@ export default defineConfig({ SUPABASE_SERVICE_ROLE_KEY, FRONTEND_URL, REFRESH_COOLDOWN_SECONDS: String(REFRESH_COOLDOWN_SECONDS), - CACHE_TTL_MINUTES: "30", // The Resend SDK throws at import time without a key. Newsletter // delivery is deliberately out of E2E scope (it would email real // people), so a dummy key is enough to boot the app. diff --git a/apps/e2e/tsconfig.json b/apps/e2e/tsconfig.json index a0d8688..32ca233 100644 --- a/apps/e2e/tsconfig.json +++ b/apps/e2e/tsconfig.json @@ -9,6 +9,6 @@ "types": ["node"], "noEmit": true }, - "include": ["tests", "fixtures", "helpers", "*.ts"], + "include": ["tests", "helpers", "*.ts"], "exclude": ["node_modules"] } diff --git a/turbo.json b/turbo.json index 03d027b..f1c48f2 100644 --- a/turbo.json +++ b/turbo.json @@ -23,7 +23,6 @@ "dependsOn": ["^build"] }, "test:e2e": { - "dependsOn": ["^build"], "cache": false }, "typecheck": { From c51b3e9c892ca432be70e63f8158d5cd55e73c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 17:14:47 +0200 Subject: [PATCH 07/15] feat(cache): move watchlist cache to Cloudflare R2 --- README.md | 34 +- apps/backend/package.json | 1 + .../src/__tests__/catalog-fallback.test.ts | 116 +++++- .../__tests__/helpers/mock-watchlist-cache.ts | 73 ++++ apps/backend/src/__tests__/meta-cache.test.ts | 23 +- apps/backend/src/__tests__/refresh.test.ts | 67 ++-- .../src/__tests__/watchlist-crud.test.ts | 4 + apps/backend/src/lib/r2.ts | 31 ++ apps/backend/src/routes/catalog.ts | 41 +- .../src/scripts/cleanup-invalid-users.ts | 25 +- .../__tests__/catalog-routing.test.ts | 21 +- .../__tests__/watchlist-cache.test.ts | 208 ++++++++++ apps/backend/src/services/stremio-catalogs.ts | 2 + apps/backend/src/services/user.ts | 13 + apps/backend/src/services/watchlist-cache.ts | 370 ++++++++++++++++++ apps/backend/src/services/watchlist.ts | 230 +++-------- apps/backend/src/utils.ts | 25 +- docs/r2-cache-migration.md | 68 ++++ packages/shared/package.json | 3 - .../shared/src/database.types.extended.ts | 35 -- packages/shared/src/database.types.ts | 35 -- packages/shared/src/index.ts | 2 +- packages/shared/src/stremio.types.ts | 4 + pnpm-lock.yaml | 312 ++++++++++++++- ...60826000000_drop_cache_tables_after_r2.sql | 6 + 25 files changed, 1394 insertions(+), 355 deletions(-) create mode 100644 apps/backend/src/__tests__/helpers/mock-watchlist-cache.ts create mode 100644 apps/backend/src/lib/r2.ts create mode 100644 apps/backend/src/services/__tests__/watchlist-cache.test.ts create mode 100644 apps/backend/src/services/watchlist-cache.ts create mode 100644 docs/r2-cache-migration.md delete mode 100644 packages/shared/src/database.types.extended.ts create mode 100644 supabase/migrations/20260826000000_drop_cache_tables_after_r2.sql diff --git a/README.md b/README.md index 43cb578..a9549e7 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Stremlist is a Stremio addon that turns your IMDb watchlist into a Stremio catal - Optional Rating Poster Database (RPDB) poster support via API key - Simple install flow through a hosted configuration UI - Cache-first watchlist serving with periodic auto-refresh and a manual "Refresh now" control -- Lightweight backend with Supabase for user management and watchlist caching +- Lightweight backend with Supabase for user configuration and Cloudflare R2 for watchlist caching - Monorepo architecture with Turborepo (`apps` + `packages`) ## Monorepo Structure @@ -34,7 +34,8 @@ This repository follows the Turborepo recommended structure: - Frontend and backend are deployed on [Vercel](https://vercel.com) - Backend serves Stremio addon endpoints and configuration flow -- Supabase stores user configuration and cached watchlist data +- Supabase stores user configuration +- Cloudflare R2 stores gzip-compressed watchlist cache objects ## Getting Started @@ -106,16 +107,20 @@ http://localhost:7001/manifest.json Set backend env vars in `apps/backend/.env`. -| Variable | Required | Description | Default | -| --- | --- | --- | --- | -| `PORT` | No | Backend HTTP port | `7001` | -| `FRONTEND_URL` | No | URL used for `/:userId/configure` redirect | `https://stremlist.com` | -| `SUPABASE_URL` | Yes | Supabase project URL | - | -| `SUPABASE_SERVICE_ROLE_KEY` | Yes | Supabase service role key | - | -| `CACHE_TTL_MINUTES` | No | How long a cached watchlist is served before it is refreshed on the next request | `30` | -| `REFRESH_COOLDOWN_SECONDS` | No | Minimum time between manual "Refresh now" requests per user | `60` | -| `RESEND_API_KEY` | No | Resend API key for newsletter subscription endpoint | - | -| `RESEND_AUDIENCE_ID` | No | Resend audience ID for newsletter subscription endpoint | - | +| Variable | Required | Description | Default | +| --------------------------- | -------- | -------------------------------------------------------------------------------- | ----------------------- | +| `PORT` | No | Backend HTTP port | `7001` | +| `FRONTEND_URL` | No | URL used for `/:userId/configure` redirect | `https://stremlist.com` | +| `SUPABASE_URL` | Yes | Supabase project URL | - | +| `SUPABASE_SERVICE_ROLE_KEY` | Yes | Supabase service role key | - | +| `R2_ACCOUNT_ID` | Yes | Cloudflare account ID used by the R2 S3 endpoint | - | +| `R2_ACCESS_KEY_ID` | Yes | Bucket-scoped R2 API token access key | - | +| `R2_SECRET_ACCESS_KEY` | Yes | Bucket-scoped R2 API token secret | - | +| `R2_BUCKET` | Yes | Private R2 cache bucket name | - | +| `CACHE_TTL_MINUTES` | No | How long a cached watchlist is served before it is refreshed on the next request | `30` | +| `REFRESH_COOLDOWN_SECONDS` | No | Minimum time between manual "Refresh now" requests per user | `60` | +| `RESEND_API_KEY` | No | Resend API key for newsletter subscription endpoint | - | +| `RESEND_AUDIENCE_ID` | No | Resend audience ID for newsletter subscription endpoint | - | ## Type Generation @@ -127,10 +132,13 @@ pnpm generate:types This updates `packages/shared/src/database.types.ts`. +The production R2 rollout and cleanup procedure is documented in +[`docs/r2-cache-migration.md`](docs/r2-cache-migration.md). + ## License ISC ## Disclaimer -This project is not affiliated with IMDb or Stremio. \ No newline at end of file +This project is not affiliated with IMDb or Stremio. diff --git a/apps/backend/package.json b/apps/backend/package.json index 48a5bdb..4c3c614 100644 --- a/apps/backend/package.json +++ b/apps/backend/package.json @@ -24,6 +24,7 @@ "cleanup:invalid-users": "tsx src/scripts/cleanup-invalid-users.ts" }, "dependencies": { + "@aws-sdk/client-s3": "^3.1118.0", "@hono/zod-validator": "^0.4.3", "@stremlist/shared": "workspace:*", "@supabase/supabase-js": "^2.95.3", diff --git a/apps/backend/src/__tests__/catalog-fallback.test.ts b/apps/backend/src/__tests__/catalog-fallback.test.ts index b26f757..6478dd9 100644 --- a/apps/backend/src/__tests__/catalog-fallback.test.ts +++ b/apps/backend/src/__tests__/catalog-fallback.test.ts @@ -1,9 +1,14 @@ +import type { StremioMeta } from "@stremlist/shared"; import { describe, it, expect, beforeEach, vi } from "vitest"; vi.mock("../lib/supabase", async () => { return await import("./helpers/mock-supabase.js"); }); +vi.mock("../services/watchlist-cache", async () => { + return await import("./helpers/mock-watchlist-cache.js"); +}); + vi.mock("../lib/resend", () => ({ resend: { contacts: { create: vi.fn() } }, })); @@ -11,6 +16,7 @@ vi.mock("../lib/resend", () => ({ import app from "../index.js"; import * as scraper from "../services/imdb-scraper"; import { db } from "./helpers/mock-supabase.js"; +import { cache } from "./helpers/mock-watchlist-cache.js"; const OWNER = "ur216216210"; const UUID_1 = "6bde5e3d-617f-4912-950a-2f9acf815b7e"; @@ -26,13 +32,13 @@ function seedUser(imdbUserId: string) { }); } -function seedWatchlist(id: string) { +function seedWatchlist(id: string, sortOption = "added_at-asc") { db.getTable("user_watchlists").push({ id, owner_user_id: OWNER, imdb_user_id: OWNER, catalog_title: "", - sort_option: "added_at-asc", + sort_option: sortOption, position: 0, created_at: new Date().toISOString(), updated_at: new Date().toISOString(), @@ -41,25 +47,14 @@ function seedWatchlist(id: string) { function seedCache( watchlistId: string, - metas: { id: string; type: string }[], + metas: StremioMeta[], cachedAt?: string, ) { - // An empty `metas` seeds zero rows — the normalised equivalent of an empty - // blob: the next read sees no rows and treats it as a cache miss. - const at = cachedAt ?? new Date().toISOString(); - metas.forEach((meta, i) => { - db.getTable("watchlist_cache_items").push({ - watchlist_id: watchlistId, - item_id: meta.id, - type: meta.type, - position: i, - data: meta, - cached_at: at, - }); - }); + if (metas.length === 0) return; + cache.seed(watchlistId, metas, cachedAt ? new Date(cachedAt) : new Date()); } -const CACHED_MOVIE = { +const CACHED_MOVIE: StremioMeta = { id: "tt0111161", type: "movie", name: "The Shawshank Redemption", @@ -86,6 +81,7 @@ function requestMovieCatalog() { beforeEach(() => { db.reset(); + cache.reset(); vi.restoreAllMocks(); }); @@ -173,3 +169,89 @@ describe("catalog route degrades gracefully on fetch failure", () => { expect((await res.json()) as CatalogResponse).toEqual({ metas: [] }); }); }); + +describe("catalog pagination", () => { + it("serves Stremio pages of at most 100 items using the skip extra", async () => { + seedUser(OWNER); + seedWatchlist(UUID_1); + seedCache( + UUID_1, + Array.from( + { length: 205 }, + (_, index): StremioMeta => ({ + ...CACHED_MOVIE, + id: `tt${String(index).padStart(7, "0")}`, + name: `Movie ${index}`, + }), + ), + ); + + const first = await requestMovieCatalog(); + const second = await app.request( + `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`, + ); + const last = await app.request( + `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=200.json`, + ); + + const firstBody = (await first.json()) as CatalogResponse; + const secondBody = (await second.json()) as CatalogResponse; + const lastBody = (await last.json()) as CatalogResponse; + expect(firstBody.metas).toHaveLength(100); + expect(secondBody.metas).toHaveLength(100); + expect(lastBody.metas).toHaveLength(5); + expect(firstBody.metas[0].id).toBe("tt0000000"); + expect(secondBody.metas[0].id).toBe("tt0000100"); + expect(lastBody.metas[0].id).toBe("tt0000200"); + expect(first.headers.get("Cache-Control")).toBe("no-store"); + expect(first.headers.get("Vercel-CDN-Cache-Control")).toBeNull(); + }); + + it("keeps random pages stable and non-overlapping within a cache generation", async () => { + seedUser(OWNER); + seedWatchlist(UUID_1, "random"); + seedCache( + UUID_1, + Array.from( + { length: 205 }, + (_, index): StremioMeta => ({ + ...CACHED_MOVIE, + id: `tt${String(index).padStart(7, "0")}`, + name: `Movie ${index}`, + }), + ), + ); + + const first = await requestMovieCatalog(); + const second = await app.request( + `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=100.json`, + ); + const repeatedFirst = await requestMovieCatalog(); + + const firstIds = ((await first.json()) as CatalogResponse).metas.map( + (meta) => meta.id, + ); + const secondIds = ((await second.json()) as CatalogResponse).metas.map( + (meta) => meta.id, + ); + const repeatedFirstIds = ( + (await repeatedFirst.json()) as CatalogResponse + ).metas.map((meta) => meta.id); + + expect(repeatedFirstIds).toEqual(firstIds); + expect(new Set([...firstIds, ...secondIds]).size).toBe(200); + }); + + it("rejects an invalid skip value", async () => { + seedUser(OWNER); + seedWatchlist(UUID_1); + seedCache(UUID_1, [CACHED_MOVIE]); + + const res = await app.request( + `/${OWNER}/catalog/movie/wl-${UUID_1}-movie/skip=wat.json`, + ); + + expect(res.status).toBe(400); + expect((await res.json()) as CatalogResponse).toEqual({ metas: [] }); + }); +}); diff --git a/apps/backend/src/__tests__/helpers/mock-watchlist-cache.ts b/apps/backend/src/__tests__/helpers/mock-watchlist-cache.ts new file mode 100644 index 0000000..a40462c --- /dev/null +++ b/apps/backend/src/__tests__/helpers/mock-watchlist-cache.ts @@ -0,0 +1,73 @@ +import type { StremioMeta, WatchlistData } from "@stremlist/shared"; + +interface Entry { + data: WatchlistData; + cachedAt: Date; + generation: string; +} + +class InMemoryWatchlistCache { + private entries = new Map(); + + reset(): void { + this.entries.clear(); + } + + seed(watchlistId: string, metas: StremioMeta[], cachedAt = new Date()): void { + this.entries.set(watchlistId, { + data: { metas: structuredClone(metas) }, + cachedAt, + generation: `${watchlistId}:${cachedAt.toISOString()}`, + }); + } + + get(watchlistId: string): Entry | null { + return this.entries.get(watchlistId) ?? null; + } + + delete(watchlistId: string): void { + this.entries.delete(watchlistId); + } +} + +export const cache = new InMemoryWatchlistCache(); + +export function getCachedWatchlist(watchlistId: string): Promise { + return Promise.resolve(cache.get(watchlistId)); +} + +export function writeCachedWatchlist( + watchlistId: string, + watchlistData: WatchlistData, + cachedAt = new Date(), +): Promise { + const seen = new Set(); + const metas = watchlistData.metas.filter((meta) => { + const key = `${meta.type}:${meta.id}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + if (metas.length === 0) cache.delete(watchlistId); + else cache.seed(watchlistId, metas, cachedAt); + return Promise.resolve(`${watchlistId}:${cachedAt.toISOString()}`); +} + +export function findCachedMeta( + watchlistIds: string[], + type: string, + id: string, +): Promise { + for (const watchlistId of watchlistIds) { + const found = cache + .get(watchlistId) + ?.data.metas.find((meta) => meta.type === type && meta.id === id); + if (found) return Promise.resolve(found); + } + return Promise.resolve(null); +} + +export function deleteCachedWatchlist(watchlistId: string): Promise { + cache.delete(watchlistId); + return Promise.resolve(); +} diff --git a/apps/backend/src/__tests__/meta-cache.test.ts b/apps/backend/src/__tests__/meta-cache.test.ts index 5bae7ca..d3ab2be 100644 --- a/apps/backend/src/__tests__/meta-cache.test.ts +++ b/apps/backend/src/__tests__/meta-cache.test.ts @@ -1,3 +1,4 @@ +import type { StremioMeta } from "@stremlist/shared"; import { describe, it, expect, beforeEach, vi } from "vitest"; import app from "../index.js"; @@ -6,12 +7,17 @@ vi.mock("../lib/supabase", async () => { return await import("./helpers/mock-supabase.js"); }); +vi.mock("../services/watchlist-cache", async () => { + return await import("./helpers/mock-watchlist-cache.js"); +}); + vi.mock("../lib/resend", () => ({ resend: { contacts: { create: vi.fn() } }, })); import * as watchlistSvc from "../services/watchlist"; import { db } from "./helpers/mock-supabase.js"; +import { cache } from "./helpers/mock-watchlist-cache.js"; // --------------------------------------------------------------------------- // Helpers @@ -46,23 +52,13 @@ function seedWatchlist(id: string, imdbUserId = OWNER) { function seedCache( watchlistId: string, - metas: { id: string; type: string }[], + metas: StremioMeta[], cachedAt?: string, ) { - const at = cachedAt ?? new Date().toISOString(); - metas.forEach((meta, i) => { - db.getTable("watchlist_cache_items").push({ - watchlist_id: watchlistId, - item_id: meta.id, - type: meta.type, - position: i, - data: meta, - cached_at: at, - }); - }); + cache.seed(watchlistId, metas, cachedAt ? new Date(cachedAt) : new Date()); } -const SHAWSHANK = { +const SHAWSHANK: StremioMeta = { id: "tt0111161", type: "movie", name: "The Shawshank Redemption", @@ -82,6 +78,7 @@ interface MetaResponse { beforeEach(() => { db.reset(); + cache.reset(); vi.restoreAllMocks(); }); diff --git a/apps/backend/src/__tests__/refresh.test.ts b/apps/backend/src/__tests__/refresh.test.ts index cf174d1..c3d6513 100644 --- a/apps/backend/src/__tests__/refresh.test.ts +++ b/apps/backend/src/__tests__/refresh.test.ts @@ -5,6 +5,10 @@ vi.mock("../lib/supabase", async () => { return await import("./helpers/mock-supabase.js"); }); +vi.mock("../services/watchlist-cache", async () => { + return await import("./helpers/mock-watchlist-cache.js"); +}); + vi.mock("../lib/resend", () => ({ resend: { contacts: { create: vi.fn() } }, })); @@ -12,6 +16,7 @@ vi.mock("../lib/resend", () => ({ import app from "../index.js"; import * as scraper from "../services/imdb-scraper"; import { db } from "./helpers/mock-supabase.js"; +import { cache } from "./helpers/mock-watchlist-cache.js"; const OWNER = "ur216216210"; const UUID_1 = "6bde5e3d-617f-4912-950a-2f9acf815b7e"; @@ -44,17 +49,7 @@ function seedWatchlist(id: string) { } function seedCache(watchlistId: string, metas: { id: string; type: string }[]) { - const at = new Date().toISOString(); - metas.forEach((meta, i) => { - db.getTable("watchlist_cache_items").push({ - watchlist_id: watchlistId, - item_id: meta.id, - type: meta.type, - position: i, - data: meta, - cached_at: at, - }); - }); + cache.seed(watchlistId, metas as StremioMeta[]); } const CACHED_MOVIE: StremioMeta = { @@ -83,6 +78,7 @@ function requestRefresh() { beforeEach(() => { db.reset(); + cache.reset(); vi.restoreAllMocks(); }); @@ -125,29 +121,19 @@ describe("manual refresh reports honest success/failure counts", () => { expect(body.lastFetchedAt).not.toBe(TEN_MINUTES_AGO); }); - it("prunes the previous generation, leaving only fresh non-duplicated rows", async () => { + it("replaces the previous cache with fresh non-duplicated items", async () => { seedUser(TEN_MINUTES_AGO); seedWatchlist(UUID_1); - // Previous generation: an item that gets dropped (OLD) and one that stays - // (SHARED). Seed them with an older timestamp so the prune sees them as stale. + // Previous cache: an item that gets dropped (OLD) and one that stays + // (SHARED). Seed them with an older timestamp so a refresh is required. const anHourAgo = new Date(Date.now() - 60 * 60 * 1000).toISOString(); - db.getTable("watchlist_cache_items").push( - { - watchlist_id: UUID_1, - item_id: "tt0000001", - type: "movie", - position: 0, - data: { id: "tt0000001", type: "movie" }, - cached_at: anHourAgo, - }, - { - watchlist_id: UUID_1, - item_id: "tt0111161", - type: "movie", - position: 1, - data: { id: "tt0111161", type: "movie" }, - cached_at: anHourAgo, - }, + cache.seed( + UUID_1, + [ + { ...CACHED_MOVIE, id: "tt0000001" }, + { ...CACHED_MOVIE, id: "tt0111161" }, + ], + new Date(anHourAgo), ); // Fresh fetch keeps SHARED (tt0111161) and adds NEW (tt0000002); OLD is gone. const NEW: StremioMeta = { @@ -166,20 +152,17 @@ describe("manual refresh reports honest success/failure counts", () => { const res = await requestRefresh(); expect(res.status).toBe(200); - const rows = db - .getTable("watchlist_cache_items") - .filter((r) => r.watchlist_id === UUID_1); - const ids = rows.map((r) => r.item_id).sort(); + const ids = (cache.get(UUID_1)?.data.metas ?? []) + .map((meta) => meta.id) + .sort(); // OLD dropped, SHARED kept once (not duplicated), NEW added. expect(ids).toEqual(["tt0000002", "tt0111161"]); }); - it("de-duplicates repeated ids before caching (no ON CONFLICT failure)", async () => { + it("de-duplicates repeated ids before caching", async () => { seedUser(TEN_MINUTES_AGO); seedWatchlist(UUID_1); - // IMDb lists aren't guaranteed sets: this one carries tt0111161 twice. The - // cache write must dedupe, otherwise the upsert fails with - // "ON CONFLICT DO UPDATE command cannot affect row a second time". + // IMDb lists aren't guaranteed sets: this one carries tt0111161 twice. const GODFATHER: StremioMeta = { ...CACHED_MOVIE, id: "tt0068646", @@ -196,10 +179,8 @@ describe("manual refresh reports honest success/failure counts", () => { expect(body.refreshed).toBe(1); expect(body.failed).toBe(0); // tt0111161 stored exactly once (first occurrence kept), alongside the other. - const ids = db - .getTable("watchlist_cache_items") - .filter((r) => r.watchlist_id === UUID_1) - .map((r) => r.item_id) + const ids = (cache.get(UUID_1)?.data.metas ?? []) + .map((meta) => meta.id) .sort(); expect(ids).toEqual(["tt0068646", "tt0111161"]); }); diff --git a/apps/backend/src/__tests__/watchlist-crud.test.ts b/apps/backend/src/__tests__/watchlist-crud.test.ts index c46b488..ea6a435 100644 --- a/apps/backend/src/__tests__/watchlist-crud.test.ts +++ b/apps/backend/src/__tests__/watchlist-crud.test.ts @@ -8,6 +8,10 @@ vi.mock("../lib/supabase", async () => { return await import("./helpers/mock-supabase.js"); }); +vi.mock("../services/watchlist-cache", async () => { + return await import("./helpers/mock-watchlist-cache.js"); +}); + vi.mock("../lib/resend", () => ({ resend: { contacts: { create: vi.fn() } }, })); diff --git a/apps/backend/src/lib/r2.ts b/apps/backend/src/lib/r2.ts new file mode 100644 index 0000000..d71118a --- /dev/null +++ b/apps/backend/src/lib/r2.ts @@ -0,0 +1,31 @@ +import { S3Client } from "@aws-sdk/client-s3"; + +let client: S3Client | null = null; + +function requiredEnv(name: string): string { + const value = process.env[name]; + if (!value) { + throw new Error(`Missing ${name} environment variable`); + } + return value; +} + +export function getR2Client(): S3Client { + if (client) return client; + + const accountId = requiredEnv("R2_ACCOUNT_ID"); + client = new S3Client({ + region: "auto", + endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + credentials: { + accessKeyId: requiredEnv("R2_ACCESS_KEY_ID"), + secretAccessKey: requiredEnv("R2_SECRET_ACCESS_KEY"), + }, + }); + + return client; +} + +export function getR2Bucket(): string { + return requiredEnv("R2_BUCKET"); +} diff --git a/apps/backend/src/routes/catalog.ts b/apps/backend/src/routes/catalog.ts index 6534a84..07cf9a3 100644 --- a/apps/backend/src/routes/catalog.ts +++ b/apps/backend/src/routes/catalog.ts @@ -1,5 +1,6 @@ import type { ConfigWatchlist, StremioMeta } from "@stremlist/shared"; import { Hono } from "hono"; +import type { Context } from "hono"; import { parseCatalogId } from "../services/catalog-id"; import { getUserRpdbApiKey, getUserWatchlistById } from "../services/user"; import { @@ -8,6 +9,7 @@ import { } from "../services/watchlist"; const catalog = new Hono(); +const CATALOG_PAGE_SIZE = 100; // A single informational card Stremio renders inside the catalog row, so the // user sees *why* it's empty (e.g. their IMDb list is private) instead of a @@ -40,13 +42,27 @@ function buildUnavailableMeta( }; } -catalog.get("/:userId/catalog/:type/:id.json", async (c) => { +function parseSkip(c: Context): number | null { + const extra = routeParam(c, "extra"); + const value = extra + ? new URLSearchParams(extra.replace(/\.json$/u, "")).get("skip") + : c.req.query("skip"); + if (value === undefined || value === null || value === "") return 0; + + const skip = Number(value); + return Number.isSafeInteger(skip) && skip >= 0 ? skip : null; +} + +function routeParam(c: Context, name: string): string | undefined { + const params = c.req.param() as Record; + return params[name] ?? params[`${name}.json`]; +} + +async function serveCatalog(c: Context) { + c.header("Cache-Control", "no-store"); const userId = c.req.param("userId"); const requestedType = c.req.param("type"); - const catalogId = (c.req.param("id") ?? c.req.param("id.json")).replace( - /\.json$/u, - "", - ); + const catalogId = (routeParam(c, "id") ?? "").replace(/\.json$/u, ""); try { if (!userId || !requestedType || !catalogId) { @@ -65,6 +81,11 @@ catalog.get("/:userId/catalog/:type/:id.json", async (c) => { return c.json({ metas: [] }); } + const skip = parseSkip(c); + if (skip === null) { + return c.json({ metas: [] }, 400); + } + const parsedCatalog = parseCatalogId(catalogId); if (!parsedCatalog?.type || parsedCatalog.type !== requestedType) { console.warn( @@ -95,12 +116,13 @@ catalog.get("/:userId/catalog/:type/:id.json", async (c) => { rpdbApiKey, }); - const metas = watchlistData.metas.filter( + const matchingMetas = watchlistData.metas.filter( (item) => item.type === requestedType, ); + const metas = matchingMetas.slice(skip, skip + CATALOG_PAGE_SIZE); console.log( - `Serving catalog for user ${userId}, type: ${requestedType}, watchlist: ${watchlistConfig.id}, items: ${metas.length}`, + `Serving catalog for user ${userId}, type: ${requestedType}, watchlist: ${watchlistConfig.id}, skip: ${skip}, page items: ${metas.length}, total items: ${matchingMetas.length}`, ); return c.json({ metas }); @@ -131,6 +153,9 @@ catalog.get("/:userId/catalog/:type/:id.json", async (c) => { ); return c.json({ metas: [] }, 500); } -}); +} + +catalog.get("/:userId/catalog/:type/:id/:extra.json", serveCatalog); +catalog.get("/:userId/catalog/:type/:id.json", serveCatalog); export default catalog; diff --git a/apps/backend/src/scripts/cleanup-invalid-users.ts b/apps/backend/src/scripts/cleanup-invalid-users.ts index fbfe2e3..e5afb8d 100644 --- a/apps/backend/src/scripts/cleanup-invalid-users.ts +++ b/apps/backend/src/scripts/cleanup-invalid-users.ts @@ -5,6 +5,7 @@ import { config } from "dotenv"; import { readFileSync, writeFileSync } from "fs"; import path from "path"; import { getImdbWatchlist } from "../services/imdb-scraper"; +import { deleteCachedWatchlist } from "../services/watchlist-cache"; console.log("Starting cleanup..."); @@ -135,7 +136,29 @@ async function deleteUsers(userIds: string[]): Promise { for (let i = 0; i < userIds.length; i += BATCH_SIZE) { const batch = userIds.slice(i, i + BATCH_SIZE); - // Dependent tables (user_watchlists, watchlist_cache) cascade on delete. + const { data: watchlists, error: watchlistsError } = await supabase + .from("user_watchlists") + .select("id") + .in("owner_user_id", batch); + if (watchlistsError) { + throw new Error( + `Failed fetching watchlists before user deletion: ${watchlistsError.message}`, + ); + } + + const cacheDeletes = await Promise.allSettled( + watchlists.map(({ id }) => deleteCachedWatchlist(id)), + ); + const cacheDeleteFailure = cacheDeletes.find( + (result) => result.status === "rejected", + ); + if (cacheDeleteFailure?.status === "rejected") { + throw new Error( + `Failed deleting R2 caches before user deletion: ${String(cacheDeleteFailure.reason)}`, + ); + } + + // user_watchlists cascades from users; its R2 objects were removed above. const { error: userError } = await supabase .from("users") .delete() diff --git a/apps/backend/src/services/__tests__/catalog-routing.test.ts b/apps/backend/src/services/__tests__/catalog-routing.test.ts index 5970f15..53d9163 100644 --- a/apps/backend/src/services/__tests__/catalog-routing.test.ts +++ b/apps/backend/src/services/__tests__/catalog-routing.test.ts @@ -2,6 +2,12 @@ import { describe, expect, it } from "vitest"; import { buildCatalogId, parseCatalogId } from "../catalog-id"; import { buildManifestCatalogs } from "../stremio-catalogs"; +function catalogsWithoutExtra( + catalogs: ReturnType, +) { + return catalogs.map(({ id, name, type }) => ({ id, name, type })); +} + describe("catalog id helpers", () => { it("builds and parses movie ids", () => { const watchlistId = "77e10eda-0e07-4c60-8ec7-23fb1b1d0573"; @@ -41,7 +47,10 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect( + catalogs.every((catalog) => catalog.extra?.[0]?.name === "skip"), + ).toBe(true); + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-movie", name: "Stremlist Leo Picks", @@ -77,7 +86,7 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-movie", name: "Stremlist", @@ -111,7 +120,7 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-movie", name: "Stremlist 1", @@ -147,7 +156,7 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-movie", name: "Stremlist Leo Picks", @@ -168,7 +177,7 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-series", name: "Stremlist Leo Picks", @@ -189,7 +198,7 @@ describe("manifest catalog generation", () => { }, ]); - expect(catalogs).toEqual([ + expect(catalogsWithoutExtra(catalogs)).toEqual([ { id: "wl-77e10eda-0e07-4c60-8ec7-23fb1b1d0573-movie", name: "Stremlist Leo Picks", diff --git a/apps/backend/src/services/__tests__/watchlist-cache.test.ts b/apps/backend/src/services/__tests__/watchlist-cache.test.ts new file mode 100644 index 0000000..cd2795b --- /dev/null +++ b/apps/backend/src/services/__tests__/watchlist-cache.test.ts @@ -0,0 +1,208 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, +} from "@aws-sdk/client-s3"; +import type { StremioMeta } from "@stremlist/shared"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +function requiredKey(key: string | undefined): string { + if (!key) throw new Error("R2 test command is missing a key"); + return key; +} + +const r2 = vi.hoisted(() => ({ + objects: new Map(), + failGetContaining: null as string | null, + failPutContaining: null as string | null, + send: vi.fn((command: unknown) => Promise.resolve(command)), +})); + +function handleCommand(command: unknown): unknown { + if (command instanceof PutObjectCommand) { + const key = requiredKey(command.input.Key); + if (r2.failPutContaining && key.includes(r2.failPutContaining)) { + throw new Error("simulated R2 write failure"); + } + if (!(command.input.Body instanceof Uint8Array)) { + throw new Error("R2 test expects a Uint8Array body"); + } + r2.objects.set(key, Uint8Array.from(command.input.Body)); + return {}; + } + + if (command instanceof GetObjectCommand) { + const key = requiredKey(command.input.Key); + if (r2.failGetContaining && key.includes(r2.failGetContaining)) { + throw new Error("simulated R2 read failure"); + } + const body = r2.objects.get(key); + if (!body) { + throw Object.assign(new Error("missing"), { + name: "NoSuchKey", + $metadata: { httpStatusCode: 404 }, + }); + } + return { + Body: { + transformToByteArray: () => Promise.resolve(body), + transformToString: () => + Promise.resolve(Buffer.from(body).toString("utf8")), + }, + }; + } + + if (command instanceof DeleteObjectCommand) { + r2.objects.delete(requiredKey(command.input.Key)); + return {}; + } + + throw new Error("Unsupported R2 command"); +} + +r2.send.mockImplementation((command: unknown) => + Promise.resolve(handleCommand(command)), +); + +vi.mock("../../lib/r2", () => ({ + getR2Bucket: () => "test-bucket", + getR2Client: () => ({ send: r2.send }), +})); + +import { + deleteCachedWatchlist, + findCachedMeta, + getCachedWatchlist, + writeCachedWatchlist, +} from "../watchlist-cache"; + +const MOVIE: StremioMeta = { + id: "tt0111161", + type: "movie", + name: "The Shawshank Redemption", + poster: "https://example.com/poster.jpg", + posterShape: "poster", + genres: ["Drama"], + description: "A film.", +}; + +let sequence = 0; +function watchlistId(): string { + sequence += 1; + return `00000000-0000-4000-8000-${String(sequence).padStart(12, "0")}`; +} + +beforeEach(() => { + r2.objects.clear(); + r2.failGetContaining = null; + r2.failPutContaining = null; + r2.send.mockClear(); +}); + +describe("R2 watchlist cache", () => { + it("round-trips a compressed catalog and de-duplicates repeated items", async () => { + const id = watchlistId(); + const cachedAt = new Date("2026-08-26T10:00:00.000Z"); + + await writeCachedWatchlist(id, { metas: [MOVIE, { ...MOVIE }] }, cachedAt); + + const cached = await getCachedWatchlist(id); + expect(cached?.data).toEqual({ metas: [MOVIE] }); + expect(cached?.cachedAt).toEqual(cachedAt); + expect(cached?.generation).toMatch( + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u, + ); + expect(r2.objects.size).toBe(2); + expect([...r2.objects.keys()]).toEqual( + expect.arrayContaining([ + expect.stringMatching(/manifest\.json$/u), + expect.stringMatching(/generations\/.+\.json\.gz$/u), + ]), + ); + }); + + it("finds a meta in a cached watchlist and keeps types separate", async () => { + const id = watchlistId(); + const series: StremioMeta = { ...MOVIE, type: "series", name: "Series" }; + await writeCachedWatchlist(id, { metas: [MOVIE, series] }); + + expect(await findCachedMeta([id], "movie", MOVIE.id)).toEqual(MOVIE); + expect(await findCachedMeta([id], "series", MOVIE.id)).toEqual(series); + expect(await findCachedMeta([id], "movie", "tt9999999")).toBeNull(); + }); + + it("uses the manifest index to avoid reading catalog blobs on a meta miss", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + for (let index = 0; index <= 100; index += 1) { + await writeCachedWatchlist(watchlistId(), { metas: [MOVIE] }); + } + r2.send.mockClear(); + + expect(await findCachedMeta([id], "movie", "tt9999999")).toBeNull(); + expect(r2.send).toHaveBeenCalledTimes(1); + const command = r2.send.mock.calls[0][0]; + expect(command).toBeInstanceOf(GetObjectCommand); + expect((command as GetObjectCommand).input.Key).toMatch(/manifest\.json$/u); + }); + + it("treats R2 read failures as cache misses", async () => { + const id = watchlistId(); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + r2.failGetContaining = "manifest.json"; + + expect(await getCachedWatchlist(id)).toBeNull(); + expect(errorSpy).toHaveBeenCalledWith( + `Failed to read R2 cache for ${id}:`, + expect.any(Error), + ); + }); + + it("keeps the previous object when an overwrite fails", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + r2.failPutContaining = "/generations/"; + + await expect( + writeCachedWatchlist(id, { + metas: [{ ...MOVIE, id: "tt0068646", name: "The Godfather" }], + }), + ).rejects.toThrow("simulated R2 write failure"); + + r2.failPutContaining = null; + expect((await getCachedWatchlist(id))?.data.metas).toEqual([MOVIE]); + }); + + it("keeps the previous manifest when the atomic switch fails", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + r2.failPutContaining = "manifest.json"; + + await expect( + writeCachedWatchlist(id, { + metas: [{ ...MOVIE, id: "tt0068646", name: "The Godfather" }], + }), + ).rejects.toThrow("simulated R2 write failure"); + + r2.failPutContaining = null; + expect((await getCachedWatchlist(id))?.data.metas).toEqual([MOVIE]); + // The unreferenced generation is left for the lifecycle rule. Deleting it + // after a network error would be unsafe because the manifest PUT may have + // committed even when the client did not receive the response. + expect(r2.objects.size).toBe(3); + }); + + it("deletes the cache object", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + + await deleteCachedWatchlist(id); + + expect(await getCachedWatchlist(id)).toBeNull(); + expect( + [...r2.objects.keys()].filter((key) => + key.startsWith(`watchlists/${id}/`), + ), + ).toEqual([]); + }); +}); diff --git a/apps/backend/src/services/stremio-catalogs.ts b/apps/backend/src/services/stremio-catalogs.ts index 673c43f..9d87e93 100644 --- a/apps/backend/src/services/stremio-catalogs.ts +++ b/apps/backend/src/services/stremio-catalogs.ts @@ -42,11 +42,13 @@ export function buildManifestCatalogs( id: buildCatalogId(watchlist.id, "movie"), name: buildCatalogName(effectiveTitle), type: "movie", + extra: [{ name: "skip", isRequired: false }], }; const seriesCatalog: StremioCatalog = { id: buildCatalogId(watchlist.id, "series"), name: buildCatalogName(effectiveTitle), type: "series", + extra: [{ name: "skip", isRequired: false }], }; if (displayMode === "movie") { diff --git a/apps/backend/src/services/user.ts b/apps/backend/src/services/user.ts index 5fbff9a..29c7f78 100644 --- a/apps/backend/src/services/user.ts +++ b/apps/backend/src/services/user.ts @@ -1,6 +1,7 @@ import { DEFAULT_SORT_OPTION } from "@stremlist/shared"; import type { ConfigWatchlist, Tables } from "@stremlist/shared"; import { supabase } from "../lib/supabase"; +import { deleteCachedWatchlist } from "./watchlist-cache"; type User = Tables<"users">; type UserWatchlist = Tables<"user_watchlists">; @@ -181,6 +182,18 @@ export async function replaceUserWatchlists( ); throw deleteError; } + + const cleanupResults = await Promise.allSettled( + toDelete.map((watchlistId) => deleteCachedWatchlist(watchlistId)), + ); + cleanupResults.forEach((result, index) => { + if (result.status === "rejected") { + console.error( + `Failed to delete R2 cache for removed watchlist ${toDelete[index]}:`, + result.reason, + ); + } + }); } if (toUpdate.length > 0) { diff --git a/apps/backend/src/services/watchlist-cache.ts b/apps/backend/src/services/watchlist-cache.ts new file mode 100644 index 0000000..1dfbcac --- /dev/null +++ b/apps/backend/src/services/watchlist-cache.ts @@ -0,0 +1,370 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + PutObjectCommand, +} from "@aws-sdk/client-s3"; +import type { StremioMeta, WatchlistData } from "@stremlist/shared"; +import { z } from "zod"; +import { randomUUID } from "node:crypto"; +import { gunzipSync, gzipSync } from "node:zlib"; +import { getR2Bucket, getR2Client } from "../lib/r2"; + +const CACHE_FORMAT_VERSION = 1; +const MEMORY_CACHE_TTL_MS = 60_000; +const MEMORY_CACHE_MAX_ENTRIES = 100; + +const stremioMetaSchema = z.object({ + id: z.string(), + name: z.string(), + poster: z.string().nullable(), + posterShape: z.enum(["poster", "square", "landscape"]), + type: z.enum(["movie", "series"]), + genres: z.array(z.string()), + description: z.string(), + imdbRating: z.string().optional(), + releaseInfo: z.string().optional(), + director: z.array(z.string()).optional(), + cast: z.array(z.string()).optional(), + runtime: z.string().optional(), +}); + +const catalogObjectSchema = z.object({ + version: z.literal(CACHE_FORMAT_VERSION), + metas: z.array(stremioMetaSchema), +}); + +const cacheManifestSchema = z.object({ + version: z.literal(CACHE_FORMAT_VERSION), + generation: z.string().uuid(), + cachedAt: z.string().datetime(), + catalogKey: z.string(), + metaKeys: z.array(z.string()), +}); + +type CatalogObject = z.infer; +type CacheManifest = z.infer; + +interface MemoryEntry { + value: T; + expiresAt: number; +} + +export interface CachedWatchlist { + data: WatchlistData; + cachedAt: Date; + generation: string; +} + +const manifestMemoryCache = new Map< + string, + MemoryEntry +>(); +const catalogMemoryCache = new Map>(); + +function manifestKey(watchlistId: string): string { + return `watchlists/${watchlistId}/manifest.json`; +} + +function catalogKey(watchlistId: string, generation: string): string { + return `watchlists/${watchlistId}/generations/${generation}.json.gz`; +} + +function metaKey(meta: Pick): string { + return `${meta.type}:${meta.id}`; +} + +function isNotFound(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const candidate = error as { + name?: string; + $metadata?: { httpStatusCode?: number }; + }; + return ( + candidate.name === "NoSuchKey" || + candidate.$metadata?.httpStatusCode === 404 + ); +} + +function getMemoryValue( + cache: Map>, + key: string, +): T | undefined { + const entry = cache.get(key); + if (!entry || entry.expiresAt <= Date.now()) { + cache.delete(key); + return undefined; + } + + cache.delete(key); + cache.set(key, entry); + return entry.value; +} + +function setMemoryValue( + cache: Map>, + key: string, + value: T, +): void { + cache.delete(key); + cache.set(key, { + value, + expiresAt: Date.now() + MEMORY_CACHE_TTL_MS, + }); + + while (cache.size > MEMORY_CACHE_MAX_ENTRIES) { + const oldestKey = cache.keys().next().value; + if (!oldestKey) break; + cache.delete(oldestKey); + } +} + +async function readManifest( + watchlistId: string, +): Promise { + const cached = getMemoryValue(manifestMemoryCache, watchlistId); + if (cached !== undefined) return cached; + + try { + const response = await getR2Client().send( + new GetObjectCommand({ + Bucket: getR2Bucket(), + Key: manifestKey(watchlistId), + }), + ); + if (!response.Body) return null; + + const parsed: unknown = JSON.parse(await response.Body.transformToString()); + const manifest = cacheManifestSchema.parse(parsed); + if (manifest.catalogKey !== catalogKey(watchlistId, manifest.generation)) { + throw new Error(`Invalid R2 catalog key for ${watchlistId}`); + } + setMemoryValue(manifestMemoryCache, watchlistId, manifest); + return manifest; + } catch (error) { + if (!isNotFound(error)) throw error; + setMemoryValue(manifestMemoryCache, watchlistId, null); + return null; + } +} + +async function readCatalog( + manifest: CacheManifest, +): Promise { + const cached = getMemoryValue(catalogMemoryCache, manifest.catalogKey); + if (cached) return cached; + + try { + const response = await getR2Client().send( + new GetObjectCommand({ + Bucket: getR2Bucket(), + Key: manifest.catalogKey, + }), + ); + if (!response.Body) return null; + + const compressed = Buffer.from(await response.Body.transformToByteArray()); + const parsed: unknown = JSON.parse(gunzipSync(compressed).toString("utf8")); + const catalog = catalogObjectSchema.parse(parsed); + setMemoryValue(catalogMemoryCache, manifest.catalogKey, catalog); + return catalog; + } catch (error) { + if (!isNotFound(error)) throw error; + return null; + } +} + +function uniqueMetas(metas: StremioMeta[]): StremioMeta[] { + const seen = new Set(); + return metas.filter((meta) => { + const key = metaKey(meta); + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function hasSortedKey(keys: string[], target: string): boolean { + let low = 0; + let high = keys.length - 1; + + while (low <= high) { + const middle = Math.floor((low + high) / 2); + const candidate = keys[middle]; + if (candidate === target) return true; + if (candidate < target) low = middle + 1; + else high = middle - 1; + } + + return false; +} + +export async function getCachedWatchlist( + watchlistId: string, +): Promise { + try { + const manifest = await readManifest(watchlistId); + if (!manifest) return null; + + const catalog = await readCatalog(manifest); + if (!catalog || catalog.metas.length === 0) return null; + + return { + data: { metas: catalog.metas }, + cachedAt: new Date(manifest.cachedAt), + generation: manifest.generation, + }; + } catch (error) { + console.error(`Failed to read R2 cache for ${watchlistId}:`, error); + return null; + } +} + +export async function writeCachedWatchlist( + watchlistId: string, + watchlistData: WatchlistData, + cachedAt = new Date(), +): Promise { + const metas = uniqueMetas(watchlistData.metas); + if (metas.length === 0) { + await deleteCachedWatchlist(watchlistId); + return randomUUID(); + } + + let previousManifest: CacheManifest | null = null; + try { + previousManifest = await readManifest(watchlistId); + } catch (error) { + console.error( + `Failed to read previous R2 manifest for ${watchlistId}:`, + error, + ); + } + + const generation = randomUUID(); + const nextCatalogKey = catalogKey(watchlistId, generation); + const catalog = catalogObjectSchema.parse({ + version: CACHE_FORMAT_VERSION, + metas, + }); + const manifest: CacheManifest = { + version: CACHE_FORMAT_VERSION, + generation, + cachedAt: cachedAt.toISOString(), + catalogKey: nextCatalogKey, + metaKeys: catalog.metas.map(metaKey).sort(), + }; + + await getR2Client().send( + new PutObjectCommand({ + Bucket: getR2Bucket(), + Key: nextCatalogKey, + Body: gzipSync(Buffer.from(JSON.stringify(catalog))), + ContentType: "application/json", + ContentEncoding: "gzip", + CacheControl: "private, max-age=0, must-revalidate", + }), + ); + + await getR2Client().send( + new PutObjectCommand({ + Bucket: getR2Bucket(), + Key: manifestKey(watchlistId), + Body: Buffer.from(JSON.stringify(manifest)), + ContentType: "application/json", + CacheControl: "private, max-age=0, must-revalidate", + }), + ); + + setMemoryValue(catalogMemoryCache, nextCatalogKey, catalog); + setMemoryValue(manifestMemoryCache, watchlistId, manifest); + + if (previousManifest && previousManifest.catalogKey !== manifest.catalogKey) { + try { + await getR2Client().send( + new DeleteObjectCommand({ + Bucket: getR2Bucket(), + Key: previousManifest.catalogKey, + }), + ); + catalogMemoryCache.delete(previousManifest.catalogKey); + } catch (error) { + console.error( + `Failed to delete previous R2 generation for ${watchlistId}:`, + error, + ); + } + } + + return generation; +} + +export async function findCachedMeta( + watchlistIds: string[], + type: string, + id: string, +): Promise { + const target = `${type}:${id}`; + const manifestResults = await Promise.allSettled( + watchlistIds.map((watchlistId) => readManifest(watchlistId)), + ); + const candidates: CacheManifest[] = []; + + manifestResults.forEach((result, index) => { + if (result.status === "rejected") { + console.error( + `Failed to read R2 cache manifest for ${watchlistIds[index]}:`, + result.reason, + ); + return; + } + if (result.value && hasSortedKey(result.value.metaKeys, target)) { + candidates.push(result.value); + } + }); + + for (const manifest of candidates) { + try { + const catalog = await readCatalog(manifest); + const found = catalog?.metas.find( + (item) => item.type === type && item.id === id, + ); + if (found) return found; + } catch (error) { + console.error("Failed to read an indexed R2 catalog:", error); + } + } + + return null; +} + +export async function deleteCachedWatchlist( + watchlistId: string, +): Promise { + let previousManifest: CacheManifest | null = null; + try { + previousManifest = await readManifest(watchlistId); + } catch (error) { + console.error( + `Failed to read R2 manifest before deleting ${watchlistId}:`, + error, + ); + } + + await getR2Client().send( + new DeleteObjectCommand({ + Bucket: getR2Bucket(), + Key: manifestKey(watchlistId), + }), + ); + setMemoryValue(manifestMemoryCache, watchlistId, null); + + if (previousManifest) { + await getR2Client().send( + new DeleteObjectCommand({ + Bucket: getR2Bucket(), + Key: previousManifest.catalogKey, + }), + ); + catalogMemoryCache.delete(previousManifest.catalogKey); + } +} diff --git a/apps/backend/src/services/watchlist.ts b/apps/backend/src/services/watchlist.ts index 72d4ebc..b3905a1 100644 --- a/apps/backend/src/services/watchlist.ts +++ b/apps/backend/src/services/watchlist.ts @@ -4,11 +4,7 @@ import { isChartId, parseSortOption, } from "@stremlist/shared"; -import type { - WatchlistData, - SortOptions, - TablesInsert, -} from "@stremlist/shared"; +import type { WatchlistData, SortOptions } from "@stremlist/shared"; import { supabase } from "../lib/supabase"; import { shuffleArray } from "../utils"; import { @@ -21,6 +17,11 @@ import { } from "./imdb-scraper"; import type { WatchlistErrorReason } from "./imdb-scraper"; import { getUserRpdbApiKey, getUserWatchlists } from "./user"; +import { + findCachedMeta, + getCachedWatchlist, + writeCachedWatchlist, +} from "./watchlist-cache"; export type WatchlistUnavailableReason = WatchlistErrorReason | "unavailable"; @@ -45,136 +46,16 @@ const CACHE_TTL_MS = ? Number(process.env.CACHE_TTL_MINUTES) : 30) * 60_000; -// PostgREST caps a single SELECT at 1000 rows, so a watchlist with more items -// (the prod max is ~9951) would be silently truncated. Page through in -// 1000-row windows ordered by `position` and stitch the full list back together. -const CACHE_PAGE_SIZE = 1000; - -// Upsert in smaller batches to keep each request body modest (a 9951-item list -// is several MB) and to bound the size of each ON CONFLICT command. -const CACHE_WRITE_CHUNK_SIZE = 500; - -async function getCachedWatchlist( - watchlistId: string, -): Promise<{ data: WatchlistData; cachedAt: Date } | null> { - // Read the normalised per-item cache and reconstruct the WatchlistData blob. - // `position` preserves the canonical (added_at-asc) order the items were - // stored in, so resortCachedData applies sort + RPDB at serve time as before. - const metas: WatchlistData["metas"] = []; - let cachedAt: string | null = null; - - for (let from = 0; ; from += CACHE_PAGE_SIZE) { - const { data, error } = await supabase - .from("watchlist_cache_items") - .select("data, cached_at") - .eq("watchlist_id", watchlistId) - .order("position", { ascending: true }) - .range(from, from + CACHE_PAGE_SIZE - 1); - - if (error) { - console.error( - `Failed to get cached watchlist for ${watchlistId}:`, - error.message, - ); - return null; - } - - if (cachedAt === null && data.length > 0) cachedAt = data[0].cached_at; - for (const row of data) metas.push(row.data); - - if (data.length < CACHE_PAGE_SIZE) break; - } - - // Zero rows == miss, same contract as the old single-blob lookup. This keeps - // the "empty cache is a non-hit, always re-fetch" logic in the caller intact. - if (metas.length === 0 || cachedAt === null) return null; - - return { - data: { metas }, - cachedAt: new Date(cachedAt), - }; -} - async function upsertCache( watchlistId: string, watchlistData: WatchlistData, -): Promise { - // One shared timestamp for the whole generation. We upsert the new rows then - // prune anything older — never delete-then-insert, which would open a window - // where a concurrent catalog/refresh read sees zero rows (a false miss) and - // re-scrapes IMDb. - const cachedAt = new Date().toISOString(); - - // Empty list: clear the set so the next read is a miss (matches the old - // behaviour where an empty blob was treated as a non-hit). No upsert needed. - if (watchlistData.metas.length === 0) { - const { error } = await supabase - .from("watchlist_cache_items") - .delete() - .eq("watchlist_id", watchlistId); - if (error) { - console.error( - `Failed to clear watchlist cache for ${watchlistId}:`, - error.message, - ); - } - return; - } - - // De-duplicate by item_id, keeping the first occurrence. IMDb lists are NOT - // guaranteed to be sets — some carry the same `tt` id twice. Without this the - // upsert hits "ON CONFLICT DO UPDATE command cannot affect row a second time" - // (Postgres refuses to touch the same PK row twice in one command) and the - // whole cache write fails. Keeping the first occurrence mirrors the backfill's - // ON CONFLICT DO NOTHING. `position` stays the original index so order holds. - const seen = new Set(); - const rows: TablesInsert<"watchlist_cache_items">[] = []; - watchlistData.metas.forEach((meta, i) => { - if (seen.has(meta.id)) return; - seen.add(meta.id); - rows.push({ - watchlist_id: watchlistId, - item_id: meta.id, - type: meta.type, - position: i, - data: meta, - cached_at: cachedAt, - }); - }); - - // Chunk the write: a single upsert of a 9951-item list is a multi-MB request - // body. Bail before the prune if any chunk fails so we never wipe the - // previous generation on a partial write. - for (let i = 0; i < rows.length; i += CACHE_WRITE_CHUNK_SIZE) { - const { error: upsertError } = await supabase - .from("watchlist_cache_items") - .upsert(rows.slice(i, i + CACHE_WRITE_CHUNK_SIZE), { - onConflict: "watchlist_id,item_id", - }); - - if (upsertError) { - console.error( - `Failed to cache watchlist for ${watchlistId}:`, - upsertError.message, - ); - return; - } - } - - // Prune the previous generation: any row not touched by this upsert still - // carries an older cached_at. Items dropped from the list disappear; items - // that remain were just refreshed to `cachedAt` so they survive. - const { error: pruneError } = await supabase - .from("watchlist_cache_items") - .delete() - .eq("watchlist_id", watchlistId) - .lt("cached_at", cachedAt); - - if (pruneError) { - console.error( - `Failed to prune stale cache for ${watchlistId}:`, - pruneError.message, - ); + cachedAt: Date, +): Promise { + try { + return await writeCachedWatchlist(watchlistId, watchlistData, cachedAt); + } catch (error) { + console.error(`Failed to cache watchlist ${watchlistId} in R2:`, error); + return null; } } @@ -202,9 +83,9 @@ export async function getWatchlistByConfig( const sortOptionStr = config.sortOption ?? DEFAULT_SORT_OPTION; const sortOptions = parseSortOption(sortOptionStr); - // Cache-first happy path: a fresh cache hit is a single indexed SELECT with - // zero writes and zero IMDb calls. The cached blob is stored canonically - // (added_at-asc, raw posters), so sort + RPDB are always applied at serve time. + // Cache-first happy path: a fresh R2 hit avoids both Supabase writes and IMDb + // calls. The catalog stays canonical (added_at-asc, raw posters), so sort + + // RPDB are always applied at serve time. // // An *empty* cache (0 items) is treated as a non-hit so we always re-fetch: // it's indistinguishable from "the list went private since we cached it", and @@ -218,7 +99,12 @@ export async function getWatchlistByConfig( cached.data.metas.length > 0 && Date.now() - cached.cachedAt.getTime() < CACHE_TTL_MS ) { - return resortCachedData(cached.data, sortOptions, config.rpdbApiKey); + return resortCachedData( + cached.data, + sortOptions, + cached.generation, + config.rpdbApiKey, + ); } } @@ -230,20 +116,26 @@ export async function getWatchlistByConfig( : fetchWatchlist; // Fetch canonically so the cached blob is sort- and RPDB-key-agnostic. const fresh = await fetcher(config.imdbUserId, DEFAULT_SORT_OPTIONS, null); + const cachedAt = new Date(); - await Promise.all([ - upsertCache(config.watchlistId, fresh), + const [generation] = await Promise.all([ + upsertCache(config.watchlistId, fresh, cachedAt), // Tier 2: only stamp the analytics timestamp on a real refresh. ...(config.skipUserTimestamp ? [] : [ supabase .from("users") - .update({ last_fetched_at: new Date().toISOString() }) + .update({ last_fetched_at: cachedAt.toISOString() }) .eq("imdb_user_id", config.ownerUserId), ]), ]); - return resortCachedData(fresh, sortOptions, config.rpdbApiKey); + return resortCachedData( + fresh, + sortOptions, + generation ?? contentGeneration(config.watchlistId, fresh), + config.rpdbApiKey, + ); } catch (err) { const message = err instanceof Error ? err.message : String(err); console.error( @@ -267,7 +159,12 @@ export async function getWatchlistByConfig( .from("users") .update({ last_cache_served_at: new Date().toISOString() }) .eq("imdb_user_id", config.ownerUserId); - return resortCachedData(cached.data, sortOptions, config.rpdbApiKey); + return resortCachedData( + cached.data, + sortOptions, + cached.generation, + config.rpdbApiKey, + ); } } @@ -282,12 +179,13 @@ export async function getWatchlistByConfig( * Resolve a single meta item for a Stremio detail page using ONLY the cache. * * Unlike getWatchlistByConfig this never scrapes IMDb, never writes, and never - * throws: on a cold cache, a miss, or any DB error it returns null so the meta + * throws: on a cold cache, a miss, or any R2 error it returns null so the meta * route answers { meta: null } and Stremio falls back to Cinemeta. It also * deliberately ignores the cache TTL — opening one already-cached title must - * not trigger a refresh. The old per-request fan-out (getWatchlistByConfig for - * every list, which synchronously re-scraped IMDb on stale caches) is what - * caused the prod 500/504 storm on /:userId/meta/... + * not trigger a refresh. R2 manifests provide the membership check, so a miss + * does not download and decompress each full catalog. The old per-request + * getWatchlistByConfig fan-out synchronously re-scraped IMDb on stale caches + * and caused the prod 500/504 storm on /:userId/meta/... */ export async function findMetaInUserCache( userId: string, @@ -301,29 +199,13 @@ export async function findMetaInUserCache( ]); if (watchlists.length === 0) return null; - // Single indexed row lookup instead of pulling every list's full blob. - // Still scoped to the user's own watchlists: Stremlist meta only overrides - // Cinemeta for titles that are actually in one of the user's lists. - const { data, error } = await supabase - .from("watchlist_cache_items") - .select("data") - .eq("item_id", id) - .eq("type", type) - .in( - "watchlist_id", - watchlists.map((w) => w.id), - ) - .limit(1) - .maybeSingle(); - - if (error) { - console.error(`Failed to read meta cache for ${userId}:`, error.message); - return null; - } - - if (!data) return null; + const found = await findCachedMeta( + watchlists.map((watchlist) => watchlist.id), + type, + id, + ); + if (!found) return null; - const found = data.data; return { ...found, poster: buildPosterUrl(found.id, found.poster, rpdbApiKey), @@ -338,6 +220,7 @@ export async function findMetaInUserCache( function resortCachedData( data: WatchlistData, sortOptions: SortOptions, + generation: string, rpdbApiKey?: string | null, ): WatchlistData { const metas = [...data.metas]; @@ -352,7 +235,12 @@ function resortCachedData( } if (by === "random") { - return { metas: applyRpdbPostersToMetas(shuffleArray(metas), rpdbApiKey) }; + return { + metas: applyRpdbPostersToMetas( + shuffleArray(metas, generation), + rpdbApiKey, + ), + }; } metas.sort((a, b) => { @@ -376,6 +264,12 @@ function resortCachedData( return { metas: applyRpdbPostersToMetas(metas, rpdbApiKey) }; } +function contentGeneration(watchlistId: string, data: WatchlistData): string { + return `${watchlistId}:${data.metas + .map((meta) => `${meta.type}:${meta.id}`) + .join(",")}`; +} + function applyRpdbPostersToMetas( metas: WatchlistData["metas"], rpdbApiKey?: string | null, diff --git a/apps/backend/src/utils.ts b/apps/backend/src/utils.ts index 1428157..1810986 100644 --- a/apps/backend/src/utils.ts +++ b/apps/backend/src/utils.ts @@ -1,6 +1,27 @@ -export function shuffleArray(items: T[]): T[] { +function hashSeed(value: string): number { + let hash = 0x811c9dc5; + for (let index = 0; index < value.length; index += 1) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return hash >>> 0; +} + +function seededRandom(seed: string): () => number { + let state = hashSeed(seed); + return () => { + state += 0x6d2b79f5; + let value = state; + value = Math.imul(value ^ (value >>> 15), value | 1); + value ^= value + Math.imul(value ^ (value >>> 7), value | 61); + return ((value ^ (value >>> 14)) >>> 0) / 4_294_967_296; + }; +} + +export function shuffleArray(items: T[], seed?: string): T[] { + const random = seed ? seededRandom(seed) : Math.random; for (let i = items.length - 1; i > 0; i -= 1) { - const j = Math.floor(Math.random() * (i + 1)); + const j = Math.floor(random() * (i + 1)); [items[i], items[j]] = [items[j], items[i]]; } return items; diff --git a/docs/r2-cache-migration.md b/docs/r2-cache-migration.md new file mode 100644 index 0000000..4a721f2 --- /dev/null +++ b/docs/r2-cache-migration.md @@ -0,0 +1,68 @@ +# Cloudflare R2 cache rollout + +Stremlist keeps user configuration in Supabase, but stores IMDb catalog cache +objects in a private Cloudflare R2 bucket. Each refresh writes an immutable, +gzip-compressed catalog generation, then atomically switches a small manifest +to that generation. Readers see either the old complete catalog or the new one. +The manifest also holds a compact title index, so a metadata miss does not +download every catalog. + +This layout costs two R2 writes per watchlist refresh. Do not split a catalog +into per-item objects. Per-item writes would make Class A operations the first +free-tier constraint. + +## 1. Create the R2 bucket + +1. In Cloudflare, create a private Standard R2 bucket named `stremlist-cache`. +2. Create an R2 API token with Object Read & Write access, scoped only to that + bucket. +3. Add a lifecycle rule that deletes objects after 30 days. Active catalogs are + refreshed before then. The rule also removes inactive users and unreferenced + generations left by interrupted or concurrent refreshes. + +Set these variables in `apps/backend/.env` and in the Vercel backend project: + +```dotenv +R2_ACCOUNT_ID= +R2_ACCESS_KEY_ID= +R2_SECRET_ACCESS_KEY= +R2_BUCKET=stremlist-cache +``` + +Do not expose the R2 credentials to the frontend. The bucket does not need a +public domain. + +## 2. Deploy and verify + +Deploy the backend with all four R2 variables. The existing Supabase cache is +not copied. Catalogs will initially miss the cache and populate R2 as users +request them. + +Verify: + +1. A catalog with more than 100 titles returns 100 titles on its first page and + the next titles at `/skip=100.json`. +2. Opening a title from the catalog resolves its `/meta/...json` endpoint. +3. Manual refresh updates the catalog without errors. +4. R2 metrics show successful Class A and Class B operations. + +Rollback before the SQL cleanup is simply a deployment of the previous backend +version; the Supabase cache tables are still intact. + +## 3. Reclaim Supabase storage + +After production has been stable for at least one full cache TTL, apply +`supabase/migrations/20260826000000_drop_cache_tables_after_r2.sql`. The drop +is a normal migration so a fresh database replay matches +`packages/shared/src/database.types.ts`. The tables are dropped, not merely +emptied, so their relation storage is released immediately. + +Then run this in the Supabase SQL editor: + +```sql +SELECT pg_size_pretty(pg_database_size(current_database())) AS database_size; +``` + +Confirm the reported size is below the current Supabase Free limit, then change +the subscription to Free. Keep normal alerts on database size and R2 operations; +the two services have separate quotas. diff --git a/packages/shared/package.json b/packages/shared/package.json index 081cc2f..31a718f 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -11,9 +11,6 @@ "lint": "eslint .", "typecheck": "tsc --noEmit" }, - "dependencies": { - "type-fest": "^4.41.0" - }, "devDependencies": { "@stremlist/eslint-config": "workspace:*", "eslint": "^9.39.2", diff --git a/packages/shared/src/database.types.extended.ts b/packages/shared/src/database.types.extended.ts deleted file mode 100644 index 2be3db4..0000000 --- a/packages/shared/src/database.types.extended.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { MergeDeep } from "type-fest"; -import type { StremioMeta } from "./stremio.types"; -import type { Database as PostgresSchema } from "./database.types"; - -export type { Json } from "./database.types"; - -export type Database = MergeDeep< - PostgresSchema, - { - public: { - Tables: { - watchlist_cache_items: { - Row: { - data: StremioMeta; - }; - Insert: { - data: StremioMeta; - }; - Update: { - data?: StremioMeta; - }; - }; - }; - }; - } ->; - -export type Tables = - Database["public"]["Tables"][T]["Row"]; - -export type TablesInsert = - Database["public"]["Tables"][T]["Insert"]; - -export type TablesUpdate = - Database["public"]["Tables"][T]["Update"]; diff --git a/packages/shared/src/database.types.ts b/packages/shared/src/database.types.ts index f298669..53040f4 100644 --- a/packages/shared/src/database.types.ts +++ b/packages/shared/src/database.types.ts @@ -85,41 +85,6 @@ export type Database = { }; Relationships: []; }; - watchlist_cache_items: { - Row: { - cached_at: string; - data: Json; - item_id: string; - position: number; - type: string; - watchlist_id: string; - }; - Insert: { - cached_at?: string; - data: Json; - item_id: string; - position: number; - type: string; - watchlist_id: string; - }; - Update: { - cached_at?: string; - data?: Json; - item_id?: string; - position?: number; - type?: string; - watchlist_id?: string; - }; - Relationships: [ - { - foreignKeyName: "watchlist_cache_items_watchlist_id_fkey"; - columns: ["watchlist_id"]; - isOneToOne: false; - referencedRelation: "user_watchlists"; - referencedColumns: ["id"]; - }, - ]; - }; }; Views: { [_ in never]: never; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 7726ff0..881e147 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -4,7 +4,7 @@ export type { Tables, TablesInsert, TablesUpdate, -} from "./database.types.extended"; +} from "./database.types"; export type { WatchlistData, diff --git a/packages/shared/src/stremio.types.ts b/packages/shared/src/stremio.types.ts index 2ddef38..457cc01 100644 --- a/packages/shared/src/stremio.types.ts +++ b/packages/shared/src/stremio.types.ts @@ -53,6 +53,10 @@ export interface StremioCatalog { id: string; name: string; type: "movie" | "series"; + extra?: { + name: "skip"; + isRequired?: boolean; + }[]; } export interface StremioResource { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a99ea8a..70ff97b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: apps/backend: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1118.0 + version: 3.1118.0 '@hono/zod-validator': specifier: ^0.4.3 version: 0.4.3(hono@4.11.9)(zod@3.25.76) @@ -223,10 +226,6 @@ importers: version: 9.39.2(jiti@2.6.1) packages/shared: - dependencies: - type-fest: - specifier: ^4.41.0 - version: 4.41.0 devDependencies: '@stremlist/eslint-config': specifier: workspace:* @@ -243,6 +242,78 @@ importers: packages: + '@aws-sdk/checksums@3.1000.29': + resolution: {integrity: sha512-Dtu0gr4dnATZAPwEYbpCsG+MpLM7OAliy2gTepEFQwl1vZ6DL3QMH2FveMa3HLvPsOdhJsPRB3KtxVhph9T75A==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/client-s3@3.1118.0': + resolution: {integrity: sha512-nh46pLKeRNFvTOZAJMfYR7kZ/KzhSqzhXVIJSwl4jJ/b2D+KOXL2jeOCryOvGgwpUQo9YNWFUg63cm2qr5MX9g==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/core@3.977.9': + resolution: {integrity: sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-env@3.972.70': + resolution: {integrity: sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-http@3.972.72': + resolution: {integrity: sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-ini@3.973.15': + resolution: {integrity: sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-login@3.972.77': + resolution: {integrity: sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-node@3.972.81': + resolution: {integrity: sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-process@3.972.70': + resolution: {integrity: sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-sso@3.973.14': + resolution: {integrity: sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/credential-provider-web-identity@3.972.76': + resolution: {integrity: sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/middleware-sdk-s3@3.972.75': + resolution: {integrity: sha512-wMIsNumRVKaNMKhvU/s9VrdEwE8S6gSzXp4RygFG5BEMnGkkXf8cjh8zf7cKJBpUDpqTWqwbz5isEgp9rH6Lng==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/nested-clients@3.997.44': + resolution: {integrity: sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/signature-v4-multi-region@3.996.46': + resolution: {integrity: sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/token-providers@3.1116.0': + resolution: {integrity: sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/types@3.974.5': + resolution: {integrity: sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==} + engines: {node: '>=20.0.0'} + + '@aws-sdk/xml-builder@3.972.40': + resolution: {integrity: sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==} + engines: {node: '>=20.0.0'} + + '@aws/lambda-invoke-store@0.3.0': + resolution: {integrity: sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==} + engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.0': resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} engines: {node: '>=6.9.0'} @@ -1213,6 +1284,30 @@ packages: cpu: [x64] os: [win32] + '@smithy/core@3.33.3': + resolution: {integrity: sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==} + engines: {node: '>=18.0.0'} + + '@smithy/credential-provider-imds@4.5.2': + resolution: {integrity: sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==} + engines: {node: '>=18.0.0'} + + '@smithy/fetch-http-handler@5.7.2': + resolution: {integrity: sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==} + engines: {node: '>=18.0.0'} + + '@smithy/node-http-handler@4.11.3': + resolution: {integrity: sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==} + engines: {node: '>=18.0.0'} + + '@smithy/signature-v4@5.7.3': + resolution: {integrity: sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==} + engines: {node: '>=18.0.0'} + + '@smithy/types@4.17.2': + resolution: {integrity: sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==} + engines: {node: '>=18.0.0'} + '@stablelib/base64@1.0.1': resolution: {integrity: sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==} @@ -1693,6 +1788,9 @@ packages: boolbase@1.0.0: resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==} + bowser@2.14.1: + resolution: {integrity: sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==} + brace-expansion@1.1.12: resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} @@ -2669,10 +2767,6 @@ packages: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} - type-fest@4.41.0: - resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} - engines: {node: '>=16'} - typescript-eslint@8.55.0: resolution: {integrity: sha512-HE4wj+r5lmDVS9gdaN0/+iqNvPZwGfnJ5lZuz7s5vLlg9ODw0bIiiETaios9LvFI1U94/VBXGm3CB2Y5cNFMpw==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -2857,6 +2951,171 @@ packages: snapshots: + '@aws-sdk/checksums@3.1000.29': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/client-s3@3.1118.0': + dependencies: + '@aws-sdk/checksums': 3.1000.29 + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-node': 3.972.81 + '@aws-sdk/middleware-sdk-s3': 3.972.75 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/core@3.977.9': + dependencies: + '@aws-sdk/types': 3.974.5 + '@aws-sdk/xml-builder': 3.972.40 + '@aws/lambda-invoke-store': 0.3.0 + '@smithy/core': 3.33.3 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + bowser: 2.14.1 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-env@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-http@3.972.72': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-ini@3.973.15': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-login': 3.972.77 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-login@3.972.77': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-node@3.972.81': + dependencies: + '@aws-sdk/credential-provider-env': 3.972.70 + '@aws-sdk/credential-provider-http': 3.972.72 + '@aws-sdk/credential-provider-ini': 3.973.15 + '@aws-sdk/credential-provider-process': 3.972.70 + '@aws-sdk/credential-provider-sso': 3.973.14 + '@aws-sdk/credential-provider-web-identity': 3.972.76 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/credential-provider-imds': 4.5.2 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-process@3.972.70': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-sso@3.973.14': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/token-providers': 3.1116.0 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/credential-provider-web-identity@3.972.76': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/middleware-sdk-s3@3.972.75': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/nested-clients@3.997.44': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/signature-v4-multi-region': 3.996.46 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/fetch-http-handler': 5.7.2 + '@smithy/node-http-handler': 4.11.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/signature-v4-multi-region@3.996.46': + dependencies: + '@aws-sdk/types': 3.974.5 + '@smithy/signature-v4': 5.7.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/token-providers@3.1116.0': + dependencies: + '@aws-sdk/core': 3.977.9 + '@aws-sdk/nested-clients': 3.997.44 + '@aws-sdk/types': 3.974.5 + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/types@3.974.5': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws-sdk/xml-builder@3.972.40': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@aws/lambda-invoke-store@0.3.0': {} + '@babel/code-frame@7.29.0': dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -3637,6 +3896,39 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.57.1': optional: true + '@smithy/core@3.33.3': + dependencies: + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/credential-provider-imds@4.5.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/fetch-http-handler@5.7.2': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/node-http-handler@4.11.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/signature-v4@5.7.3': + dependencies: + '@smithy/core': 3.33.3 + '@smithy/types': 4.17.2 + tslib: 2.8.1 + + '@smithy/types@4.17.2': + dependencies: + tslib: 2.8.1 + '@stablelib/base64@1.0.1': {} '@standard-schema/spec@1.1.0': {} @@ -4074,6 +4366,8 @@ snapshots: boolbase@1.0.0: {} + bowser@2.14.1: {} + brace-expansion@1.1.12: dependencies: balanced-match: 1.0.2 @@ -5005,8 +5299,6 @@ snapshots: dependencies: prelude-ls: 1.2.1 - type-fest@4.41.0: {} - typescript-eslint@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3): dependencies: '@typescript-eslint/eslint-plugin': 8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.2(jiti@2.6.1))(typescript@5.9.3) diff --git a/supabase/migrations/20260826000000_drop_cache_tables_after_r2.sql b/supabase/migrations/20260826000000_drop_cache_tables_after_r2.sql new file mode 100644 index 0000000..bfbf67d --- /dev/null +++ b/supabase/migrations/20260826000000_drop_cache_tables_after_r2.sql @@ -0,0 +1,6 @@ +-- Apply only after the R2 deployment has been verified for one full cache TTL. +-- Keeping this drop in the migration chain makes fresh database replays match +-- the generated TypeScript schema. + +DROP TABLE IF EXISTS public.watchlist_cache_items; +DROP TABLE IF EXISTS public.watchlist_cache; From ff1c0fb861d2ea95e5b3d83012803f1996cff3cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 17:44:23 +0200 Subject: [PATCH 08/15] test(e2e): exercise R2 cache with local MinIO - Add local S3-compatible storage setup and cleanup helpers - Validate cache persistence, pagination, and watchlist removal --- .github/workflows/ci.yml | 8 +++ apps/backend/src/lib/r2.ts | 8 ++- apps/e2e/README.md | 24 ++++++-- apps/e2e/env.ts | 10 ++++ apps/e2e/helpers/api.ts | 4 +- apps/e2e/helpers/db.ts | 22 ++++--- apps/e2e/helpers/r2.ts | 98 ++++++++++++++++++++++++++++++++ apps/e2e/package.json | 1 + apps/e2e/playwright.config.ts | 8 +++ apps/e2e/preflight.ts | 21 +++++++ apps/e2e/tests/addon-api.spec.ts | 78 +++++++++++++++++++++---- docs/r2-cache-migration.md | 4 ++ pnpm-lock.yaml | 3 + 13 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 apps/e2e/helpers/r2.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 866a45b..3db3acc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,14 @@ jobs: - name: Start local Supabase (db + REST only) run: supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta + - name: Start local R2-compatible store + run: | + docker run --rm -d --name stremlist-e2e-r2 \ + -p 127.0.0.1:7431:9000 \ + -e MINIO_ROOT_USER=stremlist-e2e \ + -e MINIO_ROOT_PASSWORD=stremlist-e2e-secret \ + quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z server /data + - name: Install dependencies run: pnpm install --frozen-lockfile diff --git a/apps/backend/src/lib/r2.ts b/apps/backend/src/lib/r2.ts index d71118a..2fdaa71 100644 --- a/apps/backend/src/lib/r2.ts +++ b/apps/backend/src/lib/r2.ts @@ -13,10 +13,14 @@ function requiredEnv(name: string): string { export function getR2Client(): S3Client { if (client) return client; - const accountId = requiredEnv("R2_ACCOUNT_ID"); + const endpoint = process.env.R2_ENDPOINT?.trim(); + const accountId = endpoint ? null : requiredEnv("R2_ACCOUNT_ID"); client = new S3Client({ region: "auto", - endpoint: `https://${accountId}.r2.cloudflarestorage.com`, + endpoint: endpoint ?? `https://${accountId}.r2.cloudflarestorage.com`, + // Cloudflare supports virtual-hosted bucket URLs. Local S3-compatible + // servers normally expose buckets as path segments instead. + forcePathStyle: !!endpoint, credentials: { accessKeyId: requiredEnv("R2_ACCESS_KEY_ID"), secretAccessKey: requiredEnv("R2_SECRET_ACCESS_KEY"), diff --git a/apps/e2e/README.md b/apps/e2e/README.md index 430e99e..fe282f1 100644 --- a/apps/e2e/README.md +++ b/apps/e2e/README.md @@ -2,8 +2,9 @@ End-to-end tests that exercise Stremlist the way a real user does: the addon is installed into the **hosted Stremio Web app** (web.stremio.com) from a -backend running locally, with **live IMDb data** and a **local Supabase -stack**. The configure/onboarding pages of the frontend are covered too. +backend running locally, with **live IMDb data**, a **local Supabase stack**, +and a **local MinIO bucket** exercising the same S3 API used for Cloudflare +R2. The configure/onboarding pages of the frontend are covered too. ## How it works @@ -13,6 +14,9 @@ stack**. The configure/onboarding pages of the frontend are covered too. - The backend points at a local Supabase stack (`supabase start`), reset between tests. Functional seeding goes through the backend's own HTTP API, so tests exercise real code paths. +- The backend points at MinIO (`:7431`) through its configurable S3 endpoint. + Tests inspect the resulting manifest and compressed generation objects and + remove objects owned by E2E users between cases. - Stremio Web runs in anonymous mode: each fresh browser context has its own local addon collection. No Stremio account or shared state is involved. - Chromium is launched with `--disable-features=LocalNetworkAccessChecks,...` @@ -31,6 +35,12 @@ stack**. The configure/onboarding pages of the frontend are covered too. # One-time / per boot: start the local Supabase stack (needs Docker running) supabase start -x gotrue,realtime,storage-api,imgproxy,studio,edge-runtime,logflare,vector,supavisor,mailpit,postgres-meta +docker run --rm -d --name stremlist-e2e-r2 \ + -p 127.0.0.1:7431:9000 \ + -e MINIO_ROOT_USER=stremlist-e2e \ + -e MINIO_ROOT_PASSWORD=stremlist-e2e-secret \ + quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z server /data + # From the repo root: run every E2E project pnpm test:e2e @@ -40,16 +50,18 @@ pnpm --filter @stremlist/e2e test:e2e --project=live-smoke pnpm --filter @stremlist/e2e test:e2e --project=live-regression ``` -The suite deletes test users between cases. Foreign-key cascades clear their -watchlists and caches in the same database statement. The harness rejects any -non-loopback Supabase URL unless the caller provides the explicit destructive -confirmation described below. +The suite deletes test users between cases. It removes their R2 objects first, +then relies on foreign-key cascades for their Supabase watchlists. The harness +rejects any non-loopback Supabase URL unless the caller provides the explicit +destructive confirmation described below. ## Environment knobs | Variable | Purpose | | -------------------------------------------------------- | ----------------------------------------------------------------------------------------- | | `E2E_SUPABASE_URL` / `E2E_SUPABASE_SERVICE_ROLE_KEY` | Non-default local Supabase stack | +| `E2E_R2_ENDPOINT` / `E2E_R2_BUCKET` | Non-default S3-compatible endpoint and disposable bucket | +| `E2E_R2_ACCESS_KEY_ID` / `E2E_R2_SECRET_ACCESS_KEY` | Credentials for the disposable S3-compatible store | | `E2E_ALLOW_REMOTE_DATABASE=I_UNDERSTAND_THIS_WIPES_DATA` | Permit an isolated remote test project. Cleanup deletes every user and all dependent data | | `E2E_IMDB_USER_ID` / `E2E_IMDB_USER_ID_2` | Override the public watchlists under test | | `E2E_IMDB_LIST_ID` | Override the public `ls` list under test | diff --git a/apps/e2e/env.ts b/apps/e2e/env.ts index fcbec1d..f2265a2 100644 --- a/apps/e2e/env.ts +++ b/apps/e2e/env.ts @@ -18,6 +18,16 @@ export const SUPABASE_SERVICE_ROLE_KEY = process.env.E2E_SUPABASE_SERVICE_ROLE_KEY ?? "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJzdXBhYmFzZS1kZW1vIiwicm9sZSI6InNlcnZpY2Vfcm9sZSIsImV4cCI6MTk4MzgxMjk5Nn0.EGIM96RAZx35lJzdJsyH-qQwv8Hdp7fsn3W0YpN81IU"; +// MinIO is used as the local S3-compatible R2 test double. These credentials +// belong only to the disposable E2E container and may be overridden in CI. +export const R2_ENDPOINT = + process.env.E2E_R2_ENDPOINT ?? "http://127.0.0.1:7431"; +export const R2_ACCESS_KEY_ID = + process.env.E2E_R2_ACCESS_KEY_ID ?? "stremlist-e2e"; +export const R2_SECRET_ACCESS_KEY = + process.env.E2E_R2_SECRET_ACCESS_KEY ?? "stremlist-e2e-secret"; +export const R2_BUCKET = process.env.E2E_R2_BUCKET ?? "stremlist-e2e-cache"; + const REMOTE_DATABASE_CONFIRMATION = "I_UNDERSTAND_THIS_WIPES_DATA"; function assertSafeSupabaseTarget(): void { diff --git a/apps/e2e/helpers/api.ts b/apps/e2e/helpers/api.ts index 08c13cb..341acce 100644 --- a/apps/e2e/helpers/api.ts +++ b/apps/e2e/helpers/api.ts @@ -57,9 +57,11 @@ export async function getCatalog( userId: string, type: string, catalogId: string, + skip = 0, ): Promise<{ status: number; metas: CatalogMeta[] }> { + const extra = skip > 0 ? `/skip=${skip}` : ""; const { status, body } = await getJson<{ metas: CatalogMeta[] }>( - `/${userId}/catalog/${type}/${catalogId}.json`, + `/${userId}/catalog/${type}/${catalogId}${extra}.json`, ); return { status, metas: body.metas }; } diff --git a/apps/e2e/helpers/db.ts b/apps/e2e/helpers/db.ts index 54e6214..a19cc47 100644 --- a/apps/e2e/helpers/db.ts +++ b/apps/e2e/helpers/db.ts @@ -2,6 +2,7 @@ import { createClient } from "@supabase/supabase-js"; import type { Database } from "@stremlist/shared"; import { SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL } from "../env.js"; import { E2E_USER_IDS } from "./test-data.js"; +import { deleteCacheObjects } from "./r2.js"; // Service-role client: bypasses RLS, used only to reset and inspect state // between tests. All functional seeding goes through the backend's own HTTP @@ -10,6 +11,18 @@ const db = createClient(SUPABASE_URL, SUPABASE_SERVICE_ROLE_KEY); /** Delete only this run's test users. Foreign-key cascades reset their data. */ export async function resetDb(): Promise { + const { data: watchlists, error: watchlistError } = await db + .from("user_watchlists") + .select("id") + .in("owner_user_id", [...E2E_USER_IDS]); + if (watchlistError) { + throw new Error( + `resetDb watchlist lookup failed: ${watchlistError.message}`, + ); + } + + await deleteCacheObjects(watchlists.map((watchlist) => watchlist.id)); + const { error } = await db .from("users") .delete() @@ -26,12 +39,3 @@ export async function clearRefreshCooldown(userId: string): Promise { .eq("imdb_user_id", userId); if (error) throw new Error(`clearRefreshCooldown failed: ${error.message}`); } - -export async function countCacheItems(watchlistId: string): Promise { - const { count, error } = await db - .from("watchlist_cache_items") - .select("*", { count: "exact", head: true }) - .eq("watchlist_id", watchlistId); - if (error) throw new Error(`countCacheItems failed: ${error.message}`); - return count ?? 0; -} diff --git a/apps/e2e/helpers/r2.ts b/apps/e2e/helpers/r2.ts new file mode 100644 index 0000000..8401e94 --- /dev/null +++ b/apps/e2e/helpers/r2.ts @@ -0,0 +1,98 @@ +import { + CreateBucketCommand, + DeleteObjectsCommand, + HeadBucketCommand, + ListObjectsV2Command, + S3Client, +} from "@aws-sdk/client-s3"; +import { + R2_ACCESS_KEY_ID, + R2_BUCKET, + R2_ENDPOINT, + R2_SECRET_ACCESS_KEY, +} from "../env.js"; + +const r2 = new S3Client({ + region: "auto", + endpoint: R2_ENDPOINT, + forcePathStyle: true, + credentials: { + accessKeyId: R2_ACCESS_KEY_ID, + secretAccessKey: R2_SECRET_ACCESS_KEY, + }, +}); + +function isNotFound(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const candidate = error as { + name?: string; + $metadata?: { httpStatusCode?: number }; + }; + return ( + candidate.name === "NotFound" || + candidate.name === "NoSuchBucket" || + candidate.$metadata?.httpStatusCode === 404 + ); +} + +export async function ensureR2Bucket(): Promise { + try { + await r2.send(new HeadBucketCommand({ Bucket: R2_BUCKET })); + } catch (error) { + if (!isNotFound(error)) throw error; + await r2.send(new CreateBucketCommand({ Bucket: R2_BUCKET })); + } +} + +async function listKeys(prefix: string): Promise { + const keys: string[] = []; + let continuationToken: string | undefined; + + do { + const response = await r2.send( + new ListObjectsV2Command({ + Bucket: R2_BUCKET, + Prefix: prefix, + ContinuationToken: continuationToken, + }), + ); + keys.push( + ...(response.Contents ?? []).flatMap((object) => + object.Key ? [object.Key] : [], + ), + ); + continuationToken = response.IsTruncated + ? response.NextContinuationToken + : undefined; + } while (continuationToken); + + return keys; +} + +export async function countCacheObjects(watchlistId: string): Promise { + return (await listKeys(`watchlists/${watchlistId}/`)).length; +} + +export async function deleteCacheObjects( + watchlistIds: string[], +): Promise { + const keys = ( + await Promise.all( + [...new Set(watchlistIds)].map((watchlistId) => + listKeys(`watchlists/${watchlistId}/`), + ), + ) + ).flat(); + + for (let index = 0; index < keys.length; index += 1_000) { + await r2.send( + new DeleteObjectsCommand({ + Bucket: R2_BUCKET, + Delete: { + Objects: keys.slice(index, index + 1_000).map((Key) => ({ Key })), + Quiet: true, + }, + }), + ); + } +} diff --git a/apps/e2e/package.json b/apps/e2e/package.json index 6ca7054..83359f7 100644 --- a/apps/e2e/package.json +++ b/apps/e2e/package.json @@ -10,6 +10,7 @@ "format:check": "prettier . --check" }, "devDependencies": { + "@aws-sdk/client-s3": "^3.1118.0", "@playwright/test": "^1.62.1", "@stremlist/backend": "workspace:*", "@stremlist/eslint-config": "workspace:*", diff --git a/apps/e2e/playwright.config.ts b/apps/e2e/playwright.config.ts index 36bccb6..2a36cec 100644 --- a/apps/e2e/playwright.config.ts +++ b/apps/e2e/playwright.config.ts @@ -4,6 +4,10 @@ import { BACKEND_URL, FRONTEND_PORT, FRONTEND_URL, + R2_ACCESS_KEY_ID, + R2_BUCKET, + R2_ENDPOINT, + R2_SECRET_ACCESS_KEY, REFRESH_COOLDOWN_SECONDS, SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL, @@ -68,6 +72,10 @@ export default defineConfig({ SUPABASE_SERVICE_ROLE_KEY, FRONTEND_URL, REFRESH_COOLDOWN_SECONDS: String(REFRESH_COOLDOWN_SECONDS), + R2_ENDPOINT, + R2_ACCESS_KEY_ID, + R2_SECRET_ACCESS_KEY, + R2_BUCKET, // The Resend SDK throws at import time without a key. Newsletter // delivery is deliberately out of E2E scope (it would email real // people), so a dummy key is enough to boot the app. diff --git a/apps/e2e/preflight.ts b/apps/e2e/preflight.ts index ae9e447..8ef631b 100644 --- a/apps/e2e/preflight.ts +++ b/apps/e2e/preflight.ts @@ -1,4 +1,5 @@ import { SUPABASE_SERVICE_ROLE_KEY, SUPABASE_URL } from "./env.js"; +import { ensureR2Bucket } from "./helpers/r2.js"; // This script runs before Playwright starts its web servers, so database // failures surface immediately instead of becoming a backend startup timeout. @@ -25,3 +26,23 @@ try { `Underlying error: ${error instanceof Error ? error.message : String(error)}`, ); } + +let r2Error: unknown; +for (let attempt = 0; attempt < 20; attempt += 1) { + try { + await ensureR2Bucket(); + r2Error = undefined; + break; + } catch (error) { + r2Error = error; + await new Promise((resolve) => setTimeout(resolve, 500)); + } +} + +if (r2Error) { + throw new Error( + `The E2E R2-compatible store is not reachable.\n` + + `Start MinIO from the repository root with the command documented in apps/e2e/README.md.\n` + + `Underlying error: ${r2Error instanceof Error ? r2Error.message : String(r2Error)}`, + ); +} diff --git a/apps/e2e/tests/addon-api.spec.ts b/apps/e2e/tests/addon-api.spec.ts index f0e0a9e..6e3bfb8 100644 --- a/apps/e2e/tests/addon-api.spec.ts +++ b/apps/e2e/tests/addon-api.spec.ts @@ -13,11 +13,8 @@ import { validateUser, type CatalogMeta, } from "../helpers/api.js"; -import { - clearRefreshCooldown, - countCacheItems, - resetDb, -} from "../helpers/db.js"; +import { clearRefreshCooldown, resetDb } from "../helpers/db.js"; +import { countCacheObjects } from "../helpers/r2.js"; import { P_HANDLE, PRIVATE_LIST, @@ -116,8 +113,8 @@ test.describe("catalogs", () => { expect(meta.id).toMatch(/^tt\d+$/); expect(meta.name.length).toBeGreaterThan(0); } - // The fetch must have populated the per-item cache. - expect(await countCacheItems(config.watchlists[0].id)).toBeGreaterThan(0); + // One manifest and one compressed catalog generation are persisted. + expect(await countCacheObjects(config.watchlists[0].id)).toBe(2); }, ); @@ -213,13 +210,39 @@ test.describe("catalogs", () => { ]); const { body } = await getConfig(PUBLIC_USER); const chart = body.watchlists[0]; - const { metas } = await getCatalog( + const firstPage = await getCatalog( + PUBLIC_USER, + "movie", + `wl-${chart.id}-movie`, + ); + const secondPage = await getCatalog( + PUBLIC_USER, + "movie", + `wl-${chart.id}-movie`, + 100, + ); + const thirdPage = await getCatalog( PUBLIC_USER, "movie", `wl-${chart.id}-movie`, + 200, ); - // IMDb Top 250 — allow slack for titles Stremio types can't represent. + expect(firstPage.status).toBe(200); + expect(secondPage.status).toBe(200); + expect(thirdPage.status).toBe(200); + expect(firstPage.metas).toHaveLength(100); + expect(secondPage.metas).toHaveLength(100); + expect(thirdPage.metas.length).toBeGreaterThan(0); + + const metas = [ + ...firstPage.metas, + ...secondPage.metas, + ...thirdPage.metas, + ]; + // IMDb Top 250. Allow slack for titles Stremio types cannot represent, + // but make sure pagination neither duplicates nor drops a whole page. expect(metas.length).toBeGreaterThan(200); + expect(new Set(metas.map((meta) => meta.id)).size).toBe(metas.length); for (const meta of metas.slice(0, 10)) { expect(meta.type).toBe("movie"); expect(meta.id).toMatch(/^tt\d+$/); @@ -310,6 +333,41 @@ test.describe("catalogs", () => { expect(badType.metas).toEqual([]); }, ); + + test( + "removing a watchlist deletes its R2 cache objects", + { tag: "@live-regression" }, + async () => { + const config = await bootstrapUser(PUBLIC_USER); + const removed = config.watchlists[0]; + const created = await postConfig(PUBLIC_USER, [ + { + id: removed.id, + imdbUserId: removed.imdbUserId, + sortOption: removed.sortOption, + }, + { imdbUserId: PUBLIC_LIST, sortOption: "added_at-asc" }, + ]); + expect(created.status).toBe(200); + + const current = (await getConfig(PUBLIC_USER)).body.watchlists; + const kept = current.find((watchlist) => watchlist.id !== removed.id); + expect(kept).toBeDefined(); + + await getCatalog(PUBLIC_USER, "movie", `wl-${removed.id}-movie`); + expect(await countCacheObjects(removed.id)).toBe(2); + + const updated = await postConfig(PUBLIC_USER, [ + { + id: kept!.id, + imdbUserId: kept!.imdbUserId, + sortOption: kept!.sortOption, + }, + ]); + expect(updated.status).toBe(200); + expect(await countCacheObjects(removed.id)).toBe(0); + }, + ); }); test.describe("meta", () => { @@ -438,7 +496,7 @@ test.describe("refresh", () => { expect(first.body.refreshed).toBe(1); expect(first.body.failed).toBe(0); expect(first.body.total).toBe(1); - expect(await countCacheItems(config.watchlists[0].id)).toBeGreaterThan(0); + expect(await countCacheObjects(config.watchlists[0].id)).toBe(2); const second = await refresh(PUBLIC_USER); expect(second.body.throttled).toBe(true); diff --git a/docs/r2-cache-migration.md b/docs/r2-cache-migration.md index 4a721f2..b62e0ce 100644 --- a/docs/r2-cache-migration.md +++ b/docs/r2-cache-migration.md @@ -29,6 +29,10 @@ R2_SECRET_ACCESS_KEY= R2_BUCKET=stremlist-cache ``` +`R2_ENDPOINT` is reserved for local S3-compatible test servers. Leave it unset +in production so the backend derives the Cloudflare endpoint from +`R2_ACCOUNT_ID`. + Do not expose the R2 credentials to the frontend. The bucket does not need a public domain. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1fcc93d..ed73b83 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -87,6 +87,9 @@ importers: apps/e2e: devDependencies: + '@aws-sdk/client-s3': + specifier: ^3.1118.0 + version: 3.1118.0 '@playwright/test': specifier: ^1.62.1 version: 1.62.1 From 5fd0c0a0d4b93c17dd892c1e4862e0ba64423296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:01:45 +0200 Subject: [PATCH 09/15] fix: address Greptile review feedback --- .github/workflows/ci.yml | 2 +- .../__tests__/watchlist-cache.test.ts | 16 +++++++++++ apps/backend/src/services/watchlist-cache.ts | 27 ------------------- 3 files changed, 17 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3db3acc..275f666 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: node-version: 20 cache: pnpm - - uses: supabase/setup-cli@ab058987d8d6c725971f6cf9d0b5c98467e30bd1 # v1 + - uses: supabase/setup-cli@v1 with: version: 2.98.2 diff --git a/apps/backend/src/services/__tests__/watchlist-cache.test.ts b/apps/backend/src/services/__tests__/watchlist-cache.test.ts index cd2795b..79f49f5 100644 --- a/apps/backend/src/services/__tests__/watchlist-cache.test.ts +++ b/apps/backend/src/services/__tests__/watchlist-cache.test.ts @@ -192,6 +192,22 @@ describe("R2 watchlist cache", () => { expect(r2.objects.size).toBe(3); }); + it("keeps the previous generation readable after replacement", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + const previousCatalogKey = [...r2.objects.keys()].find((key) => + key.includes("/generations/"), + ); + if (!previousCatalogKey) throw new Error("Missing previous generation"); + + await writeCachedWatchlist(id, { + metas: [{ ...MOVIE, id: "tt0068646", name: "The Godfather" }], + }); + + expect(r2.objects.has(previousCatalogKey)).toBe(true); + expect(r2.objects.size).toBe(3); + }); + it("deletes the cache object", async () => { const id = watchlistId(); await writeCachedWatchlist(id, { metas: [MOVIE] }); diff --git a/apps/backend/src/services/watchlist-cache.ts b/apps/backend/src/services/watchlist-cache.ts index 1dfbcac..c4bac20 100644 --- a/apps/backend/src/services/watchlist-cache.ts +++ b/apps/backend/src/services/watchlist-cache.ts @@ -230,16 +230,6 @@ export async function writeCachedWatchlist( return randomUUID(); } - let previousManifest: CacheManifest | null = null; - try { - previousManifest = await readManifest(watchlistId); - } catch (error) { - console.error( - `Failed to read previous R2 manifest for ${watchlistId}:`, - error, - ); - } - const generation = randomUUID(); const nextCatalogKey = catalogKey(watchlistId, generation); const catalog = catalogObjectSchema.parse({ @@ -278,23 +268,6 @@ export async function writeCachedWatchlist( setMemoryValue(catalogMemoryCache, nextCatalogKey, catalog); setMemoryValue(manifestMemoryCache, watchlistId, manifest); - if (previousManifest && previousManifest.catalogKey !== manifest.catalogKey) { - try { - await getR2Client().send( - new DeleteObjectCommand({ - Bucket: getR2Bucket(), - Key: previousManifest.catalogKey, - }), - ); - catalogMemoryCache.delete(previousManifest.catalogKey); - } catch (error) { - console.error( - `Failed to delete previous R2 generation for ${watchlistId}:`, - error, - ); - } - } - return generation; } From 12f05d8bf4a4452879cac38de9f647bba15fe74a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:09:39 +0200 Subject: [PATCH 10/15] fix(cache): make cache deletion race-safe --- .../__tests__/watchlist-cache.test.ts | 73 ++++++++++- apps/backend/src/services/watchlist-cache.ts | 113 ++++++++++++++---- docs/r2-cache-migration.md | 4 +- 3 files changed, 162 insertions(+), 28 deletions(-) diff --git a/apps/backend/src/services/__tests__/watchlist-cache.test.ts b/apps/backend/src/services/__tests__/watchlist-cache.test.ts index 79f49f5..ac14969 100644 --- a/apps/backend/src/services/__tests__/watchlist-cache.test.ts +++ b/apps/backend/src/services/__tests__/watchlist-cache.test.ts @@ -15,9 +15,23 @@ const r2 = vi.hoisted(() => ({ objects: new Map(), failGetContaining: null as string | null, failPutContaining: null as string | null, + beforeConditionalPut: null as (() => void) | null, send: vi.fn((command: unknown) => Promise.resolve(command)), })); +function etagFor(body: Uint8Array): string { + return `"${Buffer.from(body).toString("base64")}"`; +} + +function storedObject( + objects: Map, + key: string, +): Uint8Array { + const body = objects.get(key); + if (!body) throw new Error(`Missing R2 test object: ${key}`); + return body; +} + function handleCommand(command: unknown): unknown { if (command instanceof PutObjectCommand) { const key = requiredKey(command.input.Key); @@ -27,6 +41,18 @@ function handleCommand(command: unknown): unknown { if (!(command.input.Body instanceof Uint8Array)) { throw new Error("R2 test expects a Uint8Array body"); } + if (command.input.IfMatch) { + const beforeConditionalPut = r2.beforeConditionalPut; + r2.beforeConditionalPut = null; + beforeConditionalPut?.(); + const current = r2.objects.get(key); + if (!current || etagFor(current) !== command.input.IfMatch) { + throw Object.assign(new Error("precondition failed"), { + name: "PreconditionFailed", + $metadata: { httpStatusCode: 412 }, + }); + } + } r2.objects.set(key, Uint8Array.from(command.input.Body)); return {}; } @@ -44,6 +70,7 @@ function handleCommand(command: unknown): unknown { }); } return { + ETag: etagFor(body), Body: { transformToByteArray: () => Promise.resolve(body), transformToString: () => @@ -96,6 +123,7 @@ beforeEach(() => { r2.objects.clear(); r2.failGetContaining = null; r2.failPutContaining = null; + r2.beforeConditionalPut = null; r2.send.mockClear(); }); @@ -208,7 +236,48 @@ describe("R2 watchlist cache", () => { expect(r2.objects.size).toBe(3); }); - it("deletes the cache object", async () => { + it("does not delete a concurrently replaced manifest", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + const previousObjects = new Map(r2.objects); + + const replacement = { + ...MOVIE, + id: "tt0068646", + name: "The Godfather", + }; + await writeCachedWatchlist(id, { metas: [replacement] }); + const replacementObjects = new Map(r2.objects); + const currentManifestKey = [...replacementObjects.keys()].find((key) => + key.endsWith("/manifest.json"), + ); + const replacementCatalogKey = [...replacementObjects.keys()].find( + (key) => key.includes("/generations/") && !previousObjects.has(key), + ); + if (!currentManifestKey || !replacementCatalogKey) { + throw new Error("Missing replacement R2 test objects"); + } + + r2.objects.clear(); + previousObjects.forEach((body, key) => r2.objects.set(key, body)); + r2.beforeConditionalPut = () => { + r2.objects.set( + currentManifestKey, + storedObject(replacementObjects, currentManifestKey), + ); + r2.objects.set( + replacementCatalogKey, + storedObject(replacementObjects, replacementCatalogKey), + ); + }; + + await deleteCachedWatchlist(id); + + expect((await getCachedWatchlist(id))?.data.metas).toEqual([replacement]); + expect(r2.objects.has(replacementCatalogKey)).toBe(true); + }); + + it("invalidates the cache and deletes its current catalog", async () => { const id = watchlistId(); await writeCachedWatchlist(id, { metas: [MOVIE] }); @@ -219,6 +288,6 @@ describe("R2 watchlist cache", () => { [...r2.objects.keys()].filter((key) => key.startsWith(`watchlists/${id}/`), ), - ).toEqual([]); + ).toEqual([`watchlists/${id}/manifest.json`]); }); }); diff --git a/apps/backend/src/services/watchlist-cache.ts b/apps/backend/src/services/watchlist-cache.ts index c4bac20..58b5d96 100644 --- a/apps/backend/src/services/watchlist-cache.ts +++ b/apps/backend/src/services/watchlist-cache.ts @@ -41,6 +41,17 @@ const cacheManifestSchema = z.object({ metaKeys: z.array(z.string()), }); +const deletedManifestSchema = z.object({ + version: z.literal(CACHE_FORMAT_VERSION), + deleted: z.literal(true), + deletedAt: z.string().datetime(), +}); + +const storedManifestSchema = z.union([ + cacheManifestSchema, + deletedManifestSchema, +]); + type CatalogObject = z.infer; type CacheManifest = z.infer; @@ -49,6 +60,11 @@ interface MemoryEntry { expiresAt: number; } +interface ManifestRead { + manifest: CacheManifest | null; + etag?: string; +} + export interface CachedWatchlist { data: WatchlistData; cachedAt: Date; @@ -85,6 +101,18 @@ function isNotFound(error: unknown): boolean { ); } +function isPreconditionFailed(error: unknown): boolean { + if (!error || typeof error !== "object") return false; + const candidate = error as { + name?: string; + $metadata?: { httpStatusCode?: number }; + }; + return ( + candidate.name === "PreconditionFailed" || + candidate.$metadata?.httpStatusCode === 412 + ); +} + function getMemoryValue( cache: Map>, key: string, @@ -118,12 +146,7 @@ function setMemoryValue( } } -async function readManifest( - watchlistId: string, -): Promise { - const cached = getMemoryValue(manifestMemoryCache, watchlistId); - if (cached !== undefined) return cached; - +async function readManifestFromR2(watchlistId: string): Promise { try { const response = await getR2Client().send( new GetObjectCommand({ @@ -131,22 +154,36 @@ async function readManifest( Key: manifestKey(watchlistId), }), ); - if (!response.Body) return null; + if (!response.Body) return { manifest: null, etag: response.ETag }; const parsed: unknown = JSON.parse(await response.Body.transformToString()); - const manifest = cacheManifestSchema.parse(parsed); + const storedManifest = storedManifestSchema.parse(parsed); + if ("deleted" in storedManifest) { + return { manifest: null, etag: response.ETag }; + } + + const manifest = storedManifest; if (manifest.catalogKey !== catalogKey(watchlistId, manifest.generation)) { throw new Error(`Invalid R2 catalog key for ${watchlistId}`); } - setMemoryValue(manifestMemoryCache, watchlistId, manifest); - return manifest; + return { manifest, etag: response.ETag }; } catch (error) { if (!isNotFound(error)) throw error; - setMemoryValue(manifestMemoryCache, watchlistId, null); - return null; + return { manifest: null }; } } +async function readManifest( + watchlistId: string, +): Promise { + const cached = getMemoryValue(manifestMemoryCache, watchlistId); + if (cached !== undefined) return cached; + + const { manifest } = await readManifestFromR2(watchlistId); + setMemoryValue(manifestMemoryCache, watchlistId, manifest); + return manifest; +} + async function readCatalog( manifest: CacheManifest, ): Promise { @@ -313,31 +350,57 @@ export async function findCachedMeta( export async function deleteCachedWatchlist( watchlistId: string, ): Promise { - let previousManifest: CacheManifest | null = null; + let current: ManifestRead; try { - previousManifest = await readManifest(watchlistId); + current = await readManifestFromR2(watchlistId); } catch (error) { console.error( `Failed to read R2 manifest before deleting ${watchlistId}:`, error, ); + throw error; } - await getR2Client().send( - new DeleteObjectCommand({ - Bucket: getR2Bucket(), - Key: manifestKey(watchlistId), - }), - ); - setMemoryValue(manifestMemoryCache, watchlistId, null); + if (!current.manifest) { + setMemoryValue(manifestMemoryCache, watchlistId, null); + return; + } + if (!current.etag) { + throw new Error(`R2 manifest for ${watchlistId} is missing an ETag`); + } - if (previousManifest) { + try { await getR2Client().send( - new DeleteObjectCommand({ + new PutObjectCommand({ Bucket: getR2Bucket(), - Key: previousManifest.catalogKey, + Key: manifestKey(watchlistId), + Body: Buffer.from( + JSON.stringify({ + version: CACHE_FORMAT_VERSION, + deleted: true, + deletedAt: new Date().toISOString(), + }), + ), + ContentType: "application/json", + CacheControl: "private, max-age=0, must-revalidate", + IfMatch: current.etag, }), ); - catalogMemoryCache.delete(previousManifest.catalogKey); + } catch (error) { + if (isPreconditionFailed(error)) { + manifestMemoryCache.delete(watchlistId); + return; + } + throw error; } + + setMemoryValue(manifestMemoryCache, watchlistId, null); + + await getR2Client().send( + new DeleteObjectCommand({ + Bucket: getR2Bucket(), + Key: current.manifest.catalogKey, + }), + ); + catalogMemoryCache.delete(current.manifest.catalogKey); } diff --git a/docs/r2-cache-migration.md b/docs/r2-cache-migration.md index b62e0ce..9f3b9d6 100644 --- a/docs/r2-cache-migration.md +++ b/docs/r2-cache-migration.md @@ -5,7 +5,9 @@ objects in a private Cloudflare R2 bucket. Each refresh writes an immutable, gzip-compressed catalog generation, then atomically switches a small manifest to that generation. Readers see either the old complete catalog or the new one. The manifest also holds a compact title index, so a metadata miss does not -download every catalog. +download every catalog. Cache deletion conditionally replaces the current +manifest with a tombstone before removing its catalog, so it cannot delete a +generation published concurrently by another backend instance. This layout costs two R2 writes per watchlist refresh. Do not split a catalog into per-item objects. Per-item writes would make Class A operations the first From 674537e4946438e1d1186060e5b2c201dcc28a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:19:19 +0200 Subject: [PATCH 11/15] fix(cache): refresh stale R2 manifests --- .../__tests__/watchlist-cache.test.ts | 53 ++++++++++++ apps/backend/src/services/watchlist-cache.ts | 85 ++++++++++++++++--- 2 files changed, 126 insertions(+), 12 deletions(-) diff --git a/apps/backend/src/services/__tests__/watchlist-cache.test.ts b/apps/backend/src/services/__tests__/watchlist-cache.test.ts index ac14969..adac16c 100644 --- a/apps/backend/src/services/__tests__/watchlist-cache.test.ts +++ b/apps/backend/src/services/__tests__/watchlist-cache.test.ts @@ -16,6 +16,7 @@ const r2 = vi.hoisted(() => ({ failGetContaining: null as string | null, failPutContaining: null as string | null, beforeConditionalPut: null as (() => void) | null, + afterConditionalPut: null as (() => Promise) | null, send: vi.fn((command: unknown) => Promise.resolve(command)), })); @@ -35,6 +36,7 @@ function storedObject( function handleCommand(command: unknown): unknown { if (command instanceof PutObjectCommand) { const key = requiredKey(command.input.Key); + let afterConditionalPut: (() => Promise) | null = null; if (r2.failPutContaining && key.includes(r2.failPutContaining)) { throw new Error("simulated R2 write failure"); } @@ -44,6 +46,8 @@ function handleCommand(command: unknown): unknown { if (command.input.IfMatch) { const beforeConditionalPut = r2.beforeConditionalPut; r2.beforeConditionalPut = null; + afterConditionalPut = r2.afterConditionalPut; + r2.afterConditionalPut = null; beforeConditionalPut?.(); const current = r2.objects.get(key); if (!current || etagFor(current) !== command.input.IfMatch) { @@ -54,6 +58,9 @@ function handleCommand(command: unknown): unknown { } } r2.objects.set(key, Uint8Array.from(command.input.Body)); + if (afterConditionalPut) { + return afterConditionalPut().then(() => ({})); + } return {}; } @@ -124,6 +131,7 @@ beforeEach(() => { r2.failGetContaining = null; r2.failPutContaining = null; r2.beforeConditionalPut = null; + r2.afterConditionalPut = null; r2.send.mockClear(); }); @@ -277,6 +285,51 @@ describe("R2 watchlist cache", () => { expect(r2.objects.has(replacementCatalogKey)).toBe(true); }); + it("keeps a local replacement cached when it follows the tombstone", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + const replacement = { + ...MOVIE, + id: "tt0068646", + name: "The Godfather", + }; + r2.afterConditionalPut = () => + writeCachedWatchlist(id, { metas: [replacement] }).then(() => undefined); + + await deleteCachedWatchlist(id); + r2.failGetContaining = "manifest.json"; + + expect((await getCachedWatchlist(id))?.data.metas).toEqual([replacement]); + }); + + it("refreshes a stale manifest when its generation is missing", async () => { + const id = watchlistId(); + await writeCachedWatchlist(id, { metas: [MOVIE] }); + const staleObjects = new Map(r2.objects); + + r2.objects.clear(); + const replacement = { + ...MOVIE, + id: "tt0068646", + name: "The Godfather", + }; + await writeCachedWatchlist(id, { metas: [replacement] }); + const replacementObjects = new Map(r2.objects); + + for (let index = 0; index <= 100; index += 1) { + await writeCachedWatchlist(watchlistId(), { metas: [MOVIE] }); + } + + r2.objects.clear(); + staleObjects.forEach((body, key) => r2.objects.set(key, body)); + expect(await findCachedMeta([id], "movie", "tt9999999")).toBeNull(); + + r2.objects.clear(); + replacementObjects.forEach((body, key) => r2.objects.set(key, body)); + + expect((await getCachedWatchlist(id))?.data.metas).toEqual([replacement]); + }); + it("invalidates the cache and deletes its current catalog", async () => { const id = watchlistId(); await writeCachedWatchlist(id, { metas: [MOVIE] }); diff --git a/apps/backend/src/services/watchlist-cache.ts b/apps/backend/src/services/watchlist-cache.ts index 58b5d96..3645200 100644 --- a/apps/backend/src/services/watchlist-cache.ts +++ b/apps/backend/src/services/watchlist-cache.ts @@ -65,6 +65,11 @@ interface ManifestRead { etag?: string; } +interface CatalogRead { + manifest: CacheManifest; + catalog: CatalogObject; +} + export interface CachedWatchlist { data: WatchlistData; cachedAt: Date; @@ -146,6 +151,24 @@ function setMemoryValue( } } +function cacheDeletedManifest( + watchlistId: string, + deletedGeneration: string, +): void { + const entry = manifestMemoryCache.get(watchlistId); + if (entry?.value && entry.value.generation !== deletedGeneration) return; + setMemoryValue(manifestMemoryCache, watchlistId, null); +} + +function evictManifestGeneration( + watchlistId: string, + staleGeneration: string, +): void { + const entry = manifestMemoryCache.get(watchlistId); + if (entry?.value && entry.value.generation !== staleGeneration) return; + manifestMemoryCache.delete(watchlistId); +} + async function readManifestFromR2(watchlistId: string): Promise { try { const response = await getR2Client().send( @@ -179,7 +202,13 @@ async function readManifest( const cached = getMemoryValue(manifestMemoryCache, watchlistId); if (cached !== undefined) return cached; + const entryBeforeRead = manifestMemoryCache.get(watchlistId); const { manifest } = await readManifestFromR2(watchlistId); + const concurrentEntry = manifestMemoryCache.get(watchlistId); + if (concurrentEntry !== entryBeforeRead) { + const concurrentValue = getMemoryValue(manifestMemoryCache, watchlistId); + if (concurrentValue !== undefined) return concurrentValue; + } setMemoryValue(manifestMemoryCache, watchlistId, manifest); return manifest; } @@ -210,6 +239,27 @@ async function readCatalog( } } +async function readCatalogWithManifestRefresh( + watchlistId: string, + manifest: CacheManifest, +): Promise { + const catalog = await readCatalog(manifest); + if (catalog) return { manifest, catalog }; + + evictManifestGeneration(watchlistId, manifest.generation); + const refreshedManifest = await readManifest(watchlistId); + if ( + !refreshedManifest || + refreshedManifest.generation === manifest.generation + ) { + return null; + } + + const refreshedCatalog = await readCatalog(refreshedManifest); + if (!refreshedCatalog) return null; + return { manifest: refreshedManifest, catalog: refreshedCatalog }; +} + function uniqueMetas(metas: StremioMeta[]): StremioMeta[] { const seen = new Set(); return metas.filter((meta) => { @@ -242,13 +292,13 @@ export async function getCachedWatchlist( const manifest = await readManifest(watchlistId); if (!manifest) return null; - const catalog = await readCatalog(manifest); - if (!catalog || catalog.metas.length === 0) return null; + const current = await readCatalogWithManifestRefresh(watchlistId, manifest); + if (!current || current.catalog.metas.length === 0) return null; return { - data: { metas: catalog.metas }, - cachedAt: new Date(manifest.cachedAt), - generation: manifest.generation, + data: { metas: current.catalog.metas }, + cachedAt: new Date(current.manifest.cachedAt), + generation: current.manifest.generation, }; } catch (error) { console.error(`Failed to read R2 cache for ${watchlistId}:`, error); @@ -317,7 +367,10 @@ export async function findCachedMeta( const manifestResults = await Promise.allSettled( watchlistIds.map((watchlistId) => readManifest(watchlistId)), ); - const candidates: CacheManifest[] = []; + const candidates: { + watchlistId: string; + manifest: CacheManifest; + }[] = []; manifestResults.forEach((result, index) => { if (result.status === "rejected") { @@ -328,14 +381,23 @@ export async function findCachedMeta( return; } if (result.value && hasSortedKey(result.value.metaKeys, target)) { - candidates.push(result.value); + candidates.push({ + watchlistId: watchlistIds[index], + manifest: result.value, + }); } }); - for (const manifest of candidates) { + for (const candidate of candidates) { try { - const catalog = await readCatalog(manifest); - const found = catalog?.metas.find( + const current = await readCatalogWithManifestRefresh( + candidate.watchlistId, + candidate.manifest, + ); + if (!current || !hasSortedKey(current.manifest.metaKeys, target)) { + continue; + } + const found = current.catalog.metas.find( (item) => item.type === type && item.id === id, ); if (found) return found; @@ -388,13 +450,12 @@ export async function deleteCachedWatchlist( ); } catch (error) { if (isPreconditionFailed(error)) { - manifestMemoryCache.delete(watchlistId); return; } throw error; } - setMemoryValue(manifestMemoryCache, watchlistId, null); + cacheDeletedManifest(watchlistId, current.manifest.generation); await getR2Client().send( new DeleteObjectCommand({ From cb7af10437c49fd2e777d6b7c05c9a5d490a98d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:40:50 +0200 Subject: [PATCH 12/15] fix(backend): externalize S3 client from server bundle --- .github/workflows/ci.yml | 8 ++++++++ apps/backend/build.ts | 1 + 2 files changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 275f666..677921a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,14 @@ jobs: - name: Lint, build, and test run: pnpm turbo run lint build test + - name: Smoke-test backend bundle + env: + SUPABASE_URL: https://example.invalid + SUPABASE_SERVICE_ROLE_KEY: test + RESEND_API_KEY: re_test + RESEND_AUDIENCE_ID: test + run: node --input-type=module --eval 'await import("./apps/backend/dist/index.js")' + e2e: name: E2E if: github.event_name == 'pull_request' diff --git a/apps/backend/build.ts b/apps/backend/build.ts index f103551..2fc7474 100644 --- a/apps/backend/build.ts +++ b/apps/backend/build.ts @@ -7,6 +7,7 @@ await esbuild.build({ platform: "node", format: "esm", bundle: true, + external: ["@aws-sdk/client-s3"], }); // Build the client entry (used by frontend for RPC types) without bundling From ce132e13543119479c883ce31e209d4d0b6ac59a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:43:11 +0200 Subject: [PATCH 13/15] ci: allow manual pull request validation --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 677921a..86c805a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + workflow_dispatch: push: branches: [main] pull_request: @@ -40,7 +41,7 @@ jobs: e2e: name: E2E - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 01b478f59c57ce04401490b7c21756d8e15cef7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:48:58 +0200 Subject: [PATCH 14/15] test(e2e): assert cache deletion tombstone --- apps/e2e/helpers/r2.ts | 20 ++++++++++++++++++++ apps/e2e/tests/addon-api.spec.ts | 16 +++++++++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/apps/e2e/helpers/r2.ts b/apps/e2e/helpers/r2.ts index 8401e94..59b9768 100644 --- a/apps/e2e/helpers/r2.ts +++ b/apps/e2e/helpers/r2.ts @@ -1,6 +1,7 @@ import { CreateBucketCommand, DeleteObjectsCommand, + GetObjectCommand, HeadBucketCommand, ListObjectsV2Command, S3Client, @@ -73,6 +74,25 @@ export async function countCacheObjects(watchlistId: string): Promise { return (await listKeys(`watchlists/${watchlistId}/`)).length; } +export async function getCacheObjectKeys( + watchlistId: string, +): Promise { + return listKeys(`watchlists/${watchlistId}/`); +} + +export async function getCacheManifest( + watchlistId: string, +): Promise { + const response = await r2.send( + new GetObjectCommand({ + Bucket: R2_BUCKET, + Key: `watchlists/${watchlistId}/manifest.json`, + }), + ); + if (!response.Body) throw new Error("R2 cache manifest has no body"); + return JSON.parse(await response.Body.transformToString()); +} + export async function deleteCacheObjects( watchlistIds: string[], ): Promise { diff --git a/apps/e2e/tests/addon-api.spec.ts b/apps/e2e/tests/addon-api.spec.ts index 6e3bfb8..6361720 100644 --- a/apps/e2e/tests/addon-api.spec.ts +++ b/apps/e2e/tests/addon-api.spec.ts @@ -14,7 +14,11 @@ import { type CatalogMeta, } from "../helpers/api.js"; import { clearRefreshCooldown, resetDb } from "../helpers/db.js"; -import { countCacheObjects } from "../helpers/r2.js"; +import { + countCacheObjects, + getCacheManifest, + getCacheObjectKeys, +} from "../helpers/r2.js"; import { P_HANDLE, PRIVATE_LIST, @@ -335,7 +339,7 @@ test.describe("catalogs", () => { ); test( - "removing a watchlist deletes its R2 cache objects", + "removing a watchlist deletes cached generations and leaves a tombstone", { tag: "@live-regression" }, async () => { const config = await bootstrapUser(PUBLIC_USER); @@ -365,7 +369,13 @@ test.describe("catalogs", () => { }, ]); expect(updated.status).toBe(200); - expect(await countCacheObjects(removed.id)).toBe(0); + expect(await getCacheObjectKeys(removed.id)).toEqual([ + `watchlists/${removed.id}/manifest.json`, + ]); + expect(await getCacheManifest(removed.id)).toMatchObject({ + version: 1, + deleted: true, + }); }, ); }); From 89326e6048bc1fc3207e8a59ccea3b49052d9f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9o=20Mathurin?= Date: Wed, 26 Aug 2026 18:55:09 +0200 Subject: [PATCH 15/15] ci: validate staging branch pushes --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86c805a..3678486 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,7 @@ name: CI on: workflow_dispatch: push: - branches: [main] + branches: [main, staging] pull_request: branches: [main, staging]