From 8c676738a44b43bc7f4afb7448789ccfc9f376a5 Mon Sep 17 00:00:00 2001 From: bballdavis Date: Tue, 8 Sep 2026 15:44:27 -0500 Subject: [PATCH 01/27] feat: add external API control plane --- .env.example | 4 + .github/workflows/ci.yml | 3 + docker-compose.dev.yml | 1 + docker-compose.external-db.yml | 2 + docker-compose.yml | 2 + docs/ENVIRONMENT_VARIABLES.md | 21 +- docs/EXTERNAL_API.md | 437 ++++++ docs/PRD_EXTERNAL_REQUEST_API.md | 766 +++++++++ fixtures/external-api-v1/SHA256SUMS | 1 + fixtures/external-api-v1/contract.json | 566 +++++++ package.json | 4 +- scripts/external-api-contract-server.js | 31 + server/__tests__/logger.test.js | 12 + .../server.additional-routes.test.js | 5 + server/__tests__/server.apikeys.test.js | 23 + server/__tests__/server.auth-sessions.test.js | 5 + server/logger.js | 9 + server/middleware/externalApiAuth.js | 111 ++ server/models/__tests__/apikey.model.test.js | 30 + .../__tests__/channelvideo.model.test.js | 13 + server/models/channelvideo.js | 4 +- server/models/index.js | 2 + .../apiKeyChannelGrantModule.test.js | 91 ++ server/modules/__tests__/apiKeyModule.test.js | 169 ++ .../__tests__/externalCatalogService.test.js | 416 +++++ .../externalContractFixtures.test.js | 110 ++ .../__tests__/externalEligibility.test.js | 40 + .../__tests__/externalPermissions.test.js | 73 + .../__tests__/externalQuotaService.test.js | 153 ++ .../__tests__/externalRequestService.test.js | 939 +++++++++++ .../__tests__/externalThumbnailProxy.test.js | 51 + .../__tests__/safeCommandLogging.test.js | 38 + server/modules/apiKeyChannelGrantModule.js | 97 ++ server/modules/apiKeyModule.js | 221 ++- .../modules/channel/channelYtdlpExecutor.js | 29 +- server/modules/downloadModule.js | 17 +- server/modules/externalApiResponse.js | 37 + server/modules/externalCatalogService.js | 823 ++++++++++ server/modules/externalEligibility.js | 63 + server/modules/externalPermissions.js | 67 + server/modules/externalQuotaService.js | 239 +++ server/modules/externalRequestService.js | 1382 +++++++++++++++++ server/modules/externalThumbnailProxy.js | 68 + server/modules/externalWorkLimiter.js | 59 + server/modules/jobModule.js | 16 +- server/modules/playlistModule.js | 10 +- server/modules/ratingMapper.js | 2 +- server/modules/safeCommandLogging.js | 106 ++ .../subscriptionImport/cookiesFetcher.js | 2 +- server/modules/videoDeletionModule.js | 21 +- server/modules/ytDlpRunner.js | 5 +- .../routes/__tests__/apikeys.policy.test.js | 245 +++ server/routes/__tests__/externalApi.test.js | 543 +++++++ .../externalApiContractHarness.test.js | 67 + .../routes/__tests__/externalRequests.test.js | 98 ++ server/routes/apikeys.js | 260 +++- server/routes/externalApi.js | 500 ++++++ server/routes/externalRequests.js | 181 +++ server/routes/index.js | 49 + server/server.js | 136 +- server/swagger.js | 99 +- server/testing/externalApiContractApp.js | 296 ++++ 62 files changed, 9797 insertions(+), 73 deletions(-) create mode 100644 docs/EXTERNAL_API.md create mode 100644 docs/PRD_EXTERNAL_REQUEST_API.md create mode 100644 fixtures/external-api-v1/SHA256SUMS create mode 100644 fixtures/external-api-v1/contract.json create mode 100755 scripts/external-api-contract-server.js create mode 100644 server/middleware/externalApiAuth.js create mode 100644 server/models/__tests__/apikey.model.test.js create mode 100644 server/models/__tests__/channelvideo.model.test.js create mode 100644 server/modules/__tests__/apiKeyChannelGrantModule.test.js create mode 100644 server/modules/__tests__/apiKeyModule.test.js create mode 100644 server/modules/__tests__/externalCatalogService.test.js create mode 100644 server/modules/__tests__/externalContractFixtures.test.js create mode 100644 server/modules/__tests__/externalEligibility.test.js create mode 100644 server/modules/__tests__/externalPermissions.test.js create mode 100644 server/modules/__tests__/externalQuotaService.test.js create mode 100644 server/modules/__tests__/externalRequestService.test.js create mode 100644 server/modules/__tests__/externalThumbnailProxy.test.js create mode 100644 server/modules/__tests__/safeCommandLogging.test.js create mode 100644 server/modules/apiKeyChannelGrantModule.js create mode 100644 server/modules/externalApiResponse.js create mode 100644 server/modules/externalCatalogService.js create mode 100644 server/modules/externalEligibility.js create mode 100644 server/modules/externalPermissions.js create mode 100644 server/modules/externalQuotaService.js create mode 100644 server/modules/externalRequestService.js create mode 100644 server/modules/externalThumbnailProxy.js create mode 100644 server/modules/externalWorkLimiter.js create mode 100644 server/modules/safeCommandLogging.js create mode 100644 server/routes/__tests__/apikeys.policy.test.js create mode 100644 server/routes/__tests__/externalApi.test.js create mode 100644 server/routes/__tests__/externalApiContractHarness.test.js create mode 100644 server/routes/__tests__/externalRequests.test.js create mode 100644 server/routes/externalApi.js create mode 100644 server/routes/externalRequests.js create mode 100644 server/testing/externalApiContractApp.js diff --git a/.env.example b/.env.example index 337155a6f..b9b78d59e 100644 --- a/.env.example +++ b/.env.example @@ -78,6 +78,10 @@ YOUTUBE_OUTPUT_DIR=./downloads # Default: true (authentication enabled) AUTH_ENABLED=true +# Versioned external API. This API always requires an external API key even +# when AUTH_ENABLED=false. Set to false to disable the API namespace. +EXTERNAL_API_ENABLED=true + # Initial Admin Credentials (for headless/automated deployments) # If not set, you'll create credentials through the web UI on first access # And they will be saved to config/config.json (password hashed) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d928669a8..5053f9573 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -121,6 +121,9 @@ jobs: - name: Install dependencies run: npm ci --ignore-scripts + - name: Validate external API fixture and checksum + run: npm run test:external-fixture + - name: Run backend tests with coverage run: npm run test:backend -- --coverage --coverageReporters=json-summary --coverageReporters=lcov --coverageReporters=text env: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 7daf9b6c0..0c37179f3 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -49,6 +49,7 @@ services: - DB_PASSWORD=${DB_PASSWORD:-youtarrpassword} - YOUTUBE_OUTPUT_DIR=/usr/src/app/data - AUTH_ENABLED=${AUTH_ENABLED:-true} + - EXTERNAL_API_ENABLED=${EXTERNAL_API_ENABLED:-true} - AUTH_PRESET_USERNAME=${AUTH_PRESET_USERNAME:-} - AUTH_PRESET_PASSWORD=${AUTH_PRESET_PASSWORD:-} - TRUST_PROXY=${TRUST_PROXY:-} diff --git a/docker-compose.external-db.yml b/docker-compose.external-db.yml index 61383bd4a..497aae2eb 100644 --- a/docker-compose.external-db.yml +++ b/docker-compose.external-db.yml @@ -39,6 +39,8 @@ services: DB_NAME: ${DB_NAME:-youtarr} # Optional: Disable authentication (for platforms with external auth like Elfhosted) AUTH_ENABLED: ${AUTH_ENABLED:-} + # Optional: Disable the separately authenticated versioned external API + EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-true} # Optional: Seed initial admin credentials for headless deployments AUTH_PRESET_USERNAME: ${AUTH_PRESET_USERNAME:-} AUTH_PRESET_PASSWORD: ${AUTH_PRESET_PASSWORD:-} diff --git a/docker-compose.yml b/docker-compose.yml index 9fdde676b..ed8e10910 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,6 +67,8 @@ services: DB_NAME: ${DB_NAME:-youtarr} # Optional: Disable authentication (for platforms with external auth like Elfhosted) AUTH_ENABLED: ${AUTH_ENABLED:-} + # Optional: Disable the separately authenticated versioned external API + EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-true} # Optional: Seed initial admin credentials for headless deployments AUTH_PRESET_USERNAME: ${AUTH_PRESET_USERNAME:-} AUTH_PRESET_PASSWORD: ${AUTH_PRESET_PASSWORD:-} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 38ab2d5b8..0d329f600 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -99,6 +99,21 @@ To use an external database: - When using reverse proxy with authentication - Platform deployments with external auth (e.g., Cloudflare Access) +### EXTERNAL_API_ENABLED +**Required**: No +**Default**: `true` +**Options**: `true`, `false` +**Description**: Enables the versioned `/external-api/v1` integration API by default +**Security**: External API routes always require a non-legacy `x-api-key`, even when `AUTH_ENABLED=false` + +Set this to `false` only when the versioned API namespace should be disabled. +The routes remain protected until an administrator has assigned an API key an +external role and policy. Existing API keys migrate as `legacy_download` and +cannot use the versioned API. For endpoint-only public access, keep the Youtarr +origin port private and publish only the exact v1 path through a TLS reverse +proxy. Exposing the complete port also exposes unrelated application routes; see +[Youtarr External API](EXTERNAL_API.md). + ### AUTH_PRESET_USERNAME **Required**: No **Default**: None @@ -125,8 +140,10 @@ To use an external database: **Recommendations**: - Set `TRUST_PROXY=false` when Youtarr is exposed directly without a reverse proxy - Leave unset only if you want the historical Express proxy-header trust behavior; Youtarr's rate-limit, session, and setup audit IPs will still key on the direct peer IP until `TRUST_PROXY` is explicitly configured -- Set `TRUST_PROXY=1` when Youtarr is behind one trusted reverse proxy and you want per-client rate limits -- Prefer a specific hop count or trusted subnet over broad `true` when exposing Youtarr through a proxy you control +- Set `TRUST_PROXY=1` only when Youtarr is behind exactly one trusted reverse + proxy and you want per-client rate limits +- Prefer the exact proxy IP/subnet over broad `true`; the origin must not be + reachable in a way that lets a client bypass the trusted proxy ## YouTube Cookies diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md new file mode 100644 index 000000000..995e06312 --- /dev/null +++ b/docs/EXTERNAL_API.md @@ -0,0 +1,437 @@ +# Youtarr External API + +`/external-api/v1` is Youtarr's constrained integration boundary for approved +clients. It is enabled by default and never accepts a Youtarr browser session +or a legacy download key. Set `EXTERNAL_API_ENABLED=false` to disable the +namespace for a deployment. + +## Enablement and authentication + +Create an external-role key in **Settings → API Keys → API Keys & External +Access**. The key is revealed once. Send it only in the `x-api-key` header over +HTTPS. Set `EXTERNAL_API_ENABLED=false` and restart Youtarr if the namespace +should be disabled. + +Existing keys migrate as `legacy_download`. They continue to work only with +`POST /api/videos/download`. Conversely, external-role keys cannot use that +legacy endpoint, so its direct-download behavior cannot bypass external +policies or approval. + +Every external key includes catalog and owner-request reads. Request +capabilities are independently configurable: + +| Permission | Added scope | +| --- | --- | +| Request videos | `video:request` | +| Request channels | `channel:request` | +| Delete downloaded videos | `video:delete` | + +The `role` value remains as a backward-compatible summary for older clients, +but `capabilities.scopes` is authoritative. Clients must not infer a +permission that is absent from the scopes array. `admin` remains a policy +summary only and does not authorize remote approval. Review always requires a +normal Youtarr administrator session. + +## Policy and grants + +Every external key has: + +- independent video-request, channel-request, and downloaded-video-deletion + permissions; catalog and owner-status reads are always included; +- a maximum allowed movie/TV rating; +- an allow/disallow decision for unrated or unrecognized ratings; +- allowed media types (`video`, `short`, and/or `livestream`); +- separate auto-approval decisions that are valid only when their parent + permission is enabled; +- durable workload ceilings, defaulting to 5 active jobs, 30 accepted writes + per UTC hour, and 200 per UTC day (administrators may select lower values); +- an explicit set of granted Youtarr channel database IDs. + +The HTTP contract retains a compact `maxRatingLevel` value, but the +administrator UI presents the actual rating ceilings: + +| Level | Administrator label | Movie ratings | TV ratings | +| --- | --- | --- | --- | +| 1 | General audiences | G | TV-Y, TV-G | +| 2 | Parental guidance | PG | TV-Y7, TV-PG | +| 3 | Teen | PG-13 | TV-14 | +| 4 | Mature | R, NC-17 | TV-MA | + +YouTube usually does not supply a useful content rating. Youtarr therefore +uses the video's explicit/manual rating when present, then the channel's +manually assigned default rating. If neither is recognized, the key's +`allowUnrated` decision applies. The same calculation is used for catalog +visibility, counts, assets, catalog rows, request creation, and approval +revalidation. + +Youtarr applies one server-side eligibility decision to lists, counts, direct +IDs, assets, catalog feeds, request creation, and approval revalidation. +Disabled or terminated channels never qualify. A missing grant is treated the +same as a nonexistent target. + +## Error contract + +All JSON responses under `/external-api/*`, including feature-off and +unknown-route responses, use private, no-store cache headers and vary on +`x-api-key`. Errors use: + +```json +{ + "error": { + "code": "not_found", + "message": "External API route not found", + "requestId": "server-generated-identifier" + } +} +``` + +Clients must treat unknown additive fields and unknown enum values as +forward-compatible. They must not infer target existence from 403/404 +differences. + +The standardized statuses are 400 (malformed or invalid request), 401 +(authentication), 403 (missing scope), 404 (unknown or hidden resource), 405 +(method), 413 (body size), 415 (content type), 429 (rate/quota), 500 +(unexpected failure), and 503 (bounded downstream capacity). Compressed +request bodies are not accepted. CORS is disabled. + +The sanitized cross-client decoding fixture is +`fixtures/external-api-v1/contract.json`; its reviewable checksum is recorded +beside it in `SHA256SUMS`. Fixture version 3 covers capabilities, channels, +cursor-paginated mixed-media catalog pages, rich and sparse video details, +every request type and status, video-request outcomes, and forward-compatible +unknown enum samples. The fixture contains no credentials or private +deployment address. Consumer repositories should vendor the JSON and checksum +unchanged and pin the producing Youtarr commit used for synchronization. + +## Endpoints + +### Capabilities + +`GET /external-api/v1/capabilities` + +Returns the API/server version, effective role/scopes, policy, feature flags, +and effective quota limits/remaining allowance. Clients should call this +before presenting optional actions. + +### Granted catalog + +`GET /external-api/v1/channels` + +Query parameters: + +- `cursor` (opaque) or `page` (default 1), plus `pageSize` (1–100); +- `search` (maximum 200 characters); +- `subfolder` (exact match, maximum 255 characters); +- `sortBy`: `title`, `videoCount`, `downloadedCount`, or `id`; +- `sortOrder`: `asc` or `desc`. + +`GET /external-api/v1/channels/{channelDatabaseId}/videos` + +Query parameters: + +- `cursor` (preferred) or `page` (compatibility only), plus `pageSize` and + `search`; +- `tabType`: `videos`, `shorts`, or `streams`; +- `status`: `all`, `requestable`, `available`, `downloaded`, or `requested`; +- `minDuration`, `maxDuration`, `dateFrom`, and `dateTo`; +- `sortBy`: `date`, `title`, or `duration`; +- `sortOrder`: `asc` or `desc`. + +Video status has these exact meanings: + +- `all` (or omitted): every policy-eligible cached row; +- `requestable`: not downloaded and without a `pending`, `approved`, or + `processing` video request owned by the calling key; +- `available`: not downloaded, including rows with an active request; +- `downloaded`: a `Videos` row exists with `removed=false`; +- `requested`: an active request exists for the calling key. + +Responses include an opaque `nextCursor` while more rows remain. Video cursors +are stable keyset cursors bound to the calling key, endpoint, filters, and +sorting used to create them. A client must restart at the first page after +changing any filter, sort, or page size. Cursors avoid offset drift but do not +freeze a database snapshot, so a newly indexed or state-changing row may +appear only on a later refresh. + +Existing numeric `page` parameters remain supported through page 100 for +compatibility. Use cursors whenever completeness matters. Clients must not use +`total` to preallocate a fixed snapshot. Catalog endpoints read Youtarr's cache +and do not trigger YouTube network fetches. + +### Cross-channel video catalog + +`GET /external-api/v1/videos` + +Returns the complete policy-filtered catalog across every channel granted to +the calling key. One cursor chain combines dozens of channels; clients must not +list channels and request each channel's videos unless they are rendering +channel-specific screens. `pageSize` is limited to 100. It accepts `search`, +`tabType`, `status`, duration/date bounds, `sortBy`, and `sortOrder` with the +same meanings as the channel video catalog. + +Complete catalog traversal: + +```http +GET /external-api/v1/videos?pageSize=100 +GET /external-api/v1/videos?pageSize=100&cursor={nextCursor} +``` + +Only immediately requestable videos: + +```http +GET /external-api/v1/videos?status=requestable&pageSize=100 +``` + +Not downloaded, including pending/approved/processing requests: + +```http +GET /external-api/v1/videos?status=available&pageSize=100 +``` + +Downloaded inventory: + +```http +GET /external-api/v1/videos?status=downloaded&pageSize=100 +``` + +Pass each response's `nextCursor` back with the same filters and sorting until +it is `null`. Request creation remains idempotent and revalidates current +download/request state, closing the race between reading a page and submitting +a request. + +Youtarr supplies candidates only. Client-side history, scoring signals, and +ranking state remain outside Youtarr and must never be sent to it. + +### Video detail + +`GET /external-api/v1/videos/{youtubeId}` + +Returns one policy-filtered video with the catalog identity and request status +plus the full curated metadata used by Youtarr's video detail modal. This +includes the complete description, engagement counts, tags, categories, +availability, upload data, technical video fields, related-file summaries, and +available resolutions. The lookup is intentionally one video at a time and may +populate Youtarr's metadata cache when no cached `.info.json` exists. These +lookups share Youtarr's bounded external-work queue and return 503 when that +queue is full. + +Downloaded-video details include timestamps, sizes, protection state, and +resolution. Youtarr filesystem paths are never exposed because they are neither +safe nor usable by a remote client. Missing, hidden, ungranted, and +policy-ineligible videos all return the same 404 contract. + +### Assets + +- `GET /external-api/v1/assets/channels/{channelDatabaseId}/thumbnail` +- `GET /external-api/v1/assets/videos/{youtubeId}/thumbnail` + +Local assets require the same key, grant, channel state, rating, and media +policy as their catalog row. Youtarr rejects unsafe identifiers, traversal, +symlinks, and resources outside its image directory. Responses are private and +do not redirect. + +Catalog responses always return these Youtarr API asset paths, never a direct +Google/YouTube image URL. The client fetches the image bytes with the same +`x-api-key` header used for JSON requests. Youtarr serves its optimized local +JPEG when available; otherwise the video asset endpoint securely fetches an +allow-listed YouTube/Google thumbnail without redirecting the client. Upstream +responses are bounded by timeout, content type, and size. + +### Create requests + +Video: + +```http +POST /external-api/v1/requests/videos +``` + +```json +{ + "youtubeId": "abcdefghijk", + "channelId": 8, + "idempotencyKey": "optional, 1–200 characters" +} +``` + +Channel: + +```http +POST /external-api/v1/requests/channels +``` + +```json +{ + "channelUrl": "https://www.youtube.com/@example", + "idempotencyKey": "optional, 1–200 characters" +} +``` + +Only canonical YouTube handle, `/channel/`, `/c/`, and `/user/` URLs are +accepted. Approval provisions/enables the channel. By default it also grants +the resulting channel to the requesting key; the administrator may turn that +off during approval. That decision is stored with the request so an +idempotent recovery cannot broaden the grant after interrupted execution. +Uncached channel metadata resolution terminates after two minutes. + +Downloaded-video deletion: + +```http +POST /external-api/v1/requests/delete-videos +``` + +```json +{ + "youtubeId": "abcdefghijk", + "channelId": 8, + "idempotencyKey": "optional, 1–200 characters" +} +``` + +Deletion applies only to the downloaded video asset. It never removes or +disables a channel subscription. Missing and already-removed targets complete +idempotently and retain a terminal request record for audit. + +Created requests return HTTP 202 and `outcome: "created"`. Duplicates and +already-terminal targets return HTTP 200 with `duplicate`, +`already_downloaded`, or `already_deleted`. + +### Owner-scoped request reads + +- `GET /external-api/v1/requests` +- `GET /external-api/v1/requests/{requestId}` + +Reads expose only records owned by the calling key. List paging accepts either +an opaque `cursor` or the compatible `page` parameter, is limited to 100 rows, +rejects pages beyond 100, and accepts an exact `status` filter. + +Statuses are `pending`, `approved`, `processing`, `completed`, `rejected`, +`failed`, or `cancelled`. Youtarr reconciles downloader state before returning +results. A terminal downloader job without a produced video becomes `failed`; +it does not remain stuck in `processing`. Video deletion is reconciled to +completed when the asset is already absent. Interrupted channel/deletion +execution can be reclaimed after a five-minute stale window and rerun through +its idempotent operation. + +## Administrator review + +The web queue uses session-only endpoints under `/api/external-requests`: + +- list with `status`, `requestType`, and `apiKeyId` filters; +- request detail; +- approve; +- reject with a 1–300 character reason. + +Approval locks the current request, reloads the current key policy, and +revalidates authorization and target state immediately before execution. +Revocation or a policy/grant change therefore takes effect immediately. + +## Rate limits + +External reads are limited to 120 requests per minute per key. Request writes +are limited to 10 per minute per key. Review actions have a separate +session-only limit of 30 per minute. + +## Deployment and operations + +Youtarr authenticates every external request with `x-api-key` and enforces the +key's role, independent request permissions, channel grants, content policy, +quotas, and request limits. External-key authentication remains active when +`AUTH_ENABLED=false`; browser sessions, legacy download keys, +`Authorization`, and `x-access-token` do not authenticate this namespace. + +The operator owns TLS, the public hostname, firewalling the Youtarr origin, +and the reverse proxy. Keep the Youtarr origin port private and publish only +`/external-api/v1` and its descendants. Do not expose the complete application +port: that would also publish the SPA, setup and health routes, Swagger, +images, WebSocket, session management, and legacy APIs. + +Set `TRUST_PROXY` to the exact trusted proxy hop or subnet. Never use an +unrestricted setting on an origin reachable by untrusted clients. + +The proxy must: + +- allow only `GET`, `HEAD`, and `POST` for `/external-api/v1` paths; +- limit request bodies to 16 KiB and enforce connection/header/read timeouts; +- forward `x-api-key` and `Content-Type`, while removing `Cookie`, + `Authorization`, and `x-access-token`; +- reject WebSocket upgrades, unmatched hosts, other versions, and all other + application paths; +- disable response caching and hide `Set-Cookie`; +- apply TLS, connection, and edge rate limits appropriate for the deployment. + +Start with auto-approval disabled, a database and configuration backup, and a +view-only key with the minimum channel grants. Confirm the private Youtarr +application and proxy rules before exposing the service. Revoke a compromised +key immediately; to shut down the whole namespace, set +`EXTERNAL_API_ENABLED=false` and restart Youtarr. + +Each key defaults to at most 5 active jobs, 30 accepted writes per UTC hour, +and 200 per UTC day. Administrators may select lower limits. A shared +application ceiling also bounds downloader, deletion, and channel-provisioning +work. `/external-api/v1/capabilities` reports effective limits and remaining +allowance. + +Monitor 401, 403, 404, 429, 500, and 503 rates without logging API keys. +Youtarr sanitizes downloader command logging, including proxy credentials, +cookies, headers, and paths; rotate credentials and expire older logs if they +may contain secrets. + +## Configuration and administration + +1. In **Settings → API Keys → API Keys & External Access**, create an external + key and save the raw secret; it is shown only once. +2. The session-authenticated `GET /api/keys` management response includes + `channel_grant_count`: the number of grants whose channels are currently + enabled and non-terminated. It is not part of the `/external-api/v1` contract. +3. Choose the smallest request permissions, rating ceiling, media types, + quotas, and enabled channel grants. +4. Use the administrator **External Requests** queue to approve or reject + pending requests when auto-approval is not enabled. + +Saving an external key with zero approved channels is intentional and does not +backfill or grant access. That key fails closed: catalog reads and requests +return no granted content until an administrator adds enabled, non-terminated +channel grants. + +Existing keys migrate as `legacy_download` and remain limited to +`POST /api/videos/download`. They cannot use `/external-api/v1`. External +keys cannot use the legacy direct-download endpoint. + +## Synthetic consumer contract + +The canonical, production-free consumer dataset lives in +`fixtures/external-api-v1/contract.json`. Validate its published checksum with +`npm run test:external-fixture`. + +Its `representativeProfile` records only sanitized structural observations +from a deployed catalog: channel/page scale, media-type mix, common rating and +nullability patterns, and authenticated artwork URL shape. It contains no +production titles, identifiers, descriptions, request records, or credentials. +The ordinary catalog rows follow that profile; deliberately unusual enum, +duration, rating, sparse-detail, and failure cases remain separate edge data. +Fixture tests derive the profile back from the rows and fail when they drift. + +Run the lightweight contract server with: + +```sh +npm run test:external-contract-server +``` + +The executable mounts the real `/external-api/v1` router and authentication +middleware with deterministic catalog, request, quota, and artwork adapters. +It prints a JSON object containing its loopback base URL and synthetic API key; +it never starts the full Youtarr application or requires MySQL. Route tests +exercise the same server and keep the fixture, checksum, response envelopes, +query validation, headers, status codes, and request behavior release-gated. + +The `/__contract/scenario` and `/__contract/state` endpoints exist only on this +test executable. They let consumer UI suites select deterministic empty, +safety-filtered, unauthorized, transport, unsupported-version, malformed, +delayed, and server-error responses and inspect accepted synthetic writes. They +are not mounted by Youtarr itself. + +For normal Youtarr API integrations such as bookmarklets and mobile shortcuts, +see [API Integration](API_INTEGRATION.md). This document is the authoritative +reference for the versioned external API contract, endpoint behavior, and +deployment requirements. diff --git a/docs/PRD_EXTERNAL_REQUEST_API.md b/docs/PRD_EXTERNAL_REQUEST_API.md new file mode 100644 index 000000000..54a4ced22 --- /dev/null +++ b/docs/PRD_EXTERNAL_REQUEST_API.md @@ -0,0 +1,766 @@ +# PRD: Youtarr External API and Request Management + +## Document Status + +- Status: Draft for maintainer review +- Target project: Youtarr +- Last updated: 2026-07-29 +- Audience: Any external client integrating with a Youtarr instance +- Primary use case: Provide a stable, documented, API-key-protected Youtarr API for browsing eligible content and submitting approval-backed requests. + +This API is a Youtarr product capability. Its contract, authorization model, documentation, and delivery plan must remain client-agnostic and must not depend on any one downstream application. + +## Background + +Youtarr already supports: + +- session-authenticated web UI access +- API key creation in the **API Keys & External Access** settings section +- a REST API with Swagger/OpenAPI docs +- a limited API-key workflow for `POST /api/videos/download` + +Today, API keys are intentionally restricted to a single endpoint that queues individual video downloads. That is a good security baseline, but it is too narrow for integrations that need to: + +- show available channels +- show videos within those channels, including thumbnails and download state +- request videos to be downloaded +- optionally request new channels +- operate without exposing the full Youtarr web UI + +This PRD defines a safety-first expansion of Youtarr so it can support general-purpose external integrations while keeping the main app protected behind separate middleware if desired. + +## Current Implementation Status + +As of 2026-07-29: + +- The upstream API enhancement is tracked in issue `#568`; no implementation PR is open. +- The current `api-expansion` branch is a design branch. Relative to upstream `dev`, it adds this PRD and clarifies the authenticated thumbnail/asset contract; it does not contain the broader API implementation. +- Existing API keys, their management UI, and the legacy `POST /api/videos/download` flow provide the baseline to evolve. +- The dedicated `/external-api/v1` namespace, expanded roles and policies, channel grants, catalog endpoints, durable request lifecycle, administrator queue, and operational hardening remain to be implemented. + +Implementation therefore begins with the foundation phase below. Later phases must not be described as complete until their individual PRs are merged and their acceptance checks pass. + +## Problem Statement + +Instance owners and integration authors need a limited, API-key-protected surface of Youtarr without exposing the full admin UI. External clients need read access to eligible content and limited request access, while instance owners need: + +- strict API-key enforcement +- simple reverse-proxy isolation +- approval controls for risky actions +- auditing and revocation +- predictable paging and filtering for very large channels + +## Goals + +1. Add a dedicated external API namespace that can be exposed separately from the main app. +2. Require API-key authentication for every external API request. +3. Support read access to channels and channel videos for external clients. +4. Support request-based workflows for downloading videos and optionally adding channels. +5. Add role/scoped API keys with approval-related behavior. +6. Add an in-app request queue for approving or rejecting outstanding requests. +7. Document safe reverse-proxy deployment patterns clearly. +8. Break implementation into manageable PR-sized phases. +9. Keep the public contract generic so browser extensions, native apps, automations, dashboards, and other third-party clients can integrate without consumer-specific behavior. + +## Non-Goals + +- Replacing session auth for the main Youtarr web UI in this implementation program +- Building multi-user household accounts +- Exposing every existing Youtarr endpoint to API keys +- Full external CRUD over all settings and channel management +- Solving public-internet hardening beyond Youtarr's documented safe deployment patterns +- Adding behavior, fields, or terminology that exists only for one named downstream client + +## Users and Personas + +### Instance Admin + +The Youtarr owner who manages channels, downloads, settings, and approvals. + +### External Viewer + +A client/device allowed to browse approved channels and videos only. + +### External Requester + +A client/device allowed to browse approved content and submit requests for download or new channels/videos, subject to auto-approval or admin review. + +### External Admin + +A future phase role that can use a constrained web UI backed by API keys instead of session auth. + +## Product Principles + +### Safety First + +External access must default to least privilege. The new API should be easy to place behind a separate reverse-proxy path rule such as `/external-api/*`, while the rest of Youtarr remains behind stronger middleware or private-only access. + +### Explicit Separation + +External API routes should live under their own prefix instead of reusing the mixed existing route shape. This simplifies: + +- reverse-proxy rules +- auth policy +- logging +- documentation +- future rate-limiting and abuse controls + +### Requests, Not Direct Power + +Risky actions should be modeled as requests first unless a key's policy explicitly allows auto-approval. + +### Client-Agnostic Contract + +The external API exposes Youtarr resources and workflows, not consumer-specific screens or recommendation logic. Examples may illustrate how a client could use the API, but no named integration defines the contract. + +## Proposed Solution Overview + +Add a new external API namespace: + +- Proposed prefix: `/external-api/v1` + +All routes under this prefix: + +- require `x-api-key` +- reject session-token-only access +- are governed by API key scopes/role policies +- are documented separately in Swagger/OpenAPI + +Program capabilities: + +1. Read channel list +2. Read channel video lists with paging and filters +3. Submit video download requests +4. Submit new channel requests +5. Submit approval-backed downloaded-video deletion requests +6. Read a bounded, policy-filtered cross-channel video feed +7. Review and process pending requests in a new Requests section in the main web UI + +## Authentication and Authorization + +### External API Authentication + +- All `/external-api/v1/*` endpoints require `x-api-key`. +- Requests without a valid API key return `401`. +- Session tokens (`x-access-token`) do not grant access to this namespace. +- No external endpoint is anonymous. + +### API Key Model + +Expand the existing API key model to support role/scope metadata. + +#### Access levels + +- `view`: read-only access to allowed external browse endpoints +- `request`: includes `view` plus ability to create requests +- `delete`: includes `request` plus delete requests where allowed by policy +- `admin`: the broadest external API policy scope; it does not grant session administration or request-review authority + +#### Key policy flags + +- `autoApproveVideoRequests` +- `autoApproveDeleteRequests` +- `autoApproveChannelRequests` + +These flags should only be available where they make sense for the selected access level. + +#### Notes + +- Existing keys should migrate safely with a legacy-compatible default, likely equivalent to a narrowly scoped request key for the current single-video endpoint. +- Keys remain hashed at rest. +- Keys remain one-time-view on creation. +- Keys should display metadata such as scope, last used, usage count, and created date. + +## External API Surface + +The exact path names can change during tech design, but the namespace and capabilities should remain stable. + +### 1. Channels List + +Purpose: allow external clients to display approved channels. + +Example: + +- `GET /external-api/v1/channels` + +Suggested query support: + +- `page` +- `pageSize` +- `search` +- `subfolder` +- `sortBy` +- `sortOrder` + +Suggested response fields: + +- `id` +- `channelId` +- `title` +- `thumbnailUrl` +- `description` or truncated summary +- `subfolder` +- `videoCount` +- `downloadedCount` +- `lastFetchedAt` + +Notes: + +- Reuse existing internal channel pagination semantics where possible. +- Return enough metadata for a client to render a browsable channel list without needing extra follow-up calls for basic display. +- Thumbnail references must be usable by authenticated external clients without requiring a Youtarr session cookie. + +### 2. Channel Videos + +Purpose: allow external clients to browse a channel's videos, thumbnails, metadata, and download state. + +Example: + +- `GET /external-api/v1/channels/:channelId/videos` + +Suggested query support: + +- `page` +- `pageSize` +- `offset` +- `limit` +- `status=all|downloaded|not_downloaded|requested|pending` +- `search` +- `sortBy` +- `sortOrder` +- `tabType=videos|shorts|streams` +- `dateFrom` +- `dateTo` +- `minDuration` +- `maxDuration` +- `maxRating` + +Suggested response fields: + +- `youtubeId` +- `title` +- `thumbnailUrl` +- `publishedAt` +- `duration` +- `description` +- `isDownloaded` +- `isRequested` +- `requestStatus` +- `rating` +- `channelId` +- `channelTitle` + +#### Thumbnail and media asset delivery + +This needs to be explicit because external clients may not be logged into the main Youtarr web UI. + +Requirements: + +- Thumbnail and related image assets exposed to external clients must be retrievable with API-key-based access, not only browser session auth. +- The external API should not return asset URLs that silently depend on existing Youtarr login cookies. +- The asset delivery contract must work for native clients that can attach an API key to each request. + +Recommended approach: + +- Return external-API-scoped asset URLs such as `/external-api/v1/assets/...` or `/external-api/v1/thumbnails/...`. +- Protect those asset endpoints with the same `x-api-key` requirement as the rest of the external namespace. +- Keep those asset URLs stable enough for clients to render list views without custom scraping or cookie-sharing. + +Compatibility note: + +- If some thumbnail values currently point directly to YouTube-hosted images, that is acceptable as a fallback for non-sensitive public imagery. +- If Youtarr serves locally cached thumbnails, posters, or protected assets, the PRD should assume those must be reachable through authenticated external asset endpoints. + +#### Paging requirements + +This area is critical because some channels have thousands of videos. + +Requirements: + +- Support deterministic paging over stable, explicit sort orders. +- Prefer `page + pageSize` as the public contract for consistency with current Youtarr behavior. +- Also consider `offset + limit` support if that improves client UX for "videos 30-40" style access. +- Response must include paging metadata: +- `page` +- `pageSize` +- `total` +- `totalPages` +- optionally `offset` and `limit` if supported + +#### Data sourcing behavior + +The implementation should not require a full fresh YouTube fetch for every external browse request. + +Expected behavior: + +- External browse endpoints primarily serve from Youtarr's cached/database-backed channel video records. +- Existing channel refresh behavior remains separate from external browsing. +- If the channel has not yet been fully indexed, the API should return a truthful partial-state indicator rather than blocking for a long-running scrape. + +Potential response metadata: + +- `dataSource=cache|partial_cache` +- `isFullyIndexed` +- `lastIndexedAt` +- `indexingHint` + +This keeps response latency predictable and avoids coupling player UX to yt-dlp fetch timing. + +### 3. Video Download Request + +Purpose: allow external clients to request a video download. + +Example: + +- `POST /external-api/v1/requests/videos` + +Request body: + +- `youtubeId` or canonical YouTube URL +- optional `channelId` +- optional client metadata + +Behavior: + +- If the key policy allows auto-approval, create and execute the download immediately. +- Otherwise, create a pending request record for approval. +- Duplicate requests should be idempotent or return a clear "already requested/downloaded" response. + +Response should indicate: + +- `status=approved|pending|duplicate|already_downloaded|rejected` +- `requestId` +- `message` + +### 4. Channel Request + +Purpose: allow external clients to request that a new channel be added. + +Example: + +- `POST /external-api/v1/requests/channels` + +Request body: + +- channel URL + +Behavior: + +- validate URL +- resolve channel metadata when practical +- create pending or auto-approved request based on key policy + +This supports the future simplified requester UI and direct client integrations. + +### 5. Request Status + +Purpose: allow external clients to check the status of their requests. + +Examples: + +- `GET /external-api/v1/requests` +- `GET /external-api/v1/requests/:requestId` + +Suggested filters: + +- `status=pending|approved|rejected|completed` +- `type=video|channel|delete` + +This is especially useful for mobile apps that need to reflect whether a request is still waiting on admin approval. + +### 6. Downloaded-Video Deletion Request + +Purpose: allow an authorized external client to request deletion of an already-downloaded video asset. + +Example: + +- `POST /external-api/v1/requests/delete-videos` + +Behavior: + +- always use the durable request lifecycle +- never directly delete through the external API +- revalidate role, grants, content policy, and downloaded state before approval +- never remove a channel subscription +- complete idempotently when the asset is already absent + +### 7. Cross-Channel Video Feed + +Purpose: allow external clients to discover eligible videos across their granted channels without defining recommendation behavior inside Youtarr. + +Example: + +- `GET /external-api/v1/videos` + +Behavior: + +- use the same grant, channel-state, rating, and media-type policy as all other catalog endpoints +- return stable, bounded pages from cached Youtarr data +- expose generic candidate records only; do not accept client profiles, playback history, or ranking signals + +## Request Approval Workflow + +Add a new Requests area to the main Youtarr web UI for session-authenticated admins. + +### UI requirements + +- show pending requests +- show approved/rejected/completed history as useful follow-up scope +- filter by request type and status +- display requester key name/prefix for auditing +- approve or reject individual requests +- bulk approve/reject may be added later, but not required for initial delivery + +### Initial request types + +- video download request +- channel add request + +### Later request type in this implementation program + +- delete downloaded video + +### Potential future request types + +- delete channel +- refresh channel + +## Configuration UX Changes + +Extend the existing **API Keys & External Access** section instead of creating a parallel configuration area. + +### Additions + +- access level selector: `view`, `request`, `delete`, `admin` +- auto-approve toggles appropriate to the role +- clearer description of what each access level can do +- visibility into rate limits, last used, and request policy +- migration/labeling for legacy keys + +### Recommended UX constraints + +- `view` cannot create requests +- `request` can create requests but cannot directly delete content +- `delete` only enables delete request flows, not unrestricted deletion +- `admin` should be clearly labeled as high-risk + +## Security Requirements + +### Namespace Separation + +The external API must have its own top-level prefix, recommended as `/external-api/v1`, to enable safe reverse-proxy handling. + +### Mandatory API-Key Protection + +- Every endpoint under the external prefix requires a valid API key. +- No endpoint under the external prefix is accessible with no auth. +- No fallback to session auth on the external prefix. + +### Reverse Proxy Guidance + +Documentation must include safe examples for: + +- Traefik +- Caddy +- Nginx + +Recommended pattern: + +- protect the main Youtarr UI with stronger middleware, private network access, or SSO +- optionally expose only `/external-api/*` +- keep Swagger/docs for the external API behind admin access unless intentionally exposed + +### Logging and Auditing + +Log for each external request: + +- API key id/prefix/name +- path +- method +- result +- request id where applicable + +### Rate Limiting + +The foundation PR should keep per-key rate limiting and extend it to the new external namespace. + +Consider: + +- different limits for browse vs write endpoints +- lower limits for request-creation endpoints + +### Error Handling + +Use clear API responses for: + +- invalid key +- insufficient scope +- duplicate request +- already downloaded +- resource not found +- approval required +- rate limited + +## Future: API-Key-Backed Simplified Web UI + +This is intentionally outside the initial implementation program and must not block the external API. + +Goal: allow API keys to authenticate into a simplified Youtarr web experience. + +### Role-based UI behavior + +- `view`: videos-only UI, browse existing videos and metadata +- `request`: channel list + channel detail pages + ability to request videos and request new channels +- `admin`: near-full UI access + +### Restrictions + +- request users should not gain access to settings +- request users should not gain ignore/delete/admin controls unless explicitly authorized +- UI should be capability-driven from server-provided key scope, not hardcoded only in the client + +This work should be treated as a separate PRD or technical design. + +## Functional Requirements Summary + +1. Youtarr must expose a dedicated external API prefix. +2. All external API endpoints must require API keys. +3. The API key model must support access levels and approval-related policy flags. +4. External clients must be able to list channels. +5. External clients must be able to list channel videos with paging, filters, download-state metadata, and usable authenticated thumbnail references. +6. External clients must be able to create video download requests. +7. External clients must be able to create channel-add requests. +8. Authorized external clients must be able to create approval-backed downloaded-video deletion requests. +9. External clients must be able to read a bounded, policy-filtered cross-channel video feed without consumer-specific ranking behavior. +10. Admins must be able to approve or reject pending requests in the web UI. +11. Documentation must explain safe reverse-proxy deployment. +12. Existing API key behavior must remain backward compatible during migration. + +## Success Metrics + +- Admin can safely expose only the external API via reverse proxy. +- External client can render channels and videos without session auth. +- External client can fetch thumbnail/image assets needed for those views using API-key-protected routes. +- External requester can submit a video request that appears in the approval queue. +- Admin can approve the request and the download proceeds normally. +- No external endpoint is accessible without an API key. +- Existing bookmarklet/single-video flows continue to work. + +## Sequential Implementation Strategy + +Implementation will use three sequential, independently reviewable PRs. PRs will not be stacked: after each merge, the next PR will be rebased onto the updated upstream branch and its focused and regression checks will be rerun. + +Documentation and tests ship with the behavior they describe. Each PR must be independently testable at its own boundary, and later PRs must not be required to prove an earlier PR safe. + +### PR 1: Preparation, Data Model, and Migrations + +**Purpose** + +Create the persistence and configuration foundation required by the API without exposing new runtime routes. + +**Includes** + +- extend `ApiKeys` with role, policy, media, auto-approval, revocation, and legacy-compatibility fields +- migrate existing keys to a `legacy_download` scope that preserves the current `POST /api/videos/download` behavior +- add `ApiKeyChannelGrants` with unique key/channel membership and indexed foreign keys +- add `ExternalRequests` with opaque ID, owner, type, status, idempotency/deduplication data, nullable video/channel targets, persisted grant decision, job linkage, messages, and timestamps +- add indexes for owner/status reads, management filters, active-target deduplication, and bounded catalog queries +- add Sequelize models, associations, shared enums/constants, and validation primitives +- add `EXTERNAL_API_ENABLED=false` configuration and environment documentation +- document upgrade, rollback, re-application, data-retention, and destructive rollback implications + +**Excludes** + +- `/external-api/v1` routes +- external authentication and authorization middleware +- request execution or approval behavior +- API-key policy management endpoints +- administrator GUI + +**Required test coverage** + +- migration unit tests for every `up` and `down` path, including missing/existing table and column behavior +- isolated MariaDB integration tests for fresh install, pre-feature upgrade, current-schema restart, idempotent re-run, rollback, and re-application +- model and association Jest tests for defaults, enum validation, foreign keys, uniqueness, ownership, and timestamps +- API-key module regression tests proving existing key creation, hashing, one-time reveal, usage tracking, and legacy downloads remain unchanged +- negative tests for invalid roles, request types/statuses, malformed policy data, duplicate grants, duplicate active requests, and orphaned references +- rollback tests proving request/grant rows are removed only as documented and that database rollback does not claim to restore deleted media + +**Acceptance** + +- all migration paths pass against a real supported database, not only mocked Sequelize calls +- existing installations and existing API keys upgrade without losing access to the legacy endpoint +- the application can start safely with the new schema while the external namespace remains disabled +- no new public or management API behavior is reachable + +### PR 2: External API Surface and Backend Control Plane + +**Purpose** + +Deliver the complete headless integration contract and all session-authenticated backend controls. After this PR, the feature must be fully usable and testable through an ordinary API client without the new GUI. + +**Includes** + +- disabled-by-default `/external-api/v1` namespace and normalized error envelope +- API-key-only authentication that rejects missing, invalid, revoked, legacy-only, and session-only credentials +- cumulative role checks, per-key channel grants, rating/media policy, auto-approval policy, and separate read/write rate limits +- capabilities, granted-channel, channel-video, cross-channel-video, and authenticated asset endpoints +- owner-scoped video, channel, and downloaded-video deletion request creation/list/detail endpoints +- idempotency, concurrent deduplication, reconciliation, bounded stale-work recovery, and terminal-state handling +- approval-time revalidation and integration with existing downloader, channel provisioning, and downloaded-video deletion operations +- session-authenticated management endpoints for listing/detailing/approving/rejecting requests +- session-authenticated API-key endpoints for transactional role, policy, auto-approval, and grant management +- immediate key revocation and namespace-wide feature shutdown +- OpenAPI contract, operator documentation, proxy examples, migration/rollback runbook, legacy deprecation guidance, correlation IDs, and redacted structured logging + +**Excludes** + +- Requests page +- API-key policy/grant editor UI +- external-client-specific recommendation behavior +- API-key access to session administration or request-review authority + +**Required test coverage** + +- route-level Jest/Supertest coverage for every external and management endpoint, success response, validation error, and normalized error envelope +- authentication matrix covering missing, malformed, invalid, revoked, legacy, insufficient-role, ungranted, session-only, and mixed session/API-key credentials +- authorization matrix covering roles, grants, enabled/terminated channels, rating ceilings, unrated behavior, media types, direct IDs, counts, lists, assets, candidates, request creation, and approval revalidation +- pagination/filter tests for stable ordering, tie-breakers, maximum page size, empty pages, invalid parameters, and large channels +- asset tests covering public thumbnails, authenticated local assets, traversal, symlinks, redirects, missing files, unauthorized resources, and secret-free URLs +- request lifecycle Jest tests for pending, auto-approved, processing, completed, rejected, failed, cancelled, already-downloaded, missing-output, and already-deleted behavior +- idempotency and concurrency tests proving duplicate submissions cannot create multiple active requests +- recovery tests for interrupted channel and delete work, stale claims, repeated approval, and terminal-state reconciliation +- integration tests proving video requests reach the downloader, channel approval provisions/enables the canonical channel, and delete approval removes only the downloaded asset +- management API tests proving API keys cannot review requests or manage keys even when a browser session is also present +- rate-limit, feature-off `404`, immediate-revocation, log-redaction, correlation-ID, proxy allow/deny, and legacy-endpoint regression tests +- performance tests with thousands of cached videos covering query plans, bounded pages, timing, and memory +- OpenAPI validation proving every published endpoint, schema, status, security requirement, and example matches runtime behavior + +**Acceptance** + +- the complete contract works without the GUI through documented API calls +- every externally visible resource applies one shared authorization and eligibility policy +- all request types deterministically reach a correct terminal or recoverable state +- the external namespace can be shut down without exposing the main UI or other private routes +- existing session routes and the legacy download endpoint retain regression coverage + +### PR 3: Administrator GUI + +**Purpose** + +Add the Youtarr administrator experience on top of the stable management API from PR 2 without introducing new backend contract or authorization behavior. + +**Includes** + +- responsive, session-only Requests page with filters, history, details, current-state warnings, approve/reject actions, bounded rejection reasons, requester identity, target/job state, and audit information +- desktop table and tablet/mobile card presentations without compressed columns or horizontal overflow +- video/delete thumbnails and established video-chip treatments +- channel identity, thumbnail/handle, and concise canonical target presentation +- empty, loading, error, retry, long-content, stale-state, media-failure, and action-in-progress states +- role, human-readable rating, media-type, auto-approval, and channel-grant controls in the existing API Keys & External Access section +- transactional create/edit flows, privilege-increase confirmation, one-time secret reveal, and immediate-revocation guidance +- Requests routing/navigation and direct links between request review and API-key management +- administrator/user documentation and responsive screenshots + +**Excludes** + +- migrations +- new backend endpoints +- new authorization decisions or request-state transitions +- external-client UI + +**Required Jest coverage** + +- request-list rendering, paging, filters, sorting, empty/loading/error/retry states, and response-shape validation +- request-detail rendering for video, channel, and delete requests +- approve/reject flows, confirmation, bounded rejection reason, optimistic/in-progress behavior, API failures, stale-state conflicts, and post-action refresh +- role/policy/grant creation and editing, validation, privilege-increase confirmation, transactional failures, secret reveal/copy guidance, and revocation +- route and navigation coverage for `/requests`, direct links, browser back/forward behavior, and protected session-only rendering +- responsive rendering assertions for desktop table and mobile card selection +- keyboard navigation, focus placement/restoration, dialog behavior, screen-reader names, reduced-motion behavior, and touch-target semantics +- regression coverage for the existing bookmarklet, legacy key display, Configuration/Settings routes, navigation shell, and supported themes + +**Required Storybook coverage** + +- stories for the Requests page and its independently meaningful table, card, detail, and action-dialog components +- desktop, tablet, and mobile viewports +- every request type and lifecycle status +- empty, loading, error, retry, stale, long-content, media-failure, and action-in-progress states +- API-key policy cards/editor states, including least privilege, elevated privilege, validation errors, and revoked keys +- light/dark and supported theme-token coverage +- `play` interactions for filtering, opening details, approving, rejecting, retrying, editing policy/grants, confirmation, and revocation +- every critical story added to the existing Jest Storybook parity suite through `runStoryWithPlay` +- successful static Storybook build +- CI-enforced accessibility assertions; merely loading the a11y addon without a failing check is not sufficient + +**Acceptance** + +- backend behavior from PR 2 is consumed without contract changes +- desktop and mobile workflows complete without overflow or inaccessible controls +- Jest component/interaction tests and Storybook interaction coverage pass for every critical state +- keyboard, screen-reader, focus, contrast, reduced-motion, and responsive touch-target checks pass + +## Test Strategy and Merge Gates + +Test ownership follows the implementation: + +- PR 1 owns schema, migration, model, and legacy compatibility tests. +- PR 2 owns backend module, route, authorization, lifecycle, integration, performance, OpenAPI, operational, and proxy-isolation tests. +- PR 3 owns frontend Jest, Storybook, responsive, interaction, and accessibility tests. + +The repository currently provides backend Jest, frontend Jest, a static Storybook build, and Jest-driven Storybook `play` coverage. The implementation must extend those real harnesses rather than creating parallel one-off test systems. + +At minimum, every PR must pass the checks relevant to its diff: + +```text +npm run lint +npm run lint:ts +npm run test:backend +npm run test:frontend +npm run test:coverage +cd client && npm run build +cd client && npm run build-storybook +cd client && npm test -- src/tests/storybook_coverage.test.js --passWithNoTests +``` + +PR 1 must additionally run the isolated real-database migration matrix. PR 2 must additionally run deterministic request-worker, proxy allow/deny, large-catalog performance, and API contract validation suites. PR 3 must additionally run the blocking accessibility and responsive browser validation introduced for the feature. + +Coverage percentage alone is not sufficient. Each requirement and state transition must map to at least one named test, and every defect found during implementation or review must receive a regression test at the lowest useful layer. + +## Definition of Done for Every PR + +Each PR must: + +- state its purpose, included behavior, and explicit exclusions +- update the authoritative API contract and the relevant operator or administrator guide +- describe API, data, authorization, migration, rollback, and compatibility implications +- include focused acceptance tests plus regression coverage for existing behavior +- include responsive screenshots or Storybook states when UI changes +- report only the phase that is actually complete +- identify dependent follow-ups and known limitations +- remain small enough to review as one cohesive ownership seam + +## Open Questions + +1. Should the public external browse contract use `page/pageSize` only, or support both `page/pageSize` and `offset/limit`? +2. Should a `request` key be able to auto-approve downloads only for videos belonging to already-subscribed channels? +3. Should channel requests always require approval even when video requests can auto-approve? +4. Are there additional safeguards required beyond the approval-backed, downloaded-asset-only deletion model defined here? +5. Should external clients be allowed to browse only certain subfolders/libraries per key in a later phase? +6. Should Swagger expose external endpoints publicly, or only when the main admin UI is accessible? +7. Should the role currently named `admin` be renamed before implementation to avoid implying external request-review authority? + +## Recommendations + +- Use `/external-api/v1` as the dedicated prefix. +- Deliver the API through the sequential Youtarr PR program above; do not bundle consumer integrations into those reviews. +- Treat destructive operations as approval-backed requests, not direct actions. +- Reuse existing channel/video pagination and filtering logic where possible, but define a cleaner external contract. +- Make documentation part of the feature, not follow-up work. +- Evaluate every endpoint and response field as a reusable Youtarr capability rather than against one consuming application. + +## References + +- Existing Youtarr API key docs currently describe API keys as limited to single-video downloads. +- Existing Youtarr channel/video routes already support paging and filters for session-authenticated use. +- Existing **API Keys & External Access** settings UX is the right place to extend key capabilities. diff --git a/fixtures/external-api-v1/SHA256SUMS b/fixtures/external-api-v1/SHA256SUMS new file mode 100644 index 000000000..212b99f94 --- /dev/null +++ b/fixtures/external-api-v1/SHA256SUMS @@ -0,0 +1 @@ +5a175ad47f088c91723df3907dc80f3a5c13b4d5cd51962793fab266fdd88ac2 contract.json diff --git a/fixtures/external-api-v1/contract.json b/fixtures/external-api-v1/contract.json new file mode 100644 index 000000000..6d2588451 --- /dev/null +++ b/fixtures/external-api-v1/contract.json @@ -0,0 +1,566 @@ +{ + "fixtureVersion": 4, + "capabilities": { + "apiVersion": "1", + "serverVersion": "1.77.0", + "role": "request", + "scopes": [ + "catalog:read", + "requests:read", + "video:request" + ], + "policy": { + "allowVideoRequests": true, + "allowChannelRequests": false, + "allowDeleteVideoRequests": false, + "autoApproveVideoRequests": false, + "autoApproveChannelRequests": false, + "autoApproveDeleteRequests": false, + "maxRatingLevel": 2, + "allowUnrated": true, + "allowedMediaTypes": ["video", "short", "livestream"] + }, + "features": { + "catalog": true, + "requests": true, + "channelRequests": true, + "deleteRequests": true, + "recommendations": true, + "authenticatedAssets": true, + "videoDetails": true + }, + "quota": { + "limits": { + "maxActiveJobs": 5, + "hourlyWriteLimit": 30, + "dailyWriteLimit": 200 + }, + "remaining": { + "activeJobs": 5, + "hourlyWrites": 30, + "dailyWrites": 200 + } + } + }, + "representativeProfile": { + "basis": "sanitized-live-shape", + "channelCount": 4, + "catalogFirstPageCount": 12, + "mediaTypeCounts": { + "video": 7, + "short": 4, + "livestream": 1 + }, + "commonRatings": ["TV-Y", "TV-Y7"], + "typicalNullFields": ["description", "requestStatus"], + "shortsMayOmitPublishedAt": true, + "artworkUsesAuthenticatedSameOriginPaths": true + }, + "channelsPage": { + "data": [ + { + "id": 8, + "channelId": "UC0000000000000000000001", + "title": "Fixture channel", + "descriptionSummary": "A populated channel contract record.", + "thumbnailUrl": "/external-api/v1/assets/channels/8/thumbnail", + "subfolder": "Learning", + "videoCount": 4, + "downloadedCount": 1, + "lastFetchedAt": "2026-07-31T11:30:00.000Z" + }, + { + "id": 9, + "channelId": "UC0000000000000000000002", + "title": "Science fixture channel", + "descriptionSummary": null, + "thumbnailUrl": "/external-api/v1/assets/channels/9/thumbnail", + "subfolder": null, + "videoCount": 3, + "downloadedCount": 0, + "lastFetchedAt": null + }, + { + "id": 10, + "channelId": "UC0000000000000000000003", + "title": "Story fixture channel", + "descriptionSummary": null, + "thumbnailUrl": "/external-api/v1/assets/channels/10/thumbnail", + "subfolder": null, + "videoCount": 3, + "downloadedCount": 0, + "lastFetchedAt": "2026-07-31T11:20:00.000Z" + }, + { + "id": 11, + "channelId": "UC0000000000000000000004", + "title": "Music fixture channel", + "descriptionSummary": "A second populated synthetic channel.", + "thumbnailUrl": "/external-api/v1/assets/channels/11/thumbnail", + "subfolder": null, + "videoCount": 3, + "downloadedCount": 0, + "lastFetchedAt": "2026-07-31T11:10:00.000Z" + } + ], + "pagination": { + "page": 1, + "pageSize": 100, + "total": 4, + "totalPages": 1, + "nextCursor": null + }, + "dataSource": "cache" + }, + "error": { + "error": { + "code": "not_found", + "message": "External API route not found", + "requestId": "00000000-0000-4000-8000-000000000001" + } + }, + "requests": [ + { + "id": "00000000-0000-4000-8000-000000000010", + "type": "video", + "status": "processing", + "target": { + "youtubeId": "abcdefghijk", + "channelId": 8, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:00:00.000Z", + "updatedAt": "2026-07-26T12:01:00.000Z" + }, + { + "id": "00000000-0000-4000-8000-000000000013", + "type": "video", + "status": "approved", + "target": { + "youtubeId": "shortvid001", + "channelId": 8, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:04:00.000Z", + "updatedAt": "2026-07-26T12:04:30.000Z", + "decidedAt": "2026-07-26T12:04:30.000Z" + }, + { + "id": "00000000-0000-4000-8000-000000000014", + "type": "video", + "status": "rejected", + "target": { + "youtubeId": "streamvid01", + "channelId": 9, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:05:00.000Z", + "updatedAt": "2026-07-26T12:05:30.000Z", + "decidedAt": "2026-07-26T12:05:30.000Z" + }, + { + "id": "00000000-0000-4000-8000-000000000015", + "type": "video", + "status": "failed", + "target": { + "youtubeId": "unrated0001", + "channelId": 9, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:06:00.000Z", + "updatedAt": "2026-07-26T12:06:30.000Z", + "completedAt": "2026-07-26T12:06:30.000Z", + "message": "Sanitized fixture failure" + }, + { + "id": "00000000-0000-4000-8000-000000000016", + "type": "video", + "status": "cancelled", + "target": { + "youtubeId": "cancelled01", + "channelId": 9, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:07:00.000Z", + "updatedAt": "2026-07-26T12:07:30.000Z", + "completedAt": "2026-07-26T12:07:30.000Z" + }, + { + "id": "00000000-0000-4000-8000-000000000011", + "type": "channel", + "status": "pending", + "target": { + "youtubeId": null, + "channelId": null, + "channelUrl": "https://www.youtube.com/@example" + }, + "createdAt": "2026-07-26T12:02:00.000Z", + "updatedAt": "2026-07-26T12:02:00.000Z", + "grantToRequestingKey": true + }, + { + "id": "00000000-0000-4000-8000-000000000012", + "type": "delete_video", + "status": "completed", + "target": { + "youtubeId": "lmnopqrstuv", + "channelId": 9, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:03:00.000Z", + "updatedAt": "2026-07-26T12:03:00.000Z", + "decidedAt": "2026-07-26T12:03:00.000Z", + "completedAt": "2026-07-26T12:03:00.000Z", + "message": "Video is already deleted" + } + ], + "catalogPage": { + "data": [ + { + "youtubeId": "abcdefghijk", + "title": "Fixture candidate", + "thumbnailUrl": "/external-api/v1/assets/videos/abcdefghijk/thumbnail", + "publishedAt": "2026-07-20T00:00:00.000Z", + "duration": 120, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 8, + "channelId": "UC0000000000000000000001", + "channelTitle": "Fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "shortvid001", + "title": "Fixture short", + "thumbnailUrl": "/external-api/v1/assets/videos/shortvid001/thumbnail", + "publishedAt": null, + "duration": 45, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 8, + "channelId": "UC0000000000000000000001", + "channelTitle": "Fixture channel", + "mediaType": "short" + }, + { + "youtubeId": "streamvid01", + "title": "Fixture livestream", + "thumbnailUrl": "/external-api/v1/assets/videos/streamvid01/thumbnail", + "publishedAt": "2026-07-19T00:00:00.000Z", + "duration": 3600, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 9, + "channelId": "UC0000000000000000000002", + "channelTitle": "Science fixture channel", + "mediaType": "livestream" + }, + { + "youtubeId": "lessonvid02", + "title": "Fixture lesson two", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid02/thumbnail", + "publishedAt": "2026-07-18T00:00:00.000Z", + "duration": 742, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 9, + "channelId": "UC0000000000000000000002", + "channelTitle": "Science fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "lessonvid03", + "title": "Fixture lesson three", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid03/thumbnail", + "publishedAt": "2026-07-17T00:00:00.000Z", + "duration": 615, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 10, + "channelId": "UC0000000000000000000003", + "channelTitle": "Story fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "lessonvid04", + "title": "Fixture lesson four", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid04/thumbnail", + "publishedAt": "2026-07-16T00:00:00.000Z", + "duration": 488, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 11, + "channelId": "UC0000000000000000000004", + "channelTitle": "Music fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "lessonvid05", + "title": "Fixture lesson five", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid05/thumbnail", + "publishedAt": "2026-07-15T00:00:00.000Z", + "duration": 901, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 8, + "channelId": "UC0000000000000000000001", + "channelTitle": "Fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "lessonvid06", + "title": "Fixture lesson six", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid06/thumbnail", + "publishedAt": "2026-07-14T00:00:00.000Z", + "duration": 533, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 9, + "channelId": "UC0000000000000000000002", + "channelTitle": "Science fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "lessonvid07", + "title": "Fixture lesson seven", + "thumbnailUrl": "/external-api/v1/assets/videos/lessonvid07/thumbnail", + "publishedAt": "2026-07-13T00:00:00.000Z", + "duration": 676, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 10, + "channelId": "UC0000000000000000000003", + "channelTitle": "Story fixture channel", + "mediaType": "video" + }, + { + "youtubeId": "shortvid002", + "title": "Fixture short two", + "thumbnailUrl": "/external-api/v1/assets/videos/shortvid002/thumbnail", + "publishedAt": null, + "duration": 51, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 11, + "channelId": "UC0000000000000000000004", + "channelTitle": "Music fixture channel", + "mediaType": "short" + }, + { + "youtubeId": "shortvid003", + "title": "Fixture short three", + "thumbnailUrl": "/external-api/v1/assets/videos/shortvid003/thumbnail", + "publishedAt": null, + "duration": 38, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "channelDatabaseId": 9, + "channelId": "UC0000000000000000000002", + "channelTitle": "Science fixture channel", + "mediaType": "short" + }, + { + "youtubeId": "shortvid004", + "title": "Fixture short four", + "thumbnailUrl": "/external-api/v1/assets/videos/shortvid004/thumbnail", + "publishedAt": null, + "duration": 57, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y", + "channelDatabaseId": 10, + "channelId": "UC0000000000000000000003", + "channelTitle": "Story fixture channel", + "mediaType": "short" + } + ], + "pagination": { + "page": 1, + "pageSize": 100, + "total": 13, + "totalPages": 2, + "nextCursor": "eyJ2IjoxLCJwYWdlIjoyfQ" + }, + "dataSource": "cache", + "isFullyIndexed": true, + "lastIndexedAt": null, + "indexingHint": null + }, + "catalogNextPage": { + "data": [ + { + "youtubeId": "unrated0001", + "title": "Final fixture page", + "thumbnailUrl": null, + "publishedAt": "2026-07-18T00:00:00.000Z", + "duration": 3600, + "description": null, + "isDownloaded": false, + "isRequested": false, + "requestStatus": "rejected", + "rating": null, + "channelDatabaseId": 9, + "channelId": "UC0000000000000000000002", + "channelTitle": "Science fixture channel", + "mediaType": "video" + } + ], + "pagination": { + "page": 2, + "pageSize": 100, + "total": 4, + "totalPages": 2, + "nextCursor": null + }, + "dataSource": "cache", + "isFullyIndexed": true, + "lastIndexedAt": null, + "indexingHint": null + }, + "videoDetail": { + "youtubeId": "abcdefghijk", + "title": "Fixture candidate", + "thumbnailUrl": "/external-api/v1/assets/videos/abcdefghijk/thumbnail", + "publishedAt": "2026-07-20T00:00:00.000Z", + "duration": 120, + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": "TV-Y7", + "ratingSource": null, + "channelDatabaseId": 8, + "channelId": "UC0000000000000000000001", + "channelTitle": "Fixture channel", + "mediaType": "video", + "availability": "public", + "downloadedAt": null, + "fileSize": null, + "audioFileSize": null, + "isProtected": false, + "videoResolution": null, + "metadata": { + "description": "Complete fixture description for the one-video detail response.", + "viewCount": 1000, + "likeCount": 50, + "commentCount": null, + "tags": ["fixture", "video"], + "categories": ["Education"], + "uploadDate": "20260720", + "resolution": "1920x1080", + "width": 1920, + "height": 1080, + "fps": 30, + "aspectRatio": 1.78, + "language": "en", + "isLive": false, + "wasLive": false, + "availability": "public", + "channelFollowerCount": 10000, + "ageLimit": 0, + "webpageUrl": "https://www.youtube.com/watch?v=abcdefghijk", + "relatedFiles": null, + "availableResolutions": [360, 720, 1080] + } + }, + "sparseVideoDetail": { + "youtubeId": "shortvid001", + "title": "Fixture short", + "thumbnailUrl": null, + "publishedAt": null, + "duration": "0:45", + "isDownloaded": false, + "isRequested": false, + "requestStatus": null, + "rating": 1, + "ratingSource": null, + "channelDatabaseId": 8, + "channelId": "UC0000000000000000000001", + "channelTitle": "Fixture channel", + "mediaType": "short", + "availability": null, + "downloadedAt": null, + "fileSize": null, + "audioFileSize": null, + "isProtected": null, + "videoResolution": null, + "metadata": null + }, + "videoRequestResponses": [ + { + "outcome": "created", + "request": { + "id": "00000000-0000-4000-8000-000000000010", + "type": "video", + "status": "processing", + "target": { + "youtubeId": "abcdefghijk", + "channelId": 8, + "channelUrl": null + }, + "createdAt": "2026-07-26T12:00:00.000Z", + "updatedAt": "2026-07-26T12:01:00.000Z" + } + }, + { + "outcome": "duplicate", + "request": null + }, + { + "outcome": "already_downloaded", + "request": null + } + ], + "unknownEnumSamples": { + "request": { + "id": "00000000-0000-4000-8000-000000000099", + "type": "future_request_type", + "status": "future_status", + "target": { + "youtubeId": null, + "channelId": null, + "channelUrl": null + }, + "createdAt": "2026-07-31T12:00:00.000Z", + "updatedAt": "2026-07-31T12:00:00.000Z" + }, + "videoRequestResponse": { + "outcome": "future_outcome", + "request": null + } + } +} diff --git a/package.json b/package.json index 626467679..9399f72eb 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,9 @@ "docs:generate": "npm --prefix website run generate", "docs:check": "npm --prefix website run check", "docs:build": "npm --prefix website run build", - "docs:serve": "npm --prefix website run serve" + "docs:serve": "npm --prefix website run serve", + "test:external-fixture": "jest --config jest.config.js server/modules/__tests__/externalContractFixtures.test.js --runInBand", + "test:external-contract-server": "node scripts/external-api-contract-server.js" }, "lint-staged": { "./client/src/*.{ts,tsx},./server/*.js": [ diff --git a/scripts/external-api-contract-server.js b/scripts/external-api-contract-server.js new file mode 100755 index 000000000..c43a84ab5 --- /dev/null +++ b/scripts/external-api-contract-server.js @@ -0,0 +1,31 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { + SYNTHETIC_API_KEY, + createExternalApiContractApp, +} = require('../server/testing/externalApiContractApp'); + +const contract = JSON.parse(fs.readFileSync( + path.join(__dirname, '../fixtures/external-api-v1/contract.json'), + 'utf8' +)); +const scenario = process.env.YOUTARR_CONTRACT_SCENARIO || 'normal'; +const delayMs = Number(process.env.YOUTARR_CONTRACT_DELAY_MS || 0); +const port = Number(process.env.YOUTARR_CONTRACT_PORT || 0); +const { app } = createExternalApiContractApp({ contract, scenario, delayMs }); + +const server = app.listen(port, '127.0.0.1', () => { + const address = server.address(); + process.stdout.write(`${JSON.stringify({ + baseURL: `http://127.0.0.1:${address.port}`, + apiKey: SYNTHETIC_API_KEY, + scenario, + })}\n`); +}); + +for (const signal of ['SIGINT', 'SIGTERM']) { + process.on(signal, () => server.close(() => process.exit(0))); +} diff --git a/server/__tests__/logger.test.js b/server/__tests__/logger.test.js index b2bf05b60..d61302bbd 100644 --- a/server/__tests__/logger.test.js +++ b/server/__tests__/logger.test.js @@ -107,6 +107,9 @@ describe('logger.js', () => { expect(config.redact.paths).toContain('plexAuthToken'); expect(config.redact.paths).toContain('session_token'); expect(config.redact.paths).toContain('plexApiKey'); + expect(config.redact.paths).toContain('key'); + expect(config.redact.paths).toContain('key_hash'); + expect(config.redact.paths).toContain('req.body.idempotencyKey'); }); it('should configure redaction for authorization headers', () => { @@ -182,6 +185,15 @@ describe('logger.js', () => { }); describe('Environment-specific behavior', () => { + it('does not create the pretty-print worker transport in test mode', () => { + process.env.NODE_ENV = 'test'; + + require('../logger'); + + const config = mockPino.mock.calls[0][0]; + expect(config.transport).toBeUndefined(); + }); + it('should handle different log levels via LOG_LEVEL', () => { const logLevels = ['trace', 'debug', 'info', 'warn', 'error', 'fatal']; diff --git a/server/__tests__/server.additional-routes.test.js b/server/__tests__/server.additional-routes.test.js index 027572b4c..0033bb2d1 100644 --- a/server/__tests__/server.additional-routes.test.js +++ b/server/__tests__/server.additional-routes.test.js @@ -354,6 +354,11 @@ const createServerModule = ({ jest.doMock('fs', () => ({ readFileSync: jest.fn(() => '') })); jest.doMock('child_process', () => childProcessMock); jest.doMock('pino-http', () => pinoHttpMock); + jest.doMock('swagger-jsdoc', () => jest.fn(() => ({ + openapi: '3.0.0', + paths: {}, + components: { schemas: {} }, + }))); const setupTokenModuleMock = { setTokenPath: jest.fn(), diff --git a/server/__tests__/server.apikeys.test.js b/server/__tests__/server.apikeys.test.js index 3411402d3..1860cc84a 100644 --- a/server/__tests__/server.apikeys.test.js +++ b/server/__tests__/server.apikeys.test.js @@ -346,6 +346,11 @@ const createServerModule = ({ describe: jest.fn(() => ''), })); jest.doMock('express-rate-limit', () => jest.fn(() => (req, res, next) => next())); + jest.doMock('swagger-jsdoc', () => jest.fn(() => ({ + openapi: '3.0.0', + paths: {}, + components: { schemas: {} }, + }))); jest.doMock('https', () => ({ get: jest.fn() })); jest.doMock('fs', () => ({ readFileSync: jest.fn(() => ''), @@ -879,6 +884,24 @@ describe('API Key Authentication - Security Tests', () => { expect(validated).toBeNull(); }); + test('rejects an external-role key before the legacy download handler', async () => { + const apiKeyModuleMock = createApiKeyModuleMock(); + apiKeyModuleMock.validateApiKey.mockResolvedValue({ + id: 99, name: 'External', key_prefix: '12345678', role: 'request', + }); + const { app } = await createServerModule({ apiKeyModuleMock }); + const [verifyToken] = findRouteHandlers(app, 'post', '/api/videos/download'); + const req = createMockRequest({ + method: 'POST', path: '/api/videos/download', headers: { 'x-api-key': '12345678external' }, + }); + const res = createMockResponse(); + + await verifyToken(req, res, jest.fn()); + + expect(res.statusCode).toBe(403); + expect(res.body).toEqual({ error: 'External API keys cannot access the download endpoint' }); + }); + test('rejects request without URL', async () => { const apiKeyModuleMock = createApiKeyModuleMock(); const created = await apiKeyModuleMock.createApiKey('Download Key'); diff --git a/server/__tests__/server.auth-sessions.test.js b/server/__tests__/server.auth-sessions.test.js index 738a17033..f32d53d5b 100644 --- a/server/__tests__/server.auth-sessions.test.js +++ b/server/__tests__/server.auth-sessions.test.js @@ -196,6 +196,11 @@ const createServerModule = ({ jest.doMock('../logger', () => loggerMock); jest.doMock('child_process', () => childProcessMock); jest.doMock('pino-http', () => pinoHttpMock); + jest.doMock('swagger-jsdoc', () => jest.fn(() => ({ + openapi: '3.0.0', + paths: {}, + components: { schemas: {} }, + }))); jest.doMock('../modules/channelModule', () => ({ subscribe: jest.fn(), diff --git a/server/logger.js b/server/logger.js index bd38d392f..bec4eccdd 100644 --- a/server/logger.js +++ b/server/logger.js @@ -79,7 +79,13 @@ const pinoConfig = { 'jellyfinApiKey', 'embyApiKey', 'youtubeApiKey', + 'apiKey', + 'key', + 'key_hash', + 'idempotencyKey', 'req.body.apiKey', + 'req.body.key', + 'req.body.idempotencyKey', 'req.headers.authorization', 'req.headers["x-access-token"]', 'req.headers["x-api-key"]', @@ -106,6 +112,9 @@ const pinoConfig = { }, }; +// Tests use a synchronous destination without opening transport workers. +if (process.env.NODE_ENV === 'test') delete pinoConfig.transport; + const logger = pino(pinoConfig); if (logDirectoryError) { diff --git a/server/middleware/externalApiAuth.js b/server/middleware/externalApiAuth.js new file mode 100644 index 000000000..ae195ce9b --- /dev/null +++ b/server/middleware/externalApiAuth.js @@ -0,0 +1,111 @@ +const EXTERNAL_ROLES = ['view', 'request', 'delete', 'admin']; +const ALLOWED_MEDIA_TYPES = ['video', 'short', 'livestream']; +const { sendExternalError } = require('../modules/externalApiResponse'); +const { + normalizeExternalPermissions, + roleForExternalPermissions, +} = require('../modules/externalPermissions'); + +function normalizeAllowedMediaTypes(value) { + if (!Array.isArray(value)) return null; + if (value.some((type) => !ALLOWED_MEDIA_TYPES.includes(type))) return null; + const types = [...new Set(value)]; + return types.length > 0 ? types : null; +} + +function normalizeExternalApiKey(apiKey) { + if (!apiKey || apiKey.is_active !== true || apiKey.revoked_at || + !EXTERNAL_ROLES.includes(apiKey.role)) { + return null; + } + const permissionFields = [ + 'allow_video_requests', + 'allow_channel_requests', + 'allow_delete_video_requests', + ]; + if (permissionFields.some((field) => typeof apiKey[field] !== 'boolean')) { + return null; + } + const allowedMediaTypes = normalizeAllowedMediaTypes(apiKey.allowed_media_types); + const permissions = normalizeExternalPermissions(apiKey); + if (!allowedMediaTypes || !permissions || + roleForExternalPermissions(permissions, apiKey.role) !== apiKey.role || + !Number.isInteger(apiKey.max_rating_level) || + apiKey.max_rating_level < 1 || apiKey.max_rating_level > 4 || + typeof apiKey.allow_unrated !== 'boolean' || + !Number.isInteger(apiKey.max_active_jobs) || + apiKey.max_active_jobs < 1 || apiKey.max_active_jobs > 5 || + !Number.isInteger(apiKey.hourly_write_limit) || + apiKey.hourly_write_limit < 1 || apiKey.hourly_write_limit > 30 || + !Number.isInteger(apiKey.daily_write_limit) || + apiKey.daily_write_limit < 1 || apiKey.daily_write_limit > 200) { + return null; + } + const autoApproveFields = [ + ['auto_approve_video_requests', permissions.allowVideoRequests], + ['auto_approve_channel_requests', permissions.allowChannelRequests], + ['auto_approve_delete_requests', permissions.allowDeleteVideoRequests], + ]; + if (autoApproveFields.some(([field, permission]) => + typeof apiKey[field] !== 'boolean' || (apiKey[field] && !permission))) { + return null; + } + return { + id: apiKey.id, + name: apiKey.name, + role: apiKey.role, + autoApproveVideoRequests: apiKey.auto_approve_video_requests, + autoApproveChannelRequests: apiKey.auto_approve_channel_requests, + autoApproveDeleteRequests: apiKey.auto_approve_delete_requests, + ...permissions, + maxRatingLevel: apiKey.max_rating_level, + allowUnrated: apiKey.allow_unrated, + allowedMediaTypes, + maxActiveJobs: apiKey.max_active_jobs, + hourlyWriteLimit: apiKey.hourly_write_limit, + dailyWriteLimit: apiKey.daily_write_limit, + }; +} + +function createExternalApiAuth({ validateApiKey }) { + return async function externalApiAuth(req, res, next) { + const key = req.headers['x-api-key']; + if (!key || typeof key !== 'string') { + return sendExternalError(res, 401, 'x-api-key is required', { + code: 'missing_api_key', + requestId: req.id, + }); + } + + try { + const apiKey = await validateApiKey(key); + if (!apiKey) { + return sendExternalError(res, 401, 'Invalid external API key', { + code: 'invalid_api_key', + requestId: req.id, + }); + } + const normalized = normalizeExternalApiKey(apiKey); + if (!normalized) { + return sendExternalError(res, 401, 'Invalid external API key policy', { + code: 'invalid_key_policy', + requestId: req.id, + }); + } + req.externalApiKey = normalized; + return next(); + } catch (error) { + req.log?.error({ err: error }, 'External API key verification failed'); + return sendExternalError(res, 500, 'External API authentication error', { + requestId: req.id, + }); + } + }; +} + +module.exports = { + createExternalApiAuth, + EXTERNAL_ROLES, + normalizeAllowedMediaTypes, + normalizeExternalApiKey, +}; diff --git a/server/models/__tests__/apikey.model.test.js b/server/models/__tests__/apikey.model.test.js new file mode 100644 index 000000000..b3bf21eb1 --- /dev/null +++ b/server/models/__tests__/apikey.model.test.js @@ -0,0 +1,30 @@ +const ApiKey = require('../apikey'); + +describe('ApiKey model', () => { + const getter = ApiKey.rawAttributes.allowed_media_types.get; + + test('normalizes MariaDB JSON text into the policy media-type array', () => { + const record = { + getDataValue: jest.fn().mockReturnValue('["video","short"]'), + }; + + expect(getter.call(record)).toEqual(['video', 'short']); + }); + + test('preserves corrupt JSON so authentication fails closed', () => { + const record = { + getDataValue: jest.fn().mockReturnValue('not-json'), + }; + + expect(getter.call(record)).toBe('not-json'); + }); + + test('passes through JSON values already decoded by the dialect', () => { + const mediaTypes = ['video']; + const record = { + getDataValue: jest.fn().mockReturnValue(mediaTypes), + }; + + expect(getter.call(record)).toBe(mediaTypes); + }); +}); diff --git a/server/models/__tests__/channelvideo.model.test.js b/server/models/__tests__/channelvideo.model.test.js new file mode 100644 index 000000000..61525e114 --- /dev/null +++ b/server/models/__tests__/channelvideo.model.test.js @@ -0,0 +1,13 @@ +const models = require('../index'); + +describe('ChannelVideo model registration', () => { + test('is exported for services that use the shared model registry', () => { + expect(models.ChannelVideo).toBeDefined(); + expect(models.ChannelVideo.tableName).toBe('channelvideos'); + }); + + test('matches nullable cached metadata columns in the database schema', () => { + expect(models.ChannelVideo.rawAttributes.title.allowNull).toBe(true); + expect(models.ChannelVideo.rawAttributes.thumbnail.allowNull).toBe(true); + }); +}); diff --git a/server/models/channelvideo.js b/server/models/channelvideo.js index 96f99dc82..2104742b9 100644 --- a/server/models/channelvideo.js +++ b/server/models/channelvideo.js @@ -21,11 +21,11 @@ ChannelVideo.init( }, title: { type: DataTypes.STRING, - allowNull: false, + allowNull: true, }, thumbnail: { type: DataTypes.STRING, - allowNull: false, + allowNull: true, }, duration: { type: DataTypes.INTEGER, diff --git a/server/models/index.js b/server/models/index.js index 9bd394aed..fe67be226 100644 --- a/server/models/index.js +++ b/server/models/index.js @@ -4,6 +4,7 @@ const JobVideo = require('./jobvideo'); const JobVideoDownload = require('./jobvideodownload'); const Video = require('./video'); const Channel = require('./channel'); +const ChannelVideo = require('./channelvideo'); const Session = require('./session'); const ApiKey = require('./apikey'); const Playlist = require('./playlist'); @@ -56,6 +57,7 @@ module.exports = { JobVideoDownload, Video, Channel, + ChannelVideo, Session, ApiKey, Playlist, diff --git a/server/modules/__tests__/apiKeyChannelGrantModule.test.js b/server/modules/__tests__/apiKeyChannelGrantModule.test.js new file mode 100644 index 000000000..b5c65d477 --- /dev/null +++ b/server/modules/__tests__/apiKeyChannelGrantModule.test.js @@ -0,0 +1,91 @@ +jest.mock('../../db', () => ({ + sequelize: { + transaction: jest.fn(async (callback) => callback({ id: 'transaction' })), + }, +})); +const { Op } = require('sequelize'); +jest.mock('../../models', () => ({ + ApiKey: { findByPk: jest.fn() }, + ApiKeyChannelGrant: { + findAll: jest.fn(), + destroy: jest.fn(), + bulkCreate: jest.fn(), + }, + Channel: { count: jest.fn() }, +})); + +const { ApiKey, ApiKeyChannelGrant, Channel } = require('../../models'); +const grants = require('../apiKeyChannelGrantModule'); + +describe('API key channel grants', () => { + beforeEach(() => jest.clearAllMocks()); + + test('normalizes and transactionally replaces the exact enabled set', async () => { + ApiKey.findByPk.mockResolvedValue({ id: 7, role: 'view', is_active: true, revoked_at: null }); + Channel.count.mockResolvedValue(2); + const result = await grants.replaceChannelGrants(7, [9, 3, 9]); + expect(result).toEqual({ keyId: 7, channelIds: [3, 9] }); + expect(ApiKeyChannelGrant.destroy).toHaveBeenCalledWith({ + where: { api_key_id: 7 }, transaction: { id: 'transaction' }, + }); + expect(ApiKeyChannelGrant.bulkCreate).toHaveBeenCalledWith( + [{ api_key_id: 7, channel_id: 3 }, { api_key_id: 7, channel_id: 9 }], + { transaction: { id: 'transaction' } } + ); + }); + + test('lists only grants whose channels are still enabled and non-terminated', async () => { + ApiKey.findByPk.mockResolvedValue({ + id: 7, role: 'view', is_active: true, revoked_at: null, + }); + ApiKeyChannelGrant.findAll.mockResolvedValue([{ channel_id: 3 }]); + + await expect(grants.getChannelGrants(7)).resolves.toEqual({ + keyId: 7, + channelIds: [3], + }); + expect(ApiKeyChannelGrant.findAll).toHaveBeenCalledWith(expect.objectContaining({ + where: { api_key_id: 7 }, + include: [expect.objectContaining({ + model: Channel, + as: 'channel', + required: true, + where: { enabled: true, terminated_at: { [Op.is]: null } }, + })], + })); + }); + + test('counts only grants whose channels are still enabled and non-terminated', async () => { + ApiKeyChannelGrant.findAll.mockResolvedValue([ + { api_key_id: 7, channel_id: 3 }, + { api_key_id: 7, channel_id: 5 }, + { api_key_id: 9, channel_id: 8 }, + ]); + + await expect(grants.getEffectiveChannelGrantCounts([7, 9, 12])).resolves.toEqual( + new Map([[7, 2], [9, 1]]) + ); + expect(ApiKeyChannelGrant.findAll).toHaveBeenCalledWith(expect.objectContaining({ + where: { api_key_id: [7, 9, 12] }, + include: [expect.objectContaining({ + model: Channel, + as: 'channel', + required: true, + where: { enabled: true, terminated_at: { [Op.is]: null } }, + })], + })); + }); + + test('rejects legacy keys before changing grants', async () => { + ApiKey.findByPk.mockResolvedValue({ id: 7, role: 'legacy_download', is_active: true, revoked_at: null }); + await expect(grants.replaceChannelGrants(7, [3])).rejects.toThrow('Only active external API keys'); + expect(ApiKeyChannelGrant.destroy).not.toHaveBeenCalled(); + }); + + test('rejects disabled or unknown channel IDs before changing grants', async () => { + ApiKey.findByPk.mockResolvedValue({ id: 7, role: 'view', is_active: true, revoked_at: null }); + Channel.count.mockResolvedValue(1); + await expect(grants.replaceChannelGrants(7, [3, 9])).rejects.toThrow('Every channel ID'); + expect(ApiKeyChannelGrant.destroy).not.toHaveBeenCalled(); + }); +}); diff --git a/server/modules/__tests__/apiKeyModule.test.js b/server/modules/__tests__/apiKeyModule.test.js new file mode 100644 index 000000000..584e914ba --- /dev/null +++ b/server/modules/__tests__/apiKeyModule.test.js @@ -0,0 +1,169 @@ +jest.mock('../../models/apikey', () => ({ + count: jest.fn(), create: jest.fn(), findAll: jest.fn(), findByPk: jest.fn(), +})); +jest.mock('../../modules/apiKeyChannelGrantModule', () => ({ + getEffectiveChannelGrantCounts: jest.fn(), +})); +jest.mock('../../logger', () => ({ info: jest.fn(), debug: jest.fn() })); + +const ApiKey = require('../../models/apikey'); +const { getEffectiveChannelGrantCounts } = require('../../modules/apiKeyChannelGrantModule'); +const apiKeyModule = require('../apiKeyModule'); + +describe('external API key policies', () => { + beforeEach(() => jest.clearAllMocks()); + + test('creates legacy-compatible keys when policy is omitted', async () => { + ApiKey.count.mockResolvedValue(0); + ApiKey.create.mockResolvedValue({ id: 1, name: 'Legacy' }); + const created = await apiKeyModule.createApiKey('Legacy'); + expect(created).toMatchObject({ id: 1, name: 'Legacy' }); + expect(ApiKey.create.mock.calls[0][0].role).toBeUndefined(); + }); + + test('strictly validates policy fields and defaults', async () => { + expect(() => apiKeyModule.validatePolicy({ role: 'view', unsupported: true })).toThrow('Unsupported'); + expect(() => apiKeyModule.validatePolicy({ role: 'view', maxRatingLevel: 5 })).toThrow('maxRatingLevel'); + expect(() => apiKeyModule.validatePolicy({ role: 'view', allowedMediaTypes: ['audio'] })).toThrow('allowedMediaTypes'); + expect(apiKeyModule.validatePolicy({ role: 'view' })).toEqual(expect.objectContaining({ + role: 'view', max_rating_level: 4, allowed_media_types: ['video'], + })); + expect(apiKeyModule.validatePolicy({ + role: 'view', allowedMediaTypes: ['video', 'short', 'livestream'], + }).allowed_media_types).toEqual(['video', 'short', 'livestream']); + expect(() => apiKeyModule.validatePolicy({ + role: 'view', + allowVideoRequests: false, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + autoApproveVideoRequests: true, + autoApproveChannelRequests: true, + autoApproveDeleteRequests: true, + })).toThrow('requires'); + expect(() => apiKeyModule.validatePolicy({ + role: 'request', + allowVideoRequests: true, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + autoApproveVideoRequests: true, + autoApproveChannelRequests: true, + })).toThrow('autoApproveChannelRequests'); + expect(apiKeyModule.validatePolicy({ + role: 'request', + allowVideoRequests: true, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + autoApproveVideoRequests: true, + autoApproveChannelRequests: false, + })).toEqual(expect.objectContaining({ + role: 'request', + allow_video_requests: true, + allow_channel_requests: false, + allow_delete_video_requests: false, + auto_approve_video_requests: true, + auto_approve_channel_requests: false, + })); + }); + + test('updates an existing key policy', async () => { + const key = { + id: 1, name: 'External', key_hash: 'must-not-leak', key_prefix: '12345678', + role: 'view', is_active: true, revoked_at: null, update: jest.fn().mockResolvedValue(), + }; + ApiKey.findByPk.mockResolvedValue(key); + const result = await apiKeyModule.updateApiKey(1, { role: 'request' }); + expect(result).toEqual(expect.objectContaining({ id: 1, name: 'External', key_prefix: '12345678' })); + expect(result).not.toHaveProperty('key_hash'); + expect(key.update).toHaveBeenCalledWith(expect.objectContaining({ role: 'request' })); + }); + + test('does not convert between legacy and constrained key types', async () => { + ApiKey.findByPk.mockResolvedValue({ + id: 1, + role: 'legacy_download', + is_active: true, + revoked_at: null, + }); + await expect(apiKeyModule.updateApiKey(1, { role: 'view' })) + .rejects.toThrow('cannot be converted'); + + ApiKey.findByPk.mockResolvedValue({ + id: 2, + role: 'view', + is_active: true, + revoked_at: null, + }); + await expect(apiKeyModule.updateApiKey(2, { role: 'legacy_download' })) + .rejects.toThrow('cannot be converted'); + }); + + test('regenerates an active key in place and returns the new secret once', async () => { + const key = { + id: 1, + name: 'External', + is_active: true, + revoked_at: null, + update: jest.fn().mockResolvedValue(), + }; + ApiKey.findByPk.mockResolvedValue(key); + + const result = await apiKeyModule.regenerateApiKey(1); + + expect(result).toEqual(expect.objectContaining({ + id: 1, + name: 'External', + key: expect.stringMatching(/^[a-f0-9]{64}$/), + prefix: expect.stringMatching(/^[a-f0-9]{8}$/), + })); + expect(result.key.startsWith(result.prefix)).toBe(true); + expect(key.update).toHaveBeenCalledWith({ + key_hash: require('crypto').createHash('sha256').update(result.key).digest('hex'), + key_prefix: result.prefix, + last_used_at: null, + }); + }); + + test('does not regenerate missing or revoked keys', async () => { + ApiKey.findByPk.mockResolvedValueOnce(null); + await expect(apiKeyModule.regenerateApiKey(1)).resolves.toBeNull(); + ApiKey.findByPk.mockResolvedValueOnce({ + id: 2, is_active: false, revoked_at: new Date(), + }); + await expect(apiKeyModule.regenerateApiKey(2)).resolves.toBeNull(); + }); + + test('uses the management response contract for lists without key hashes', async () => { + ApiKey.findAll.mockResolvedValue([{ id: 1, name: 'Safe', key_hash: 'must-not-leak', key_prefix: '12345678' }]); + getEffectiveChannelGrantCounts.mockResolvedValue(new Map([[1, 2]])); + await expect(apiKeyModule.listApiKeys()).resolves.toEqual([ + { id: 1, name: 'Safe', key_prefix: '12345678', channel_grant_count: 2 }, + ]); + expect(getEffectiveChannelGrantCounts).toHaveBeenCalledWith([1]); + }); + + test('retains soft-revoked keys in the management list for audit visibility', async () => { + const key = { id: 1, name: 'Revoked', key_prefix: '12345678', revoked_at: null, update: jest.fn().mockResolvedValue() }; + ApiKey.findByPk.mockResolvedValue(key); + await apiKeyModule.deleteApiKey(1); + ApiKey.findAll.mockResolvedValue([{ + id: 1, name: 'Revoked', key_prefix: '12345678', + is_active: false, revoked_at: new Date('2026-07-26T00:00:00.000Z'), + }]); + getEffectiveChannelGrantCounts.mockResolvedValue(new Map()); + await expect(apiKeyModule.listApiKeys()).resolves.toEqual([ + expect.objectContaining({ + id: 1, name: 'Revoked', is_active: false, channel_grant_count: 0, + }), + ]); + expect(ApiKey.findAll).toHaveBeenCalledWith(expect.not.objectContaining({ where: expect.anything() })); + expect(key.update).toHaveBeenCalledWith(expect.objectContaining({ revoked_at: expect.any(Date) })); + }); + + test('soft revokes instead of deleting', async () => { + const key = { id: 1, name: 'Compromised', key_prefix: '12345678', revoked_at: null, update: jest.fn().mockResolvedValue() }; + ApiKey.findByPk.mockResolvedValue(key); + await expect(apiKeyModule.deleteApiKey(1)).resolves.toBe(true); + expect(key.update).toHaveBeenCalledWith(expect.objectContaining({ is_active: false, revoked_at: expect.any(Date) })); + expect(ApiKey.destroy).toBeUndefined(); + }); +}); diff --git a/server/modules/__tests__/externalCatalogService.test.js b/server/modules/__tests__/externalCatalogService.test.js new file mode 100644 index 000000000..0d610d5f1 --- /dev/null +++ b/server/modules/__tests__/externalCatalogService.test.js @@ -0,0 +1,416 @@ +jest.mock('../../db', () => ({ + sequelize: { query: jest.fn() }, +})); +jest.mock('../configModule', () => ({ + getImagePath: jest.fn(() => '/safe/images'), +})); + +const { sequelize } = require('../../db'); +const catalog = require('../externalCatalogService'); + +const key = (overrides = {}) => ({ + id: 4, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + ...overrides, +}); + +describe('external cached catalog', () => { + beforeEach(() => jest.clearAllMocks()); + + test('lists only granted enabled channels with SQL paging and policy-filtered counts', async () => { + sequelize.query + .mockResolvedValueOnce([{ total: 1 }]) + .mockResolvedValueOnce([{ + id: 8, channel_id: 'UCsafe', title: 'Safe Channel', description: 'Description', + sub_folder: 'Kids', lastFetchedByTab: '{"video":"2026-07-20T00:00:00.000Z"}', + videoCount: '12', downloadedCount: '3', + }]); + const result = await catalog.listChannels(key(), { + page: '2', pageSize: '10', search: 'safe', sortBy: 'videoCount', sortOrder: 'desc', + }); + + expect(result.data[0]).toEqual(expect.objectContaining({ + id: 8, + channelId: 'UCsafe', + thumbnailUrl: '/external-api/v1/assets/channels/8/thumbnail', + videoCount: 12, + downloadedCount: 3, + })); + expect(result.pagination).toEqual({ + page: 2, pageSize: 10, total: 1, totalPages: 1, nextCursor: null, + }); + const listSql = sequelize.query.mock.calls[1][0]; + const options = sequelize.query.mock.calls[1][1]; + expect(listSql).toContain('INNER JOIN api_key_channel_grants'); + expect(listSql).toContain('g.api_key_id = :keyId'); + expect(listSql).toContain('LIMIT :pageSize OFFSET :offset'); + expect(listSql).toContain( + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))' + ); + expect(options.replacements).toEqual(expect.objectContaining({ keyId: 4, pageSize: 10, offset: 10 })); + expect(options.replacements.allowedRatings).toContain('TV-Y7'); + expect(options.replacements.recognizedRatings).toContain('TV-Y7'); + }); + + test('validates paging and sorting instead of interpolating user input', async () => { + await expect(catalog.listChannels(key(), { pageSize: '101' })).rejects.toThrow('pageSize'); + await expect(catalog.listChannels(key(), { sortBy: 'title; DROP TABLE channels' })).rejects.toThrow('sortBy'); + expect(sequelize.query).not.toHaveBeenCalled(); + }); + + test('enforces grants, media type, rating, duration, date, and deterministic SQL paging', async () => { + sequelize.query + .mockResolvedValueOnce([{ + id: 8, channel_id: 'UCsafe', title: 'Safe Channel', + lastFetchedByTab: '{"short":"2026-07-20T00:00:00.000Z"}', + }]) + .mockResolvedValueOnce([{ total: 1 }]) + .mockResolvedValueOnce([{ + youtube_id: 'abc', title: 'Allowed', thumbnail: 'https://i.ytimg.com/vi/abc/hqdefault.jpg', + publishedAt: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', + duration: 90, media_type: 'short', description: null, downloaded_id: null, + downloaded_removed: null, rating: 'TV-Y', request_status: 'processing', + }]); + const result = await catalog.listChannelVideos( + key({ allowedMediaTypes: ['video', 'short'] }), + '8', + { + tabType: 'shorts', minDuration: '30', maxDuration: '120', + dateFrom: '2026-07-01', dateTo: '2026-07-31', pageSize: '20', + } + ); + + expect(result.data[0]).toEqual(expect.objectContaining({ + youtubeId: 'abc', rating: 'TV-Y', channelId: 'UCsafe', mediaType: 'short', + thumbnailUrl: '/external-api/v1/assets/videos/abc/thumbnail', + isDownloaded: false, isRequested: true, requestStatus: 'processing', + })); + expect(result.isFullyIndexed).toBe(true); + const channelSql = sequelize.query.mock.calls[0][0]; + const listSql = sequelize.query.mock.calls[2][0]; + expect(channelSql).toContain('INNER JOIN api_key_channel_grants'); + expect(channelSql).toContain('c.enabled = true'); + expect(channelSql).toContain('c.terminated_at IS NULL'); + expect(listSql).toContain('cv.media_type = :mediaType'); + expect(listSql).toContain('cv.duration >= :minDuration'); + expect(listSql).toContain('cv.duration <= :maxDuration'); + expect(listSql).toContain('cv.published_at >= :dateFrom'); + expect(listSql).toContain( + 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC' + ); + expect(listSql).toContain('LIMIT :fetchLimit OFFSET :offset'); + expect(listSql).toContain('FROM external_requests er'); + expect(listSql).toContain('er.api_key_id = :keyId'); + }); + + test('returns the complete deterministic cross-channel catalog beyond the former three-page cap', async () => { + sequelize.query + .mockResolvedValueOnce([{ total: 1000 }]) + .mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', + title: 'Safe candidate', + thumbnail: '/cached/private-thumbnail.jpg', + publishedAt: '2026-07-10T00:00:00.000Z', + published_at_source: 'exact', + duration: 120, + media_type: 'video', + description: null, + downloaded_id: null, + downloaded_removed: null, + rating: 'TV-Y', + channel_database_id: 8, + channel_id: 'UCsafe', + channel_title: 'Safe Channel', + request_status: 'failed', + }]); + + const result = await catalog.listVideos(key(), { + page: '3', + pageSize: '100', + status: 'available', + }); + + expect(result).toMatchObject({ + data: [{ + youtubeId: 'abcdefghijk', + channelDatabaseId: 8, + thumbnailUrl: '/external-api/v1/assets/videos/abcdefghijk/thumbnail', + isRequested: false, + requestStatus: 'failed', + }], + pagination: { + page: 3, pageSize: 100, total: 1000, totalPages: 10, nextCursor: null, + }, + dataSource: 'cache', + isFullyIndexed: true, + }); + const listSql = sequelize.query.mock.calls[1][0]; + expect(listSql).toContain('INNER JOIN api_key_channel_grants'); + expect(listSql).toContain('c.terminated_at IS NULL'); + expect(listSql).toContain( + 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC' + ); + expect(listSql).toContain('LIMIT :fetchLimit OFFSET :offset'); + expect(sequelize.query.mock.calls[1][1].replacements).toMatchObject({ + keyId: 4, + fetchLimit: 101, + offset: 200, + }); + + sequelize.query.mockClear(); + sequelize.query.mockResolvedValueOnce([{ total: 1000 }]).mockResolvedValueOnce([]); + await expect(catalog.listVideos(key(), { page: '4', pageSize: '100' })) + .resolves.toMatchObject({ pagination: { page: 4, totalPages: 10 } }); + sequelize.query.mockClear(); + await expect(catalog.listVideos(key(), { page: '101', pageSize: '100' })) + .rejects.toThrow('page must be between 1 and 100'); + expect(sequelize.query).not.toHaveBeenCalled(); + }); + + test('filters requestable rows server-side and keeps active requests in available results', async () => { + sequelize.query.mockResolvedValueOnce([{ total: 0 }]).mockResolvedValueOnce([]); + await catalog.listVideos(key(), { + status: 'requestable', + minDuration: '30', + maxDuration: '600', + dateFrom: '2026-01-01', + dateTo: '2026-07-31', + }); + const requestableSql = sequelize.query.mock.calls[1][0]; + expect(requestableSql).toContain('(v.id IS NULL OR v.removed = true)'); + expect(requestableSql).toContain('NOT EXISTS'); + expect(requestableSql).toContain('er2.status IN (\'pending\', \'approved\', \'processing\')'); + expect(requestableSql).toContain('cv.duration >= :minDuration'); + expect(requestableSql).toContain('cv.duration <= :maxDuration'); + expect(requestableSql).toContain('cv.published_at >= :dateFrom'); + expect(requestableSql).toContain('cv.published_at <= :dateTo'); + + sequelize.query.mockClear(); + sequelize.query.mockResolvedValueOnce([{ total: 0 }]).mockResolvedValueOnce([]); + await catalog.listVideos(key(), { status: 'available' }); + const availableSql = sequelize.query.mock.calls[1][0]; + expect(availableSql).toContain('(v.id IS NULL OR v.removed = true)'); + expect(availableSql).not.toContain('NOT EXISTS'); + }); + + test('uses filter-bound keyset cursors and rejects changed filters or sorting', async () => { + const first = { + youtube_id: 'aaaaaaaaaaa', + title: 'Same title', + publishedAt: null, + published_at_source: 'approximate', + duration: null, + media_type: 'video', + downloaded_id: null, + downloaded_removed: null, + rating: 'TV-Y', + channel_database_id: 8, + channel_id: 'UCsafe', + channel_title: 'Safe Channel', + request_status: null, + cursor_sort_value: '', + }; + const lookahead = { + ...first, + youtube_id: 'bbbbbbbbbbb', + channel_database_id: 9, + channel_id: 'UCother', + channel_title: 'Other Channel', + }; + sequelize.query + .mockResolvedValueOnce([{ total: 2 }]) + .mockResolvedValueOnce([first, lookahead]); + + const firstPage = await catalog.listVideos(key(), { + pageSize: '1', + status: 'all', + sortBy: 'date', + sortOrder: 'desc', + }); + expect(firstPage.data).toHaveLength(1); + expect(firstPage.pagination.nextCursor).toEqual(expect.any(String)); + + sequelize.query.mockClear(); + sequelize.query + .mockResolvedValueOnce([{ total: 2 }]) + .mockResolvedValueOnce([lookahead]); + const secondPage = await catalog.listVideos(key(), { + pageSize: '1', + status: 'all', + sortBy: 'date', + sortOrder: 'desc', + cursor: firstPage.pagination.nextCursor, + }); + expect(secondPage.data[0].youtubeId).toBe('bbbbbbbbbbb'); + const seekSql = sequelize.query.mock.calls[1][0]; + expect(seekSql).toContain('COALESCE(cv.published_at, \'\') < :cursorSortValue'); + expect(seekSql).toContain('c.id > :cursorChannelDatabaseId'); + expect(seekSql).not.toContain('OFFSET :offset'); + + await expect(catalog.listVideos(key(), { + cursor: firstPage.pagination.nextCursor, + status: 'requestable', + })).rejects.toThrow('cursor is invalid'); + await expect(catalog.listVideos(key(), { + cursor: firstPage.pagination.nextCursor, + status: 'all', + sortOrder: 'asc', + })).rejects.toThrow('cursor is invalid'); + await expect(catalog.listVideos(key(), { + cursor: Buffer.from(JSON.stringify({ v: 1, page: 2 })).toString('base64url'), + })).rejects.toThrow('cursor is invalid'); + await expect(catalog.listVideos(key(), { + cursor: 'not-a-cursor', + })).rejects.toThrow('cursor is invalid'); + await expect(catalog.listVideos(key(), { + cursor: firstPage.pagination.nextCursor, + page: '2', + pageSize: '1', + status: 'all', + })).rejects.toThrow('cursor and page cannot be used together'); + }); + + test('fails closed for invalid policy and disallowed media without querying', async () => { + await expect(catalog.listChannelVideos(key({ maxRatingLevel: 99 }), '8', {})) + .rejects.toMatchObject({ status: 401 }); + await expect(catalog.listChannelVideos(key(), '8', { tabType: 'shorts' })) + .rejects.toMatchObject({ status: 403 }); + expect(sequelize.query).not.toHaveBeenCalled(); + }); + + test('keeps TV-Y7 in the rated policy branch when unrated content is allowed', async () => { + sequelize.query + .mockResolvedValueOnce([{ total: 0 }]) + .mockResolvedValueOnce([]); + + await catalog.listChannels(key({ allowUnrated: true }), {}); + + const options = sequelize.query.mock.calls[1][1]; + expect(options.replacements.allowedRatings).toContain('TV-Y7'); + expect(options.replacements.recognizedRatings).toContain('TV-Y7'); + }); + + test('returns the same 404 for missing and ungranted channels', async () => { + sequelize.query.mockResolvedValueOnce([]); + await expect(catalog.listChannelVideos(key(), '99', {})) + .rejects.toMatchObject({ status: 404, message: 'Channel not found' }); + }); + + test('returns one eligible video with full modal metadata and no filesystem paths', async () => { + sequelize.query.mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', + title: 'Detailed video', + publishedAt: '2026-07-10T00:00:00.000Z', + published_at_source: 'exact', + duration: 120, + media_type: 'video', + availability: 'public', + description: 'database fallback', + downloaded_id: 42, + downloaded_removed: false, + last_downloaded_at: '2026-07-20T12:00:00.000Z', + fileSize: '12345', + audioFileSize: '678', + protected: true, + rating_source: 'manual', + video_resolution: '1920x1080', + rating: 'TV-Y7', + channel_database_id: 8, + channel_id: 'UCsafe', + channel_title: 'Safe Channel', + request_status: 'approved', + }]); + const metadataService = { + getVideoMetadata: jest.fn().mockResolvedValue({ + description: 'full description', + viewCount: 1000, + likeCount: 50, + tags: ['one', 'two'], + relatedFiles: [{ fileName: 'captions.srt', fileSize: 30, type: 'Subtitles' }], + }), + }; + + const result = await catalog.getVideoDetail(key(), 'abcdefghijk', metadataService); + + expect(result).toMatchObject({ + youtubeId: 'abcdefghijk', + thumbnailUrl: '/external-api/v1/assets/videos/abcdefghijk/thumbnail', + isDownloaded: true, + isRequested: true, + requestStatus: 'approved', + downloadedAt: '2026-07-20T12:00:00.000Z', + fileSize: 12345, + audioFileSize: 678, + isProtected: true, + videoResolution: '1920x1080', + metadata: { + description: 'full description', + viewCount: 1000, + relatedFiles: [{ fileName: 'captions.srt', fileSize: 30, type: 'Subtitles' }], + }, + }); + expect(result).not.toHaveProperty('filePath'); + expect(result).not.toHaveProperty('audioFilePath'); + expect(metadataService.getVideoMetadata).toHaveBeenCalledWith('abcdefghijk'); + expect(sequelize.query.mock.calls[0][0]).toContain('INNER JOIN api_key_channel_grants'); + }); + + test('returns 404 without fetching metadata when video is not eligible', async () => { + sequelize.query.mockResolvedValueOnce([]); + const metadataService = { getVideoMetadata: jest.fn() }; + + await expect(catalog.getVideoDetail(key(), 'abcdefghijk', metadataService)) + .rejects.toMatchObject({ status: 404, message: 'Video not found' }); + expect(metadataService.getVideoMetadata).not.toHaveBeenCalled(); + }); + + test('falls back to the stored description when extended metadata has none', async () => { + sequelize.query.mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', + title: 'Detailed video', + media_type: 'video', + description: 'database fallback', + channel_database_id: 8, + channel_id: 'UCsafe', + channel_title: 'Safe Channel', + }]); + const metadataService = { + getVideoMetadata: jest.fn().mockResolvedValue({ description: null }), + }; + + const result = await catalog.getVideoDetail(key(), 'abcdefghijk', metadataService); + expect(result.metadata.description).toBe('database fallback'); + }); + + test('rejects unsafe cached thumbnail identifiers without exposing a path', async () => { + sequelize.query.mockResolvedValueOnce([{ channel_id: '../../secret' }]); + await expect(catalog.getChannelThumbnail(key(), '8')) + .rejects.toMatchObject({ status: 404, message: 'Thumbnail not found' }); + }); + + test('only exposes HTTPS thumbnail URLs from known YouTube image hosts', () => { + expect(catalog.publicVideoThumbnail('https://i.ytimg.com/vi/abc/hqdefault.jpg')).toContain('ytimg.com'); + expect(catalog.publicVideoThumbnail('http://i.ytimg.com/vi/abc/hqdefault.jpg')).toBeNull(); + expect(catalog.publicVideoThumbnail('https://example.com/tracker.jpg')).toBeNull(); + expect(catalog.publicVideoThumbnail('/images/local-secret.jpg')).toBeNull(); + }); + + test('uses the API asset route for every catalog video thumbnail', () => { + expect(catalog.videoThumbnailUrl('abcdefghijk')) + .toBe('/external-api/v1/assets/videos/abcdefghijk/thumbnail'); + }); + + test('falls back through Youtarr to a safe upstream thumbnail when no local image exists', async () => { + sequelize.query.mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', + thumbnail: 'https://i.ytimg.com/vi/abcdefghijk/mqdefault.jpg', + }]); + + await expect(catalog.getVideoThumbnail(key(), 'abcdefghijk')).resolves.toEqual({ + source: 'upstream', + url: 'https://i.ytimg.com/vi/abcdefghijk/mqdefault.jpg', + }); + }); +}); diff --git a/server/modules/__tests__/externalContractFixtures.test.js b/server/modules/__tests__/externalContractFixtures.test.js new file mode 100644 index 000000000..e7aa0832d --- /dev/null +++ b/server/modules/__tests__/externalContractFixtures.test.js @@ -0,0 +1,110 @@ +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const { externalErrorBody } = require('../externalApiResponse'); +const { dto } = require('../externalRequestService'); + +const fixtureDirectory = path.join( + __dirname, + '../../../fixtures/external-api-v1' +); +const fixtureBytes = fs.readFileSync(path.join(fixtureDirectory, 'contract.json')); +const canonicalFixtureBytes = Buffer.from(fixtureBytes.toString('utf8').replace(/\r\n/g, '\n')); +const fixture = JSON.parse(fixtureBytes.toString('utf8')); + +describe('external API shared contract fixture', () => { + test('covers the complete Plinx-consumed v1 surface', () => { + expect(fixture.fixtureVersion).toBe(4); + expect(fixture.capabilities).toEqual(expect.objectContaining({ + apiVersion: '1', + policy: expect.objectContaining({ + allowedMediaTypes: expect.arrayContaining(['video', 'short', 'livestream']), + }), + quota: expect.any(Object), + features: expect.objectContaining({ videoDetails: true }), + })); + expect(fixture.channelsPage.data).toHaveLength(4); + expect(fixture.catalogPage.pagination.nextCursor).toEqual(expect.any(String)); + expect(fixture.catalogNextPage.pagination.nextCursor).toBeNull(); + expect(new Set(fixture.catalogPage.data.map((video) => video.mediaType))) + .toEqual(new Set(['video', 'short', 'livestream'])); + expect(fixture.videoDetail.metadata).toEqual(expect.any(Object)); + expect(fixture.sparseVideoDetail.metadata).toBeNull(); + expect(fixture.videoRequestResponses.map((response) => response.outcome)) + .toEqual(['created', 'duplicate', 'already_downloaded']); + expect(new Set(fixture.requests.map((request) => request.status))) + .toEqual(new Set([ + 'pending', 'approved', 'processing', 'completed', + 'rejected', 'failed', 'cancelled', + ])); + }); + + test('keeps the ordinary catalog representative of sanitized live shapes', () => { + const profile = fixture.representativeProfile; + const catalog = fixture.catalogPage.data; + const countBy = (field) => catalog.reduce((counts, row) => ({ + ...counts, + [row[field]]: (counts[row[field]] || 0) + 1, + }), {}); + + expect(profile).toEqual(expect.objectContaining({ + basis: 'sanitized-live-shape', + channelCount: fixture.channelsPage.data.length, + catalogFirstPageCount: catalog.length, + commonRatings: ['TV-Y', 'TV-Y7'], + artworkUsesAuthenticatedSameOriginPaths: true, + })); + expect(countBy('mediaType')).toEqual(profile.mediaTypeCounts); + expect(new Set(catalog.map((video) => video.rating))) + .toEqual(new Set(profile.commonRatings)); + expect(catalog.every((video) => video.description === null)).toBe(true); + expect(catalog.every((video) => video.requestStatus === null)).toBe(true); + expect(catalog.every((video) => Number.isFinite(video.duration))).toBe(true); + expect(catalog.filter((video) => video.mediaType === 'short') + .every((video) => video.publishedAt === null)).toBe(true); + expect(catalog.every((video) => video.thumbnailUrl === + `/external-api/v1/assets/videos/${video.youtubeId}/thumbnail`)).toBe(true); + expect(fixture.channelsPage.data.every((channel) => channel.thumbnailUrl === + `/external-api/v1/assets/channels/${channel.id}/thumbnail`)).toBe(true); + }); + + test('contains no credential-shaped fixture fields', () => { + const serialized = JSON.stringify(fixture); + expect(serialized).not.toMatch(/x-api-key|x-access-token|key_hash|session-token/i); + }); + + test('matches the published cross-repository checksum', () => { + const expected = fs.readFileSync( + path.join(fixtureDirectory, 'SHA256SUMS'), + 'utf8' + ).trim().split(/\s+/)[0]; + const actual = crypto.createHash('sha256').update(canonicalFixtureBytes).digest('hex'); + expect(actual).toBe(expected); + }); + + test('matches the versioned external error envelope', () => { + expect(externalErrorBody( + 404, + 'External API route not found', + { requestId: '00000000-0000-4000-8000-000000000001' } + )).toEqual(fixture.error); + }); + + test.each(fixture.requests)('matches the $type request DTO', (expected) => { + const record = { + id: expected.id, + request_type: expected.type, + status: expected.status, + youtube_id: expected.target.youtubeId, + channel_id: expected.target.channelId, + channel_url: expected.target.channelUrl, + created_at: expected.createdAt, + updated_at: expected.updatedAt, + decided_at: expected.decidedAt, + completed_at: expected.completedAt, + message: expected.message, + grant_to_requesting_key: expected.grantToRequestingKey, + }; + expect(dto(record)).toEqual(expected); + }); +}); diff --git a/server/modules/__tests__/externalEligibility.test.js b/server/modules/__tests__/externalEligibility.test.js new file mode 100644 index 000000000..ac2cf438f --- /dev/null +++ b/server/modules/__tests__/externalEligibility.test.js @@ -0,0 +1,40 @@ +const { + effectiveRating, + isMediaTypeEligible, + isRatingEligible, + normalizeExternalPolicy, + ratingPolicy, +} = require('../externalEligibility'); + +describe('shared external eligibility', () => { + const policy = { + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video', 'short'], + }; + + test('uses the channel default only when the video rating is absent', () => { + expect(effectiveRating(null, 'TV-Y7')).toBe('TV-Y7'); + expect(effectiveRating('', 'TV-Y7')).toBe('TV-Y7'); + expect(effectiveRating('TV-PG', 'TV-Y')).toBe('TV-PG'); + }); + + test('drives the same rating allow-list used by SQL and request validation', () => { + const ratings = ratingPolicy(policy); + expect(ratings.allowedRatings).toContain('TV-Y7'); + expect(ratings.allowedRatings).not.toContain('TV-14'); + expect(isRatingEligible(policy, null, 'TV-Y7')).toBe(true); + expect(isRatingEligible(policy, 'TV-14', 'TV-Y')).toBe(false); + expect(isRatingEligible(policy, 'unknown', null)).toBe(false); + }); + + test('fails closed on invalid policy and unsupported media', () => { + expect(() => normalizeExternalPolicy({ + ...policy, + allowedMediaTypes: ['audio'], + })).toThrow('Invalid external API key policy'); + expect(isMediaTypeEligible(policy, 'short')).toBe(true); + expect(isMediaTypeEligible(policy, 'livestream')).toBe(false); + expect(isMediaTypeEligible(policy, 'audio')).toBe(false); + }); +}); diff --git a/server/modules/__tests__/externalPermissions.test.js b/server/modules/__tests__/externalPermissions.test.js new file mode 100644 index 000000000..795480e06 --- /dev/null +++ b/server/modules/__tests__/externalPermissions.test.js @@ -0,0 +1,73 @@ +const { + normalizeExternalPermissions, + scopesForExternalKey, + hasExternalScope, + roleForExternalPermissions, +} = require('../externalPermissions'); + +describe('external API key permissions', () => { + test('falls back to cumulative roles for pre-migration-shaped keys', () => { + expect(normalizeExternalPermissions({ role: 'request' })).toEqual({ + allowVideoRequests: true, + allowChannelRequests: true, + allowDeleteVideoRequests: false, + }); + expect(scopesForExternalKey({ role: 'delete' })).toEqual([ + 'catalog:read', + 'requests:read', + 'video:request', + 'channel:request', + 'video:delete', + ]); + }); + + test('uses explicit permissions independently of the summary role', () => { + const key = { + role: 'request', + allowVideoRequests: true, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + }; + expect(scopesForExternalKey(key)).toEqual([ + 'catalog:read', + 'requests:read', + 'video:request', + ]); + expect(hasExternalScope(key, 'channel:request')).toBe(false); + }); + + test('defaults omitted granular permissions off instead of expanding from the summary role', () => { + expect(normalizeExternalPermissions({ + role: 'delete', + allowVideoRequests: true, + })).toEqual({ + allowVideoRequests: true, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + }); + }); + + test('fails closed for invalid permission values', () => { + expect(normalizeExternalPermissions({ + role: 'request', + allowVideoRequests: 'yes', + })).toBeNull(); + expect(scopesForExternalKey({ + role: 'request', + allowVideoRequests: 'yes', + })).toBeNull(); + }); + + test('derives the backward-compatible summary role', () => { + expect(roleForExternalPermissions({ + allowVideoRequests: false, + allowChannelRequests: true, + allowDeleteVideoRequests: false, + })).toBe('request'); + expect(roleForExternalPermissions({ + allowVideoRequests: false, + allowChannelRequests: false, + allowDeleteVideoRequests: true, + })).toBe('delete'); + }); +}); diff --git a/server/modules/__tests__/externalQuotaService.test.js b/server/modules/__tests__/externalQuotaService.test.js new file mode 100644 index 000000000..a9c1660d5 --- /dev/null +++ b/server/modules/__tests__/externalQuotaService.test.js @@ -0,0 +1,153 @@ +const { + createExternalQuotaService, + QuotaError, + startOfHour, + startOfDay, +} = require('../externalQuotaService'); + +const now = new Date('2026-07-28T15:42:30.000Z'); + +function key(overrides = {}) { + return { + id: 4, + name: 'Client', + role: 'request', + is_active: true, + revoked_at: null, + allow_video_requests: true, + allow_channel_requests: true, + allow_delete_video_requests: false, + auto_approve_video_requests: false, + auto_approve_channel_requests: false, + auto_approve_delete_requests: false, + max_rating_level: 3, + allow_unrated: false, + allowed_media_types: ['video'], + max_active_jobs: 5, + hourly_write_limit: 30, + daily_write_limit: 200, + ...overrides, + }; +} + +function fixture({ active = 0, hourWrites = 0, dayWrites = 0, keyOverrides = {} } = {}) { + const hour = { + id: 1, + accepted_writes: hourWrites, + update: jest.fn(async function update(values) { Object.assign(this, values); }), + }; + const day = { + id: 2, + accepted_writes: dayWrites, + update: jest.fn(async function update(values) { Object.assign(this, values); }), + }; + const transaction = { LOCK: { UPDATE: 'UPDATE' } }; + const models = { + ApiKey: { findByPk: jest.fn().mockResolvedValue(key(keyOverrides)) }, + ExternalRequest: { + findAll: jest.fn().mockResolvedValue([]), + count: jest.fn().mockResolvedValue(active), + }, + Video: { findAll: jest.fn().mockResolvedValue([]) }, + Job: { findAll: jest.fn().mockResolvedValue([]) }, + ExternalApiUsageBucket: { + findOrCreate: jest.fn() + .mockResolvedValueOnce([hour, hourWrites === 0]) + .mockResolvedValueOnce([day, dayWrites === 0]), + findByPk: jest.fn() + .mockResolvedValueOnce(hour) + .mockResolvedValueOnce(day), + findOne: jest.fn() + .mockResolvedValueOnce(hour) + .mockResolvedValueOnce(day), + }, + }; + const sequelize = { + transaction: jest.fn(async (callback) => callback(transaction)), + }; + return { + service: createExternalQuotaService({ models, sequelize, now: () => now }), + models, + hour, + day, + transaction, + }; +} + +describe('external API durable quotas', () => { + test('uses stable UTC hour and day buckets', () => { + expect(startOfHour(now).toISOString()).toBe('2026-07-28T15:00:00.000Z'); + expect(startOfDay(now).toISOString()).toBe('2026-07-28T00:00:00.000Z'); + }); + + test('atomically reserves hourly and daily allowance', async () => { + const { service, hour, day } = fixture({ hourWrites: 2, dayWrites: 9 }); + const authorized = await service.reserveWrite(4, 'video:request'); + expect(authorized.id).toBe(4); + expect(hour.update).toHaveBeenCalledWith( + expect.objectContaining({ accepted_writes: 3 }), + expect.objectContaining({ transaction: expect.anything() }) + ); + expect(day.update).toHaveBeenCalledWith( + expect.objectContaining({ accepted_writes: 10 }), + expect.objectContaining({ transaction: expect.anything() }) + ); + }); + + test('fails closed when a durable limit is exhausted', async () => { + const { service } = fixture({ + hourWrites: 3, + keyOverrides: { hourly_write_limit: 3 }, + }); + await expect(service.reserveWrite(4, 'video:request')).rejects.toEqual( + expect.objectContaining({ + name: 'QuotaError', + code: 'hourly_write_limit', + status: 429, + }) + ); + }); + + test('rejects changed authority before an execution side effect', async () => { + const { service, models } = fixture(); + models.ApiKey.findByPk.mockResolvedValue(key({ + allow_video_requests: false, + role: 'view', + })); + await expect( + service.assertExecutionCapacity(4, 'video:request', 'request-id') + ).rejects.toBeInstanceOf(QuotaError); + }); + + test('reconciles completed downloads before reporting active capacity', async () => { + const completedRequest = { + id: 'request-1', + request_type: 'video', + youtube_id: 'abcdefghijk', + job_id: 'job-1', + status: 'processing', + update: jest.fn(async function update(values) { + Object.assign(this, values); + }), + }; + const { service, models } = fixture({ active: 1 }); + models.ExternalRequest.findAll.mockResolvedValue([completedRequest]); + models.Video.findAll.mockResolvedValue([{ youtubeId: 'abcdefghijk' }]); + models.ExternalRequest.count.mockImplementation(async () => + completedRequest.status === 'processing' ? 1 : 0 + ); + + const result = await service.status({ + id: 4, + maxActiveJobs: 5, + hourlyWriteLimit: 30, + dailyWriteLimit: 200, + }); + + expect(completedRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed', active_dedupe_key: null }), + {} + ); + expect(result.remaining.activeJobs).toBe(5); + }); +}); diff --git a/server/modules/__tests__/externalRequestService.test.js b/server/modules/__tests__/externalRequestService.test.js new file mode 100644 index 000000000..563c1f54c --- /dev/null +++ b/server/modules/__tests__/externalRequestService.test.js @@ -0,0 +1,939 @@ +const { createExternalRequestService, RequestError } = require('../externalRequestService'); + +const timestamp = new Date('2026-07-26T12:00:00.000Z'); +const youtubeId = 'abcdefghijk'; + +function record(overrides = {}) { + return { + id: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + api_key_id: 4, + channel_id: 8, + youtube_id: youtubeId, + request_type: 'video', + status: 'pending', + created_at: timestamp, + updated_at: timestamp, + decided_at: null, + completed_at: null, + message: null, + update: jest.fn(async function update(values) { + Object.assign(this, values); + }), + ...overrides, + }; +} + +function fixture(overrides = {}) { + const created = record(); + const transaction = { LOCK: { UPDATE: 'UPDATE' } }; + const models = { + ExternalRequest: { + create: jest.fn(async (values) => Object.assign(created, values)), + update: jest.fn().mockResolvedValue([1]), + findOne: jest.fn().mockResolvedValue(null), + findByPk: jest.fn().mockResolvedValue(null), + findAndCountAll: jest.fn().mockResolvedValue({ rows: [], count: 0 }), + }, + ApiKeyChannelGrant: { + findOne: jest.fn().mockResolvedValue({ id: 1 }), + findOrCreate: jest.fn().mockResolvedValue([{ id: 1 }, true]), + }, + Channel: { + findByPk: jest.fn().mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, default_rating: 'TV-Y', + }), + findOne: jest.fn().mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, + terminated_at: null, default_rating: 'TV-Y', + }), + }, + ChannelVideo: { + findOne: jest.fn().mockResolvedValue({ + youtube_id: youtubeId, media_type: 'video', youtube_removed: false, ignored: false, + }), + findAll: jest.fn().mockResolvedValue([]), + }, + Video: { findOne: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, + Job: { findAll: jest.fn().mockResolvedValue([]) }, + ApiKey: { + findByPk: jest.fn().mockResolvedValue({ + id: 4, + name: 'External Client', + role: 'request', + is_active: true, + revoked_at: null, + allow_video_requests: true, + allow_channel_requests: true, + allow_delete_video_requests: false, + auto_approve_video_requests: false, + auto_approve_channel_requests: false, + auto_approve_delete_requests: false, + max_rating_level: 2, + allow_unrated: false, + allowed_media_types: ['video'], + max_active_jobs: 5, + hourly_write_limit: 30, + daily_write_limit: 200, + }), + findAll: jest.fn().mockResolvedValue([]), + }, + ...overrides.models, + }; + const sequelize = { + transaction: jest.fn(async (callback) => callback(transaction)), + }; + const executor = jest.fn().mockResolvedValue(undefined); + const channelProvisioner = overrides.channelProvisioner || { + getChannelInfo: jest.fn().mockResolvedValue({ channel_id: 'UC1234567890123456789012' }), + }; + const videoDeleter = overrides.videoDeleter || { + deleteVideosByYoutubeIds: jest.fn().mockResolvedValue({ success: true, failed: [] }), + deleteVideoById: jest.fn().mockResolvedValue({ success: true }), + }; + const quotaService = overrides.quotaService || { + reserveWrite: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + isActive: true, + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + reloadAuthorizedKey: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + assertExecutionCapacity: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + }; + const service = createExternalRequestService({ + models, + executor: overrides.executor || executor, + channelProvisioner, + videoDeleter, + now: overrides.now || (() => timestamp), + sequelize: overrides.sequelize || sequelize, + quotaService, + workLimiter: overrides.workLimiter, + }); + const key = { + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: false, + autoApproveChannelRequests: false, + autoApproveDeleteRequests: false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + ...overrides.key, + }; + return { + service, models, executor, key, created, sequelize, transaction, + channelProvisioner, videoDeleter, quotaService, + }; +} + +describe('external video request service', () => { + test('persists a pending request without executing a download', async () => { + const { service, models, executor, key, transaction, quotaService } = fixture(); + const result = await service.createVideoRequest(key, { youtubeId, channelId: 8 }); + expect(result).toMatchObject({ + outcome: 'created', + request: { type: 'video', status: 'pending', target: { youtubeId, channelId: 8 } }, + }); + expect(models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ + api_key_id: 4, + active_dedupe_key: `4:video:${youtubeId}`, + status: 'pending', + }), { transaction }); + expect(quotaService.reserveWrite).toHaveBeenCalledWith( + 4, + 'video:request', + transaction + ); + expect(executor).not.toHaveBeenCalled(); + }); + + test('auto-approval queues only the canonical URL and server-owned channel mapping', async () => { + const { service, models, executor, key, transaction } = fixture({ + key: { autoApproveVideoRequests: true }, + }); + const result = await service.createVideoRequest(key, { + youtubeId, channelId: 8, idempotencyKey: 'external-client-1', + }); + expect(models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ + status: 'pending', + decided_at: timestamp, + idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), + }), { transaction }); + expect(executor).toHaveBeenCalledWith({ + body: { + urls: [`https://www.youtube.com/watch?v=${youtubeId}`], + channelId: 'UC1234567890123456789012', + ownerChannelMap: { [youtubeId]: 'UC1234567890123456789012' }, + initiatedBy: { type: 'api_key', name: 'External Client' }, + jobLabel: 'External video request', + externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + }, + }); + expect(result.request.status).toBe('processing'); + }); + + test('returns service unavailable when the shared work queue is full', async () => { + const workError = Object.assign(new Error('full'), { + name: 'ExternalWorkLimitError', + }); + const limited = fixture({ + key: { autoApproveVideoRequests: true }, + workLimiter: { run: jest.fn().mockRejectedValue(workError) }, + }); + await expect(limited.service.createVideoRequest(limited.key, { + youtubeId, + channelId: 8, + })).rejects.toMatchObject({ + name: 'RequestError', + status: 503, + code: 'work_queue_full', + }); + expect(limited.created.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + })); + }); + + test('revalidates a channel grant inside the work slot before enqueue', async () => { + const raced = fixture({ + key: { autoApproveVideoRequests: true }, + }); + raced.models.ApiKeyChannelGrant.findOne + .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) + .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) + .mockResolvedValueOnce(null); + + const result = await raced.service.createVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.executor).not.toHaveBeenCalled(); + expect(raced.created.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + })); + expect(result.request.status).toBe('failed'); + }); + + test('does not enqueue when video auto-approval is disabled before execution', async () => { + const raced = fixture({ key: { autoApproveVideoRequests: true } }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + allowVideoRequests: true, + autoApproveVideoRequests: false, + }); + + const result = await raced.service.createVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.executor).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('resumes an interrupted auto-approved dispatch with the stable request job id', async () => { + const interrupted = record({ status: 'pending', decided_at: timestamp, job_id: null }); + const retryExecutor = jest.fn().mockResolvedValue(interrupted.id); + const retry = fixture({ + key: { autoApproveVideoRequests: true }, + executor: retryExecutor, + }); + retry.models.ExternalRequest.findOne.mockResolvedValue(interrupted); + const result = await retry.service.createVideoRequest( + retry.key, + { youtubeId, channelId: 8, idempotencyKey: 'same-request' } + ); + expect(retry.models.ExternalRequest.create).not.toHaveBeenCalled(); + expect(retryExecutor).toHaveBeenCalledWith(expect.objectContaining({ + body: expect.objectContaining({ externalRequestId: interrupted.id }), + })); + expect(interrupted.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'processing', + job_id: interrupted.id, + })); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { status: 'processing' }, + }); + }); + + test('fails closed for missing grants, unknown media, and ratings above policy', async () => { + const missingGrant = fixture(); + missingGrant.models.ApiKeyChannelGrant.findOne.mockResolvedValue(null); + await expect(missingGrant.service.createVideoRequest( + missingGrant.key, { youtubeId, channelId: 8 } + )).rejects.toMatchObject({ status: 404 }); + + const unknownMedia = fixture(); + unknownMedia.models.ChannelVideo.findOne.mockResolvedValue({ + youtube_id: youtubeId, media_type: 'unknown', youtube_removed: false, ignored: false, + }); + await expect(unknownMedia.service.createVideoRequest( + unknownMedia.key, { youtubeId, channelId: 8 } + )).rejects.toMatchObject({ status: 404, message: 'Video not found' }); + + const mature = fixture(); + mature.models.Channel.findByPk.mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, default_rating: 'R', + }); + await expect(mature.service.createVideoRequest( + mature.key, { youtubeId, channelId: 8 } + )).rejects.toMatchObject({ status: 404, message: 'Video not found' }); + }); + + test('returns downloaded and concurrent duplicate outcomes without a second row', async () => { + const downloaded = fixture(); + downloaded.models.Video.findOne.mockResolvedValue({ + youtubeId, normalized_rating: 'TV-Y', removed: false, + }); + await expect(downloaded.service.createVideoRequest( + downloaded.key, { youtubeId, channelId: 8 } + )).resolves.toEqual({ outcome: 'already_downloaded', request: null }); + expect(downloaded.models.ExternalRequest.create).not.toHaveBeenCalled(); + + const duplicateRecord = record({ status: 'processing' }); + const duplicate = fixture(); + duplicate.models.ExternalRequest.findOne.mockResolvedValue(duplicateRecord); + const result = await duplicate.service.createVideoRequest( + duplicate.key, { youtubeId, channelId: 8 } + ); + expect(result).toMatchObject({ outcome: 'duplicate', request: { status: 'processing' } }); + expect(duplicate.models.ExternalRequest.create).not.toHaveBeenCalled(); + }); + + test('returns the winning row when the database unique key resolves a create race', async () => { + const duplicateRecord = record({ status: 'pending' }); + const concurrent = fixture(); + concurrent.models.ExternalRequest.findOne + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(duplicateRecord); + concurrent.models.ExternalRequest.create.mockRejectedValue( + Object.assign(new Error('duplicate'), { name: 'SequelizeUniqueConstraintError' }) + ); + await expect(concurrent.service.createVideoRequest( + concurrent.key, { youtubeId, channelId: 8 } + )).resolves.toMatchObject({ outcome: 'duplicate', request: { id: duplicateRecord.id } }); + }); + + test('creates a canonical approval-backed channel request', async () => { + const pending = fixture(); + const result = await pending.service.createChannelRequest(pending.key, { + channelUrl: 'youtube.com/@Safe Family', + idempotencyKey: 'channel-1', + }); + + expect(pending.models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ + api_key_id: 4, + channel_id: null, + youtube_id: null, + channel_url: 'https://www.youtube.com/@Safe%20Family', + request_type: 'channel', + status: 'pending', + idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), + }), { transaction: pending.transaction }); + expect(result).toMatchObject({ + outcome: 'created', + request: { + type: 'channel', + status: 'pending', + target: { channelUrl: 'https://www.youtube.com/@Safe%20Family' }, + }, + }); + expect(pending.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); + }); + + test('auto-approved channel requests provision and grant the resulting channel', async () => { + const approved = fixture({ key: { autoApproveChannelRequests: true } }); + const result = await approved.service.createChannelRequest(approved.key, { + channelUrl: 'https://www.youtube.com/channel/UC1234567890123456789012', + }); + + expect(approved.channelProvisioner.getChannelInfo).toHaveBeenCalledWith( + 'https://www.youtube.com/channel/UC1234567890123456789012', + false, + true, + {}, + { skipTabDetection: true } + ); + expect(approved.models.ApiKeyChannelGrant.findOrCreate).toHaveBeenCalledWith({ + where: { api_key_id: 4, channel_id: 8 }, + defaults: { api_key_id: 4, channel_id: 8 }, + transaction: approved.transaction, + }); + expect(result.request).toMatchObject({ + type: 'channel', + status: 'completed', + target: { channelId: 8 }, + }); + }); + + test('does not provision when channel auto-approval is disabled before execution', async () => { + const raced = fixture({ key: { autoApproveChannelRequests: true } }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + allowChannelRequests: true, + autoApproveChannelRequests: false, + }); + + const result = await raced.service.createChannelRequest(raced.key, { + channelUrl: 'https://www.youtube.com/@safe', + }); + + expect(raced.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('returns the winning channel request when a concurrent create wins', async () => { + const winner = record({ + request_type: 'channel', + channel_id: null, + youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', + }); + const concurrent = fixture(); + concurrent.models.ExternalRequest.findOne + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(winner); + concurrent.models.ExternalRequest.create.mockRejectedValue( + Object.assign(new Error('duplicate'), { name: 'SequelizeUniqueConstraintError' }) + ); + + await expect(concurrent.service.createChannelRequest(concurrent.key, { + channelUrl: 'https://www.youtube.com/@safe', + })).resolves.toMatchObject({ + outcome: 'duplicate', + request: { id: winner.id, type: 'channel' }, + }); + }); + + test('resumes an interrupted auto-approved channel request on idempotent retry', async () => { + const interrupted = record({ + request_type: 'channel', + channel_id: null, + youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', + status: 'pending', + decided_at: timestamp, + }); + const recovering = fixture({ + key: { autoApproveChannelRequests: true }, + }); + recovering.models.ExternalRequest.findOne.mockResolvedValue(interrupted); + + const result = await recovering.service.createChannelRequest(recovering.key, { + channelUrl: 'https://www.youtube.com/@safe', + }); + + expect(recovering.channelProvisioner.getChannelInfo).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { type: 'channel', status: 'completed' }, + }); + }); + + test('delete requests are role-bound and hide absent targets', async () => { + const forbidden = fixture(); + await expect(forbidden.service.createDeleteVideoRequest(forbidden.key, { + youtubeId, + channelId: 8, + })).rejects.toMatchObject({ status: 403 }); + + const absent = fixture({ key: { role: 'delete' } }); + await expect(absent.service.createDeleteVideoRequest(absent.key, { + youtubeId, + channelId: 8, + })).rejects.toMatchObject({ status: 404, message: 'Video not found' }); + expect(absent.models.ExternalRequest.create).not.toHaveBeenCalled(); + expect(absent.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); + }); + + test('auto-approved deletion reaches a completed terminal state', async () => { + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + idempotencyKey: 'delete-1', + }); + + expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( + 91, + expect.objectContaining({ + transaction: deleting.transaction, + video: expect.objectContaining({ id: 91, youtubeId }), + }) + ); + expect(result).toMatchObject({ + outcome: 'created', + request: { type: 'delete_video', status: 'completed' }, + }); + }); + + test('does not delete when delete auto-approval is disabled before execution', async () => { + const raced = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + raced.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + role: 'delete', + allowDeleteVideoRequests: true, + autoApproveDeleteRequests: false, + }); + + const result = await raced.service.createDeleteVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('deletion completes idempotently when the target disappears during execution', async () => { + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + videoDeleter: { + deleteVideoById: jest.fn().mockResolvedValue({ + success: false, + error: 'Video not found in database', + }), + }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + }); + + expect(result.request).toMatchObject({ + status: 'completed', + message: 'Video is already deleted', + }); + }); + + test('recovers a stale processing deletion idempotently', async () => { + const stale = record({ + request_type: 'delete_video', + status: 'processing', + channel_url: null, + decided_at: new Date('2026-07-26T11:00:00.000Z'), + updated_at: new Date('2026-07-26T11:00:00.000Z'), + }); + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + deleting.models.ExternalRequest.findOne.mockResolvedValue(stale); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + }); + + expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( + 91, + expect.objectContaining({ transaction: deleting.transaction }) + ); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { type: 'delete_video', status: 'completed' }, + }); + }); + + test('reconciles a processing deletion after its video is absent', async () => { + const deleting = fixture({ key: { role: 'delete' } }); + const processing = record({ + request_type: 'delete_video', + status: 'processing', + decided_at: timestamp, + }); + deleting.models.ExternalRequest.findAndCountAll.mockResolvedValue({ + rows: [processing], + count: 1, + }); + deleting.models.Video.findAll.mockResolvedValue([]); + + const result = await deleting.service.listRequests(deleting.key); + + expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', + active_dedupe_key: null, + completed_at: timestamp, + })); + expect(result.data[0]).toMatchObject({ + type: 'delete_video', + status: 'completed', + }); + }); + + test('marks queue failures terminal so a retry is possible', async () => { + const queueError = new Error('secret executor failure'); + const failed = fixture({ + key: { autoApproveVideoRequests: true }, + executor: jest.fn().mockRejectedValue(queueError), + }); + const result = await failed.service.createVideoRequest( + failed.key, { youtubeId, channelId: 8 } + ); + expect(failed.created.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + message: 'Download could not be queued', + })); + expect(result.request).toMatchObject({ + status: 'failed', + message: 'Download could not be queued', + }); + expect(result.request.message).not.toContain('secret'); + }); + + test('lists only the calling key and lazily reconciles completed downloads', async () => { + const processing = record({ status: 'processing' }); + const own = fixture(); + own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); + own.models.Video.findAll.mockResolvedValue([{ youtubeId }]); + const result = await own.service.listRequests(own.key, { + page: '1', pageSize: '10', + }); + expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith(expect.objectContaining({ + where: { api_key_id: 4 }, + limit: 10, + offset: 0, + })); + expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', + active_dedupe_key: null, + completed_at: timestamp, + })); + expect(result.data[0].status).toBe('completed'); + }); + + test.each(['Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings'])( + 'reconciles a %s downloader job to a retryable failed request', + async (jobStatus) => { + const processing = record({ status: 'processing', job_id: 'job-123' }); + const own = fixture(); + own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); + own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: jobStatus }]); + const result = await own.service.listRequests(own.key, {}); + expect(own.models.Job.findAll).toHaveBeenCalledWith({ + where: { id: ['job-123'] }, + attributes: ['id', 'status'], + }); + expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + message: 'Download did not complete', + })); + expect(result.data[0]).toMatchObject({ + status: 'failed', + message: 'Download did not complete', + }); + } + ); + + test('reconciles a status-filtered page and requeries its count', async () => { + const stale = record({ status: 'processing', job_id: 'job-123' }); + const own = fixture(); + own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: 'Error' }]); + own.models.ExternalRequest.findAndCountAll + .mockResolvedValueOnce({ rows: [stale], count: 1 }) + .mockResolvedValueOnce({ rows: [], count: 0 }); + + const result = await own.service.listRequests(own.key, { status: 'processing' }); + + expect(stale.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( + expect.objectContaining({ where: { api_key_id: 4, status: 'processing' } }) + ); + expect(result).toMatchObject({ + data: [], + pagination: { total: 0, totalPages: 0 }, + }); + }); + + test('validates scope, body fields, paging, and status allowlists', async () => { + const { service, key } = fixture(); + await expect(service.createVideoRequest( + { ...key, role: 'view' }, { youtubeId, channelId: 8 } + )).rejects.toBeInstanceOf(RequestError); + await expect(service.createVideoRequest( + { ...key, allowVideoRequests: false }, { youtubeId, channelId: 8 } + )).rejects.toThrow('video:request scope is required'); + await expect(service.createChannelRequest( + { ...key, allowChannelRequests: false }, { channelUrl: 'youtube.com/@safe' } + )).rejects.toThrow('channel:request scope is required'); + await expect(service.createDeleteVideoRequest( + { ...key, role: 'delete', allowDeleteVideoRequests: false }, + { youtubeId, channelId: 8 } + )).rejects.toThrow('video:delete scope is required'); + await expect(service.createVideoRequest( + key, { youtubeId, channelId: 8, resolution: '2160' } + )).rejects.toThrow('unsupported'); + await expect(service.createVideoRequest(key, { youtubeId })).rejects.toThrow('channelId is required'); + await expect(service.listRequests(key, { status: 'DROP TABLE' })).rejects.toThrow('status'); + await expect(service.listRequests(key, { pageSize: '101' })).rejects.toThrow('pageSize'); + }); + + test('lists administrator requests with safe joined metadata and filters', async () => { + const listed = record({ + status: 'pending', + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', key_hash: 'must-not-leak', + role: 'request', is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + default_rating: 'TV-PG', + }, + job: null, + }); + const admin = fixture(); + admin.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [listed], count: 1 }); + admin.models.ApiKey.findAll.mockResolvedValue([listed.apiKey]); + admin.models.ChannelVideo.findAll.mockResolvedValue([{ + youtube_id: youtubeId, + channel_id: 'UC1234567890123456789012', + title: 'Safe video', + media_type: 'video', + }]); + admin.models.Video.findAll.mockResolvedValue([{ + youtubeId, + normalized_rating: 'TV-Y', + }]); + + const result = await admin.service.listAdminRequests({ + page: '2', pageSize: '10', status: 'pending', apiKeyId: '4', + }); + + expect(admin.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( + expect.objectContaining({ + where: { status: 'pending', api_key_id: 4 }, + limit: 10, + offset: 10, + distinct: true, + }) + ); + const keyInclude = admin.models.ExternalRequest.findAndCountAll.mock.calls[0][0] + .include.find((item) => item.as === 'apiKey'); + expect(keyInclude.attributes).not.toContain('key_hash'); + expect(result.data[0]).toMatchObject({ + requester: { id: 4, name: 'External Client', keyPrefix: 'abcd1234' }, + target: { + youtubeId, + channelId: 8, + youtubeChannelId: 'UC1234567890123456789012', + channelTitle: 'Safe Channel', + title: 'Safe video', + mediaType: 'video', + rating: 'TV-PG', + contentRating: 'TV-Y', + }, + job: null, + }); + expect(JSON.stringify(result)).not.toContain('must-not-leak'); + expect(admin.models.Video.findAll).toHaveBeenCalledWith({ + where: { youtubeId: [youtubeId] }, + attributes: ['youtubeId', 'normalized_rating'], + }); + }); + + test('approves a pending request only after current policy revalidation and queue acceptance', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: { + id: pending.id, status: 'In Progress', jobType: 'External video request', + timeCreated: timestamp, timeInitiated: timestamp, + }, + }); + const approved = fixture(); + approved.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(hydrated); + approved.models.ChannelVideo.findAll.mockResolvedValue([{ + youtube_id: youtubeId, + channel_id: 'UC1234567890123456789012', + title: 'Safe video', + media_type: 'video', + }]); + + const result = await approved.service.reviewVideoRequest(pending.id, 'approve', {}); + + expect(approved.models.ApiKey.findByPk).toHaveBeenCalledWith(4, expect.objectContaining({ + transaction: approved.transaction, + lock: 'UPDATE', + attributes: expect.not.arrayContaining(['key_hash']), + })); + expect(approved.executor).toHaveBeenCalledWith({ + body: expect.objectContaining({ + urls: [`https://www.youtube.com/watch?v=${youtubeId}`], + channelId: 'UC1234567890123456789012', + externalRequestId: pending.id, + }), + }); + expect(pending.update.mock.calls.map(([values]) => values.status)).toEqual([ + 'approved', 'processing', + ]); + expect(result.status).toBe('processing'); + expect(result.job.id).toBe(pending.id); + }); + + test('fails approval closed when the requester key is revoked', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: false, revoked_at: timestamp, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: null, + }); + const revoked = fixture(); + revoked.models.ApiKey.findByPk.mockResolvedValue({ + id: 4, + name: 'External Client', + role: 'request', + is_active: false, + revoked_at: timestamp, + max_rating_level: 2, + allow_unrated: false, + allowed_media_types: ['video'], + }); + revoked.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(hydrated); + revoked.models.ChannelVideo.findAll.mockResolvedValue([]); + + const result = await revoked.service.reviewVideoRequest(pending.id, 'approve', {}); + + expect(revoked.executor).not.toHaveBeenCalled(); + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + }), { transaction: revoked.transaction }); + expect(result.status).toBe('failed'); + }); + + test('rejects pending requests with a bounded sanitized reason and clears dedupe', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: null, + }); + const rejected = fixture(); + rejected.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(hydrated); + rejected.models.ChannelVideo.findAll.mockResolvedValue([]); + + const result = await rejected.service.reviewVideoRequest( + pending.id, + 'reject', + { reason: ' Not\u0000 eligible \n right now. ' } + ); + + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'rejected', + active_dedupe_key: null, + message: 'Not eligible right now.', + decided_at: timestamp, + }), { transaction: rejected.transaction }); + expect(result.status).toBe('rejected'); + }); + + test('rejects non-monotonic review transitions and malformed reasons', async () => { + const completed = fixture(); + completed.models.ExternalRequest.findOne.mockResolvedValue(record({ status: 'completed' })); + await expect(completed.service.reviewVideoRequest( + '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + 'approve', + {} + )).rejects.toMatchObject({ status: 409 }); + await expect(completed.service.reviewVideoRequest( + '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + 'reject', + { reason: 'x'.repeat(301) } + )).rejects.toThrow('reason'); + }); +}); diff --git a/server/modules/__tests__/externalThumbnailProxy.test.js b/server/modules/__tests__/externalThumbnailProxy.test.js new file mode 100644 index 000000000..9cf99b88c --- /dev/null +++ b/server/modules/__tests__/externalThumbnailProxy.test.js @@ -0,0 +1,51 @@ +const { + fetchExternalThumbnail, + ThumbnailProxyError, + THUMBNAIL_MAX_BYTES, +} = require('../externalThumbnailProxy'); + +function response({ body = Buffer.from('jpeg'), contentType = 'image/jpeg', length, ok = true } = {}) { + return new Response(body, { + status: ok ? 200 : 404, + headers: { + 'content-type': contentType, + ...(length === undefined ? {} : { 'content-length': String(length) }), + }, + }); +} + +describe('external thumbnail proxy', () => { + test('returns bounded image bytes without following redirects', async () => { + const fetchImpl = jest.fn().mockResolvedValue(response()); + + const result = await fetchExternalThumbnail('https://i.ytimg.com/image.jpg', fetchImpl); + + expect(result).toEqual({ + body: Buffer.from('jpeg'), + contentType: 'image/jpeg', + }); + expect(fetchImpl).toHaveBeenCalledWith( + 'https://i.ytimg.com/image.jpg', + expect.objectContaining({ redirect: 'error' }) + ); + }); + + test.each([ + ['non-image content', response({ contentType: 'text/html' })], + ['oversized declared body', response({ length: THUMBNAIL_MAX_BYTES + 1 })], + ['upstream failure', response({ ok: false })], + ])('rejects %s', async (_name, upstreamResponse) => { + await expect(fetchExternalThumbnail( + 'https://i.ytimg.com/image.jpg', + jest.fn().mockResolvedValue(upstreamResponse) + )).rejects.toBeInstanceOf(ThumbnailProxyError); + }); + + test('rejects an oversized streamed body', async () => { + const body = Buffer.alloc(THUMBNAIL_MAX_BYTES + 1); + await expect(fetchExternalThumbnail( + 'https://i.ytimg.com/image.jpg', + jest.fn().mockResolvedValue(response({ body })) + )).rejects.toBeInstanceOf(ThumbnailProxyError); + }); +}); diff --git a/server/modules/__tests__/safeCommandLogging.test.js b/server/modules/__tests__/safeCommandLogging.test.js new file mode 100644 index 000000000..d3d32b912 --- /dev/null +++ b/server/modules/__tests__/safeCommandLogging.test.js @@ -0,0 +1,38 @@ +const { + summarizeCommandArgs, + redactSensitiveText, +} = require('../safeCommandLogging'); + +describe('safe command logging', () => { + test('summarizes command arguments without retaining values', () => { + const summary = summarizeCommandArgs([ + '--proxy', 'https://user:secret@proxy.example:8443', + '--cookies', '/private/cookies.txt', + '--add-header', 'Authorization: Bearer sentinel', + 'https://www.youtube.com/watch?v=abcdefghijk', + ]); + expect(summary).toEqual({ + argumentCount: 7, + flags: ['--add-header', '--cookies', '--proxy'], + targetHosts: ['www.youtube.com'], + }); + expect(JSON.stringify(summary)).not.toMatch(/secret|cookies\.txt|sentinel/); + }); + + test('redacts proxy, cookie, header, query, and filesystem sentinels', () => { + const result = redactSensitiveText( + 'failed --proxy https://user:secret@proxy.example ' + + '--cookies /private/cookies.txt --add-header "Cookie: cookie-sentinel" ' + + 'Authorization: Bearer bearer-sentinel\n' + + '--client-certificate-password cert-sentinel ' + + '--client-certificate-key=relative/private-key.pem ' + + 'https://x.test/a?token=query-sentinel&ok=1 ' + + 'Destination: /downloads/private/path-sentinel/video.mp4' + ); + expect(result).not.toMatch( + /user:secret|cookies\.txt|cookie-sentinel|bearer-sentinel|cert-sentinel|private-key|query-sentinel|path-sentinel/ + ); + expect(result).toContain('[REDACTED]'); + expect(result).toContain('[REDACTED_PATH]'); + }); +}); diff --git a/server/modules/apiKeyChannelGrantModule.js b/server/modules/apiKeyChannelGrantModule.js new file mode 100644 index 000000000..29bbf1491 --- /dev/null +++ b/server/modules/apiKeyChannelGrantModule.js @@ -0,0 +1,97 @@ +const { Op } = require('sequelize'); +const { sequelize } = require('../db'); +const { ApiKey, ApiKeyChannelGrant, Channel } = require('../models'); + +function normalizeChannelIds(channelIds) { + if (!Array.isArray(channelIds)) throw new Error('channelIds must be an array'); + const normalized = [...new Set(channelIds)]; + if (normalized.some((id) => !Number.isSafeInteger(id) || id < 1)) { + throw new Error('channelIds must contain only positive integer database IDs'); + } + return normalized.sort((a, b) => a - b); +} + +async function requireExternalKey(keyId, transaction) { + const key = await ApiKey.findByPk(keyId, { transaction }); + if (!key) return null; + if (!key.is_active || key.revoked_at || key.role === 'legacy_download') { + throw new Error('Only active external API keys can receive channel grants'); + } + return key; +} + +async function getChannelGrants(keyId) { + const key = await requireExternalKey(keyId); + if (!key) return null; + const grants = await ApiKeyChannelGrant.findAll({ + where: { api_key_id: keyId }, + attributes: ['channel_id'], + include: [{ + model: Channel, + as: 'channel', + attributes: [], + required: true, + where: { enabled: true, terminated_at: { [Op.is]: null } }, + }], + order: [['channel_id', 'ASC']], + raw: true, + }); + return { keyId, channelIds: grants.map((grant) => grant.channel_id) }; +} + +async function getEffectiveChannelGrantCounts(keyIds) { + if (keyIds.length === 0) return new Map(); + const grants = await ApiKeyChannelGrant.findAll({ + where: { api_key_id: keyIds }, + attributes: ['api_key_id'], + include: [{ + model: Channel, + as: 'channel', + attributes: [], + required: true, + where: { enabled: true, terminated_at: { [Op.is]: null } }, + }], + raw: true, + }); + return grants.reduce((counts, grant) => { + counts.set(grant.api_key_id, (counts.get(grant.api_key_id) || 0) + 1); + return counts; + }, new Map()); +} + +async function replaceChannelGrants(keyId, channelIds, { transaction: existingTransaction } = {}) { + const normalized = normalizeChannelIds(channelIds); + const replace = async (transaction) => { + const key = await requireExternalKey(keyId, transaction); + if (!key) return null; + + if (normalized.length > 0) { + const enabledCount = await Channel.count({ + where: { id: normalized, enabled: true, terminated_at: { [Op.is]: null } }, + transaction, + }); + if (enabledCount !== normalized.length) { + throw new Error('Every channel ID must identify an enabled, non-terminated channel'); + } + } + + await ApiKeyChannelGrant.destroy({ where: { api_key_id: keyId }, transaction }); + if (normalized.length > 0) { + await ApiKeyChannelGrant.bulkCreate( + normalized.map((channelId) => ({ api_key_id: keyId, channel_id: channelId })), + { transaction } + ); + } + return { keyId, channelIds: normalized }; + }; + return existingTransaction + ? replace(existingTransaction) + : sequelize.transaction(replace); +} + +module.exports = { + getChannelGrants, + getEffectiveChannelGrantCounts, + replaceChannelGrants, + normalizeChannelIds, +}; diff --git a/server/modules/apiKeyModule.js b/server/modules/apiKeyModule.js index ede1174f4..c9bd3f4b4 100644 --- a/server/modules/apiKeyModule.js +++ b/server/modules/apiKeyModule.js @@ -1,8 +1,100 @@ const crypto = require('crypto'); const ApiKey = require('../models/apikey'); const logger = require('../logger'); +const { + normalizeExternalPermissions, + roleForExternalPermissions, +} = require('./externalPermissions'); +const { getEffectiveChannelGrantCounts } = require('./apiKeyChannelGrantModule'); const MAX_API_KEYS = 20; +const ROLES = ['legacy_download', 'view', 'request', 'delete', 'admin']; +const MEDIA_TYPES = ['video', 'short', 'livestream']; +const POLICY_FIELDS = [ + 'role', 'autoApproveVideoRequests', 'autoApproveChannelRequests', + 'autoApproveDeleteRequests', 'allowVideoRequests', 'allowChannelRequests', + 'allowDeleteVideoRequests', 'maxRatingLevel', 'allowUnrated', 'allowedMediaTypes', + 'maxActiveJobs', 'hourlyWriteLimit', 'dailyWriteLimit', +]; +const MANAGEMENT_ATTRIBUTES = [ + 'id', 'name', 'key_prefix', 'created_at', 'last_used_at', 'is_active', 'usage_count', 'role', + 'auto_approve_video_requests', 'auto_approve_channel_requests', 'auto_approve_delete_requests', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'max_rating_level', 'allow_unrated', 'allowed_media_types', 'revoked_at', + 'max_active_jobs', 'hourly_write_limit', 'daily_write_limit', +]; + +function serializeApiKey(apiKey) { + const source = typeof apiKey?.get === 'function' ? apiKey.get({ plain: true }) : apiKey; + return MANAGEMENT_ATTRIBUTES.reduce((result, attribute) => { + if (source && Object.prototype.hasOwnProperty.call(source, attribute)) { + result[attribute] = source[attribute]; + } + return result; + }, {}); +} + +function validatePolicy(policy) { + if (!policy || typeof policy !== 'object' || Array.isArray(policy)) { + throw new Error('Policy must be an object'); + } + const unknown = Object.keys(policy).filter((field) => !POLICY_FIELDS.includes(field)); + if (unknown.length > 0) throw new Error(`Unsupported policy field: ${unknown[0]}`); + if (!ROLES.includes(policy.role)) throw new Error('Invalid API key role'); + for (const field of [ + 'autoApproveVideoRequests', 'autoApproveChannelRequests', 'autoApproveDeleteRequests', + 'allowVideoRequests', 'allowChannelRequests', 'allowDeleteVideoRequests', 'allowUnrated', + ]) { + if (field in policy && typeof policy[field] !== 'boolean') throw new Error(`Invalid ${field}`); + } + if ('maxRatingLevel' in policy && (!Number.isInteger(policy.maxRatingLevel) || policy.maxRatingLevel < 1 || policy.maxRatingLevel > 4)) { + throw new Error('maxRatingLevel must be an integer from 1 to 4'); + } + if ('allowedMediaTypes' in policy && (!Array.isArray(policy.allowedMediaTypes) || policy.allowedMediaTypes.length === 0 || + policy.allowedMediaTypes.some((type) => !MEDIA_TYPES.includes(type)))) { + throw new Error('allowedMediaTypes must contain only video, short, or livestream'); + } + const quotaFields = [ + ['maxActiveJobs', 5], + ['hourlyWriteLimit', 30], + ['dailyWriteLimit', 200], + ]; + for (const [field, maximum] of quotaFields) { + if (field in policy && + (!Number.isInteger(policy[field]) || policy[field] < 1 || policy[field] > maximum)) { + throw new Error(`${field} must be an integer from 1 to ${maximum}`); + } + } + const permissions = normalizeExternalPermissions(policy); + if (!permissions) throw new Error('Invalid external request permissions'); + for (const [autoField, permissionField] of [ + ['autoApproveVideoRequests', 'allowVideoRequests'], + ['autoApproveChannelRequests', 'allowChannelRequests'], + ['autoApproveDeleteRequests', 'allowDeleteVideoRequests'], + ]) { + if (policy[autoField] === true && permissions[permissionField] !== true) { + throw new Error(`${autoField} requires ${permissionField}`); + } + } + return { + role: roleForExternalPermissions(permissions, policy.role), + allow_video_requests: permissions.allowVideoRequests, + allow_channel_requests: permissions.allowChannelRequests, + allow_delete_video_requests: permissions.allowDeleteVideoRequests, + auto_approve_video_requests: + permissions.allowVideoRequests && (policy.autoApproveVideoRequests ?? false), + auto_approve_channel_requests: + permissions.allowChannelRequests && (policy.autoApproveChannelRequests ?? false), + auto_approve_delete_requests: + permissions.allowDeleteVideoRequests && (policy.autoApproveDeleteRequests ?? false), + max_rating_level: policy.maxRatingLevel ?? 4, + allow_unrated: policy.allowUnrated ?? false, + allowed_media_types: policy.allowedMediaTypes ?? ['video'], + max_active_jobs: policy.maxActiveJobs ?? 5, + hourly_write_limit: policy.hourlyWriteLimit ?? 30, + daily_write_limit: policy.dailyWriteLimit ?? 200, + }; +} class ApiKeyModule { /** @@ -10,9 +102,12 @@ class ApiKeyModule { * @param {string} name - Human-readable name for the key * @returns {Object} { id, name, key, prefix } - key is only returned once! */ - async createApiKey(name) { + async createApiKey(name, policy, { transaction, logEvent = true } = {}) { // Check max keys limit - const existingCount = await ApiKey.count({ where: { is_active: true } }); + const existingCount = await ApiKey.count({ + where: { is_active: true }, + ...(transaction ? { transaction } : {}), + }); if (existingCount >= MAX_API_KEYS) { throw new Error(`Maximum number of API keys reached (${MAX_API_KEYS})`); } @@ -22,20 +117,19 @@ class ApiKeyModule { const prefix = rawKey.substring(0, 8); const keyHash = crypto.createHash('sha256').update(rawKey).digest('hex'); - const apiKey = await ApiKey.create({ + const values = { name, key_hash: keyHash, key_prefix: prefix, created_at: new Date(), is_active: true, - }); + ...(policy === undefined ? {} : validatePolicy(policy)), + }; + const apiKey = transaction + ? await ApiKey.create(values, { transaction }) + : await ApiKey.create(values); - logger.info({ - keyId: apiKey.id, - name, - prefix, - event: 'api_key_created' - }, 'API key created'); + if (logEvent) this.logApiKeyCreated({ id: apiKey.id, name, prefix }); return { id: apiKey.id, @@ -45,12 +139,21 @@ class ApiKeyModule { }; } + logApiKeyCreated({ id, name, prefix }) { + logger.info({ + keyId: id, + name, + prefix, + event: 'api_key_created', + }, 'API key created'); + } + /** * Validate an API key using timing-safe comparison * @param {string} key - The raw API key to validate * @returns {Object|null} The API key record if valid, null otherwise */ - async validateApiKey(key) { + async validateApiKey(key, { recordUsage = true } = {}) { if (!key || typeof key !== 'string' || key.length < 8) { return null; } @@ -60,7 +163,7 @@ class ApiKeyModule { // Find potential matches by prefix const candidates = await ApiKey.findAll({ - where: { key_prefix: prefix, is_active: true }, + where: { key_prefix: prefix, is_active: true, revoked_at: null }, }); for (const candidate of candidates) { @@ -70,8 +173,7 @@ class ApiKeyModule { if (storedHashBuffer.length === providedHashBuffer.length && crypto.timingSafeEqual(storedHashBuffer, providedHashBuffer)) { - // Update last_used_at (download_count is incremented separately on successful download) - await candidate.update({ last_used_at: new Date() }); + if (recordUsage) await this.markApiKeyUsed(candidate.id); return candidate; } } @@ -79,6 +181,23 @@ class ApiKeyModule { return null; } + async markApiKeyUsed(id, timestamp = new Date()) { + const { Op } = require('sequelize'); + const cutoff = new Date(timestamp.getTime() - 60 * 1000); + await ApiKey.update( + { last_used_at: timestamp }, + { + where: { + id, + [Op.or]: [ + { last_used_at: null }, + { last_used_at: { [Op.lt]: cutoff } }, + ], + }, + } + ); + } + /** * Increment the usage count for an API key * @param {number} id - API key ID @@ -96,31 +215,85 @@ class ApiKeyModule { * @returns {Array} List of API key records */ async listApiKeys() { - return ApiKey.findAll({ - attributes: ['id', 'name', 'key_prefix', 'created_at', 'last_used_at', 'is_active', 'usage_count'], + const keys = await ApiKey.findAll({ + attributes: MANAGEMENT_ATTRIBUTES, order: [['created_at', 'DESC']], }); + const serializedKeys = keys.map(serializeApiKey); + const grantCounts = await getEffectiveChannelGrantCounts( + serializedKeys.map((key) => key.id) + ); + return serializedKeys.map((key) => ({ + ...key, + channel_grant_count: grantCounts.get(key.id) || 0, + })); + } + + async updateApiKey(id, policy, { transaction } = {}) { + const apiKey = transaction + ? await ApiKey.findByPk(id, { transaction }) + : await ApiKey.findByPk(id); + if (!apiKey || !apiKey.is_active || apiKey.revoked_at) return null; + if (apiKey.role === 'legacy_download' || policy?.role === 'legacy_download') { + throw new Error('Legacy and external API key types cannot be converted'); + } + const values = validatePolicy(policy); + if (transaction) await apiKey.update(values, { transaction }); + else await apiKey.update(values); + return serializeApiKey(apiKey); + } + + /** + * Replace an active key's secret while preserving its identity, policy, + * channel grants, and audit history. The returned plaintext is shown once. + */ + async regenerateApiKey(id) { + const apiKey = await ApiKey.findByPk(id); + if (!apiKey || !apiKey.is_active || apiKey.revoked_at) return null; + + const rawKey = crypto.randomBytes(32).toString('hex'); + const prefix = rawKey.substring(0, 8); + const keyHash = crypto.createHash('sha256').update(rawKey).digest('hex'); + + await apiKey.update({ + key_hash: keyHash, + key_prefix: prefix, + last_used_at: null, + }); + logger.info({ + keyId: apiKey.id, + name: apiKey.name, + prefix, + event: 'api_key_regenerated', + }, 'API key regenerated'); + + return { + id: apiKey.id, + name: apiKey.name, + key: rawKey, + prefix, + }; } /** - * Delete an API key permanently + * Revoke an API key while retaining an audit record. * @param {number} id - API key ID * @returns {boolean} True if deleted, false if not found */ async deleteApiKey(id) { - // Get key info before deletion for audit log + // Get key info before revocation for audit log const apiKey = await ApiKey.findByPk(id); const keyName = apiKey?.name; const keyPrefix = apiKey?.key_prefix; - const result = await ApiKey.destroy({ where: { id } }); - if (result > 0) { + if (apiKey && !apiKey.revoked_at) { + await apiKey.update({ is_active: false, revoked_at: new Date() }); logger.info({ keyId: id, name: keyName, prefix: keyPrefix, - event: 'api_key_deleted' - }, 'API key deleted'); + event: 'api_key_revoked' + }, 'API key revoked'); return true; } return false; @@ -128,4 +301,6 @@ class ApiKeyModule { } module.exports = new ApiKeyModule(); - +module.exports.validatePolicy = validatePolicy; +module.exports.serializeApiKey = serializeApiKey; +module.exports.managementAttributes = MANAGEMENT_ATTRIBUTES; diff --git a/server/modules/channel/channelYtdlpExecutor.js b/server/modules/channel/channelYtdlpExecutor.js index 0112de9b4..57855c15a 100644 --- a/server/modules/channel/channelYtdlpExecutor.js +++ b/server/modules/channel/channelYtdlpExecutor.js @@ -6,6 +6,7 @@ const { pipeline } = require('stream/promises'); const { v4: uuidv4 } = require('uuid'); const { spawnYtDlp } = require('../ytdlpProcess'); const tempPathManager = require('../download/tempPathManager'); +const { redactSensitiveText } = require('../safeCommandLogging'); class ChannelYtdlpExecutor { /** @@ -18,7 +19,7 @@ class ChannelYtdlpExecutor { * @param {Function} options.onStdoutData - Called with each raw stdout chunk as it arrives * @returns {Promise} - Output content if outputFile provided */ - async executeYtDlpCommand(args, outputFile = null, { onStdoutData } = {}) { + async executeYtDlpCommand(args, outputFile = null, { onStdoutData, timeoutMs = 120000 } = {}) { const ytDlp = spawnYtDlp(args, { env: { ...process.env, @@ -50,15 +51,30 @@ class ChannelYtdlpExecutor { }); const exited = new Promise((resolve, reject) => { + let settled = false; + const finish = (callback, value) => { + if (settled) return; + settled = true; + clearTimeout(timeout); + callback(value); + }; + const timeout = setTimeout(() => { + const error = new Error(`yt-dlp channel metadata request timed out after ${timeoutMs}ms`); + error.code = 'YT_DLP_TIMEOUT'; + if (typeof ytDlp.kill === 'function') ytDlp.kill('SIGTERM'); + finish(reject, error); + }, timeoutMs); + timeout.unref?.(); + ytDlp.on('exit', (code) => { // Check for bot detection if (stderrBuffer.includes('Sign in to confirm you\'re not a bot') || stderrBuffer.includes('Sign in to confirm that you\'re not a bot')) { const error = new Error('Bot detection encountered. Please set cookies in your Configuration or try different cookies to resolve this issue.'); error.code = 'COOKIES_REQUIRED'; - reject(error); + finish(reject, error); } else if (code === 0) { - resolve(); + finish(resolve); } else { // Check for common error patterns in stderr let errorMessage = `yt-dlp exited with code ${code}`; @@ -77,11 +93,12 @@ class ChannelYtdlpExecutor { const error = new Error(errorMessage); error.code = errorCode; - error.stderr = stderrBuffer; - reject(error); + error.stderr = redactSensitiveText(stderrBuffer); + finish(reject, error); } }); - ytDlp.on('error', reject); + ytDlp.on('error', (error) => finish(reject, error)); + }); await Promise.all([exited, outputWritten]); diff --git a/server/modules/downloadModule.js b/server/modules/downloadModule.js index 0ea3ffe16..515536afb 100644 --- a/server/modules/downloadModule.js +++ b/server/modules/downloadModule.js @@ -777,7 +777,17 @@ class DownloadModule { const requestedIds = requestedUrls.flatMap(url => { try { return [normalizeUrlToVideoId(url).id]; } catch { return []; } }); - const jobId = await jobModule.addOrUpdateJob( + const externalRequestId = this.getJobDataValue(jobData, 'externalRequestId'); + if (!isNextJob && externalRequestId) { + const existingExternalJob = jobModule.getJob(externalRequestId); + if (existingExternalJob) { + if (['Error', 'Killed', 'Terminated'].includes(existingExternalJob.status)) { + throw new Error('Existing external request job is terminal'); + } + return externalRequestId; + } + } + const addJobArgs = [ { jobType: jobType, status: '', @@ -787,7 +797,9 @@ class DownloadModule { action: this.doSpecificDownloads.bind(this), }, isNextJob - ); + ]; + if (externalRequestId) addJobArgs.push(externalRequestId); + const jobId = await jobModule.addOrUpdateJob(...addJobArgs); if (!jobId) return { queued: 0, acceptedIds: [], alreadyActiveIds: [...new Set(requestedIds)] }; const job = jobModule.getJob(jobId); @@ -960,6 +972,7 @@ class DownloadModule { await jobModule.startNextJob(); throw err; } + if (externalRequestId) return jobId; return { queued: urls.length, acceptedIds: admission.acceptedIds, alreadyActiveIds: admission.alreadyActiveIds }; } diff --git a/server/modules/externalApiResponse.js b/server/modules/externalApiResponse.js new file mode 100644 index 000000000..1594229ae --- /dev/null +++ b/server/modules/externalApiResponse.js @@ -0,0 +1,37 @@ +const DEFAULT_CODES = { + 400: 'invalid_request', + 401: 'unauthorized', + 403: 'forbidden', + 404: 'not_found', + 405: 'method_not_allowed', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 500: 'internal_error', + 503: 'service_unavailable', +}; + +function externalErrorBody(status, message, { code, requestId } = {}) { + return { + error: { + code: code || DEFAULT_CODES[status] || 'request_failed', + message, + ...(requestId ? { requestId } : {}), + }, + }; +} + +function setExternalSecurityHeaders(res) { + res.set('Cache-Control', 'private, no-store'); + res.set('Pragma', 'no-cache'); + res.vary('x-api-key'); + res.set('X-Content-Type-Options', 'nosniff'); +} + +function sendExternalError(res, status, message, options) { + setExternalSecurityHeaders(res); + return res.status(status).json(externalErrorBody(status, message, options)); +} + +module.exports = { externalErrorBody, sendExternalError, setExternalSecurityHeaders }; diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js new file mode 100644 index 000000000..b374d83e3 --- /dev/null +++ b/server/modules/externalCatalogService.js @@ -0,0 +1,823 @@ +const fs = require('fs').promises; +const path = require('path'); +const crypto = require('crypto'); +const { QueryTypes } = require('sequelize'); +const { sequelize } = require('../db'); +const configModule = require('./configModule'); +const { normalizeExternalPolicy, ratingPolicy } = require('./externalEligibility'); + +const TAB_MEDIA_TYPES = { videos: 'video', shorts: 'short', streams: 'livestream' }; +const SAFE_THUMBNAIL_HOSTS = ['ytimg.com', 'ggpht.com', 'googleusercontent.com']; +const ACTIVE_REQUEST_STATUSES = ['pending', 'approved', 'processing']; +const MAX_PAGE = 100; +const MAX_DESCRIPTION_LENGTH = 2000; +const MAX_TITLE_LENGTH = 500; +const MAX_PUBLIC_URL_LENGTH = 2048; + +function boundedString(value, maximum) { + if (value === null || value === undefined) return null; + return String(value).slice(0, maximum); +} + +class CatalogError extends Error { + constructor(message, status = 400) { + super(message); + this.name = 'CatalogError'; + this.status = status; + } +} + +function parseInteger(value, fallback, minimum, maximum, name) { + if (value === undefined) return fallback; + if (!/^\d+$/.test(String(value))) throw new CatalogError(`${name} must be an integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new CatalogError(`${name} must be between ${minimum} and ${maximum}`); + } + return parsed; +} + +function normalizeSearch(value) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 200) { + throw new CatalogError('search must be a string of 200 characters or fewer'); + } + const normalized = value.trim().toLowerCase(); + if (!normalized) throw new CatalogError('search must not be blank'); + return normalized; +} + +function normalizeDate(value, name, endOfDay = false) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 40) throw new CatalogError(`${name} must be a date`); + const candidate = /^\d{4}-\d{2}-\d{2}$/.test(value) + ? `${value}T${endOfDay ? '23:59:59.999' : '00:00:00.000'}Z` + : value; + const parsed = new Date(candidate); + if (Number.isNaN(parsed.getTime())) throw new CatalogError(`${name} must be a valid date`); + return parsed.toISOString(); +} + +function normalizePolicy(policy) { + return normalizeExternalPolicy(policy, CatalogError); +} + +function ratingSql(policy, effectiveRatingSql) { + const { recognizedRatings, allowedRatings } = ratingPolicy(policy); + const clauses = [`${effectiveRatingSql} IN (:allowedRatings)`]; + if (policy.allowUnrated) { + clauses.push(`(${effectiveRatingSql} IS NULL OR ${effectiveRatingSql} NOT IN (:recognizedRatings))`); + } + return { + sql: `(${clauses.join(' OR ')})`, + replacements: { allowedRatings, recognizedRatings }, + }; +} + +function decodePageCursor(value, maximumPage) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 200) { + throw new CatalogError('cursor is invalid'); + } + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); + if (parsed?.v !== 1 || !Number.isSafeInteger(parsed.page) || + parsed.page < 1 || parsed.page > maximumPage) { + throw new Error('invalid cursor'); + } + return parsed.page; + } catch (_error) { + throw new CatalogError('cursor is invalid'); + } +} + +function encodePageCursor(page) { + return Buffer.from(JSON.stringify({ v: 1, page }), 'utf8').toString('base64url'); +} + +function pagination(query, maximumPage = MAX_PAGE) { + if (query.cursor !== undefined && query.page !== undefined) { + throw new CatalogError('cursor and page cannot be used together'); + } + const cursorPage = decodePageCursor(query.cursor, maximumPage); + const page = cursorPage || parseInteger(query.page, 1, 1, maximumPage, 'page'); + const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); + return { page, pageSize, offset: (page - 1) * pageSize }; +} + +function paginationDto(page, pageSize, total, maximumPage = MAX_PAGE) { + const totalPages = total === 0 ? 0 : Math.min(maximumPage, Math.ceil(total / pageSize)); + return { + page, + pageSize, + total, + totalPages, + nextCursor: page < totalPages ? encodePageCursor(page + 1) : null, + }; +} + +function catalogCursorFingerprint(endpoint, filters) { + return crypto.createHash('sha256') + .update(JSON.stringify({ endpoint, ...filters })) + .digest('base64url') + .slice(0, 32); +} + +function decodeCatalogCursor(value, expected) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 500) { + throw new CatalogError('cursor is invalid'); + } + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); + if (parsed?.v !== 2 || + parsed.endpoint !== expected.endpoint || + parsed.sortBy !== expected.sortBy || + parsed.sortOrder !== expected.sortOrder || + parsed.fingerprint !== expected.fingerprint || + !Number.isSafeInteger(parsed.page) || parsed.page < 2 || + !Number.isSafeInteger(parsed.channelDatabaseId) || parsed.channelDatabaseId < 1 || + typeof parsed.youtubeId !== 'string' || parsed.youtubeId.length === 0 || + parsed.youtubeId.length > 32 || + !Object.prototype.hasOwnProperty.call(parsed, 'sortValue')) { + throw new Error('invalid cursor'); + } + return parsed; + } catch (_error) { + throw new CatalogError('cursor is invalid'); + } +} + +function encodeCatalogCursor({ + endpoint, sortBy, sortOrder, fingerprint, page, + sortValue, channelDatabaseId, youtubeId, +}) { + return Buffer.from(JSON.stringify({ + v: 2, + endpoint, + sortBy, + sortOrder, + fingerprint, + page, + sortValue, + channelDatabaseId, + youtubeId, + }), 'utf8').toString('base64url'); +} + +function catalogPagination(query, cursorSpec) { + if (query.cursor !== undefined && query.page !== undefined) { + throw new CatalogError('cursor and page cannot be used together'); + } + const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); + if (query.cursor !== undefined) { + const cursor = decodeCatalogCursor(query.cursor, cursorSpec); + return { page: cursor.page, pageSize, offset: null, cursor }; + } + const page = parseInteger(query.page, 1, 1, MAX_PAGE, 'page'); + return { page, pageSize, offset: (page - 1) * pageSize, cursor: null }; +} + +function catalogSeekSql(sortExpression, sortOrder, cursor) { + if (!cursor) return ''; + const operator = sortOrder === 'asc' ? '>' : '<'; + return `AND ( + ${sortExpression} ${operator} :cursorSortValue + OR ( + ${sortExpression} = :cursorSortValue + AND ( + c.id > :cursorChannelDatabaseId + OR (c.id = :cursorChannelDatabaseId AND cv.youtube_id > :cursorYoutubeId) + ) + ) + )`; +} + +function catalogPaginationDto({ + page, pageSize, total, hasMore, lastRow, cursorSpec, +}) { + return { + page, + pageSize, + total, + totalPages: total === 0 ? 0 : Math.ceil(total / pageSize), + nextCursor: hasMore && lastRow + ? encodeCatalogCursor({ + ...cursorSpec, + page: page + 1, + sortValue: lastRow.cursor_sort_value, + channelDatabaseId: Number(lastRow.channel_database_id), + youtubeId: lastRow.youtube_id, + }) + : null, + }; +} + +function lastFetched(channel, mediaType = null) { + try { + const values = JSON.parse(channel.lastFetchedByTab || '{}'); + if (mediaType) return values[mediaType] || null; + const dates = Object.values(values).filter(Boolean).map((value) => new Date(value)); + if (dates.length === 0 || dates.some((value) => Number.isNaN(value.getTime()))) return null; + return new Date(Math.max(...dates.map((value) => value.getTime()))).toISOString(); + } catch (_error) { + return null; + } +} + +function publicVideoThumbnail(value) { + if (typeof value !== 'string' || value.length > MAX_PUBLIC_URL_LENGTH) return null; + try { + const url = new URL(value); + const hostname = url.hostname.toLowerCase(); + if (url.protocol !== 'https:' || + !SAFE_THUMBNAIL_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`))) { + return null; + } + return url.toString(); + } catch (_error) { + return null; + } +} + +function videoThumbnailUrl(youtubeId) { + return `/external-api/v1/assets/videos/${youtubeId}/thumbnail`; +} + +async function listChannels(key, query = {}) { + const policy = normalizePolicy(key); + const { page, pageSize, offset } = pagination(query); + const search = normalizeSearch(query.search); + const sortColumns = { + title: 'title', + videoCount: 'videoCount', + downloadedCount: 'downloadedCount', + id: 'id', + }; + const sortBy = query.sortBy || 'title'; + if (!Object.prototype.hasOwnProperty.call(sortColumns, sortBy)) { + throw new CatalogError('sortBy must be title, videoCount, downloadedCount, or id'); + } + const sortOrder = (query.sortOrder || 'asc').toLowerCase(); + if (!['asc', 'desc'].includes(sortOrder)) throw new CatalogError('sortOrder must be asc or desc'); + + const effectiveRating = + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; + const ratings = ratingSql(policy, effectiveRating); + const videoPolicy = `cv.youtube_removed = false AND cv.ignored = false + AND cv.media_type IN (:allowedMediaTypes) AND ${ratings.sql}`; + const whereSearch = search + ? `AND ( + LOCATE(:search, LOWER(COALESCE(c.title, c.uploader, ''))) > 0 + OR LOCATE(:search, LOWER(COALESCE(c.description, ''))) > 0 + )` + : ''; + const subfolder = query.subfolder === undefined || query.subfolder === '' + ? null + : String(query.subfolder); + if (subfolder && subfolder.length > 255) { + throw new CatalogError('subfolder must be 255 characters or fewer'); + } + const whereSubfolder = subfolder ? 'AND c.sub_folder = :subfolder' : ''; + const replacements = { + keyId: key.id, allowedMediaTypes: policy.allowedMediaTypes, search, + subfolder, pageSize, offset, ...ratings.replacements, + }; + + const countRows = await sequelize.query( + `SELECT COUNT(*) AS total + FROM channels c + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + WHERE c.enabled = true AND c.terminated_at IS NULL ${whereSearch} ${whereSubfolder}`, + { replacements, type: QueryTypes.SELECT } + ); + const rows = await sequelize.query( + `SELECT c.id, c.channel_id, COALESCE(c.title, c.uploader, '') AS title, + c.description, c.sub_folder, c.last_fetched_by_tab AS lastFetchedByTab, + (SELECT COUNT(*) FROM channelvideos cv + LEFT JOIN videos v ON v.youtube_id = cv.youtube_id + WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS videoCount, + (SELECT COUNT(*) FROM channelvideos cv + INNER JOIN videos v ON v.youtube_id = cv.youtube_id AND v.removed = false + WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS downloadedCount + FROM channels c + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + WHERE c.enabled = true AND c.terminated_at IS NULL ${whereSearch} ${whereSubfolder} + ORDER BY ${sortColumns[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC + LIMIT :pageSize OFFSET :offset`, + { replacements, type: QueryTypes.SELECT } + ); + const total = Number(countRows[0]?.total || 0); + return { + data: rows.map((row) => ({ + id: row.id, + channelId: row.channel_id, + title: boundedString(row.title, MAX_TITLE_LENGTH), + descriptionSummary: row.description ? row.description.trim().slice(0, 240) : null, + thumbnailUrl: `/external-api/v1/assets/channels/${row.id}/thumbnail`, + subfolder: boundedString(row.sub_folder, 255), + videoCount: Number(row.videoCount || 0), + downloadedCount: Number(row.downloadedCount || 0), + lastFetchedAt: lastFetched(row), + })), + pagination: paginationDto(page, pageSize, total), + dataSource: 'cache', + }; +} + +async function listChannelVideos(key, channelDatabaseId, query = {}) { + const policy = normalizePolicy(key); + const id = parseInteger(channelDatabaseId, null, 1, Number.MAX_SAFE_INTEGER, 'channel id'); + const search = normalizeSearch(query.search); + const tabType = query.tabType || 'videos'; + const mediaType = TAB_MEDIA_TYPES[tabType]; + if (!mediaType) throw new CatalogError('tabType must be videos, shorts, or streams'); + if (!policy.allowedMediaTypes.includes(mediaType)) throw new CatalogError('Media type is not allowed', 403); + const sortColumns = { date: 'cv.published_at', title: 'cv.title', duration: 'cv.duration' }; + const sortExpressions = { + date: 'COALESCE(cv.published_at, \'\')', + title: 'LOWER(COALESCE(cv.title, \'\'))', + duration: 'COALESCE(cv.duration, -1)', + }; + const sortBy = query.sortBy || 'date'; + if (!sortColumns[sortBy]) throw new CatalogError('sortBy must be date, title, or duration'); + const sortOrder = (query.sortOrder || 'desc').toLowerCase(); + if (!['asc', 'desc'].includes(sortOrder)) throw new CatalogError('sortOrder must be asc or desc'); + const minDuration = parseInteger(query.minDuration, null, 0, 604800, 'minDuration'); + const maxDuration = parseInteger(query.maxDuration, null, 0, 604800, 'maxDuration'); + if (minDuration !== null && maxDuration !== null && minDuration > maxDuration) { + throw new CatalogError('minDuration cannot exceed maxDuration'); + } + const dateFrom = normalizeDate(query.dateFrom, 'dateFrom'); + const dateTo = normalizeDate(query.dateTo, 'dateTo', true); + if (dateFrom && dateTo && dateFrom > dateTo) throw new CatalogError('dateFrom cannot exceed dateTo'); + const status = query.status && query.status !== 'all' ? query.status : null; + if (status && !['downloaded', 'available', 'requested', 'requestable'].includes(status)) { + throw new CatalogError('status must be all, downloaded, available, requested, or requestable'); + } + const cursorSpec = { + endpoint: `channel:${id}:videos`, + sortBy, + sortOrder, + fingerprint: catalogCursorFingerprint(`channel:${id}:videos`, { + keyId: key.id, + search, + tabType, + status: status || 'all', + minDuration, + maxDuration, + dateFrom, + dateTo, + pageSize: parseInteger(query.pageSize, 50, 1, 100, 'pageSize'), + }), + }; + const { page, pageSize, offset, cursor } = catalogPagination(query, cursorSpec); + + const channelRows = await sequelize.query( + `SELECT c.id, c.channel_id, COALESCE(c.title, c.uploader, '') AS title, c.last_fetched_by_tab AS lastFetchedByTab + FROM channels c + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + WHERE c.id = :channelDatabaseId AND c.enabled = true AND c.terminated_at IS NULL + LIMIT 1`, + { replacements: { keyId: key.id, channelDatabaseId: id }, type: QueryTypes.SELECT } + ); + if (channelRows.length === 0) throw new CatalogError('Channel not found', 404); + const channel = channelRows[0]; + const effectiveRating = + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; + const ratings = ratingSql(policy, effectiveRating); + const filters = [ + 'cv.channel_id = c.channel_id', + 'cv.media_type = :mediaType', + 'cv.youtube_removed = false', + 'cv.ignored = false', + ratings.sql, + ]; + if (minDuration !== null) filters.push('cv.duration >= :minDuration'); + if (maxDuration !== null) filters.push('cv.duration <= :maxDuration'); + if (search) filters.push('LOCATE(:search, LOWER(COALESCE(cv.title, \'\'))) > 0'); + if (dateFrom) filters.push('cv.published_at >= :dateFrom'); + if (dateTo) filters.push('cv.published_at <= :dateTo'); + if (status === 'downloaded') filters.push('v.id IS NOT NULL AND v.removed = false'); + if (status === 'available') filters.push('(v.id IS NULL OR v.removed = true)'); + const activeRequestSql = `EXISTS ( + SELECT 1 FROM external_requests er + WHERE er.api_key_id = :keyId + AND er.request_type = 'video' + AND er.youtube_id = cv.youtube_id + AND er.status IN ('pending', 'approved', 'processing') + )`; + if (status === 'requested') filters.push(activeRequestSql); + if (status === 'requestable') { + filters.push('(v.id IS NULL OR v.removed = true)'); + filters.push(`NOT ${activeRequestSql}`); + } + const replacements = { + keyId: key.id, channelDatabaseId: id, mediaType, minDuration, maxDuration, search, dateFrom, dateTo, + fetchLimit: pageSize + 1, offset, + cursorSortValue: cursor?.sortValue, + cursorChannelDatabaseId: cursor?.channelDatabaseId, + cursorYoutubeId: cursor?.youtubeId, + ...ratings.replacements, + }; + const from = `FROM channelvideos cv + INNER JOIN channels c ON c.id = :channelDatabaseId AND c.channel_id = cv.channel_id + LEFT JOIN videos v ON v.youtube_id = cv.youtube_id + WHERE ${filters.join(' AND ')}`; + const seekSql = catalogSeekSql(sortExpressions[sortBy], sortOrder, cursor); + const countRows = await sequelize.query( + `SELECT COUNT(*) AS total ${from}`, + { replacements, type: QueryTypes.SELECT } + ); + const fetchedRows = await sequelize.query( + `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at AS publishedAt, cv.published_at_source, + cv.duration, cv.media_type, v.description, v.id AS downloaded_id, + v.removed AS downloaded_removed, ${effectiveRating} AS rating, + c.id AS channel_database_id, + ${sortExpressions[sortBy]} AS cursor_sort_value, + (SELECT er.status + FROM external_requests er + WHERE er.api_key_id = :keyId + AND er.request_type = 'video' + AND er.youtube_id = cv.youtube_id + ORDER BY er.created_at DESC, er.id DESC + LIMIT 1) AS request_status + ${from} + ${seekSql} + ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC + LIMIT :fetchLimit${cursor ? '' : ' OFFSET :offset'}`, + { replacements, type: QueryTypes.SELECT } + ); + const hasMore = fetchedRows.length > pageSize; + const rows = fetchedRows.slice(0, pageSize); + const total = Number(countRows[0]?.total || 0); + const lastIndexedAt = lastFetched(channel, mediaType); + return { + data: rows.map((row) => ({ + youtubeId: row.youtube_id, + title: boundedString(row.title, MAX_TITLE_LENGTH), + thumbnailUrl: videoThumbnailUrl(row.youtube_id), + publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + duration: row.duration, + description: boundedString(row.description, MAX_DESCRIPTION_LENGTH), + isDownloaded: Boolean(row.downloaded_id) && !row.downloaded_removed, + isRequested: ACTIVE_REQUEST_STATUSES.includes(row.request_status), + requestStatus: row.request_status || null, + rating: row.rating || null, + channelId: channel.channel_id, + channelTitle: channel.title, + mediaType: row.media_type, + })), + pagination: catalogPaginationDto({ + page, + pageSize, + total, + hasMore, + lastRow: rows[rows.length - 1], + cursorSpec, + }), + dataSource: 'cache', + isFullyIndexed: Boolean(lastIndexedAt), + lastIndexedAt, + indexingHint: lastIndexedAt ? null : 'This channel tab has not been indexed yet.', + }; +} + +async function listVideos(key, query = {}) { + const policy = normalizePolicy(key); + const search = normalizeSearch(query.search); + const tabType = query.tabType || null; + const mediaType = tabType ? TAB_MEDIA_TYPES[tabType] : null; + if (tabType && !mediaType) throw new CatalogError('tabType must be videos, shorts, or streams'); + if (mediaType && !policy.allowedMediaTypes.includes(mediaType)) { + throw new CatalogError('Media type is not allowed', 403); + } + const sortColumns = { date: 'cv.published_at', title: 'cv.title', duration: 'cv.duration' }; + const sortExpressions = { + date: 'COALESCE(cv.published_at, \'\')', + title: 'LOWER(COALESCE(cv.title, \'\'))', + duration: 'COALESCE(cv.duration, -1)', + }; + const sortBy = query.sortBy || 'date'; + if (!sortColumns[sortBy]) throw new CatalogError('sortBy must be date, title, or duration'); + const sortOrder = (query.sortOrder || 'desc').toLowerCase(); + if (!['asc', 'desc'].includes(sortOrder)) throw new CatalogError('sortOrder must be asc or desc'); + const minDuration = parseInteger(query.minDuration, null, 0, 604800, 'minDuration'); + const maxDuration = parseInteger(query.maxDuration, null, 0, 604800, 'maxDuration'); + if (minDuration !== null && maxDuration !== null && minDuration > maxDuration) { + throw new CatalogError('minDuration cannot exceed maxDuration'); + } + const dateFrom = normalizeDate(query.dateFrom, 'dateFrom'); + const dateTo = normalizeDate(query.dateTo, 'dateTo', true); + if (dateFrom && dateTo && dateFrom > dateTo) throw new CatalogError('dateFrom cannot exceed dateTo'); + const status = query.status && query.status !== 'all' ? query.status : null; + if (status && !['downloaded', 'available', 'requested', 'requestable'].includes(status)) { + throw new CatalogError('status must be all, downloaded, available, requested, or requestable'); + } + const cursorSpec = { + endpoint: 'videos', + sortBy, + sortOrder, + fingerprint: catalogCursorFingerprint('videos', { + keyId: key.id, + search, + tabType, + status: status || 'all', + minDuration, + maxDuration, + dateFrom, + dateTo, + pageSize: parseInteger(query.pageSize, 50, 1, 100, 'pageSize'), + }), + }; + const { page, pageSize, offset, cursor } = catalogPagination(query, cursorSpec); + const effectiveRating = + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; + const ratings = ratingSql(policy, effectiveRating); + const filters = [ + 'c.enabled = true', + 'c.terminated_at IS NULL', + 'cv.youtube_removed = false', + 'cv.ignored = false', + 'cv.media_type IN (:allowedMediaTypes)', + ratings.sql, + ]; + if (mediaType) filters.push('cv.media_type = :mediaType'); + if (search) filters.push('LOCATE(:search, LOWER(COALESCE(cv.title, \'\'))) > 0'); + if (minDuration !== null) filters.push('cv.duration >= :minDuration'); + if (maxDuration !== null) filters.push('cv.duration <= :maxDuration'); + if (dateFrom) filters.push('cv.published_at >= :dateFrom'); + if (dateTo) filters.push('cv.published_at <= :dateTo'); + if (status === 'downloaded') filters.push('v.id IS NOT NULL AND v.removed = false'); + if (status === 'available') filters.push('(v.id IS NULL OR v.removed = true)'); + const activeRequestSql = `EXISTS ( + SELECT 1 FROM external_requests er2 + WHERE er2.api_key_id = :keyId + AND er2.request_type = 'video' + AND er2.youtube_id = cv.youtube_id + AND er2.status IN ('pending', 'approved', 'processing') + )`; + if (status === 'requested') filters.push(activeRequestSql); + if (status === 'requestable') { + filters.push('(v.id IS NULL OR v.removed = true)'); + filters.push(`NOT ${activeRequestSql}`); + } + const replacements = { + keyId: key.id, + allowedMediaTypes: policy.allowedMediaTypes, + mediaType, + search, + minDuration, + maxDuration, + dateFrom, + dateTo, + fetchLimit: pageSize + 1, + offset, + cursorSortValue: cursor?.sortValue, + cursorChannelDatabaseId: cursor?.channelDatabaseId, + cursorYoutubeId: cursor?.youtubeId, + ...ratings.replacements, + }; + const from = `FROM channelvideos cv + INNER JOIN channels c ON c.channel_id = cv.channel_id + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + LEFT JOIN videos v ON v.youtube_id = cv.youtube_id + WHERE ${filters.join(' AND ')}`; + const seekSql = catalogSeekSql(sortExpressions[sortBy], sortOrder, cursor); + const countRows = await sequelize.query( + `SELECT COUNT(*) AS total ${from}`, + { replacements, type: QueryTypes.SELECT } + ); + const fetchedRows = await sequelize.query( + `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at AS publishedAt, cv.published_at_source, + cv.duration, cv.media_type, v.description, v.id AS downloaded_id, + v.removed AS downloaded_removed, ${effectiveRating} AS rating, + c.id AS channel_database_id, c.channel_id, COALESCE(c.title, c.uploader, '') AS channel_title, + ${sortExpressions[sortBy]} AS cursor_sort_value, + (SELECT er.status + FROM external_requests er + WHERE er.api_key_id = :keyId + AND er.request_type = 'video' + AND er.youtube_id = cv.youtube_id + ORDER BY er.created_at DESC, er.id DESC + LIMIT 1) AS request_status + ${from} + ${seekSql} + ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC + LIMIT :fetchLimit${cursor ? '' : ' OFFSET :offset'}`, + { replacements, type: QueryTypes.SELECT } + ); + const hasMore = fetchedRows.length > pageSize; + const rows = fetchedRows.slice(0, pageSize); + const total = Number(countRows[0]?.total || 0); + return { + data: rows.map((row) => ({ + youtubeId: row.youtube_id, + title: boundedString(row.title, MAX_TITLE_LENGTH), + thumbnailUrl: videoThumbnailUrl(row.youtube_id), + publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + duration: row.duration, + description: boundedString(row.description, MAX_DESCRIPTION_LENGTH), + isDownloaded: Boolean(row.downloaded_id) && !row.downloaded_removed, + isRequested: ACTIVE_REQUEST_STATUSES.includes(row.request_status), + requestStatus: row.request_status || null, + rating: row.rating || null, + channelDatabaseId: row.channel_database_id, + channelId: row.channel_id, + channelTitle: row.channel_title, + mediaType: row.media_type, + })), + pagination: catalogPaginationDto({ + page, + pageSize, + total, + hasMore, + lastRow: rows[rows.length - 1], + cursorSpec, + }), + dataSource: 'cache', + isFullyIndexed: true, + lastIndexedAt: null, + indexingHint: null, + }; +} + +async function getVideoDetail(key, youtubeId, metadataService = null) { + const policy = normalizePolicy(key); + if (typeof youtubeId !== 'string' || !/^[A-Za-z0-9_-]{11}$/.test(youtubeId)) { + throw new CatalogError('Video not found', 404); + } + const effectiveRating = + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; + const ratings = ratingSql(policy, effectiveRating); + const rows = await sequelize.query( + `SELECT cv.youtube_id, cv.title, cv.published_at AS publishedAt, cv.published_at_source, + cv.duration, cv.media_type, cv.availability, + v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, + v.last_downloaded_at, v.file_size AS fileSize, v.audio_file_size AS audioFileSize, v.protected, + v.rating_source, v.video_resolution, ${effectiveRating} AS rating, + c.id AS channel_database_id, c.channel_id, + COALESCE(c.title, c.uploader, '') AS channel_title, + (SELECT er.status + FROM external_requests er + WHERE er.api_key_id = :keyId + AND er.request_type = 'video' + AND er.youtube_id = cv.youtube_id + ORDER BY er.created_at DESC, er.id DESC + LIMIT 1) AS request_status + FROM channelvideos cv + INNER JOIN channels c ON c.channel_id = cv.channel_id + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + LEFT JOIN videos v ON v.youtube_id = cv.youtube_id + WHERE cv.youtube_id = :youtubeId + AND c.enabled = true AND c.terminated_at IS NULL + AND cv.youtube_removed = false AND cv.ignored = false + AND cv.media_type IN (:allowedMediaTypes) + AND ${ratings.sql} + ORDER BY c.id ASC + LIMIT 1`, + { + replacements: { + keyId: key.id, + youtubeId, + allowedMediaTypes: policy.allowedMediaTypes, + ...ratings.replacements, + }, + type: QueryTypes.SELECT, + } + ); + const row = rows[0]; + if (!row) throw new CatalogError('Video not found', 404); + + const metadataProvider = metadataService || require('./videoMetadataModule'); + const metadata = await metadataProvider.getVideoMetadata(youtubeId); + const isDownloaded = Boolean(row.downloaded_id) && !row.downloaded_removed; + return { + youtubeId: row.youtube_id, + title: boundedString(row.title, MAX_TITLE_LENGTH), + thumbnailUrl: videoThumbnailUrl(row.youtube_id), + publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + duration: row.duration, + isDownloaded, + isRequested: ACTIVE_REQUEST_STATUSES.includes(row.request_status), + requestStatus: row.request_status || null, + rating: row.rating || null, + ratingSource: row.rating_source || null, + channelDatabaseId: row.channel_database_id, + channelId: row.channel_id, + channelTitle: boundedString(row.channel_title, MAX_TITLE_LENGTH), + mediaType: row.media_type, + availability: metadata?.availability ?? row.availability ?? null, + downloadedAt: isDownloaded && row.last_downloaded_at + ? new Date(row.last_downloaded_at).toISOString() + : null, + fileSize: isDownloaded && row.fileSize != null ? Number(row.fileSize) : null, + audioFileSize: isDownloaded && row.audioFileSize != null ? Number(row.audioFileSize) : null, + isProtected: isDownloaded ? Boolean(row.protected) : false, + videoResolution: isDownloaded ? row.video_resolution || null : null, + metadata: { + ...(metadata || {}), + description: metadata?.description ?? row.description ?? null, + }, + }; +} + +async function getChannelThumbnail(key, channelDatabaseId) { + normalizePolicy(key); + const id = parseInteger(channelDatabaseId, null, 1, Number.MAX_SAFE_INTEGER, 'channel id'); + const rows = await sequelize.query( + `SELECT c.channel_id + FROM channels c + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + WHERE c.id = :channelDatabaseId AND c.enabled = true AND c.terminated_at IS NULL + LIMIT 1`, + { replacements: { keyId: key.id, channelDatabaseId: id }, type: QueryTypes.SELECT } + ); + const channelId = rows[0]?.channel_id; + if (!channelId || !/^[A-Za-z0-9_-]+$/.test(channelId)) throw new CatalogError('Thumbnail not found', 404); + try { + const imageDirectory = await fs.realpath(path.resolve(configModule.getImagePath())); + const candidatePath = path.resolve(imageDirectory, `channelthumb-${channelId}.jpg`); + if (path.dirname(candidatePath) !== imageDirectory) throw new Error('unsafe path'); + const stat = await fs.lstat(candidatePath); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('not a regular file'); + const absolutePath = await fs.realpath(candidatePath); + if (path.dirname(absolutePath) !== imageDirectory) throw new Error('unsafe target'); + return absolutePath; + } catch (_error) { + throw new CatalogError('Thumbnail not found', 404); + } +} + +async function getVideoThumbnail(key, youtubeId) { + const policy = normalizePolicy(key); + if (typeof youtubeId !== 'string' || !/^[A-Za-z0-9_-]{11}$/.test(youtubeId)) { + throw new CatalogError('Thumbnail not found', 404); + } + const effectiveRating = + 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; + const ratings = ratingSql(policy, effectiveRating); + const rows = await sequelize.query( + `SELECT cv.youtube_id, cv.thumbnail + FROM channelvideos cv + INNER JOIN channels c ON c.channel_id = cv.channel_id + INNER JOIN api_key_channel_grants g + ON g.channel_id = c.id AND g.api_key_id = :keyId + LEFT JOIN videos v ON v.youtube_id = cv.youtube_id + WHERE cv.youtube_id = :youtubeId + AND c.enabled = true AND c.terminated_at IS NULL + AND cv.youtube_removed = false AND cv.ignored = false + AND cv.media_type IN (:allowedMediaTypes) + AND ${ratings.sql} + LIMIT 1`, + { + replacements: { + keyId: key.id, + youtubeId, + allowedMediaTypes: policy.allowedMediaTypes, + ...ratings.replacements, + }, + type: QueryTypes.SELECT, + } + ); + if (rows.length === 0) throw new CatalogError('Thumbnail not found', 404); + try { + const imageDirectory = await fs.realpath(path.resolve(configModule.getImagePath())); + const candidatePath = path.resolve(imageDirectory, `videothumb-${youtubeId}.jpg`); + if (path.dirname(candidatePath) !== imageDirectory) throw new Error('unsafe path'); + const stat = await fs.lstat(candidatePath); + if (!stat.isFile() || stat.isSymbolicLink()) throw new Error('not a regular file'); + const absolutePath = await fs.realpath(candidatePath); + if (path.dirname(absolutePath) !== imageDirectory) throw new Error('unsafe target'); + return { source: 'local', absolutePath }; + } catch (_error) { + const upstreamUrl = publicVideoThumbnail(rows[0].thumbnail) || + `https://i.ytimg.com/vi/${youtubeId}/hqdefault.jpg`; + return { source: 'upstream', url: upstreamUrl }; + } +} + +module.exports = { + listChannels, + listChannelVideos, + listVideos, + getVideoDetail, + getChannelThumbnail, + getVideoThumbnail, + CatalogError, + normalizePolicy, + publicVideoThumbnail, + videoThumbnailUrl, + decodeCursor: decodePageCursor, + encodeCursor: encodePageCursor, + pagination, + decodeCatalogCursor, + encodeCatalogCursor, + paginationDto, +}; diff --git a/server/modules/externalEligibility.js b/server/modules/externalEligibility.js new file mode 100644 index 000000000..19c8b3be5 --- /dev/null +++ b/server/modules/externalEligibility.js @@ -0,0 +1,63 @@ +const ratingMapper = require('./ratingMapper'); + +const MEDIA_TYPES = ['video', 'short', 'livestream']; +const RATING_LABELS = [ + 'G', 'TV-Y', 'TV-Y7', 'TV-G', 'PG', 'TV-PG', + 'PG-13', 'TV-14', 'R', 'TV-MA', 'NC-17', +]; + +function normalizeExternalPolicy(policy, ErrorType = Error) { + const maxRatingLevel = policy?.maxRatingLevel; + const allowUnrated = policy?.allowUnrated; + const allowedMediaTypes = policy?.allowedMediaTypes; + if (!Number.isInteger(maxRatingLevel) || maxRatingLevel < 1 || maxRatingLevel > 4 || + typeof allowUnrated !== 'boolean' || !Array.isArray(allowedMediaTypes) || + allowedMediaTypes.length === 0 || + allowedMediaTypes.some((type) => !MEDIA_TYPES.includes(type))) { + throw new ErrorType('Invalid external API key policy', 401); + } + return { + maxRatingLevel, + allowUnrated, + allowedMediaTypes: [...new Set(allowedMediaTypes)], + }; +} + +function effectiveRating(videoRating, channelDefaultRating) { + return videoRating == null || videoRating === '' + ? (channelDefaultRating == null || channelDefaultRating === '' ? null : channelDefaultRating) + : videoRating; +} + +function isRatingEligible(policy, videoRating, channelDefaultRating) { + const rating = effectiveRating(videoRating, channelDefaultRating); + const numericRating = ratingMapper.mapToNumericRating(rating); + if (numericRating === null) return policy.allowUnrated; + return numericRating <= policy.maxRatingLevel; +} + +function isMediaTypeEligible(policy, mediaType) { + return MEDIA_TYPES.includes(mediaType) && policy.allowedMediaTypes.includes(mediaType); +} + +function ratingPolicy(policy) { + const recognizedRatings = RATING_LABELS.filter( + (rating) => ratingMapper.mapToNumericRating(rating) !== null + ); + return { + recognizedRatings, + allowedRatings: recognizedRatings.filter( + (rating) => ratingMapper.mapToNumericRating(rating) <= policy.maxRatingLevel + ), + }; +} + +module.exports = { + MEDIA_TYPES, + RATING_LABELS, + normalizeExternalPolicy, + effectiveRating, + isRatingEligible, + isMediaTypeEligible, + ratingPolicy, +}; diff --git a/server/modules/externalPermissions.js b/server/modules/externalPermissions.js new file mode 100644 index 000000000..4e99ef696 --- /dev/null +++ b/server/modules/externalPermissions.js @@ -0,0 +1,67 @@ +const BASE_SCOPES = ['catalog:read', 'requests:read']; + +const DEFINITIONS = { + allowVideoRequests: { + snake: 'allow_video_requests', + roles: ['request', 'delete', 'admin'], + scope: 'video:request', + }, + allowChannelRequests: { + snake: 'allow_channel_requests', + roles: ['request', 'delete', 'admin'], + scope: 'channel:request', + }, + allowDeleteVideoRequests: { + snake: 'allow_delete_video_requests', + roles: ['delete', 'admin'], + scope: 'video:delete', + }, +}; + +function normalizeExternalPermissions(source = {}) { + const hasExplicitPermissions = Object.entries(DEFINITIONS).some( + ([name, definition]) => + source[name] !== undefined || source[definition.snake] !== undefined, + ); + const permissions = {}; + for (const [name, definition] of Object.entries(DEFINITIONS)) { + const value = source[name] ?? source[definition.snake]; + if (value !== undefined && value !== null && typeof value !== 'boolean') { + return null; + } + permissions[name] = value ?? + (hasExplicitPermissions ? false : definition.roles.includes(source.role)); + } + return permissions; +} + +function scopesForExternalKey(source) { + const permissions = normalizeExternalPermissions(source); + if (!permissions) return null; + return [ + ...BASE_SCOPES, + ...Object.entries(DEFINITIONS) + .filter(([name]) => permissions[name]) + .map(([, definition]) => definition.scope), + ]; +} + +function hasExternalScope(source, scope) { + return scopesForExternalKey(source)?.includes(scope) === true; +} + +function roleForExternalPermissions(permissions, preferredRole) { + if (preferredRole === 'admin') return 'admin'; + if (permissions.allowDeleteVideoRequests) return 'delete'; + if (permissions.allowVideoRequests || permissions.allowChannelRequests) return 'request'; + return 'view'; +} + +module.exports = { + BASE_SCOPES, + DEFINITIONS, + normalizeExternalPermissions, + scopesForExternalKey, + hasExternalScope, + roleForExternalPermissions, +}; diff --git a/server/modules/externalQuotaService.js b/server/modules/externalQuotaService.js new file mode 100644 index 000000000..b442c9abe --- /dev/null +++ b/server/modules/externalQuotaService.js @@ -0,0 +1,239 @@ +const { Op } = require('sequelize'); +const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); +const { hasExternalScope } = require('./externalPermissions'); + +class QuotaError extends Error { + constructor(message, code = 'quota_exceeded', status = 429) { + super(message); + this.name = 'QuotaError'; + this.code = code; + this.status = status; + } +} + +function startOfHour(value) { + const date = new Date(value); + date.setUTCMinutes(0, 0, 0); + return date; +} + +function startOfDay(value) { + const date = new Date(value); + date.setUTCHours(0, 0, 0, 0); + return date; +} + +function limitsFor(key) { + return { + maxActiveJobs: key.maxActiveJobs ?? key.max_active_jobs ?? 5, + hourlyWriteLimit: key.hourlyWriteLimit ?? key.hourly_write_limit ?? 30, + dailyWriteLimit: key.dailyWriteLimit ?? key.daily_write_limit ?? 200, + }; +} + +function createExternalQuotaService({ + models = require('../models'), + sequelize = require('../db').sequelize, + now = () => new Date(), +} = {}) { + const { ApiKey, ExternalRequest, ExternalApiUsageBucket, Video, Job } = models; + + async function reloadAuthorizedKey(keyId, requiredScope, transaction = null) { + const record = await ApiKey.findByPk(keyId, { + ...(transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}), + }); + const key = normalizeExternalApiKey(record); + if (!key || !hasExternalScope(key, requiredScope)) { + throw new QuotaError('External API authority changed', 'authority_changed', 403); + } + return key; + } + + async function activeJobs(keyId, transaction = null, excludeRequestId = null) { + const processing = await ExternalRequest.findAll({ + where: { + api_key_id: keyId, + status: 'processing', + request_type: { [Op.in]: ['video', 'delete_video'] }, + }, + attributes: [ + 'id', 'request_type', 'youtube_id', 'job_id', 'status', + 'active_dedupe_key', + ], + ...(transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}), + }); + if (processing.length > 0) { + const youtubeIds = [...new Set(processing.map((record) => record.youtube_id))]; + const videos = await Video.findAll({ + where: { youtubeId: youtubeIds, removed: false }, + attributes: ['youtubeId'], + ...(transaction ? { transaction } : {}), + }); + const present = new Set(videos.map((video) => video.youtubeId)); + const terminalAt = now(); + const completed = processing.filter((record) => + (record.request_type === 'video' && present.has(record.youtube_id)) || + (record.request_type === 'delete_video' && !present.has(record.youtube_id)) + ); + await Promise.all(completed.map((record) => record.update({ + status: 'completed', + active_dedupe_key: null, + completed_at: terminalAt, + updated_at: terminalAt, + }, transaction ? { transaction } : {}))); + + const unresolved = processing.filter((record) => + record.request_type === 'video' && + !present.has(record.youtube_id) && + record.job_id && + !completed.includes(record) + ); + if (unresolved.length > 0) { + const jobs = await Job.findAll({ + where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, + attributes: ['id', 'status'], + ...(transaction ? { transaction } : {}), + }); + const terminalJobs = new Set(jobs + .filter((job) => [ + 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', + ].includes(job.status)) + .map((job) => job.id)); + await Promise.all(unresolved + .filter((record) => terminalJobs.has(record.job_id)) + .map((record) => record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Download did not complete', + updated_at: terminalAt, + }, transaction ? { transaction } : {}))); + } + } + return ExternalRequest.count({ + where: { + api_key_id: keyId, + ...(excludeRequestId ? { id: { [Op.ne]: excludeRequestId } } : {}), + [Op.or]: [ + { status: { [Op.in]: ['approved', 'processing'] } }, + { status: 'pending', decided_at: { [Op.ne]: null } }, + ], + }, + ...(transaction ? { transaction } : {}), + }); + } + + async function getBucket(keyId, windowType, windowStart, transaction = null) { + const [bucket] = await ExternalApiUsageBucket.findOrCreate({ + where: { + api_key_id: keyId, + window_type: windowType, + window_start: windowStart, + }, + defaults: { + api_key_id: keyId, + window_type: windowType, + window_start: windowStart, + accepted_writes: 0, + created_at: now(), + updated_at: now(), + }, + ...(transaction ? { transaction } : {}), + }); + if (transaction) { + return ExternalApiUsageBucket.findByPk(bucket.id, { + transaction, + lock: transaction.LOCK.UPDATE, + }); + } + return bucket; + } + + async function reserveWrite(keyId, requiredScope, transaction = null) { + const reserve = async (activeTransaction) => { + const key = await reloadAuthorizedKey(keyId, requiredScope, activeTransaction); + const limits = limitsFor(key); + const active = await activeJobs(keyId, activeTransaction); + if (active >= limits.maxActiveJobs) { + throw new QuotaError('External API active job limit exceeded', 'active_job_limit'); + } + const timestamp = now(); + const hour = await getBucket(keyId, 'hour', startOfHour(timestamp), activeTransaction); + const day = await getBucket(keyId, 'day', startOfDay(timestamp), activeTransaction); + if (hour.accepted_writes >= limits.hourlyWriteLimit) { + throw new QuotaError('External API hourly write limit exceeded', 'hourly_write_limit'); + } + if (day.accepted_writes >= limits.dailyWriteLimit) { + throw new QuotaError('External API daily write limit exceeded', 'daily_write_limit'); + } + await hour.update({ + accepted_writes: hour.accepted_writes + 1, + updated_at: timestamp, + }, { transaction: activeTransaction }); + await day.update({ + accepted_writes: day.accepted_writes + 1, + updated_at: timestamp, + }, { transaction: activeTransaction }); + return key; + }; + return transaction ? reserve(transaction) : sequelize.transaction(reserve); + } + + async function assertExecutionCapacity( + keyId, + requiredScope, + requestId, + transaction = null + ) { + const check = async (activeTransaction) => { + const key = await reloadAuthorizedKey(keyId, requiredScope, activeTransaction); + const active = await activeJobs(keyId, activeTransaction, requestId); + if (active >= limitsFor(key).maxActiveJobs) { + throw new QuotaError('External API active job limit exceeded', 'active_job_limit'); + } + return key; + }; + return transaction ? check(transaction) : sequelize.transaction(check); + } + + async function status(key) { + const limits = limitsFor(key); + const timestamp = now(); + const [active, hour, day] = await Promise.all([ + activeJobs(key.id), + ExternalApiUsageBucket.findOne({ + where: { + api_key_id: key.id, + window_type: 'hour', + window_start: startOfHour(timestamp), + }, + }), + ExternalApiUsageBucket.findOne({ + where: { + api_key_id: key.id, + window_type: 'day', + window_start: startOfDay(timestamp), + }, + }), + ]); + const hourlyUsed = hour?.accepted_writes || 0; + const dailyUsed = day?.accepted_writes || 0; + return { + limits, + remaining: { + activeJobs: Math.max(0, limits.maxActiveJobs - active), + hourlyWrites: Math.max(0, limits.hourlyWriteLimit - hourlyUsed), + dailyWrites: Math.max(0, limits.dailyWriteLimit - dailyUsed), + }, + }; + } + + return { reserveWrite, reloadAuthorizedKey, assertExecutionCapacity, status }; +} + +module.exports = { + createExternalQuotaService, + QuotaError, + startOfHour, + startOfDay, + limitsFor, +}; diff --git a/server/modules/externalRequestService.js b/server/modules/externalRequestService.js new file mode 100644 index 000000000..f075d1906 --- /dev/null +++ b/server/modules/externalRequestService.js @@ -0,0 +1,1382 @@ +const crypto = require('crypto'); +const { Op, UniqueConstraintError } = require('sequelize'); +const { normalizePolicy } = require('./externalCatalogService'); +const { isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); +const { hasExternalScope, normalizeExternalPermissions } = require('./externalPermissions'); + +const REQUEST_STATUSES = [ + 'pending', 'approved', 'processing', 'completed', + 'rejected', 'failed', 'cancelled', +]; +const ACTIVE_STATUSES = ['pending', 'approved', 'processing']; +const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; +const AUXILIARY_RECOVERY_DELAY_MS = 5 * 60 * 1000; +const MAX_PAGE = 100; + +class RequestError extends Error { + constructor(message, status = 400, code = null) { + super(message); + this.name = 'RequestError'; + this.status = status; + this.code = code; + } +} + +function rethrowWorkLimit(error) { + if (error?.name === 'ExternalWorkLimitError') { + throw new RequestError( + 'External API work capacity is temporarily unavailable', + 503, + 'work_queue_full' + ); + } +} + +function requireCurrentAutoApproval(key, requestType) { + const enabled = { + video: key.autoApproveVideoRequests, + channel: key.autoApproveChannelRequests, + delete_video: key.autoApproveDeleteRequests, + }[requestType]; + if (enabled !== true) { + throw new RequestError('Request is no longer eligible', 403); + } +} + +function parseInteger(value, fallback, minimum, maximum, name) { + if (value === undefined) return fallback; + if (!/^\d+$/.test(String(value))) throw new RequestError(`${name} must be an integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new RequestError(`${name} must be between ${minimum} and ${maximum}`); + } + return parsed; +} + +function decodeCursor(value) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 200) { + throw new RequestError('cursor is invalid'); + } + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); + if (parsed?.v !== 1 || !Number.isSafeInteger(parsed.page) || + parsed.page < 1 || parsed.page > MAX_PAGE) { + throw new Error('invalid cursor'); + } + return parsed.page; + } catch (_error) { + throw new RequestError('cursor is invalid'); + } +} + +function requestPagination(query) { + if (query.cursor !== undefined && query.page !== undefined) { + throw new RequestError('cursor and page cannot be used together'); + } + const page = decodeCursor(query.cursor) || + parseInteger(query.page, 1, 1, MAX_PAGE, 'page'); + const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); + return { page, pageSize, offset: (page - 1) * pageSize }; +} + +function requestPaginationDto(page, pageSize, total) { + const totalPages = total === 0 ? 0 : Math.min(MAX_PAGE, Math.ceil(total / pageSize)); + return { + page, + pageSize, + total, + totalPages, + nextCursor: page < totalPages + ? Buffer.from(JSON.stringify({ v: 1, page: page + 1 }), 'utf8').toString('base64url') + : null, + }; +} + +function normalizeIdempotencyKey(value) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length < 1 || value.length > 200) { + throw new RequestError('idempotencyKey must be a string of 1 to 200 characters'); + } + return crypto.createHash('sha256').update(value).digest('hex'); +} + +function normalizeChannelUrl(value) { + if (typeof value !== 'string' || value.length < 1 || value.length > 500) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + let url; + try { + url = new URL(/^https?:\/\//i.test(value.trim()) ? value.trim() : `https://${value.trim()}`); + } catch (_error) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + const hostname = url.hostname.toLowerCase().replace(/^www\./, ''); + const segments = url.pathname.split('/').filter(Boolean); + const supportedPrefix = segments[0]?.startsWith('@') || + ['channel', 'c', 'user'].includes(segments[0]); + if (!['youtube.com', 'm.youtube.com'].includes(hostname) || + !supportedPrefix || segments.length !== (segments[0].startsWith('@') ? 1 : 2)) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + try { + return `https://www.youtube.com/${segments.map((segment) => { + const decoded = decodeURIComponent(segment); + return decoded.startsWith('@') + ? `@${encodeURIComponent(decoded.slice(1))}` + : encodeURIComponent(decoded); + }).join('/')}`; + } catch (_error) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } +} + +function dto(record) { + const value = record.toJSON ? record.toJSON() : record; + return { + id: value.id, + type: value.request_type, + status: value.status, + target: { + youtubeId: value.youtube_id || null, + channelId: value.channel_id || null, + channelUrl: value.channel_url || null, + }, + createdAt: value.created_at, + updatedAt: value.updated_at, + ...(value.decided_at ? { decidedAt: value.decided_at } : {}), + ...(value.completed_at ? { completedAt: value.completed_at } : {}), + ...(value.message ? { message: value.message } : {}), + ...(value.request_type === 'channel' && value.grant_to_requesting_key !== null && + value.grant_to_requesting_key !== undefined + ? { grantToRequestingKey: value.grant_to_requesting_key === true } + : {}), + }; +} + +function normalizeStoredKey(record) { + const value = record?.toJSON ? record.toJSON() : record; + if (!value) return null; + const permissions = normalizeExternalPermissions(value); + if (!permissions) return null; + return { + id: value.id, + name: value.name, + role: value.role, + isActive: value.is_active, + revokedAt: value.revoked_at, + autoApproveVideoRequests: value.auto_approve_video_requests, + autoApproveChannelRequests: value.auto_approve_channel_requests, + autoApproveDeleteRequests: value.auto_approve_delete_requests, + ...permissions, + maxRatingLevel: value.max_rating_level, + allowUnrated: value.allow_unrated, + allowedMediaTypes: value.allowed_media_types, + }; +} + +function sanitizeReason(value) { + if (typeof value !== 'string') { + throw new RequestError('reason is required'); + } + // eslint-disable-next-line no-control-regex + const sanitized = value.replace(/[\x00-\x1F\x7F]/g, ' ').replace(/\s+/g, ' ').trim(); + if (sanitized.length < 1 || sanitized.length > 300) { + throw new RequestError('reason must be between 1 and 300 characters'); + } + return sanitized; +} + +function isUniqueConstraintError(error) { + return error instanceof UniqueConstraintError || + error?.name === 'SequelizeUniqueConstraintError'; +} + +function adminDto(record, catalogVideo = null) { + const value = record.toJSON ? record.toJSON() : record; + const key = value.apiKey || value.api_key || null; + const channel = value.channel || null; + const job = value.job || null; + return { + ...dto(record), + requester: key ? { + id: key.id, + name: key.name, + keyPrefix: key.key_prefix, + role: key.role, + isActive: key.is_active === true, + revokedAt: key.revoked_at || null, + } : null, + target: { + youtubeId: value.youtube_id || null, + channelId: value.channel_id || null, + channelUrl: value.channel_url || null, + youtubeChannelId: channel?.channel_id || null, + channelTitle: channel?.title || channel?.uploader || null, + title: catalogVideo?.title || null, + mediaType: catalogVideo?.media_type || null, + rating: channel?.default_rating || null, + contentRating: catalogVideo?.content_rating || channel?.default_rating || null, + }, + job: job ? { + id: job.id, + status: job.status, + type: job.jobType, + createdAt: job.timeCreated, + startedAt: job.timeInitiated, + } : null, + }; +} + +function createExternalRequestService({ + models = require('../models'), + executor = (jobData) => require('./downloadModule').doGroupedManualDownloads(jobData), + channelProvisioner = null, + videoDeleter = require('./videoDeletionModule'), + now = () => new Date(), + sequelize = require('../db').sequelize, + quotaService = null, + workLimiter = require('./externalWorkLimiter').sharedExternalWorkLimiter, +} = {}) { + const { + ExternalRequest, ApiKey, ApiKeyChannelGrant, Channel, ChannelVideo, Video, Job, + } = models; + const quotas = quotaService || + require('./externalQuotaService').createExternalQuotaService({ models, sequelize, now }); + + async function createReservedRequest(keyId, scope, buildValues) { + return sequelize.transaction(async (transaction) => { + const currentKey = await quotas.reserveWrite(keyId, scope, transaction); + const record = await ExternalRequest.create( + buildValues(currentKey), + { transaction } + ); + return { currentKey, record }; + }); + } + + async function failAuthorityChange(record) { + const timestamp = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + decided_at: record.decided_at || timestamp, + updated_at: timestamp, + }); + return record; + } + + async function reconcile(records) { + const processing = records.filter((record) => record.status === 'processing'); + const mediaRequests = processing.filter( + (record) => ['video', 'delete_video'].includes(record.request_type) + ); + const youtubeIds = [...new Set(mediaRequests.map((record) => record.youtube_id))]; + if (youtubeIds.length === 0) return records; + const videos = await Video.findAll({ + where: { youtubeId: youtubeIds, removed: false }, + attributes: ['youtubeId'], + }); + const completed = new Set(videos.map((video) => video.youtubeId)); + const completedAt = now(); + const completedRecords = mediaRequests.filter((record) => + (record.request_type === 'video' && completed.has(record.youtube_id)) || + (record.request_type === 'delete_video' && !completed.has(record.youtube_id)) + ); + await Promise.all(completedRecords.map(async (record) => { + await record.update({ + status: 'completed', + active_dedupe_key: null, + completed_at: completedAt, + updated_at: completedAt, + }); + })); + const unresolved = mediaRequests.filter( + (record) => record.request_type === 'video' && + !completed.has(record.youtube_id) && record.job_id + ); + if (unresolved.length > 0) { + const jobs = await Job.findAll({ + where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, + attributes: ['id', 'status'], + }); + const terminalFailures = new Set( + jobs + .filter((job) => [ + 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', + ].includes(job.status)) + .map((job) => job.id) + ); + await Promise.all(unresolved + .filter((record) => terminalFailures.has(record.job_id)) + .map((record) => record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Download did not complete', + updated_at: completedAt, + }))); + } + return records; + } + + async function claimAuxiliaryRequest(record) { + const timestamp = now(); + let whereStatus; + if (['pending', 'approved'].includes(record.status)) { + whereStatus = record.status; + } else if (record.status === 'processing') { + const updatedAt = new Date(record.updated_at).getTime(); + if (!Number.isFinite(updatedAt) || + timestamp.getTime() - updatedAt < AUXILIARY_RECOVERY_DELAY_MS) { + return false; + } + whereStatus = 'processing'; + } else { + return false; + } + const [claimed] = await ExternalRequest.update( + { status: 'processing', updated_at: timestamp }, + { where: { id: record.id, status: whereStatus } } + ); + if (claimed !== 1) return false; + record.status = 'processing'; + record.updated_at = timestamp; + return true; + } + + async function validateTarget( + key, + youtubeId, + channelId, + transaction = null, + { lockVideo = true } = {} + ) { + const policy = normalizePolicy(key); + const queryOptions = transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}; + const videoQueryOptions = transaction + ? { transaction, ...(lockVideo ? { lock: transaction.LOCK.UPDATE } : {}) } + : {}; + const grant = await ApiKeyChannelGrant.findOne({ + where: { api_key_id: key.id, channel_id: channelId }, + ...queryOptions, + }); + const channel = await Channel.findByPk(channelId, { + attributes: ['id', 'channel_id', 'enabled', 'terminated_at', 'default_rating'], + ...queryOptions, + }); + if (!grant || !channel || channel.enabled !== true || channel.terminated_at || !channel.channel_id) { + throw new RequestError('Video not found', 404); + } + const cached = await ChannelVideo.findOne({ + where: { youtube_id: youtubeId, channel_id: channel.channel_id }, + attributes: ['youtube_id', 'media_type', 'youtube_removed', 'ignored'], + ...queryOptions, + }); + if (!cached || cached.youtube_removed !== false || cached.ignored !== false) { + throw new RequestError('Video not found', 404); + } + if (!isMediaTypeEligible(policy, cached.media_type)) { + throw new RequestError('Video not found', 404); + } + const storedVideo = await Video.findOne({ + where: { youtubeId }, + // Deletion execution reuses this locked instance in the deleter, so it + // needs the complete row (including id/filePath). Read-only validation + // keeps the narrower projection. + ...(!(transaction && lockVideo) + ? { attributes: ['youtubeId', 'normalized_rating', 'removed'] } + : {}), + ...videoQueryOptions, + }); + if (storedVideo && storedVideo.removed !== true && storedVideo.removed !== false) { + throw new RequestError('Video not found', 404); + } + const downloaded = storedVideo?.removed === false ? storedVideo : null; + if (!isRatingEligible(policy, storedVideo?.normalized_rating, channel.default_rating)) { + throw new RequestError('Video not found', 404); + } + return { channel, downloaded }; + } + + const adminIncludes = () => [ + { + model: ApiKey, + as: 'apiKey', + attributes: [ + 'id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allowed_media_types', + ], + required: false, + }, + { + model: Channel, + as: 'channel', + attributes: [ + 'id', 'channel_id', 'title', 'uploader', 'url', 'default_rating', 'terminated_at', + ], + required: false, + }, + { + model: Job, + as: 'job', + attributes: ['id', 'status', 'jobType', 'timeCreated', 'timeInitiated'], + required: false, + }, + ]; + + async function catalogMetadata(records) { + const youtubeIds = [...new Set(records.map((record) => record.youtube_id).filter(Boolean))]; + if (youtubeIds.length === 0) return new Map(); + const rows = await ChannelVideo.findAll({ + where: { youtube_id: youtubeIds }, + attributes: ['youtube_id', 'channel_id', 'title', 'media_type'], + }); + const videos = await Video.findAll({ + where: { youtubeId: youtubeIds }, + attributes: ['youtubeId', 'normalized_rating'], + }); + const videoById = new Map(videos.map((row) => { + const value = row.toJSON ? row.toJSON() : row; + return [value.youtubeId, value]; + })); + return new Map(rows.map((row) => { + const value = row.toJSON ? row.toJSON() : row; + return [`${value.channel_id}:${value.youtube_id}`, value]; + }).map(([key, value]) => [key, { + ...value, + content_rating: videoById.get(value.youtube_id)?.normalized_rating || null, + }])); + } + + async function adminDtos(records) { + const metadata = await catalogMetadata(records); + return records.map((record) => { + const value = record.toJSON ? record.toJSON() : record; + const channel = value.channel; + return adminDto( + record, + metadata.get(`${channel?.channel_id || ''}:${value.youtube_id}`) || null + ); + }); + } + + async function findDuplicate(keyId, activeDedupeKey, idempotencyHash, youtubeId, channelId) { + const clauses = [{ active_dedupe_key: activeDedupeKey }]; + if (idempotencyHash) clauses.push({ api_key_id: keyId, idempotency_hash: idempotencyHash }); + const existing = await ExternalRequest.findOne({ where: { [Op.or]: clauses } }); + if (!existing) return null; + if (existing.youtube_id !== youtubeId || existing.channel_id !== channelId) { + throw new RequestError('Idempotency key was already used for another target', 409); + } + await reconcile([existing]); + return existing; + } + + async function findTypedDuplicate({ + keyId, activeDedupeKey, idempotencyHash, requestType, youtubeId = null, + channelId = null, channelUrl = null, + }) { + const clauses = [{ active_dedupe_key: activeDedupeKey }]; + if (idempotencyHash) clauses.push({ api_key_id: keyId, idempotency_hash: idempotencyHash }); + const existing = await ExternalRequest.findOne({ where: { [Op.or]: clauses } }); + if (!existing) return null; + if (existing.request_type !== requestType || + existing.youtube_id !== youtubeId || + existing.channel_id !== channelId || + existing.channel_url !== channelUrl) { + throw new RequestError('Idempotency key was already used for another target', 409); + } + return existing; + } + + async function dispatchAutoApproved(record, key) { + try { + const jobId = await workLimiter.run(() => + sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + key.id, + 'video:request', + record.id, + transaction + ); + requireCurrentAutoApproval(currentKey, 'video'); + const target = await validateTarget( + currentKey, + record.youtube_id, + record.channel_id, + transaction + ); + return executor({ + body: { + urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], + channelId: target.channel.channel_id, + ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, + initiatedBy: { type: 'api_key', name: currentKey.name }, + jobLabel: 'External video request', + // The downloader uses this UUID as the job identity. A retry after a + // crash can therefore observe/reuse the accepted job instead of + // starting the same download twice. + externalRequestId: record.id, + }, + }); + }) + ); + const acceptedAt = now(); + await record.update({ + status: 'processing', + job_id: jobId || record.id, + updated_at: acceptedAt, + }); + } catch (error) { + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Download could not be queued', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + async function createVideoRequest(key, input) { + if (!hasExternalScope(key, 'video:request')) { + throw new RequestError('video:request scope is required', 403); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + const supported = ['youtubeId', 'channelId', 'idempotencyKey']; + if (Object.keys(input).some((name) => !supported.includes(name))) { + throw new RequestError('Request body contains unsupported fields'); + } + if (typeof input.youtubeId !== 'string' || !VIDEO_ID_PATTERN.test(input.youtubeId)) { + throw new RequestError('youtubeId must be an 11-character YouTube video ID'); + } + if (input.channelId === undefined) throw new RequestError('channelId is required'); + const channelId = parseInteger(input.channelId, null, 1, Number.MAX_SAFE_INTEGER, 'channelId'); + const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); + const { downloaded } = await validateTarget(key, input.youtubeId, channelId); + if (downloaded) return { outcome: 'already_downloaded', request: null }; + + const activeDedupeKey = `${key.id}:video:${input.youtubeId}`; + const existing = await findDuplicate( + key.id, activeDedupeKey, idempotencyHash, input.youtubeId, channelId + ); + if (existing) { + // Auto-approved requests are created as pending before enqueue. If the + // process stopped in that boundary, an idempotent client retry resumes + // dispatch with the request UUID as the stable downloader job ID. + if (existing.status === 'pending' && existing.decided_at && !existing.job_id) { + try { + const currentKey = await quotas.assertExecutionCapacity( + key.id, 'video:request', existing.id + ); + const currentTarget = await validateTarget( + currentKey, + input.youtubeId, + channelId + ); + await dispatchAutoApproved(existing, currentKey, currentTarget.channel); + } catch (_error) { + await failAuthorityChange(existing); + } + } + return { outcome: 'duplicate', request: dto(existing) }; + } + + const timestamp = now(); + let currentKey; + let record; + try { + ({ currentKey, record } = await createReservedRequest( + key.id, + 'video:request', + (reservedKey) => ({ + api_key_id: key.id, + channel_id: channelId, + youtube_id: input.youtubeId, + request_type: 'video', + status: 'pending', + active_dedupe_key: activeDedupeKey, + idempotency_hash: idempotencyHash, + decided_at: reservedKey.autoApproveVideoRequests ? timestamp : null, + created_at: timestamp, + updated_at: timestamp, + }) + )); + } catch (error) { + if (!isUniqueConstraintError(error)) { + throw error; + } + const duplicate = await findDuplicate( + key.id, activeDedupeKey, idempotencyHash, input.youtubeId, channelId + ); + if (!duplicate) throw error; + return { outcome: 'duplicate', request: dto(duplicate) }; + } + + if (currentKey.autoApproveVideoRequests) { + try { + const finalKey = await quotas.assertExecutionCapacity( + key.id, 'video:request', record.id + ); + const finalTarget = await validateTarget(finalKey, input.youtubeId, channelId); + await dispatchAutoApproved(record, finalKey, finalTarget.channel); + } catch (error) { + if (error?.status === 503) throw error; + await failAuthorityChange(record); + } + } + return { outcome: 'created', request: dto(record) }; + } + + async function provisionChannelRequest( + record, + key, + { grantToRequestingKey, requireAutoApproval = false } = {} + ) { + if (!(await claimAuxiliaryRequest(record))) return record; + const shouldGrant = grantToRequestingKey ?? + (record.grant_to_requesting_key !== false); + try { + await workLimiter.run(() => sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + key.id, + 'channel:request', + record.id, + transaction + ); + if (requireAutoApproval) { + requireCurrentAutoApproval(currentKey, 'channel'); + } + const provisioner = channelProvisioner || require('./channel/channelProvisioning'); + const result = await provisioner.getChannelInfo( + record.channel_url, + false, + true, + {}, + { skipTabDetection: true } + ); + const channel = await Channel.findOne({ + where: { + [Op.or]: [ + { channel_id: result.channel_id || result.id }, + { url: record.channel_url }, + ], + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!channel || channel.enabled !== true || channel.terminated_at) { + throw new Error('Provisioned channel is unavailable'); + } + if (shouldGrant) { + await ApiKeyChannelGrant.findOrCreate({ + where: { api_key_id: currentKey.id, channel_id: channel.id }, + defaults: { api_key_id: currentKey.id, channel_id: channel.id }, + transaction, + }); + } + const completedAt = now(); + await record.update({ + channel_id: channel.id, + status: 'completed', + active_dedupe_key: null, + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + })); + } catch (error) { + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Channel could not be provisioned', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + async function createChannelRequest(key, input) { + if (!hasExternalScope(key, 'channel:request')) { + throw new RequestError('channel:request scope is required', 403); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + if (Object.keys(input).some((name) => !['channelUrl', 'idempotencyKey'].includes(name))) { + throw new RequestError('Request body contains unsupported fields'); + } + const channelUrl = normalizeChannelUrl(input.channelUrl); + const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); + const targetHash = crypto.createHash('sha256').update(channelUrl).digest('hex').slice(0, 32); + const activeDedupeKey = `${key.id}:channel:${targetHash}`; + const existing = await findTypedDuplicate({ + keyId: key.id, + activeDedupeKey, + idempotencyHash, + requestType: 'channel', + channelUrl, + }); + if (existing) { + if (existing.decided_at && + ['pending', 'approved', 'processing'].includes(existing.status)) { + try { + const currentKey = await quotas.assertExecutionCapacity( + key.id, 'channel:request', existing.id + ); + await provisionChannelRequest(existing, currentKey, { + requireAutoApproval: existing.status !== 'approved', + }); + } catch (_error) { + await failAuthorityChange(existing); + } + } + return { outcome: 'duplicate', request: dto(existing) }; + } + const timestamp = now(); + let currentKey; + let record; + try { + ({ currentKey, record } = await createReservedRequest( + key.id, + 'channel:request', + (reservedKey) => ({ + api_key_id: key.id, + channel_id: null, + youtube_id: null, + channel_url: channelUrl, + request_type: 'channel', + status: 'pending', + grant_to_requesting_key: true, + active_dedupe_key: activeDedupeKey, + idempotency_hash: idempotencyHash, + decided_at: reservedKey.autoApproveChannelRequests ? timestamp : null, + created_at: timestamp, + updated_at: timestamp, + }) + )); + } catch (error) { + if (!isUniqueConstraintError(error)) throw error; + const duplicate = await findTypedDuplicate({ + keyId: key.id, + activeDedupeKey, + idempotencyHash, + requestType: 'channel', + channelUrl, + }); + if (!duplicate) throw error; + return { outcome: 'duplicate', request: dto(duplicate) }; + } + if (currentKey.autoApproveChannelRequests) { + try { + const finalKey = await quotas.assertExecutionCapacity( + key.id, 'channel:request', record.id + ); + await provisionChannelRequest(record, finalKey, { requireAutoApproval: true }); + } catch (error) { + if (error?.status === 503) throw error; + await failAuthorityChange(record); + } + } + return { outcome: 'created', request: dto(record) }; + } + + async function executeDeleteRequest(record, { requireAutoApproval = false } = {}) { + if (!(await claimAuxiliaryRequest(record))) return record; + try { + await workLimiter.run(() => sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + record.api_key_id, + 'video:delete', + record.id, + transaction + ); + if (requireAutoApproval) { + requireCurrentAutoApproval(currentKey, 'delete_video'); + } + const target = await validateTarget( + currentKey, + record.youtube_id, + record.channel_id, + transaction + ); + if (!target.downloaded) { + throw new RequestError('Video not found', 404); + } + const result = await videoDeleter.deleteVideoById( + target.downloaded.id, + { transaction, video: target.downloaded } + ); + const alreadyAbsent = result?.success === false && + /not found|already (?:marked as )?removed/i.test(result?.error || ''); + if (result?.success === false && !alreadyAbsent) { + throw new Error('Deletion failed'); + } + const completedAt = now(); + await record.update({ + status: 'completed', + active_dedupe_key: null, + ...(alreadyAbsent ? { message: 'Video is already deleted' } : {}), + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + })); + } catch (error) { + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Video could not be deleted', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + async function createDeleteVideoRequest(key, input) { + if (!hasExternalScope(key, 'video:delete')) { + throw new RequestError('video:delete scope is required', 403); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + const supported = ['youtubeId', 'channelId', 'idempotencyKey']; + if (Object.keys(input).some((name) => !supported.includes(name))) { + throw new RequestError('Request body contains unsupported fields'); + } + if (typeof input.youtubeId !== 'string' || !VIDEO_ID_PATTERN.test(input.youtubeId)) { + throw new RequestError('youtubeId must be an 11-character YouTube video ID'); + } + const channelId = parseInteger( + input.channelId, + null, + 1, + Number.MAX_SAFE_INTEGER, + 'channelId' + ); + const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); + const target = await validateTarget(key, input.youtubeId, channelId); + if (!target.downloaded) { + // Removed and never-downloaded targets are deliberately indistinguishable + // from every other hidden target state. + throw new RequestError('Video not found', 404); + } + const activeDedupeKey = `${key.id}:delete_video:${input.youtubeId}`; + const existing = await findTypedDuplicate({ + keyId: key.id, + activeDedupeKey, + idempotencyHash, + requestType: 'delete_video', + youtubeId: input.youtubeId, + channelId, + }); + if (existing) { + if (existing.decided_at && + ['pending', 'approved', 'processing'].includes(existing.status)) { + try { + const currentKey = await quotas.assertExecutionCapacity( + key.id, 'video:delete', existing.id + ); + await validateTarget(currentKey, input.youtubeId, channelId); + await executeDeleteRequest(existing, { + requireAutoApproval: existing.status !== 'approved', + }); + } catch (_error) { + await failAuthorityChange(existing); + } + } + return { outcome: 'duplicate', request: dto(existing) }; + } + const timestamp = now(); + let currentKey; + let record; + try { + ({ currentKey, record } = await createReservedRequest( + key.id, + 'video:delete', + (reservedKey) => ({ + api_key_id: key.id, + channel_id: channelId, + youtube_id: input.youtubeId, + request_type: 'delete_video', + status: 'pending', + active_dedupe_key: activeDedupeKey, + idempotency_hash: idempotencyHash, + message: null, + decided_at: reservedKey.autoApproveDeleteRequests ? timestamp : null, + completed_at: null, + created_at: timestamp, + updated_at: timestamp, + }) + )); + } catch (error) { + if (!isUniqueConstraintError(error)) throw error; + const duplicate = await findTypedDuplicate({ + keyId: key.id, + activeDedupeKey, + idempotencyHash, + requestType: 'delete_video', + youtubeId: input.youtubeId, + channelId, + }); + if (!duplicate) throw error; + return { outcome: 'duplicate', request: dto(duplicate) }; + } + if (currentKey.autoApproveDeleteRequests) { + try { + const finalKey = await quotas.assertExecutionCapacity( + key.id, 'video:delete', record.id + ); + await validateTarget(finalKey, input.youtubeId, channelId); + await executeDeleteRequest(record, { requireAutoApproval: true }); + } catch (error) { + if (error?.status === 503) throw error; + await failAuthorityChange(record); + } + } + return { outcome: 'created', request: dto(record) }; + } + + async function listRequests(key, query = {}) { + const { page, pageSize, offset } = requestPagination(query); + const status = query.status; + if (status !== undefined && !REQUEST_STATUSES.includes(status)) { + throw new RequestError(`status must be one of: ${REQUEST_STATUSES.join(', ')}`); + } + const where = { api_key_id: key.id, ...(status ? { status } : {}) }; + const result = await ExternalRequest.findAndCountAll({ + where, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }); + await reconcile(result.rows); + const reconciledResult = status + ? await ExternalRequest.findAndCountAll({ + where, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }) + : result; + return { + data: reconciledResult.rows.map(dto), + pagination: requestPaginationDto(page, pageSize, reconciledResult.count), + }; + } + + async function getRequest(key, id) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + const record = await ExternalRequest.findOne({ where: { id, api_key_id: key.id } }); + if (!record) throw new RequestError('Request not found', 404); + await reconcile([record]); + return dto(record); + } + + async function listAdminRequests(query = {}) { + const { page, pageSize, offset } = requestPagination(query); + const status = query.status; + if (status !== undefined && !REQUEST_STATUSES.includes(status)) { + throw new RequestError(`status must be one of: ${REQUEST_STATUSES.join(', ')}`); + } + const apiKeyId = query.apiKeyId === undefined + ? null + : parseInteger(query.apiKeyId, null, 1, Number.MAX_SAFE_INTEGER, 'apiKeyId'); + const requestType = query.requestType; + if (requestType !== undefined && !['video', 'channel', 'delete_video'].includes(requestType)) { + throw new RequestError('requestType must be video, channel, or delete_video'); + } + const where = { + ...(requestType ? { request_type: requestType } : {}), + ...(status ? { status } : {}), + ...(apiKeyId ? { api_key_id: apiKeyId } : {}), + }; + const result = await ExternalRequest.findAndCountAll({ + where, + include: adminIncludes(), + distinct: true, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }); + await reconcile(result.rows); + const reconciledResult = status + ? await ExternalRequest.findAndCountAll({ + where, + include: adminIncludes(), + distinct: true, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }) + : result; + const requesters = await ApiKey.findAll({ + attributes: ['id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at'], + order: [['name', 'ASC'], ['id', 'ASC']], + }); + return { + data: await adminDtos(reconciledResult.rows), + pagination: requestPaginationDto(page, pageSize, reconciledResult.count), + filterOptions: { + requesters: requesters.map((key) => { + const value = key.toJSON ? key.toJSON() : key; + return { + id: value.id, + name: value.name, + keyPrefix: value.key_prefix, + role: value.role, + isActive: value.is_active === true, + revokedAt: value.revoked_at || null, + }; + }), + }, + }; + } + + async function getAdminRequest(id) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + const record = await ExternalRequest.findOne({ + where: { id }, + include: adminIncludes(), + }); + if (!record) throw new RequestError('Request not found', 404); + await reconcile([record]); + return (await adminDtos([record]))[0]; + } + + async function reviewVideoRequest(id, action, input = {}) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + if (!['approve', 'reject'].includes(action)) { + throw new RequestError('Unsupported review action'); + } + const allowedFields = action === 'reject' ? ['reason'] : []; + if (Object.keys(input).some((field) => !allowedFields.includes(field))) { + throw new RequestError('Request body contains unsupported fields'); + } + const reason = action === 'reject' ? sanitizeReason(input.reason) : null; + + await sequelize.transaction(async (transaction) => { + const record = await ExternalRequest.findOne({ + where: { id, request_type: 'video' }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!record) throw new RequestError('Request not found', 404); + + if (action === 'reject') { + if (record.status !== 'pending') { + throw new RequestError('Only pending requests can be rejected', 409); + } + const rejectedAt = now(); + await record.update({ + status: 'rejected', + active_dedupe_key: null, + message: reason, + decided_at: rejectedAt, + updated_at: rejectedAt, + }, { transaction }); + return; + } + + if (record.status !== 'pending' && + !(record.status === 'approved' && !record.job_id)) { + throw new RequestError('Only pending requests can be approved', 409); + } + + const failApproval = async (message) => { + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message, + decided_at: record.decided_at || failedAt, + updated_at: failedAt, + }, { transaction }); + }; + + const storedKey = await ApiKey.findByPk(record.api_key_id, { + attributes: [ + 'id', 'name', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'allowed_media_types', + ], + transaction, + lock: transaction.LOCK.UPDATE, + }); + const key = normalizeStoredKey(storedKey); + if (!key || key.isActive !== true || key.revokedAt || + !hasExternalScope(key, 'video:request')) { + await failApproval('Request is no longer eligible'); + return; + } + try { + await quotas.assertExecutionCapacity( + record.api_key_id, + 'video:request', + record.id, + transaction + ); + } catch (error) { + if (error.name !== 'QuotaError') throw error; + await failApproval('Request exceeds the active job limit'); + return; + } + + let target; + try { + target = await validateTarget(key, record.youtube_id, record.channel_id, transaction); + } catch (error) { + if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; + await failApproval('Request is no longer eligible'); + return; + } + if (target.downloaded) { + const completedAt = now(); + await record.update({ + status: 'completed', + active_dedupe_key: null, + message: 'Video is already downloaded', + decided_at: record.decided_at || completedAt, + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + return; + } + + const duplicate = await ExternalRequest.findOne({ + where: { + id: { [Op.ne]: record.id }, + active_dedupe_key: `${record.api_key_id}:video:${record.youtube_id}`, + status: { [Op.in]: ACTIVE_STATUSES }, + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (duplicate) { + await failApproval('Another active request already exists'); + return; + } + + const approvedAt = now(); + if (record.status === 'pending') { + await record.update({ + status: 'approved', + decided_at: approvedAt, + updated_at: approvedAt, + }, { transaction }); + } + try { + const jobId = await workLimiter.run(() => executor({ + body: { + urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], + channelId: target.channel.channel_id, + ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, + initiatedBy: { type: 'api_key', name: key.name }, + jobLabel: 'External video request', + externalRequestId: record.id, + }, + })); + const acceptedAt = now(); + await record.update({ + status: 'processing', + job_id: jobId || record.id, + updated_at: acceptedAt, + }, { transaction }); + } catch (_error) { + await failApproval('Download could not be queued'); + } + }); + + return getAdminRequest(id); + } + + async function reviewRequest(id, action, input = {}) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + const current = await ExternalRequest.findByPk(id); + if (!current) throw new RequestError('Request not found', 404); + if (current.request_type === 'video') return reviewVideoRequest(id, action, input); + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + if (!['approve', 'reject'].includes(action)) { + throw new RequestError('Unsupported review action'); + } + const allowedFields = action === 'reject' + ? ['reason'] + : (current.request_type === 'channel' ? ['grantToRequestingKey'] : []); + if (Object.keys(input).some((field) => !allowedFields.includes(field))) { + throw new RequestError('Request body contains unsupported fields'); + } + if (current.request_type === 'channel' && + input.grantToRequestingKey !== undefined && + typeof input.grantToRequestingKey !== 'boolean') { + throw new RequestError('grantToRequestingKey must be a boolean'); + } + const reason = action === 'reject' ? sanitizeReason(input.reason) : null; + let key; + let claimed; + await sequelize.transaction(async (transaction) => { + const record = await ExternalRequest.findByPk(id, { + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!record) throw new RequestError('Request not found', 404); + if (record.status !== 'pending') { + throw new RequestError('Only pending requests can be reviewed', 409); + } + const decisionAt = now(); + if (action === 'reject') { + await record.update({ + status: 'rejected', + active_dedupe_key: null, + message: reason, + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + const storedKey = await ApiKey.findByPk(record.api_key_id, { + attributes: [ + 'id', 'name', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'allowed_media_types', + ], + transaction, + lock: transaction.LOCK.UPDATE, + }); + key = normalizeStoredKey(storedKey); + const requiredScope = record.request_type === 'delete_video' + ? 'video:delete' + : 'channel:request'; + if (!key || key.isActive !== true || key.revokedAt || + !hasExternalScope(key, requiredScope)) { + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + try { + await quotas.assertExecutionCapacity( + record.api_key_id, + requiredScope, + record.id, + transaction + ); + } catch (error) { + if (error.name !== 'QuotaError') throw error; + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request exceeds the active job limit', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + if (record.request_type === 'delete_video') { + try { + const target = await validateTarget( + key, + record.youtube_id, + record.channel_id, + transaction + ); + if (!target.downloaded) { + await record.update({ + status: 'completed', + active_dedupe_key: null, + message: 'Video is already deleted', + decided_at: decisionAt, + completed_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + } catch (error) { + if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + } + await record.update({ + status: 'approved', + ...(record.request_type === 'channel' + ? { grant_to_requesting_key: input.grantToRequestingKey !== false } + : {}), + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + claimed = record; + }); + if (action === 'approve' && claimed) { + if (claimed.request_type === 'channel') { + await provisionChannelRequest(claimed, key, { + grantToRequestingKey: input.grantToRequestingKey !== false, + }); + } else if (claimed.request_type === 'delete_video') { + await executeDeleteRequest(claimed); + } + } + return getAdminRequest(id); + } + + return { + createVideoRequest, + createChannelRequest, + createDeleteVideoRequest, + listRequests, + getRequest, + listAdminRequests, + getAdminRequest, + reviewVideoRequest, + reviewRequest, + }; +} + +module.exports = { + createExternalRequestService, + RequestError, + REQUEST_STATUSES, + ACTIVE_STATUSES, + dto, + adminDto, + sanitizeReason, +}; diff --git a/server/modules/externalThumbnailProxy.js b/server/modules/externalThumbnailProxy.js new file mode 100644 index 000000000..8c696df05 --- /dev/null +++ b/server/modules/externalThumbnailProxy.js @@ -0,0 +1,68 @@ +const THUMBNAIL_FETCH_TIMEOUT_MS = 10000; +const THUMBNAIL_MAX_BYTES = 5 * 1024 * 1024; +const ALLOWED_IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']); + +class ThumbnailProxyError extends Error { + constructor(message) { + super(message); + this.name = 'ThumbnailProxyError'; + } +} + +async function fetchExternalThumbnail(url, fetchImpl = global.fetch) { + let response; + try { + response = await fetchImpl(url, { + headers: { Accept: 'image/jpeg,image/png,image/webp' }, + redirect: 'error', + signal: AbortSignal.timeout(THUMBNAIL_FETCH_TIMEOUT_MS), + }); + } catch (_error) { + throw new ThumbnailProxyError('Thumbnail fetch failed'); + } + if (!response.ok) throw new ThumbnailProxyError('Thumbnail fetch failed'); + + const contentType = String(response.headers.get('content-type') || '') + .split(';', 1)[0] + .trim() + .toLowerCase(); + if (!ALLOWED_IMAGE_TYPES.has(contentType)) { + throw new ThumbnailProxyError('Thumbnail response was not an image'); + } + const declaredLength = Number(response.headers.get('content-length')); + if (Number.isFinite(declaredLength) && declaredLength > THUMBNAIL_MAX_BYTES) { + throw new ThumbnailProxyError('Thumbnail response was too large'); + } + + const reader = response.body?.getReader(); + if (!reader) throw new ThumbnailProxyError('Thumbnail response had no body'); + const chunks = []; + let totalBytes = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + totalBytes += value.byteLength; + if (totalBytes > THUMBNAIL_MAX_BYTES) { + await reader.cancel(); + throw new ThumbnailProxyError('Thumbnail response was too large'); + } + chunks.push(Buffer.from(value)); + } + } catch (error) { + if (error instanceof ThumbnailProxyError) throw error; + throw new ThumbnailProxyError('Thumbnail fetch failed'); + } + + return { + body: Buffer.concat(chunks, totalBytes), + contentType, + }; +} + +module.exports = { + fetchExternalThumbnail, + ThumbnailProxyError, + THUMBNAIL_FETCH_TIMEOUT_MS, + THUMBNAIL_MAX_BYTES, +}; diff --git a/server/modules/externalWorkLimiter.js b/server/modules/externalWorkLimiter.js new file mode 100644 index 000000000..8118c2e27 --- /dev/null +++ b/server/modules/externalWorkLimiter.js @@ -0,0 +1,59 @@ +class ExternalWorkLimitError extends Error { + constructor(message = 'External API work queue is full') { + super(message); + this.name = 'ExternalWorkLimitError'; + this.code = 'work_queue_full'; + } +} + +function createExternalWorkLimiter({ + // Youtarr's downloader already permits one running download. Reserve one + // slot from the global ceiling of three for that asynchronous queue. + concurrency = Number(process.env.EXTERNAL_API_WORK_CONCURRENCY || 2), + maxQueue = Number(process.env.EXTERNAL_API_WORK_QUEUE || 20), +} = {}) { + const limit = Number.isSafeInteger(concurrency) && concurrency > 0 + ? Math.min(concurrency, 2) + : 2; + const queueLimit = Number.isSafeInteger(maxQueue) && maxQueue >= 0 + ? Math.min(maxQueue, 100) + : 20; + let active = 0; + const queue = []; + + const drain = () => { + while (active < limit && queue.length > 0) { + const item = queue.shift(); + active += 1; + Promise.resolve() + .then(item.work) + .then(item.resolve, item.reject) + .finally(() => { + active -= 1; + drain(); + }); + } + }; + + const run = (work) => new Promise((resolve, reject) => { + if (typeof work !== 'function') { + reject(new TypeError('work must be a function')); + return; + } + if (active >= limit && queue.length >= queueLimit) { + reject(new ExternalWorkLimitError()); + return; + } + queue.push({ work, resolve, reject }); + drain(); + }); + + const status = () => ({ active, queued: queue.length, concurrency: limit, maxQueue: queueLimit }); + return { run, status }; +} + +module.exports = { + createExternalWorkLimiter, + ExternalWorkLimitError, + sharedExternalWorkLimiter: createExternalWorkLimiter(), +}; diff --git a/server/modules/jobModule.js b/server/modules/jobModule.js index 3f7dffa55..93ec99672 100644 --- a/server/modules/jobModule.js +++ b/server/modules/jobModule.js @@ -584,7 +584,7 @@ class JobModule { } } - async addOrUpdateJob(jobData, isNextJob = false) { + async addOrUpdateJob(jobData, isNextJob = false, preferredJobId = null) { let jobId; const inProgressJobId = this.getInProgressJobId(); if (!isNextJob) { @@ -592,12 +592,16 @@ class JobModule { // If there is a job in progress, create a new job with status Pending logger.info({ jobType: jobData.jobType }, 'A job is already in progress. Adding job to the queue'); jobData.status = 'Pending'; - jobId = await this.addJob(jobData); + jobId = preferredJobId + ? await this.addJob(jobData, preferredJobId) + : await this.addJob(jobData); } else { // Otherwise, add a job with status In Progress logger.info({ jobType: jobData.jobType }, 'Adding job to jobs list as In Progress'); jobData.status = 'In Progress'; - jobId = await this.addJob(jobData); + jobId = preferredJobId + ? await this.addJob(jobData, preferredJobId) + : await this.addJob(jobData); } } else if (isNextJob && !inProgressJobId) { // If this is a next job and there's no job in progress, update its status to In Progress @@ -1169,8 +1173,10 @@ class JobModule { }); } - async addJob(job) { - const jobId = uuidv4(); // Generate a new UUID + async addJob(job, preferredJobId = null) { + // Trusted internal callers may supply a stable UUID so enqueueing can be + // retried safely across an external-request persistence boundary. + const jobId = preferredJobId || uuidv4(); if (isSpecificUrlDownloadJob(job.jobType) && Array.isArray(job.data?.urls)) { const admission = videoActivity.claim(jobId, job.data.urls); job.data.urls = admission.acceptedUrls; diff --git a/server/modules/playlistModule.js b/server/modules/playlistModule.js index c13a888fb..663fefcc0 100644 --- a/server/modules/playlistModule.js +++ b/server/modules/playlistModule.js @@ -6,6 +6,7 @@ const { Playlist, PlaylistVideo, Channel, Video, Job, JobVideo } = require('../m const youtubeApi = require('./youtubeApi'); const { MAX_PLAYLIST_VIDEOS } = require('./playlistConstants'); const storageGuard = require('./storageGuard'); +const { redactSensitiveText } = require('./safeCommandLogging'); // yt-dlp's flat-playlist listing still returns private/deleted/members-only // videos but strips their metadata: the title comes back null (current yt-dlp) @@ -78,13 +79,13 @@ class PlaylistModule { if (/confirm you.re not a bot|sign in|cookies/i.test(stderr)) { return reject(new Error('COOKIES_REQUIRED')); } - logger.error({ stderr, code }, 'getPlaylistInfo failed'); + logger.error({ stderr: redactSensitiveText(stderr), code }, 'getPlaylistInfo failed'); return reject(new Error('NETWORK_ERROR')); } try { resolve(JSON.parse(stdout)); } catch (err) { - logger.error({ err, stdout }, 'getPlaylistInfo parse error'); + logger.error({ error: redactSensitiveText(err?.message) }, 'getPlaylistInfo parse error'); reject(new Error('PARSE_ERROR')); } }); @@ -492,7 +493,10 @@ class PlaylistModule { child.stderr.on('data', (d) => { stderr += d.toString(); }); child.on('close', (code) => { if (code !== 0) { - logger.error({ stderr, code }, '_spawnFlatPlaylist failed'); + logger.error({ + stderr: redactSensitiveText(stderr), + code, + }, '_spawnFlatPlaylist failed'); return reject(new Error('NETWORK_ERROR')); } try { diff --git a/server/modules/ratingMapper.js b/server/modules/ratingMapper.js index f59337d6e..11ecbe952 100644 --- a/server/modules/ratingMapper.js +++ b/server/modules/ratingMapper.js @@ -246,7 +246,7 @@ function mapToNumericRating(normalizedRating) { } // Level 2: PG and PG equivalents (family with some caution) - const level2 = ['PG', 'TV-PG']; + const level2 = ['PG', 'TV-Y7', 'TV-PG']; if (level2.includes(normalizedRating)) { return 2; } diff --git a/server/modules/safeCommandLogging.js b/server/modules/safeCommandLogging.js new file mode 100644 index 000000000..b27e8cd0a --- /dev/null +++ b/server/modules/safeCommandLogging.js @@ -0,0 +1,106 @@ +const { URL } = require('url'); + +const SENSITIVE_VALUE_FLAGS = new Set([ + '--proxy', + '--cookies', + '--cookies-from-browser', + '--add-header', + '--add-headers', + '--http-header', + '--username', + '--password', + '--video-password', + '--client-certificate', + '--client-certificate-key', + '--client-certificate-password', + '--paths', + '--paths-home', + '--output', + '-o', + '--download-archive', + '--cache-dir', + '--ffmpeg-location', + '--config-locations', + '--exec', +]); +const SENSITIVE_FLAG_PATTERN = [...SENSITIVE_VALUE_FLAGS] + .sort((left, right) => right.length - left.length) + .map((flag) => flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) + .join('|'); +const SENSITIVE_VALUE_PATTERN = new RegExp( + `((${SENSITIVE_FLAG_PATTERN})(?:=|\\s+))(?:"[^"]*"|'[^']*'|\\S+)`, + 'gi' +); + +function safeUrlHost(value) { + try { + const url = new URL(value); + return ['http:', 'https:'].includes(url.protocol) ? url.hostname : null; + } catch (_error) { + return null; + } +} + +function summarizeCommandArgs(args) { + if (!Array.isArray(args)) return { argumentCount: 0, flags: [], targetHosts: [] }; + const flags = []; + const hosts = []; + let redactNext = false; + for (const value of args) { + const token = String(value); + if (redactNext) { + redactNext = false; + continue; + } + if (token.startsWith('-')) { + const flag = token.split('=', 1)[0]; + flags.push(flag); + if (SENSITIVE_VALUE_FLAGS.has(flag) && !token.includes('=')) redactNext = true; + continue; + } + const host = safeUrlHost(token); + if (host) hosts.push(host); + } + return { + argumentCount: args.length, + flags: [...new Set(flags)].sort(), + targetHosts: [...new Set(hosts)].sort(), + }; +} + +function redactSensitiveText(value) { + let text = String(value ?? ''); + text = text.replace( + /((?:--add-header|--add-headers|--http-header)(?:=|\s+))[^\r\n]*/gi, + '$1[REDACTED]' + ); + text = text.replace( + SENSITIVE_VALUE_PATTERN, + '$1[REDACTED]' + ); + text = text.replace( + /\b(Authorization|Proxy-Authorization|Cookie|Set-Cookie)\s*:\s*[^\r\n]*/gi, + '$1: [REDACTED]' + ); + text = text.replace( + /\b(https?:\/\/)([^/\s:@]+):([^@\s/]+)@/gi, + '$1[REDACTED]@' + ); + text = text.replace( + /([?&](?:token|key|api_key|apikey|auth|authorization|signature|sig)=)[^&#\s]*/gi, + '$1[REDACTED]' + ); + text = text.replace(/\bfile:\/\/[^\s"'<>]+/gi, '[REDACTED_PATH]'); + text = text.replace( + /(^|[\s"'=(])(?:\/(?:[^/\s"'<>]+\/)*[^/\s"'<>]+|[A-Za-z]:\\(?:[^\\\s"'<>]+\\)*[^\\\s"'<>]+)/g, + '$1[REDACTED_PATH]' + ); + return text; +} + +module.exports = { + SENSITIVE_VALUE_FLAGS, + summarizeCommandArgs, + redactSensitiveText, + safeUrlHost, +}; diff --git a/server/modules/subscriptionImport/cookiesFetcher.js b/server/modules/subscriptionImport/cookiesFetcher.js index 53ebdb775..2e0e1feba 100644 --- a/server/modules/subscriptionImport/cookiesFetcher.js +++ b/server/modules/subscriptionImport/cookiesFetcher.js @@ -139,7 +139,7 @@ async function fetchWithCookies(buffer) { fs.writeFileSync(cookiesPath, buffer); fs.chmodSync(cookiesPath, COOKIES_FILE_MODE); - logger.info({ tempDir }, 'Running yt-dlp with one-time cookies to fetch subscriptions'); + logger.info('Running yt-dlp with one-time cookies to fetch subscriptions'); const { stdout } = await runYtdlp(cookiesPath); const channels = parseChannelEntries(stdout); diff --git a/server/modules/videoDeletionModule.js b/server/modules/videoDeletionModule.js index 64a090102..238d13124 100644 --- a/server/modules/videoDeletionModule.js +++ b/server/modules/videoDeletionModule.js @@ -97,10 +97,17 @@ class VideoDeletionModule { * @param {number} videoId - The database ID of the video to delete * @returns {Promise<{success: boolean, videoId: number, error?: string}>} */ - async deleteVideoById(videoId) { + async deleteVideoById(videoId, options = {}) { try { + const { transaction = null, video: suppliedVideo = null } = options; // Fetch video from database - const video = await Video.findByPk(videoId); + const video = suppliedVideo || await Video.findByPk( + videoId, + ...(transaction ? [{ + transaction, + lock: transaction.LOCK.UPDATE, + }] : []) + ); if (!video) { return { @@ -125,7 +132,10 @@ class VideoDeletionModule { if (!primaryPath) { // No file path, just mark as removed in database - await video.update({ removed: true }); + await video.update( + { removed: true }, + ...(transaction ? [{ transaction }] : []) + ); return { success: true, videoId, @@ -200,7 +210,10 @@ class VideoDeletionModule { } // Mark video as removed in database - await video.update({ removed: true }); + await video.update( + { removed: true }, + ...(transaction ? [{ transaction }] : []) + ); // Best-effort cleanup of empty channel directory await this._tryCleanupChannelDirectory(primaryPath, flat); diff --git a/server/modules/ytDlpRunner.js b/server/modules/ytDlpRunner.js index 237f49e30..7fbc7a05e 100644 --- a/server/modules/ytDlpRunner.js +++ b/server/modules/ytDlpRunner.js @@ -2,6 +2,7 @@ const { spawnYtDlp } = require('./ytdlpProcess'); const path = require('path'); const fs = require('fs'); const tempPathManager = require('./download/tempPathManager'); +const { redactSensitiveText } = require('./safeCommandLogging'); const MEMBERS_ONLY_PATTERNS = [ /join this channel to get access to members-only content/i, @@ -141,7 +142,9 @@ class YtDlpRunner { timeoutError.code = 'YTDLP_TIMEOUT'; reject(timeoutError); } else { - const errorMessage = stderr || `yt-dlp process exited with code ${code}`; + const errorMessage = redactSensitiveText( + stderr || `yt-dlp process exited with code ${code}` + ); reject(new Error(errorMessage)); } }); diff --git a/server/routes/__tests__/apikeys.policy.test.js b/server/routes/__tests__/apikeys.policy.test.js new file mode 100644 index 000000000..ab2e5ea37 --- /dev/null +++ b/server/routes/__tests__/apikeys.policy.test.js @@ -0,0 +1,245 @@ +jest.mock('../../modules/apiKeyModule', () => ({ + createApiKey: jest.fn(), + listApiKeys: jest.fn(), + logApiKeyCreated: jest.fn(), + updateApiKey: jest.fn(), + regenerateApiKey: jest.fn(), + serializeApiKey: jest.fn((key) => ({ id: key.id, name: key.name, key_prefix: key.key_prefix })), +})); +jest.mock('../../modules/apiKeyChannelGrantModule', () => ({ + getChannelGrants: jest.fn(), + replaceChannelGrants: jest.fn(), +})); +jest.mock('../../db', () => ({ + sequelize: { + transaction: jest.fn(async (callback) => callback({ id: 'transaction' })), + }, +})); + +const express = require('express'); +const request = require('supertest'); +const apiKeyModule = require('../../modules/apiKeyModule'); +const grantModule = require('../../modules/apiKeyChannelGrantModule'); +const createApiKeyRoutes = require('../apikeys'); + +function createApp(authType = 'session') { + const app = express(); + app.use(express.json()); + app.use(createApiKeyRoutes({ verifyToken: (req, _res, next) => { + req.authType = authType; + next(); + } })); + return app; +} + +describe('PATCH /api/keys/:id policy management', () => { + beforeEach(() => jest.clearAllMocks()); + + test('is session-only', async () => { + await request(createApp('api_key')).patch('/api/keys/1').send({ policy: { role: 'view' } }) + .expect(403, { error: 'API keys cannot manage other API keys' }); + }); + + test('rejects an API-key header even when upstream auth is bypassed', async () => { + await request(createApp()) + .get('/api/keys') + .set('x-api-key', 'external-key') + .expect(403, { error: 'API keys cannot manage other API keys' }); + }); + + test('rejects invalid policy validation errors', async () => { + apiKeyModule.updateApiKey.mockRejectedValue(new Error('Invalid API key role')); + await request(createApp()).patch('/api/keys/1').send({ policy: { role: 'bogus' } }) + .expect(400, { error: 'Invalid API key role' }); + }); + + test('uses the shared serializer so key_hash is absent from PATCH responses', async () => { + apiKeyModule.updateApiKey.mockResolvedValue({ id: 1, name: 'Safe', key_prefix: '12345678', key_hash: 'never-send' }); + const response = await request(createApp()).patch('/api/keys/1').send({ policy: { role: 'view' } }).expect(200); + expect(response.body.key).toEqual({ id: 1, name: 'Safe', key_prefix: '12345678' }); + expect(response.body.key).not.toHaveProperty('key_hash'); + }); +}); + +describe('GET /api/keys management list', () => { + beforeEach(() => jest.clearAllMocks()); + + test('returns effective channel grant counts without exposing key hashes', async () => { + apiKeyModule.listApiKeys.mockResolvedValue([{ + id: 1, + name: 'External', + key_prefix: '12345678', + channel_grant_count: 0, + }]); + + const response = await request(createApp()).get('/api/keys').expect(200); + expect(response.body).toEqual({ + keys: [{ + id: 1, + name: 'External', + key_prefix: '12345678', + channel_grant_count: 0, + }], + }); + expect(apiKeyModule.listApiKeys).toHaveBeenCalledTimes(1); + }); +}); + +describe('POST /api/keys/:id/regenerate', () => { + beforeEach(() => jest.clearAllMocks()); + + test('is session-only', async () => { + await request(createApp('api_key')).post('/api/keys/1/regenerate') + .expect(403, { error: 'API keys cannot manage other API keys' }); + }); + + test('replaces an active key and returns the one-time secret', async () => { + apiKeyModule.regenerateApiKey.mockResolvedValue({ + id: 1, + name: 'External', + key: 'replacement-secret', + prefix: 'replacem', + }); + const response = await request(createApp()).post('/api/keys/1/regenerate').expect(200); + expect(response.body).toEqual(expect.objectContaining({ + success: true, + id: 1, + key: 'replacement-secret', + prefix: 'replacem', + })); + expect(apiKeyModule.regenerateApiKey).toHaveBeenCalledWith(1); + }); + + test('rejects invalid IDs, non-empty bodies, and inactive keys', async () => { + await request(createApp()).post('/api/keys/nope/regenerate').expect(400); + await request(createApp()).post('/api/keys/1/regenerate').send({ surprise: true }).expect(400); + apiKeyModule.regenerateApiKey.mockResolvedValue(null); + await request(createApp()).post('/api/keys/1/regenerate') + .expect(404, { error: 'Active API key not found' }); + }); +}); + +describe('API key channel grant management', () => { + beforeEach(() => jest.clearAllMocks()); + + test('is session-only', async () => { + await request(createApp('api_key')).get('/api/keys/1/channels').expect(403); + await request(createApp('api_key')).put('/api/keys/1/channels').send({ channelIds: [] }).expect(403); + }); + + test('returns and replaces only sanitized channel database IDs', async () => { + grantModule.getChannelGrants.mockResolvedValue({ keyId: 1, channelIds: [2, 4] }); + grantModule.replaceChannelGrants.mockResolvedValue({ keyId: 1, channelIds: [3, 5] }); + + await request(createApp()).get('/api/keys/1/channels') + .expect(200, { keyId: 1, channelIds: [2, 4] }); + await request(createApp()).put('/api/keys/1/channels').send({ channelIds: [5, 3] }) + .expect(200, { success: true, keyId: 1, channelIds: [3, 5] }); + }); + + test('rejects invalid grant sets and legacy keys', async () => { + grantModule.replaceChannelGrants.mockRejectedValue( + new Error('Only active external API keys can receive channel grants') + ); + await request(createApp()).put('/api/keys/1/channels').send({ channelIds: [2] }) + .expect(400, { error: 'Only active external API keys can receive channel grants' }); + }); +}); + +describe('transactional external access management', () => { + beforeEach(() => jest.clearAllMocks()); + + test('creates an external key and its exact grants in one transaction', async () => { + apiKeyModule.createApiKey.mockResolvedValue({ + id: 9, + name: 'External Client', + key: 'one-time-secret', + prefix: '12345678', + }); + grantModule.replaceChannelGrants.mockResolvedValue({ + keyId: 9, + channelIds: [2, 4], + }); + + const response = await request(createApp()).post('/api/keys').send({ + name: 'External Client', + policy: { role: 'request' }, + channelIds: [4, 2], + }).expect(200); + + expect(response.body).toEqual(expect.objectContaining({ + id: 9, + key: 'one-time-secret', + })); + expect(apiKeyModule.createApiKey).toHaveBeenCalledWith( + 'External Client', + { role: 'request' }, + { transaction: { id: 'transaction' }, logEvent: false } + ); + expect(grantModule.replaceChannelGrants).toHaveBeenCalledWith( + 9, + [4, 2], + { transaction: { id: 'transaction' } } + ); + expect(apiKeyModule.logApiKeyCreated).toHaveBeenCalledWith( + expect.objectContaining({ id: 9, prefix: '12345678' }) + ); + }); + + test('still returns the one-time key if post-commit audit logging fails', async () => { + apiKeyModule.createApiKey.mockResolvedValue({ + id: 9, + name: 'External Client', + key: 'one-time-secret', + prefix: '12345678', + }); + grantModule.replaceChannelGrants.mockResolvedValue({ + keyId: 9, + channelIds: [], + }); + apiKeyModule.logApiKeyCreated.mockImplementationOnce(() => { + throw new Error('logger unavailable'); + }); + + await request(createApp()).post('/api/keys').send({ + name: 'External Client', + policy: { role: 'view' }, + channelIds: [], + }).expect(200).expect((response) => { + expect(response.body.key).toBe('one-time-secret'); + }); + }); + + test('updates policy and grants through the atomic endpoint', async () => { + apiKeyModule.updateApiKey.mockResolvedValue({ + id: 9, + name: 'External Client', + key_prefix: '12345678', + }); + grantModule.replaceChannelGrants.mockResolvedValue({ + keyId: 9, + channelIds: [3], + }); + + const response = await request(createApp()) + .put('/api/keys/9/external-access') + .send({ policy: { role: 'delete' }, channelIds: [3] }) + .expect(200); + + expect(response.body).toEqual({ + success: true, + key: { id: 9, name: 'External Client', key_prefix: '12345678' }, + channelIds: [3], + }); + expect(apiKeyModule.updateApiKey).toHaveBeenCalledWith( + 9, + { role: 'delete' }, + { transaction: { id: 'transaction' } } + ); + expect(grantModule.replaceChannelGrants).toHaveBeenCalledWith( + 9, + [3], + { transaction: { id: 'transaction' } } + ); + }); +}); diff --git a/server/routes/__tests__/externalApi.test.js b/server/routes/__tests__/externalApi.test.js new file mode 100644 index 000000000..f6babe5d4 --- /dev/null +++ b/server/routes/__tests__/externalApi.test.js @@ -0,0 +1,543 @@ +const express = require('express'); +const request = require('supertest'); +const { createExternalApiAuth } = require('../../middleware/externalApiAuth'); +const { createExternalApiRoutes } = require('../externalApi'); + +function makeApp({ + enabled = true, + key, + catalogService, + thumbnailProxy, + externalWorkLimiter, + requestService, + writeLimiter, + ingressLimiter, +} = {}) { + const app = express(); + app.use((req, _res, next) => { + req.id = 'request-123'; + next(); + }); + const validateApiKey = jest.fn().mockResolvedValue(key || null); + if (enabled) { + app.use('/external-api/v1', createExternalApiRoutes({ + externalApiAuth: createExternalApiAuth({ validateApiKey }), + externalApiIngressLimiter: ingressLimiter || ((_req, _res, next) => next()), + externalApiLimiter: (_req, _res, next) => next(), + externalApiWriteLimiter: writeLimiter || ((_req, _res, next) => next()), + serverVersion: '1.77.0', + catalogService, + thumbnailProxy, + externalWorkLimiter: externalWorkLimiter || { run: (operation) => operation() }, + requestService, + quotaService: { + status: jest.fn().mockResolvedValue({ + limits: { maxActiveJobs: 5, hourlyWriteLimit: 30, dailyWriteLimit: 200 }, + remaining: { activeJobs: 5, hourlyWrites: 30, dailyWrites: 200 }, + }), + }, + })); + } else { + app.use('/external-api', (_req, res) => res.status(404).json({ + error: { code: 'not_found', message: 'External API route not found' }, + })); + } + return { app, validateApiKey }; +} + +const externalKey = (overrides = {}) => ({ + id: 4, name: 'External Client', role: 'request', is_active: true, revoked_at: null, + allow_video_requests: true, allow_channel_requests: true, + allow_delete_video_requests: false, + auto_approve_video_requests: true, auto_approve_channel_requests: false, + auto_approve_delete_requests: false, max_rating_level: 3, allow_unrated: false, + allowed_media_types: ['video'], max_active_jobs: 5, hourly_write_limit: 30, + daily_write_limit: 200, ...overrides, +}); + +describe('external API capabilities', () => { + test('returns 404 when feature flag is disabled', async () => { + const { app } = makeApp({ enabled: false }); + await request(app).get('/external-api/v1/capabilities').expect(404, { + error: { code: 'not_found', message: 'External API route not found' }, + }); + }); + + test('requires x-api-key even when application auth is disabled', async () => { + const old = process.env.AUTH_ENABLED; + process.env.AUTH_ENABLED = 'false'; + const { app } = makeApp({ key: externalKey() }); + await request(app).get('/external-api/v1/capabilities').expect(401, { + error: { + code: 'missing_api_key', + message: 'x-api-key is required', + requestId: 'request-123', + }, + }); + process.env.AUTH_ENABLED = old; + }); + + test.each([ + ['invalid', null], + ['legacy', externalKey({ role: 'legacy_download' })], + ['revoked', externalKey({ revoked_at: new Date() })], + ])('rejects %s external keys', async (_name, key) => { + const { app } = makeApp({ key }); + await request(app).get('/external-api/v1/capabilities').set('x-api-key', 'test-key').expect(401); + }); + + test.each([ + ['inactive', { is_active: false }], + ['invalid rating', { max_rating_level: 9 }], + ['invalid active-job quota', { max_active_jobs: 6 }], + ['invalid hourly quota', { hourly_write_limit: 0 }], + ['invalid daily quota', { daily_write_limit: '200' }], + ['missing permission', { allow_video_requests: undefined }], + ['missing media policy', { allowed_media_types: null }], + ['inconsistent role', { + role: 'view', + allow_video_requests: false, + allow_channel_requests: false, + allow_delete_video_requests: true, + }], + ['invalid unrated flag', { allow_unrated: 'yes' }], + ['invalid auto-approve flag', { auto_approve_video_requests: 1 }], + ['auto-approve without permission', { + allow_video_requests: false, + allow_channel_requests: false, + allow_delete_video_requests: false, + auto_approve_video_requests: true, + }], + ])('fails closed for %s policy', async (_name, overrides) => { + const { app } = makeApp({ key: externalKey(overrides) }); + await request(app) + .get('/external-api/v1/capabilities') + .set('x-api-key', 'test-key') + .expect(401) + .expect((response) => expect(response.body.error.code).toBe('invalid_key_policy')); + }); + + test('applies ingress limiting before database authentication', async () => { + const ingressLimiter = jest.fn((_req, res) => res.status(429).json({ blocked: true })); + const { app, validateApiKey } = makeApp({ key: externalKey(), ingressLimiter }); + await request(app).get('/external-api/v1/capabilities').expect(429); + expect(validateApiKey).not.toHaveBeenCalled(); + }); + + test('normalizes method, content type, malformed JSON, and body-size failures', async () => { + const { app } = makeApp({ key: externalKey() }); + await request(app).put('/external-api/v1/capabilities').expect(405) + .expect((response) => expect(response.body.error.code).toBe('method_not_allowed')); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .set('Content-Type', 'text/plain') + .send('not-json') + .expect(415); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .set('Content-Type', 'application/json') + .set('Content-Encoding', 'gzip') + .send('{}') + .expect(415); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .set('Content-Type', 'application/json') + .send('{"broken":') + .expect(400); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .send({ padding: 'x'.repeat(17 * 1024) }) + .expect(413); + }); + + test('keeps unknown v1 paths inside the external JSON contract', async () => { + const { app } = makeApp({ key: externalKey() }); + const response = await request(app) + .get('/external-api/v1/unknown') + .set('x-api-key', 'test-key') + .expect(404); + + expect(response.body).toEqual({ + error: { + code: 'not_found', + message: 'External API route not found', + requestId: 'request-123', + }, + }); + expect(response.headers['content-type']).toMatch(/^application\/json/); + expect(response.headers['cache-control']).toBe('private, no-store'); + }); + + test('marks authentication errors private and non-cacheable', async () => { + const { app } = makeApp({ key: externalKey() }); + const response = await request(app).get('/external-api/v1/capabilities').expect(401); + expect(response.headers['cache-control']).toBe('private, no-store'); + expect(response.headers.vary).toContain('x-api-key'); + }); + + test('returns role scopes and sanitized policy', async () => { + const { app } = makeApp({ key: externalKey() }); + const response = await request(app).get('/external-api/v1/capabilities').set('x-api-key', 'test-key').expect(200); + expect(response.body).toEqual({ + apiVersion: '1', serverVersion: '1.77.0', role: 'request', + scopes: ['catalog:read', 'requests:read', 'video:request', 'channel:request'], + policy: { + allowVideoRequests: true, + allowChannelRequests: true, + allowDeleteVideoRequests: false, + autoApproveVideoRequests: true, autoApproveChannelRequests: false, + autoApproveDeleteRequests: false, maxRatingLevel: 3, allowUnrated: false, + allowedMediaTypes: ['video'], + }, + quota: { + limits: { maxActiveJobs: 5, hourlyWriteLimit: 30, dailyWriteLimit: 200 }, + remaining: { activeJobs: 5, hourlyWrites: 30, dailyWrites: 200 }, + }, + features: { + catalog: true, requests: true, channelRequests: true, deleteRequests: true, + recommendations: true, authenticatedAssets: true, videoDetails: true, + }, + }); + }); + + test('reports independently configured request scopes', async () => { + const { app } = makeApp({ + key: externalKey({ + allow_video_requests: true, + allow_channel_requests: false, + allow_delete_video_requests: false, + }), + }); + await request(app).get('/external-api/v1/capabilities') + .set('x-api-key', 'test-key') + .expect(200) + .expect((response) => { + expect(response.body.scopes).toEqual([ + 'catalog:read', + 'requests:read', + 'video:request', + ]); + expect(response.body.policy).toEqual(expect.objectContaining({ + allowVideoRequests: true, + allowChannelRequests: false, + allowDeleteVideoRequests: false, + })); + }); + }); + + test('persists video requests behind the lower write limiter', async () => { + const requestService = { + createVideoRequest: jest.fn().mockResolvedValue({ + outcome: 'created', + request: { + id: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + type: 'video', + status: 'pending', + target: { youtubeId: 'abcdefghijk', channelId: 8 }, + createdAt: '2026-07-26T12:00:00.000Z', + updatedAt: '2026-07-26T12:00:00.000Z', + }, + }), + listRequests: jest.fn(), + getRequest: jest.fn(), + }; + const writeLimiter = jest.fn((_req, _res, next) => next()); + const { app } = makeApp({ key: externalKey(), requestService, writeLimiter }); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .send({ youtubeId: 'abcdefghijk', channelId: 8 }) + .expect(202) + .expect((response) => expect(response.body.outcome).toBe('created')); + expect(writeLimiter).toHaveBeenCalled(); + expect(requestService.createVideoRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: 4, role: 'request' }), + { youtubeId: 'abcdefghijk', channelId: 8 } + ); + }); + + test('routes candidate, channel, and delete operations through constrained services', async () => { + const catalogService = { + listVideos: jest.fn().mockResolvedValue({ + data: [], + pagination: { page: 1, pageSize: 100, total: 0, totalPages: 0 }, + dataSource: 'cache', + isFullyIndexed: true, + }), + }; + const requestService = { + createChannelRequest: jest.fn().mockResolvedValue({ + outcome: 'created', + request: { type: 'channel', status: 'pending' }, + }), + createDeleteVideoRequest: jest.fn().mockResolvedValue({ + outcome: 'already_deleted', + request: null, + }), + }; + const writeLimiter = jest.fn((_req, _res, next) => next()); + const { app } = makeApp({ + key: externalKey({ role: 'delete', allow_delete_video_requests: true }), + catalogService, + requestService, + writeLimiter, + }); + + await request(app).get('/external-api/v1/videos?page=1&pageSize=100') + .set('x-api-key', 'test-key').expect(200); + await request(app).post('/external-api/v1/requests/channels') + .set('x-api-key', 'test-key') + .send({ channelUrl: 'https://www.youtube.com/@safe' }) + .expect(202); + await request(app).post('/external-api/v1/requests/delete-videos') + .set('x-api-key', 'test-key') + .send({ youtubeId: 'abcdefghijk', channelId: 8 }) + .expect(200, { outcome: 'already_deleted', request: null }); + + expect(catalogService.listVideos).toHaveBeenCalledWith( + expect.objectContaining({ id: 4, role: 'delete' }), + expect.objectContaining({ page: '1', pageSize: '100' }) + ); + expect(requestService.createChannelRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: 4 }), + { channelUrl: 'https://www.youtube.com/@safe' } + ); + expect(requestService.createDeleteVideoRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: 4 }), + { youtubeId: 'abcdefghijk', channelId: 8 } + ); + expect(writeLimiter).toHaveBeenCalledTimes(2); + }); + + test('lists and reads only through the authenticated request service', async () => { + const requestService = { + createVideoRequest: jest.fn(), + listRequests: jest.fn().mockResolvedValue({ + data: [], pagination: { page: 1, pageSize: 50, total: 0, totalPages: 0 }, + }), + getRequest: jest.fn().mockResolvedValue({ + id: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + type: 'video', + status: 'pending', + target: { youtubeId: 'abcdefghijk', channelId: 8 }, + createdAt: '2026-07-26T12:00:00.000Z', + updatedAt: '2026-07-26T12:00:00.000Z', + }), + }; + const { app } = makeApp({ key: externalKey(), requestService }); + await request(app).get('/external-api/v1/requests?status=pending') + .set('x-api-key', 'test-key').expect(200); + await request(app) + .get('/external-api/v1/requests/9b89e5bc-8c90-4e72-b245-270fed2eacc2') + .set('x-api-key', 'test-key').expect(200); + expect(requestService.listRequests).toHaveBeenCalledWith( + expect.objectContaining({ id: 4 }), + expect.objectContaining({ status: 'pending' }) + ); + expect(requestService.getRequest).toHaveBeenCalledWith( + expect.objectContaining({ id: 4 }), + '9b89e5bc-8c90-4e72-b245-270fed2eacc2' + ); + }); + + test('normalizes duplicate media types and fails closed on unknown media policies', async () => { + const normalized = makeApp({ key: externalKey({ allowed_media_types: ['video', 'video'] }) }); + await request(normalized.app).get('/external-api/v1/capabilities').set('x-api-key', 'test-key') + .expect(200).expect((response) => expect(response.body.policy.allowedMediaTypes).toEqual(['video'])); + + const corrupt = makeApp({ key: externalKey({ allowed_media_types: ['video', 'unknown'] }) }); + await request(corrupt.app).get('/external-api/v1/capabilities').set('x-api-key', 'test-key').expect(401); + }); + + test('protects catalog routes and forwards only authenticated key policy', async () => { + const catalogService = { + listChannels: jest.fn().mockResolvedValue({ + data: [], pagination: { page: 1, pageSize: 50, total: 0, totalPages: 0 }, dataSource: 'cache', + }), + listChannelVideos: jest.fn(), + getChannelThumbnail: jest.fn(), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + await request(app).get('/external-api/v1/channels').expect(401); + await request(app).get('/external-api/v1/channels?page=1').set('x-api-key', 'test-key').expect(200); + expect(catalogService.listChannels).toHaveBeenCalledWith( + expect.objectContaining({ id: 4, maxRatingLevel: 3, allowedMediaTypes: ['video'] }), + expect.objectContaining({ page: '1' }) + ); + }); + + test('returns one authenticated policy-filtered video detail', async () => { + const detail = { + youtubeId: 'abcdefghijk', + title: 'Detailed video', + thumbnailUrl: '/external-api/v1/assets/videos/abcdefghijk/thumbnail', + metadata: { description: 'Full description', viewCount: 1000 }, + }; + const catalogService = { + getVideoDetail: jest.fn().mockResolvedValue(detail), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + + await request(app) + .get('/external-api/v1/videos/abcdefghijk') + .set('x-api-key', 'test-key') + .expect(200, detail); + expect(catalogService.getVideoDetail).toHaveBeenCalledWith( + expect.objectContaining({ id: 4, maxRatingLevel: 3 }), + 'abcdefghijk' + ); + }); + + test('keeps missing and ineligible video details indistinguishable', async () => { + const notFound = Object.assign(new Error('Video not found'), { + name: 'CatalogError', + status: 404, + }); + const catalogService = { + getVideoDetail: jest.fn().mockRejectedValue(notFound), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + + await request(app) + .get('/external-api/v1/videos/abcdefghijk') + .set('x-api-key', 'test-key') + .expect(404) + .expect((response) => expect(response.body.error.message).toBe('Video not found')); + }); + + test('bounds expensive video-detail metadata work', async () => { + const workError = Object.assign(new Error('full'), { + name: 'ExternalWorkLimitError', + code: 'work_queue_full', + }); + const externalWorkLimiter = { + run: jest.fn().mockRejectedValue(workError), + }; + const catalogService = { getVideoDetail: jest.fn() }; + const { app } = makeApp({ + key: externalKey(), + catalogService, + externalWorkLimiter, + }); + + await request(app) + .get('/external-api/v1/videos/abcdefghijk') + .set('x-api-key', 'test-key') + .expect(503) + .expect((response) => expect(response.body.error.code).toBe('work_queue_full')); + expect(catalogService.getVideoDetail).not.toHaveBeenCalled(); + }); + + test('does not reveal whether an ungranted channel exists', async () => { + const notFound = Object.assign(new Error('Channel not found'), { name: 'CatalogError', status: 404 }); + const catalogService = { + listChannels: jest.fn(), + listChannelVideos: jest.fn().mockRejectedValue(notFound), + getChannelThumbnail: jest.fn().mockRejectedValue( + Object.assign(new Error('Thumbnail not found'), { name: 'CatalogError', status: 404 }) + ), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + await request(app).get('/external-api/v1/channels/99/videos').set('x-api-key', 'test-key') + .expect(404, { + error: { + code: 'not_found', + message: 'Channel not found', + requestId: 'request-123', + }, + }); + await request(app).get('/external-api/v1/assets/channels/99/thumbnail').set('x-api-key', 'test-key') + .expect(404, { + error: { + code: 'not_found', + message: 'Thumbnail not found', + requestId: 'request-123', + }, + }); + }); + + test('serves granted channel artwork with private cache and nosniff headers', async () => { + const catalogService = { + listChannels: jest.fn(), + listChannelVideos: jest.fn(), + getChannelThumbnail: jest.fn().mockResolvedValue(__filename), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + const response = await request(app) + .get('/external-api/v1/assets/channels/8/thumbnail') + .set('x-api-key', 'test-key') + .expect(200); + expect(response.headers['cache-control']).toBe('private, no-store'); + expect(response.headers.vary).toContain('x-api-key'); + expect(response.headers['x-content-type-options']).toBe('nosniff'); + expect(response.headers['content-type']).toMatch(/^image\/jpeg/); + }); + + test('serves optimized local video artwork through the authenticated API route', async () => { + const catalogService = { + getVideoThumbnail: jest.fn().mockResolvedValue({ + source: 'local', + absolutePath: __filename, + }), + }; + const { app } = makeApp({ key: externalKey(), catalogService }); + + const response = await request(app) + .get('/external-api/v1/assets/videos/abcdefghijk/thumbnail') + .set('x-api-key', 'test-key') + .expect(200); + expect(response.headers['content-type']).toMatch(/^image\/jpeg/); + expect(catalogService.getVideoThumbnail).toHaveBeenCalledWith( + expect.objectContaining({ id: 4 }), + 'abcdefghijk' + ); + }); + + test('proxies approved upstream artwork through Youtarr when no local thumbnail exists', async () => { + const catalogService = { + getVideoThumbnail: jest.fn().mockResolvedValue({ + source: 'upstream', + url: 'https://i.ytimg.com/vi/abcdefghijk/mqdefault.jpg', + }), + }; + const thumbnailProxy = { + fetchExternalThumbnail: jest.fn().mockResolvedValue({ + body: Buffer.from('proxied-image'), + contentType: 'image/jpeg', + }), + }; + const { app } = makeApp({ key: externalKey(), catalogService, thumbnailProxy }); + + const response = await request(app) + .get('/external-api/v1/assets/videos/abcdefghijk/thumbnail') + .set('x-api-key', 'test-key') + .expect(200); + expect(response.body).toEqual(Buffer.from('proxied-image')); + expect(thumbnailProxy.fetchExternalThumbnail).toHaveBeenCalledWith( + 'https://i.ytimg.com/vi/abcdefghijk/mqdefault.jpg' + ); + }); + + test('returns the private error contract when upstream artwork cannot be fetched', async () => { + const catalogService = { + getVideoThumbnail: jest.fn().mockResolvedValue({ + source: 'upstream', + url: 'https://i.ytimg.com/vi/abcdefghijk/mqdefault.jpg', + }), + }; + const thumbnailProxy = { + fetchExternalThumbnail: jest.fn().mockRejectedValue(new Error('offline')), + }; + const { app } = makeApp({ key: externalKey(), catalogService, thumbnailProxy }); + + const response = await request(app) + .get('/external-api/v1/assets/videos/abcdefghijk/thumbnail') + .set('x-api-key', 'test-key') + .expect(404); + expect(response.body.error).toMatchObject({ + code: 'not_found', + message: 'Thumbnail not found', + requestId: 'request-123', + }); + expect(response.headers['cache-control']).toBe('private, no-store'); + }); +}); diff --git a/server/routes/__tests__/externalApiContractHarness.test.js b/server/routes/__tests__/externalApiContractHarness.test.js new file mode 100644 index 000000000..8c3cafa95 --- /dev/null +++ b/server/routes/__tests__/externalApiContractHarness.test.js @@ -0,0 +1,67 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const request = require('supertest'); +const { + SYNTHETIC_API_KEY, + createExternalApiContractApp, +} = require('../../testing/externalApiContractApp'); + +const contract = JSON.parse(fs.readFileSync( + path.join(__dirname, '../../../fixtures/external-api-v1/contract.json'), + 'utf8' +)); + +describe('external API synthetic contract harness', () => { + test('serves the complete Plinx read path through the real router', async () => { + const { app, state } = createExternalApiContractApp({ contract }); + const auth = { 'x-api-key': SYNTHETIC_API_KEY }; + + await request(app).get('/external-api/v1/capabilities').set(auth).expect(200) + .expect((response) => expect(response.body).toEqual(contract.capabilities)); + await request(app).get('/external-api/v1/channels').set(auth).expect(200) + .expect((response) => expect(response.body.data.length).toBeGreaterThan(0)); + await request(app).get('/external-api/v1/videos?status=requestable&pageSize=40').set(auth).expect(200) + .expect((response) => expect(response.body.data.map((video) => video.mediaType)) + .toEqual(expect.arrayContaining(['video', 'short', 'livestream']))); + await request(app) + .get(`/external-api/v1/videos?cursor=${encodeURIComponent(contract.catalogPage.pagination.nextCursor)}`) + .set(auth).expect(200) + .expect((response) => expect(response.body).toEqual(contract.catalogNextPage)); + await request(app).get('/external-api/v1/videos?search=Fixture%20short%20%26%20more').set(auth) + .expect(200); + expect(state.lastVideoQuery.search).toBe('Fixture short & more'); + await request(app).get('/external-api/v1/videos?page=1&cursor=not-allowed').set(auth).expect(400); + await request(app).get('/external-api/v1/videos/abcdefghijk').set(auth).expect(200) + .expect((response) => expect(response.body.metadata).toBeTruthy()); + await request(app).get('/external-api/v1/requests?page=1&pageSize=10').set(auth).expect(200) + .expect((response) => expect(response.body.pagination.total).toBe(contract.requests.length + 3)); + await request(app).get('/external-api/v1/assets/videos/abcdefghijk/thumbnail').set(auth).expect(200) + .expect('Content-Type', /image\/png/); + await request(app).get('/external-api/v1/assets/channels/8/thumbnail').set(auth).expect(200) + .expect('Content-Type', /image\/png|image\/jpeg/); + expect(state.artworkRequests).toEqual(expect.arrayContaining(['video:abcdefghijk', 'channel:8'])); + }); + + test('records a deterministic request and rejects an invalid key', async () => { + const { app } = createExternalApiContractApp({ contract }); + await request(app).get('/external-api/v1/capabilities').set('x-api-key', 'wrong').expect(401); + + const response = await request(app) + .post('/external-api/v1/requests/videos') + .set('x-api-key', SYNTHETIC_API_KEY) + .send({ + youtubeId: 'abcdefghijk', + channelId: 8, + idempotencyKey: '00000000-0000-4000-8000-000000000001', + }) + .expect(202); + expect(response.body).toMatchObject({ outcome: 'created', request: { status: 'pending' } }); + + await request(app).get('/__contract/state').set('x-api-key', SYNTHETIC_API_KEY).expect(200) + .expect((state) => expect(state.body.lastVideoRequest).toMatchObject({ + youtubeId: 'abcdefghijk', channelId: 8, + })); + }); +}); diff --git a/server/routes/__tests__/externalRequests.test.js b/server/routes/__tests__/externalRequests.test.js new file mode 100644 index 000000000..a8b362170 --- /dev/null +++ b/server/routes/__tests__/externalRequests.test.js @@ -0,0 +1,98 @@ +const express = require('express'); +const request = require('supertest'); +const createExternalRequestReviewRoutes = require('../externalRequests'); + +const requestId = '9b89e5bc-8c90-4e72-b245-270fed2eacc2'; + +function makeApp({ authType = 'session', requestService, reviewLimiter } = {}) { + const app = express(); + app.use(express.json()); + const verifyToken = (req, _res, next) => { + req.authType = authType; + req.sessionId = authType === 'session' ? 'session-1' : undefined; + next(); + }; + app.use(createExternalRequestReviewRoutes({ + verifyToken, + requestService, + reviewLimiter: reviewLimiter || ((_req, _res, next) => next()), + })); + return app; +} + +function service() { + return { + listAdminRequests: jest.fn().mockResolvedValue({ + data: [], + pagination: { page: 1, pageSize: 50, total: 0, totalPages: 0 }, + filterOptions: { requesters: [] }, + }), + getAdminRequest: jest.fn().mockResolvedValue({ id: requestId, status: 'pending' }), + reviewVideoRequest: jest.fn().mockResolvedValue({ id: requestId, status: 'processing' }), + }; +} + +describe('external request administrator routes', () => { + test('lists and reads across keys only for session-authenticated callers', async () => { + const requestService = service(); + const app = makeApp({ requestService }); + + await request(app) + .get('/api/external-requests?status=pending&apiKeyId=4&page=2') + .expect(200); + await request(app).get(`/api/external-requests/${requestId}`).expect(200); + + expect(requestService.listAdminRequests).toHaveBeenCalledWith(expect.objectContaining({ + status: 'pending', + apiKeyId: '4', + page: '2', + })); + expect(requestService.getAdminRequest).toHaveBeenCalledWith(requestId); + + await request(makeApp({ authType: 'api_key', requestService })) + .get('/api/external-requests') + .expect(403, { error: 'Session authentication is required' }); + }); + + test('runs approve and reject through the dedicated action limiter', async () => { + const requestService = service(); + const reviewLimiter = jest.fn((_req, _res, next) => next()); + const app = makeApp({ requestService, reviewLimiter }); + + await request(app) + .post(`/api/external-requests/${requestId}/approve`) + .send({}) + .expect(200); + await request(app) + .post(`/api/external-requests/${requestId}/reject`) + .send({ reason: 'Not approved' }) + .expect(200); + + expect(reviewLimiter).toHaveBeenCalledTimes(2); + expect(requestService.reviewVideoRequest).toHaveBeenNthCalledWith( + 1, requestId, 'approve', {} + ); + expect(requestService.reviewVideoRequest).toHaveBeenNthCalledWith( + 2, requestId, 'reject', { reason: 'Not approved' } + ); + }); + + test('returns bounded service errors without exposing internal failures', async () => { + const requestService = service(); + requestService.reviewVideoRequest.mockRejectedValueOnce( + Object.assign(new Error('Only pending requests can be approved'), { + name: 'RequestError', + status: 409, + }) + ); + await request(makeApp({ requestService })) + .post(`/api/external-requests/${requestId}/approve`) + .send({}) + .expect(409, { error: 'Only pending requests can be approved' }); + + requestService.getAdminRequest.mockRejectedValueOnce(new Error('database password secret')); + await request(makeApp({ requestService })) + .get(`/api/external-requests/${requestId}`) + .expect(500, { error: 'External request review operation failed' }); + }); +}); diff --git a/server/routes/apikeys.js b/server/routes/apikeys.js index 4c1839e02..30432e97d 100644 --- a/server/routes/apikeys.js +++ b/server/routes/apikeys.js @@ -1,5 +1,4 @@ const express = require('express'); -const router = express.Router(); /** * Creates API key management routes @@ -8,7 +7,34 @@ const router = express.Router(); * @returns {express.Router} */ module.exports = function createApiKeyRoutes({ verifyToken }) { + const router = express.Router(); const apiKeyModule = require('../modules/apiKeyModule'); + const apiKeyChannelGrantModule = require('../modules/apiKeyChannelGrantModule'); + const { sequelize } = require('../db'); + + const isPolicyValidationError = (error) => + error.message.includes('Invalid') || + error.message.includes('Policy') || + error.message.includes('Unsupported') || + error.message.includes('allowedMediaTypes') || + error.message.includes('maxRatingLevel') || + error.message.includes('maxActiveJobs') || + error.message.includes('hourlyWriteLimit') || + error.message.includes('dailyWriteLimit') || + error.message.includes('requires allow') || + error.message.includes('external API key types') || + error.message.includes('channelIds') || + error.message.includes('enabled channel') || + error.message.includes('Only active external API keys'); + + // AUTH_ENABLED=false delegates browser access control to the deployment, + // but it must never turn an x-api-key into a key-management credential. + router.use('/api/keys', (req, res, next) => { + if (req.headers['x-api-key']) { + return res.status(403).json({ error: 'API keys cannot manage other API keys' }); + } + return next(); + }); /** * @swagger @@ -82,6 +108,13 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { * type: string * maxLength: 100 * description: Human-readable name for the key + * policy: + * $ref: '#/components/schemas/ExternalApiKeyPolicy' + * channelIds: + * type: array + * uniqueItems: true + * items: { type: integer, minimum: 1 } + * description: Exact initial grant set for a constrained key * responses: * 200: * description: API key created successfully @@ -113,7 +146,11 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { return res.status(403).json({ error: 'API keys cannot create other API keys' }); } - const { name } = req.body; + if (!req.body || typeof req.body !== 'object' || Array.isArray(req.body) || + Object.keys(req.body).some((field) => !['name', 'policy', 'channelIds'].includes(field))) { + return res.status(400).json({ error: 'Request body contains unsupported fields' }); + } + const { name, policy, channelIds } = req.body; if (!name || typeof name !== 'string' || name.trim().length < 1 || name.length > 100) { return res.status(400).json({ error: 'Name is required (1-100 characters)' }); } @@ -128,7 +165,41 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { } try { - const result = await apiKeyModule.createApiKey(sanitizedName); + let result; + if (policy === undefined) { + if (channelIds !== undefined) { + return res.status(400).json({ + error: 'channelIds can only be supplied for an external API key', + }); + } + result = await apiKeyModule.createApiKey(sanitizedName); + } else { + result = await sequelize.transaction(async (transaction) => { + const created = await apiKeyModule.createApiKey( + sanitizedName, + policy, + { transaction, logEvent: false } + ); + await apiKeyChannelGrantModule.replaceChannelGrants( + created.id, + channelIds ?? [], + { transaction } + ); + return created; + }); + try { + apiKeyModule.logApiKeyCreated({ + id: result.id, + name: result.name, + prefix: result.prefix, + }); + } catch (logError) { + req.log?.warn( + { err: logError, keyId: result.id }, + 'API key was created but its audit log event failed' + ); + } + } res.json({ success: true, message: 'API key created. Save this key - it will not be shown again!', @@ -136,19 +207,183 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { }); } catch (error) { req.log.error({ err: error }, 'Failed to create API key'); - if (error.message.includes('Maximum number')) { + if (error.message.includes('Maximum number') || isPolicyValidationError(error)) { return res.status(400).json({ error: error.message }); } res.status(500).json({ error: 'Failed to create API key' }); } }); + router.patch('/api/keys/:id', verifyToken, async (req, res) => { + if (req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot manage other API keys' }); + } + const id = parseInt(req.params.id, 10); + if (isNaN(id)) return res.status(400).json({ error: 'Invalid API key ID' }); + try { + const key = await apiKeyModule.updateApiKey(id, req.body?.policy); + if (!key) return res.status(404).json({ error: 'API key not found' }); + return res.json({ success: true, key: apiKeyModule.serializeApiKey(key) }); + } catch (error) { + if (isPolicyValidationError(error)) { + return res.status(400).json({ error: error.message }); + } + req.log.error({ err: error }, 'Failed to update API key policy'); + return res.status(500).json({ error: 'Failed to update API key' }); + } + }); + + router.get('/api/keys/:id/channels', verifyToken, async (req, res) => { + if (req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot manage channel grants' }); + } + const id = Number(req.params.id); + if (!Number.isSafeInteger(id) || id < 1) { + return res.status(400).json({ error: 'Invalid API key ID' }); + } + try { + const grants = await apiKeyChannelGrantModule.getChannelGrants(id); + if (!grants) return res.status(404).json({ error: 'API key not found' }); + return res.json(grants); + } catch (error) { + if (error.message.includes('Only active external API keys')) { + return res.status(400).json({ error: error.message }); + } + req.log.error({ err: error }, 'Failed to list API key channel grants'); + return res.status(500).json({ error: 'Failed to list channel grants' }); + } + }); + + router.put('/api/keys/:id/channels', verifyToken, async (req, res) => { + if (req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot manage channel grants' }); + } + const id = Number(req.params.id); + if (!Number.isSafeInteger(id) || id < 1) { + return res.status(400).json({ error: 'Invalid API key ID' }); + } + try { + const grants = await apiKeyChannelGrantModule.replaceChannelGrants(id, req.body?.channelIds); + if (!grants) return res.status(404).json({ error: 'API key not found' }); + return res.json({ success: true, ...grants }); + } catch (error) { + if (error.message.includes('channelIds') || error.message.includes('enabled channel') || + error.message.includes('Only active external API keys')) { + return res.status(400).json({ error: error.message }); + } + req.log.error({ err: error }, 'Failed to replace API key channel grants'); + return res.status(500).json({ error: 'Failed to replace channel grants' }); + } + }); + + /** + * @swagger + * /api/keys/{id}/external-access: + * put: + * summary: Atomically replace a constrained key policy and channel grants + * tags: [API Keys] + * security: [{ SessionAuth: [] }] + * parameters: + * - in: path + * name: id + * required: true + * schema: { type: integer, minimum: 1 } + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [policy, channelIds] + * additionalProperties: false + * properties: + * policy: + * $ref: '#/components/schemas/ExternalApiKeyPolicy' + * channelIds: + * type: array + * uniqueItems: true + * items: { type: integer, minimum: 1 } + * responses: + * 200: { description: Policy and exact grant set committed } + * 400: { description: Invalid policy or channel grant set } + * 403: { description: Session authentication is required } + * 404: { description: Active constrained key not found } + */ + router.put('/api/keys/:id/external-access', verifyToken, async (req, res) => { + if (req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot manage other API keys' }); + } + const id = Number(req.params.id); + if (!Number.isSafeInteger(id) || id < 1) { + return res.status(400).json({ error: 'Invalid API key ID' }); + } + if (!req.body || typeof req.body !== 'object' || Array.isArray(req.body) || + Object.keys(req.body).some((field) => !['policy', 'channelIds'].includes(field))) { + return res.status(400).json({ error: 'Request body contains unsupported fields' }); + } + try { + const result = await sequelize.transaction(async (transaction) => { + const key = await apiKeyModule.updateApiKey( + id, + req.body.policy, + { transaction } + ); + if (!key) return null; + const grants = await apiKeyChannelGrantModule.replaceChannelGrants( + id, + req.body.channelIds, + { transaction } + ); + return { key, grants }; + }); + if (!result) return res.status(404).json({ error: 'API key not found' }); + return res.json({ + success: true, + key: apiKeyModule.serializeApiKey(result.key), + channelIds: result.grants.channelIds, + }); + } catch (error) { + if (isPolicyValidationError(error)) { + return res.status(400).json({ error: error.message }); + } + req.log.error({ err: error }, 'Failed to update external API access'); + return res.status(500).json({ error: 'Failed to update external access' }); + } + }); + + router.post('/api/keys/:id/regenerate', verifyToken, async (req, res) => { + if (req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot manage other API keys' }); + } + const id = Number(req.params.id); + if (!Number.isSafeInteger(id) || id < 1) { + return res.status(400).json({ error: 'Invalid API key ID' }); + } + if (req.body && (typeof req.body !== 'object' || Array.isArray(req.body) || + Object.keys(req.body).length > 0)) { + return res.status(400).json({ error: 'Request body must be empty' }); + } + + try { + const result = await apiKeyModule.regenerateApiKey(id); + if (!result) return res.status(404).json({ error: 'Active API key not found' }); + return res.json({ + success: true, + message: 'API key regenerated. Save this key - it will not be shown again!', + ...result, + }); + } catch (error) { + req.log.error({ err: error }, 'Failed to regenerate API key'); + return res.status(500).json({ error: 'Failed to regenerate API key' }); + } + }); + /** * @swagger * /api/keys/{id}: * delete: - * summary: Delete API key - * description: Permanently delete an API key. Only accessible via session auth. + * summary: Revoke API key + * description: Revoke an API key while retaining it for audit. Only accessible via session auth. * tags: [API Keys] * parameters: * - in: path @@ -159,15 +394,15 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { * description: API key ID * responses: * 200: - * description: API key deleted successfully + * description: API key revoked successfully * 403: - * description: API keys cannot delete other API keys + * description: API keys cannot revoke other API keys * 404: * description: API key not found */ router.delete('/api/keys/:id', verifyToken, async (req, res) => { if (req.authType === 'api_key') { - return res.status(403).json({ error: 'API keys cannot delete other API keys' }); + return res.status(403).json({ error: 'API keys cannot revoke other API keys' }); } const id = parseInt(req.params.id, 10); @@ -178,16 +413,15 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { try { const success = await apiKeyModule.deleteApiKey(id); if (success) { - res.json({ success: true, message: 'API key deleted' }); + res.json({ success: true, message: 'API key revoked' }); } else { res.status(404).json({ error: 'API key not found' }); } } catch (error) { - req.log.error({ err: error }, 'Failed to delete API key'); - res.status(500).json({ error: 'Failed to delete API key' }); + req.log.error({ err: error }, 'Failed to revoke API key'); + res.status(500).json({ error: 'Failed to revoke API key' }); } }); return router; }; - diff --git a/server/routes/externalApi.js b/server/routes/externalApi.js new file mode 100644 index 000000000..8be174ee4 --- /dev/null +++ b/server/routes/externalApi.js @@ -0,0 +1,500 @@ +const express = require('express'); +const { + sendExternalError, + setExternalSecurityHeaders, +} = require('../modules/externalApiResponse'); +const { scopesForExternalKey } = require('../modules/externalPermissions'); + +function createExternalApiRoutes({ + externalApiAuth, + externalApiIngressLimiter = (_req, _res, next) => next(), + externalApiLimiter, + externalApiWriteLimiter = (_req, _res, next) => next(), + recordExternalApiUse = (_req, _res, next) => next(), + externalApiJsonParser = express.json({ limit: '16kb' }), + serverVersion, + catalogService, + thumbnailProxy, + externalWorkLimiter, + requestService, + quotaService, +}) { + const router = express.Router(); + const requests = () => requestService; + const quotas = () => quotaService; + router.use((_req, res, next) => { + setExternalSecurityHeaders(res); + next(); + }); + router.use(externalApiIngressLimiter); + router.use((req, res, next) => { + if (!['GET', 'HEAD', 'POST'].includes(req.method)) { + return sendExternalError(res, 405, 'HTTP method is not allowed', { + code: 'method_not_allowed', + requestId: req.id, + }); + } + return next(); + }); + router.use(externalApiAuth, externalApiLimiter, recordExternalApiUse); + router.use((req, res, next) => { + if (req.method !== 'POST') return next(); + const contentEncoding = String(req.get('content-encoding') || 'identity').toLowerCase(); + if (contentEncoding !== 'identity') { + return sendExternalError(res, 415, 'Compressed request bodies are not supported', { + code: 'unsupported_media_type', + requestId: req.id, + }); + } + if (!req.is('application/json')) { + return sendExternalError(res, 415, 'Content-Type must be application/json', { + code: 'unsupported_media_type', + requestId: req.id, + }); + } + return externalApiJsonParser(req, res, (error) => { + if (!error) return next(); + if (error.type === 'entity.too.large') { + return sendExternalError(res, 413, 'Request body is too large', { + requestId: req.id, + }); + } + if (error instanceof SyntaxError || error.type === 'entity.parse.failed') { + return sendExternalError(res, 400, 'Request body contains invalid JSON', { + requestId: req.id, + }); + } + if (error.type === 'request.aborted') { + return sendExternalError(res, 400, 'Request body was aborted', { + requestId: req.id, + }); + } + return next(error); + }); + }); + /** + * @swagger + * /external-api/v1/capabilities: + * get: + * summary: Read the calling key's effective external capabilities + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * responses: + * 200: { description: Effective role, scopes, policy, and feature flags } + * 401: + * description: Missing, invalid, legacy, revoked, or corrupt-policy key + * content: + * application/json: + * schema: { $ref: '#/components/schemas/ExternalError' } + */ + router.get('/capabilities', async (req, res) => { + const key = req.externalApiKey; + const scopes = scopesForExternalKey(key) || []; + try { + const quota = await quotas().status(key); + return res.json({ + apiVersion: '1', + serverVersion, + role: key.role, + scopes, + policy: { + allowVideoRequests: key.allowVideoRequests, + allowChannelRequests: key.allowChannelRequests, + allowDeleteVideoRequests: key.allowDeleteVideoRequests, + autoApproveVideoRequests: key.autoApproveVideoRequests, + autoApproveChannelRequests: key.autoApproveChannelRequests, + autoApproveDeleteRequests: key.autoApproveDeleteRequests, + maxRatingLevel: key.maxRatingLevel, + allowUnrated: key.allowUnrated, + allowedMediaTypes: key.allowedMediaTypes, + }, + quota, + features: { + catalog: true, requests: true, channelRequests: true, deleteRequests: true, + recommendations: true, authenticatedAssets: true, videoDetails: true, + }, + }); + } catch (error) { + req.log?.error({ err: error }, 'External API quota status failed'); + return sendExternalError(res, 500, 'External API capability lookup failed', { + requestId: req.id, + }); + } + }); + + const sendCatalogError = (req, res, error) => { + if (error.name === 'ExternalWorkLimitError') { + return sendExternalError(res, 503, 'External API work queue is full', { + code: error.code, + requestId: req.id, + }); + } + if (error.name === 'CatalogError' && error.status >= 400 && error.status < 500) { + return sendExternalError(res, error.status, error.message, { + code: error.code, + requestId: req.id, + }); + } + req.log?.error({ err: error }, 'External catalog request failed'); + return sendExternalError(res, 500, 'External catalog request failed'); + }; + + const sendRequestError = (req, res, error) => { + if (['RequestError', 'QuotaError'].includes(error.name) && + error.status >= 400 && (error.status < 500 || error.status === 503)) { + return sendExternalError(res, error.status, error.message, { + code: error.code, + requestId: req.id, + }); + } + req.log?.error({ err: error }, 'External request operation failed'); + return sendExternalError(res, 500, 'External request operation failed'); + }; + + /** + * @swagger + * /external-api/v1/channels: + * get: + * summary: Browse cached channels granted to the calling key + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: query, name: cursor, description: Opaque cursor; mutually exclusive with page, schema: { type: string } } + * - { in: query, name: page, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: pageSize, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: search, schema: { type: string, maxLength: 200 } } + * - { in: query, name: subfolder, schema: { type: string, maxLength: 255 } } + * - { in: query, name: sortBy, schema: { type: string, enum: [title, videoCount, downloadedCount, id] } } + * - { in: query, name: sortOrder, schema: { type: string, enum: [asc, desc] } } + * responses: + * 200: { description: Paginated cached channel catalog } + * 401: { description: Missing or invalid external API key } + */ + router.get('/channels', async (req, res) => { + try { + return res.json(await catalogService.listChannels(req.externalApiKey, req.query)); + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/channels/{id}/videos: + * get: + * summary: Browse eligible cached videos in one granted channel + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: path, name: id, required: true, schema: { type: integer } } + * - { in: query, name: cursor, description: Opaque cursor; mutually exclusive with page, schema: { type: string } } + * - { in: query, name: page, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: pageSize, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: search, schema: { type: string, maxLength: 200 } } + * - { in: query, name: tabType, schema: { type: string, enum: [videos, shorts, streams] } } + * - { in: query, name: status, schema: { type: string, enum: [all, requestable, available, downloaded, requested] } } + * - { in: query, name: minDuration, schema: { type: integer, minimum: 0 } } + * - { in: query, name: maxDuration, schema: { type: integer, minimum: 0 } } + * - { in: query, name: dateFrom, schema: { type: string, format: date-time } } + * - { in: query, name: dateTo, schema: { type: string, format: date-time } } + * - { in: query, name: sortBy, schema: { type: string, enum: [date, title, duration] } } + * - { in: query, name: sortOrder, schema: { type: string, enum: [asc, desc] } } + * responses: + * 200: { description: Paginated policy-filtered cached videos } + * 404: + * description: Missing, disabled, terminated, or ungranted channel + */ + router.get('/channels/:id/videos', async (req, res) => { + try { + return res.json(await catalogService.listChannelVideos(req.externalApiKey, req.params.id, req.query)); + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/videos: + * get: + * summary: Browse the complete cached video catalog across all granted channels + * description: Follow nextCursor to traverse every policy-filtered row without fetching channels individually. Use status=requestable to omit downloaded videos and active requests. + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: query, name: cursor, description: Opaque cursor; mutually exclusive with page, schema: { type: string } } + * - { in: query, name: page, description: Compatibility paging only; use cursor for complete traversal, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: pageSize, schema: { type: integer, minimum: 1, maximum: 100 } } + * - { in: query, name: search, schema: { type: string, maxLength: 200 } } + * - { in: query, name: tabType, schema: { type: string, enum: [videos, shorts, streams] } } + * - { in: query, name: status, schema: { type: string, enum: [all, requestable, available, downloaded, requested] } } + * - { in: query, name: minDuration, schema: { type: integer, minimum: 0 } } + * - { in: query, name: maxDuration, schema: { type: integer, minimum: 0 } } + * - { in: query, name: dateFrom, schema: { type: string, format: date-time } } + * - { in: query, name: dateTo, schema: { type: string, format: date-time } } + * - { in: query, name: sortBy, schema: { type: string, enum: [date, title, duration] } } + * - { in: query, name: sortOrder, schema: { type: string, enum: [asc, desc] } } + * responses: + * 200: { description: Complete cursor-paginated cached catalog; no Plex-derived signal enters Youtarr } + */ + router.get('/videos', async (req, res) => { + try { + return res.json(await catalogService.listVideos(req.externalApiKey, req.query)); + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/videos/{youtubeId}: + * get: + * summary: Read full curated metadata for one eligible cached video + * description: Returns the catalog identity and status plus the metadata used by Youtarr's video detail modal. + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: path, name: youtubeId, required: true, schema: { type: string, minLength: 11, maxLength: 11 } } + * responses: + * 200: { description: Full policy-filtered video detail } + * 404: { description: Missing, hidden, ungranted, or ineligible video } + */ + router.get('/videos/:youtubeId', async (req, res) => { + try { + return res.json(await externalWorkLimiter.run(() => + catalogService.getVideoDetail(req.externalApiKey, req.params.youtubeId) + )); + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/assets/channels/{id}/thumbnail: + * get: + * summary: Read eligible private channel artwork + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: path, name: id, required: true, schema: { type: integer } } + * responses: + * 200: { description: Private JPEG artwork } + * 404: { description: Missing, unsafe, disabled, terminated, or ungranted asset } + */ + router.get('/assets/channels/:id/thumbnail', async (req, res) => { + try { + const absolutePath = await catalogService.getChannelThumbnail(req.externalApiKey, req.params.id); + res.set({ + 'Content-Type': 'image/jpeg', + }); + return res.sendFile(absolutePath); + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/assets/videos/{youtubeId}/thumbnail: + * get: + * summary: Read eligible private video artwork + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - { in: path, name: youtubeId, required: true, schema: { type: string } } + * responses: + * 200: { description: Private JPEG artwork } + * 404: { description: Missing, unsafe, disabled, terminated, ungranted, or ineligible asset } + */ + router.get('/assets/videos/:youtubeId/thumbnail', async (req, res) => { + try { + const asset = await catalogService.getVideoThumbnail( + req.externalApiKey, + req.params.youtubeId + ); + if (typeof asset === 'string' || asset.source === 'local') { + res.set({ 'Content-Type': 'image/jpeg' }); + return res.sendFile(typeof asset === 'string' ? asset : asset.absolutePath); + } + try { + const proxied = await thumbnailProxy.fetchExternalThumbnail(asset.url); + res.set({ 'Content-Type': proxied.contentType }); + return res.send(proxied.body); + } catch (error) { + req.log?.warn( + { err: error, youtubeId: req.params.youtubeId }, + 'External video thumbnail proxy failed' + ); + return sendExternalError(res, 404, 'Thumbnail not found', { + requestId: req.id, + }); + } + } catch (error) { + return sendCatalogError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/requests/videos: + * post: + * summary: Request a cached video from a granted channel + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [youtubeId, channelId] + * properties: + * youtubeId: { type: string } + * channelId: { type: integer } + * idempotencyKey: { type: string, maxLength: 200 } + * responses: + * 202: { description: Request persisted } + * 403: { description: Required caller scope is missing } + * 404: { description: Cached video is missing, hidden, or ineligible } + * 409: { description: Idempotency key target conflict } + */ + router.post('/requests/videos', externalApiWriteLimiter, async (req, res) => { + try { + const result = await requests().createVideoRequest(req.externalApiKey, req.body); + return res.status(result.outcome === 'created' ? 202 : 200).json(result); + } catch (error) { + return sendRequestError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/requests/channels: + * post: + * summary: Request canonical YouTube channel provisioning + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [channelUrl] + * properties: + * channelUrl: { type: string, format: uri, maxLength: 500 } + * idempotencyKey: { type: string, maxLength: 200 } + * responses: + * 202: { description: Approval-backed channel request created } + * 200: { description: Existing idempotent request returned } + */ + router.post('/requests/channels', externalApiWriteLimiter, async (req, res) => { + try { + const result = await requests().createChannelRequest(req.externalApiKey, req.body); + return res.status(result.outcome === 'created' ? 202 : 200).json(result); + } catch (error) { + return sendRequestError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/requests/delete-videos: + * post: + * summary: Request deletion of a downloaded video asset + * description: This never removes or disables the channel subscription. + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [youtubeId, channelId] + * properties: + * youtubeId: { type: string, minLength: 11, maxLength: 11 } + * channelId: { type: integer, minimum: 1 } + * idempotencyKey: { type: string, maxLength: 200 } + * responses: + * 202: { description: Approval-backed deletion request created } + * 200: { description: Duplicate or already-deleted target } + */ + router.post('/requests/delete-videos', externalApiWriteLimiter, async (req, res) => { + try { + const result = await requests().createDeleteVideoRequest(req.externalApiKey, req.body); + return res.status(result.outcome === 'created' ? 202 : 200).json(result); + } catch (error) { + return sendRequestError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/requests: + * get: + * summary: List requests created by the calling key + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - in: query + * name: cursor + * description: Opaque cursor; mutually exclusive with page + * schema: { type: string } + * - in: query + * name: page + * schema: { type: integer, minimum: 1, maximum: 100, default: 1 } + * - in: query + * name: pageSize + * schema: { type: integer, minimum: 1, maximum: 100, default: 50 } + * - in: query + * name: status + * schema: + * type: string + * enum: [pending, approved, processing, completed, rejected, failed, cancelled] + * responses: + * 200: { description: Paginated request status list } + */ + router.get('/requests', async (req, res) => { + try { + return res.json(await requests().listRequests(req.externalApiKey, req.query)); + } catch (error) { + return sendRequestError(req, res, error); + } + }); + + /** + * @swagger + * /external-api/v1/requests/{id}: + * get: + * summary: Read one request created by the calling key + * tags: [External API] + * security: [{ ExternalApiKeyAuth: [] }] + * parameters: + * - in: path + * name: id + * required: true + * schema: { type: string, format: uuid } + * responses: + * 200: { description: Request status } + * 404: { description: Request not found } + */ + router.get('/requests/:id', async (req, res) => { + try { + return res.json(await requests().getRequest(req.externalApiKey, req.params.id)); + } catch (error) { + return sendRequestError(req, res, error); + } + }); + + // Keep every request that entered the external namespace inside its JSON + // contract. Without this terminal handler, unknown GETs can fall through to + // the SPA wildcard and unknown writes can receive Express's HTML 404. + router.use((req, res) => sendExternalError(res, 404, 'External API route not found', { + requestId: req.id, + })); + + return router; +} + +module.exports = { createExternalApiRoutes }; diff --git a/server/routes/externalRequests.js b/server/routes/externalRequests.js new file mode 100644 index 000000000..17f8b3ebe --- /dev/null +++ b/server/routes/externalRequests.js @@ -0,0 +1,181 @@ +const express = require('express'); + +function createExternalRequestReviewRoutes({ + verifyToken, + reviewLimiter = (_req, _res, next) => next(), + requestService, +}) { + const router = express.Router(); + const requests = () => requestService; + + const sessionOnly = (req, res, next) => { + if (req.authType !== 'session') { + return res.status(403).json({ error: 'Session authentication is required' }); + } + return next(); + }; + + const sendError = (req, res, error) => { + if (error.name === 'RequestError' && error.status >= 400 && error.status < 500) { + return res.status(error.status).json({ error: error.message }); + } + req.log?.error({ err: error }, 'External request review operation failed'); + return res.status(500).json({ error: 'External request review operation failed' }); + }; + + /** + * @swagger + * /api/external-requests: + * get: + * summary: List external requests for administrator review + * tags: [External Requests] + * security: [{ SessionAuth: [] }] + * parameters: + * - in: query + * name: page + * schema: { type: integer, minimum: 1, default: 1 } + * - in: query + * name: pageSize + * schema: { type: integer, minimum: 1, maximum: 100, default: 50 } + * - in: query + * name: status + * schema: + * type: string + * enum: [pending, approved, processing, completed, rejected, failed, cancelled] + * - in: query + * name: apiKeyId + * schema: { type: integer, minimum: 1 } + * - in: query + * name: requestType + * schema: { type: string, enum: [video, channel, delete_video] } + * responses: + * 200: { description: Paginated requests and safe requester filter metadata } + * 403: { description: Session authentication is required } + */ + router.get('/api/external-requests', verifyToken, sessionOnly, async (req, res) => { + try { + return res.json(await requests().listAdminRequests(req.query)); + } catch (error) { + return sendError(req, res, error); + } + }); + + /** + * @swagger + * /api/external-requests/{id}: + * get: + * summary: Read one external request for administrator review + * tags: [External Requests] + * security: [{ SessionAuth: [] }] + * parameters: + * - in: path + * name: id + * required: true + * schema: { type: string, format: uuid } + * responses: + * 200: { description: Safe requester, target, and job metadata } + * 403: { description: Session authentication is required } + * 404: { description: Request not found } + */ + router.get('/api/external-requests/:id', verifyToken, sessionOnly, async (req, res) => { + try { + return res.json(await requests().getAdminRequest(req.params.id)); + } catch (error) { + return sendError(req, res, error); + } + }); + + /** + * @swagger + * /api/external-requests/{id}/approve: + * post: + * summary: Revalidate and approve a pending external request + * tags: [External Requests] + * security: [{ SessionAuth: [] }] + * parameters: + * - in: path + * name: id + * required: true + * schema: { type: string, format: uuid } + * requestBody: + * content: + * application/json: + * schema: + * type: object + * properties: + * grantToRequestingKey: + * type: boolean + * default: true + * description: Channel requests only. + * responses: + * 200: { description: Updated request } + * 403: { description: Session authentication is required } + * 404: { description: Request not found } + * 409: { description: Request is not pending } + * 429: { description: Review action rate limit exceeded } + */ + router.post( + '/api/external-requests/:id/approve', + verifyToken, + sessionOnly, + reviewLimiter, + async (req, res) => { + try { + const service = requests(); + const review = service.reviewRequest || service.reviewVideoRequest; + return res.json(await review.call(service, req.params.id, 'approve', req.body)); + } catch (error) { + return sendError(req, res, error); + } + } + ); + + /** + * @swagger + * /api/external-requests/{id}/reject: + * post: + * summary: Reject a pending external request + * tags: [External Requests] + * security: [{ SessionAuth: [] }] + * parameters: + * - in: path + * name: id + * required: true + * schema: { type: string, format: uuid } + * requestBody: + * required: true + * content: + * application/json: + * schema: + * type: object + * required: [reason] + * properties: + * reason: { type: string, minLength: 1, maxLength: 300 } + * responses: + * 200: { description: Rejected request } + * 400: { description: Invalid reason } + * 403: { description: Session authentication is required } + * 404: { description: Request not found } + * 409: { description: Request is not pending } + * 429: { description: Review action rate limit exceeded } + */ + router.post( + '/api/external-requests/:id/reject', + verifyToken, + sessionOnly, + reviewLimiter, + async (req, res) => { + try { + const service = requests(); + const review = service.reviewRequest || service.reviewVideoRequest; + return res.json(await review.call(service, req.params.id, 'reject', req.body)); + } catch (error) { + return sendError(req, res, error); + } + } + ); + + return router; +} + +module.exports = createExternalRequestReviewRoutes; diff --git a/server/routes/index.js b/server/routes/index.js index fe3daa04d..8fc6c2e73 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -19,6 +19,9 @@ const createMaintenanceRoutes = require('./maintenance'); const createSubfolderRoutes = require('./subfolders'); const createSchedulesRoutes = require('./schedules'); const createLogRoutes = require('./logs'); +const createExternalRequestReviewRoutes = require('./externalRequests'); +const { createExternalApiRoutes } = require('./externalApi'); +const { sendExternalError } = require('../modules/externalApiResponse'); const videoMetadataModule = require('../modules/videoMetadataModule'); const videoOembedEnricher = require('../modules/videoOembedEnricher'); const playlistModule = require('../modules/playlistModule'); @@ -43,6 +46,11 @@ const cookieDetails = require('../modules/cookieDetails'); const cookieTest = require('../modules/cookieTest'); const logger = require('../logger'); const logFilesModule = require('../modules/logFilesModule'); +const externalCatalogService = require('../modules/externalCatalogService'); +const externalThumbnailProxy = require('../modules/externalThumbnailProxy'); +const { sharedExternalWorkLimiter } = require('../modules/externalWorkLimiter'); +const { createExternalRequestService } = require('../modules/externalRequestService'); +const { createExternalQuotaService } = require('../modules/externalQuotaService'); /** * Registers all route modules with the Express app @@ -50,6 +58,8 @@ const logFilesModule = require('../modules/logFilesModule'); * @param {Object} deps - Dependencies to inject into route modules */ function registerRoutes(app, deps) { + const externalRequestService = createExternalRequestService(); + const externalQuotaService = createExternalQuotaService(); const { verifyToken, loginLimiter, @@ -74,6 +84,13 @@ function registerRoutes(app, deps) { setupTokenModule, getClientAddress, isWslEnvironment, + externalApiAuth, + externalApiIngressLimiter, + externalApiLimiter, + externalApiWriteLimiter, + recordExternalApiUse, + externalRequestReviewLimiter, + serverVersion, } = deps; // Health routes (no auth required for health checks, but yt-dlp endpoints are authenticated) @@ -122,6 +139,13 @@ function registerRoutes(app, deps) { // API Key routes app.use(createApiKeyRoutes({ verifyToken })); + // Session-authenticated administrator review of external requests + app.use(createExternalRequestReviewRoutes({ + verifyToken, + reviewLimiter: externalRequestReviewLimiter, + requestService: externalRequestService, + })); + // Subscription import routes app.use(createSubscriptionRoutes({ verifyToken, subscriptionImportModule })); @@ -147,6 +171,31 @@ function registerRoutes(app, deps) { // Log file download app.use(createLogRoutes({ verifyToken, logFilesModule, configModule })); + // The versioned external API is available by default. Keep an explicit + // false opt-out for deployments that do not want to expose the namespace. + if (process.env.EXTERNAL_API_ENABLED !== 'false') { + app.use('/external-api/v1', createExternalApiRoutes({ + externalApiAuth, + externalApiIngressLimiter, + externalApiLimiter, + externalApiWriteLimiter, + recordExternalApiUse, + serverVersion, + catalogService: externalCatalogService, + thumbnailProxy: externalThumbnailProxy, + externalWorkLimiter: sharedExternalWorkLimiter, + requestService: externalRequestService, + quotaService: externalQuotaService, + })); + } + // Do not allow unknown or disabled external routes to fall through to the + // SPA. Keep the public namespace on the same versioned error contract. + app.use('/external-api', (req, res) => + sendExternalError(res, 404, 'External API route not found', { + code: 'not_found', + requestId: req.id, + }) + ); // Defensive redirect: /channels -> /subscriptions (frontend handles client-side routing, // this fallback covers direct server-side hits during the transition period) diff --git a/server/server.js b/server/server.js index d01ef9340..6b565c088 100644 --- a/server/server.js +++ b/server/server.js @@ -15,8 +15,10 @@ const { isAuthConfigured } = require('./modules/authState'); // Start the channel tab count catch-up after the startup rescan has begun. const STARTUP_TAB_COUNT_REFRESH_DELAY_MS = 2 * 60 * 1000; - +const { createExternalApiAuth } = require('./middleware/externalApiAuth'); +const { sendExternalError } = require('./modules/externalApiResponse'); const app = express(); +app.disable('x-powered-by'); app.set('trust proxy', parseTrustProxySetting(process.env.TRUST_PROXY)); if (process.env.TRUST_PROXY === undefined || process.env.TRUST_PROXY === '') { logger.info('TRUST_PROXY is unset; defaulting Express proxy trust to true for backwards compatibility. Rate limits use the direct peer IP until TRUST_PROXY is explicitly configured.'); @@ -53,11 +55,7 @@ app.use(pinoHttp({ } }, // Generate unique request ID for correlation - genReqId: (req) => { - const existingId = req.id || req.headers['x-request-id']; - if (existingId) return existingId; - return require('crypto').randomUUID(); - }, + genReqId: () => require('crypto').randomUUID(), // Only log warnings and errors at info level, success at debug customLogLevel: (req, res, err) => { if (res.statusCode >= 400 && res.statusCode < 500) { @@ -92,7 +90,27 @@ app.use(pinoHttp({ }, })); -app.use(express.json()); +const defaultJsonParser = express.json(); +app.use((req, res, next) => { + if (req.path.startsWith('/external-api')) return next(); + return defaultJsonParser(req, res, next); +}); +// Express otherwise renders malformed/oversized JSON as an HTML error. Keep +// every response in the public external namespace on the versioned envelope. +app.use((error, req, res, next) => { + if (!req.path.startsWith('/external-api')) return next(error); + if (error?.type === 'entity.too.large') { + return sendExternalError(res, 413, 'Request body is too large', { + requestId: req.id, + }); + } + if (error instanceof SyntaxError && error?.type === 'entity.parse.failed') { + return sendExternalError(res, 400, 'Request body contains invalid JSON', { + requestId: req.id, + }); + } + return next(error); +}); const path = require('path'); const fs = require('fs'); const { execSync } = require('child_process'); @@ -354,7 +372,7 @@ const initialize = async () => { subfolderModule, }); - logger.info({ directoryPath: configModule.directoryPath }, 'YouTube downloads directory configured'); + logger.info('YouTube downloads directory configured'); // Degraded mode middleware - checks database health before processing requests const checkDatabaseHealth = function(req, res, next) { @@ -408,6 +426,12 @@ const initialize = async () => { errorMessage = 'A database error has occurred. Please check the logs for details.'; } + if (req.path.startsWith('/external-api')) { + return sendExternalError(res, 503, 'External API is temporarily unavailable', { + code: 'service_unavailable', + requestId: req.id, + }); + } return res.status(503).json({ error: errorType, message: errorMessage, @@ -472,6 +496,17 @@ const initialize = async () => { }); } + // External API roles are deliberately not accepted by the legacy + // download endpoint. Missing role keeps pre-migration records and + // older test doubles compatible. + if (validKey.role !== undefined && validKey.role !== 'legacy_download') { + req.log.warn({ + event: 'api_key_legacy_access_denied', keyId: validKey.id, + keyPrefix: validKey.key_prefix, role: validKey.role, + }, 'External API key attempted legacy download access'); + return res.status(403).json({ error: 'External API keys cannot access the download endpoint' }); + } + req.log.info({ event: 'api_key_auth_success', keyId: validKey.id, @@ -667,6 +702,76 @@ const initialize = async () => { }); }, }); + const externalApiIngressLimiter = rateLimit({ + windowMs: 60 * 1000, + max: 300, + standardHeaders: true, + legacyHeaders: false, + validate: { trustProxy: false, ip: false }, + keyGenerator: (req) => `external-api-ingress:${getRateLimitAddress(req)}`, + handler: (req, res) => sendExternalError( + res, + 429, + 'External API ingress rate limit exceeded', + { code: 'rate_limited', requestId: req.id } + ), + }); + const externalApiLimiter = rateLimit({ + windowMs: 60 * 1000, + max: 120, + standardHeaders: true, + legacyHeaders: false, + validate: { trustProxy: false, ip: false }, + keyGenerator: (req) => `external-api:${req.externalApiKey?.id || 'unknown'}`, + handler: (req, res) => sendExternalError( + res, + 429, + 'External API rate limit exceeded', + { code: 'rate_limited', requestId: req.id } + ), + }); + const externalApiWriteLimiter = rateLimit({ + windowMs: 60 * 1000, + max: 10, + standardHeaders: true, + legacyHeaders: false, + validate: { trustProxy: false, ip: false }, + keyGenerator: (req) => `external-api-write:${req.externalApiKey?.id || 'unknown'}`, + handler: (req, res) => sendExternalError( + res, + 429, + 'External API write rate limit exceeded', + { code: 'rate_limited', requestId: req.id } + ), + }); + const externalRequestReviewLimiter = rateLimit({ + windowMs: 60 * 1000, + max: 30, + standardHeaders: true, + legacyHeaders: false, + validate: { trustProxy: false, ip: false }, + keyGenerator: (req) => `external-request-review:${req.sessionId || getRateLimitAddress(req)}`, + handler: (_req, res) => res.status(429).json({ + error: 'External request review rate limit exceeded', + }), + }); + const externalApiAuth = createExternalApiAuth({ + // Resolve the model-backed module only if an external request arrives. + // Startup must not initialize ApiKey outside the normal DB lifecycle. + validateApiKey: (key) => require('./modules/apiKeyModule') + .validateApiKey(key, { recordUsage: false }), + }); + const recordExternalApiUse = async (req, res, next) => { + try { + await require('./modules/apiKeyModule').markApiKeyUsed(req.externalApiKey.id); + return next(); + } catch (error) { + req.log?.error({ err: error }, 'External API usage timestamp update failed'); + return sendExternalError(res, 500, 'External API authentication error', { + requestId: req.id, + }); + } + }; /**** ONLY ROUTES BELOW THIS LINE *********/ @@ -741,6 +846,21 @@ const initialize = async () => { setupTokenModule, getClientAddress, isWslEnvironment, + externalApiAuth, + externalApiIngressLimiter, + externalApiLimiter, + externalApiWriteLimiter, + recordExternalApiUse, + externalRequestReviewLimiter, + serverVersion: require('../package.json').version, + }); + + app.use((error, req, res, next) => { + if (!req.path.startsWith('/external-api')) return next(error); + req.log?.error({ err: error }, 'Unhandled external API request error'); + return sendExternalError(res, 500, 'External API request failed', { + requestId: req.id, + }); }); // Handle any requests that don't match the ones above diff --git a/server/swagger.js b/server/swagger.js index c865afcbc..03826f0fe 100644 --- a/server/swagger.js +++ b/server/swagger.js @@ -75,7 +75,96 @@ const options = { type: 'apiKey', in: 'header', name: 'x-api-key', - description: 'API key for external integrations (bookmarklets, shortcuts). Only works for /api/videos/download endpoint.', + description: 'API key sent to legacy download endpoints or the versioned external API.', + }, + ExternalApiKeyAuth: { + type: 'apiKey', + in: 'header', + name: 'x-api-key', + description: 'External-role API key. Channel catalog access also requires an explicit per-key grant.', + }, + }, + schemas: { + ExternalError: { + type: 'object', + required: ['error'], + properties: { + error: { + type: 'object', + required: ['code', 'message'], + properties: { + code: { type: 'string', example: 'not_found' }, + message: { type: 'string' }, + requestId: { type: 'string' }, + }, + }, + }, + }, + ExternalRequest: { + type: 'object', + required: ['id', 'type', 'status', 'target', 'createdAt', 'updatedAt'], + properties: { + id: { type: 'string', format: 'uuid' }, + type: { type: 'string', enum: ['video', 'channel', 'delete_video'] }, + status: { + type: 'string', + enum: ['pending', 'approved', 'processing', 'completed', 'rejected', 'failed', 'cancelled'], + }, + target: { + type: 'object', + properties: { + youtubeId: { type: 'string', nullable: true }, + channelId: { type: 'integer', nullable: true }, + channelUrl: { type: 'string', format: 'uri', nullable: true }, + }, + }, + createdAt: { type: 'string', format: 'date-time' }, + updatedAt: { type: 'string', format: 'date-time' }, + decidedAt: { type: 'string', format: 'date-time' }, + completedAt: { type: 'string', format: 'date-time' }, + message: { type: 'string', maxLength: 500 }, + grantToRequestingKey: { type: 'boolean' }, + }, + }, + ExternalApiKeyPolicy: { + type: 'object', + required: ['role'], + properties: { + role: { + type: 'string', + enum: ['view', 'request', 'delete', 'admin'], + description: 'Backward-compatible summary; clients should use explicit permissions.', + }, + allowVideoRequests: { type: 'boolean', default: false }, + allowChannelRequests: { type: 'boolean', default: false }, + allowDeleteVideoRequests: { type: 'boolean', default: false }, + autoApproveVideoRequests: { type: 'boolean', default: false }, + autoApproveChannelRequests: { type: 'boolean', default: false }, + autoApproveDeleteRequests: { type: 'boolean', default: false }, + maxRatingLevel: { + type: 'integer', + minimum: 1, + maximum: 4, + default: 4, + description: 'Maximum content-rating band: 1 = G / TV-Y / TV-G; ' + + '2 = PG / TV-Y7 / TV-PG; 3 = PG-13 / TV-14; ' + + '4 = R / NC-17 / TV-MA.', + }, + allowUnrated: { type: 'boolean', default: false }, + allowedMediaTypes: { + type: 'array', + minItems: 1, + uniqueItems: true, + items: { + type: 'string', + enum: ['video', 'short', 'livestream'], + }, + default: ['video'], + }, + maxActiveJobs: { type: 'integer', minimum: 1, maximum: 5, default: 5 }, + hourlyWriteLimit: { type: 'integer', minimum: 1, maximum: 30, default: 30 }, + dailyWriteLimit: { type: 'integer', minimum: 1, maximum: 200, default: 200 }, + }, }, }, }, @@ -124,6 +213,14 @@ const options = { name: 'API Keys', description: 'API key management for external integrations', }, + { + name: 'External API', + description: 'Versioned, API-key-authenticated integration endpoints', + }, + { + name: 'External Requests', + description: 'Session-authenticated administrator review of external requests', + }, { name: 'Playlists', description: 'YouTube playlist subscriptions and downloads', diff --git a/server/testing/externalApiContractApp.js b/server/testing/externalApiContractApp.js new file mode 100644 index 000000000..6389ce8a1 --- /dev/null +++ b/server/testing/externalApiContractApp.js @@ -0,0 +1,296 @@ +'use strict'; + +const express = require('express'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { createExternalApiAuth } = require('../middleware/externalApiAuth'); +const { createExternalApiRoutes } = require('../routes/externalApi'); +const { CatalogError, pagination: validatePagination } = require('../modules/externalCatalogService'); + +const SYNTHETIC_API_KEY = 'plinx-synthetic-key'; +const LANDSCAPE_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAJCAIAAAC0SDtlAAAAZklEQVR4nI3GMQrCQBAF0H+udJJmxkpSmSLgCioraMIWG0nALby4f27w4RUPU2fXPjDKcT8apUNQjudgNw+McuST0XsMyvE6W0lOjHKs2alcgnLsi9dHYJRjm53aJyjHt/qvBUb5H8Jgm8kDX1f1AAAAAElFTkSuQmCC', + 'base64' +); +const SQUARE_PNG = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAsklEQVR4nI3GywoBARgG0O/RLO3IQlIiNferuUSkNFm4rKbZuCwkC01KpEiShbKRd/I9wl9ncVAodanSjEhyFKtxXQ2JkRxtN6ByIyLJYcadmhISIzladkDB0CfJoYed/tgjRnIkc5f8gU+SY5Y5vcQjRnKMpi5lK5skxyR11luLGMmxy01KFzZJjsPJWG4sYiTHdm/S5aaT5MiPxuOlESM53h+VzledJMf3p9yfGjGS/wEEcYEIZzOCRwAAAABJRU5ErkJggg==', + 'base64' +); + +function clone(value) { + return JSON.parse(JSON.stringify(value)); +} + +function rawSyntheticKey() { + return { + id: 9001, + name: 'Plinx synthetic contract', + role: 'request', + is_active: true, + revoked_at: null, + allow_video_requests: true, + allow_channel_requests: false, + allow_delete_video_requests: false, + auto_approve_video_requests: false, + auto_approve_channel_requests: false, + auto_approve_delete_requests: false, + max_rating_level: 2, + allow_unrated: true, + allowed_media_types: ['video', 'short', 'livestream'], + max_active_jobs: 5, + hourly_write_limit: 30, + daily_write_limit: 200, + }; +} + +function requestPage(requests, query = {}) { + const page = Math.max(1, Number(query.page) || 1); + const pageSize = Math.min(100, Math.max(1, Number(query.pageSize) || 50)); + const filtered = query.status + ? requests.filter((request) => request.status === query.status) + : requests; + const data = filtered.slice((page - 1) * pageSize, page * pageSize); + const totalPages = filtered.length === 0 ? 0 : Math.ceil(filtered.length / pageSize); + return { + data, + pagination: { + page, + pageSize, + total: filtered.length, + totalPages, + nextCursor: page < totalPages ? `synthetic-request-page-${page + 1}` : null, + }, + }; +} + +function createExternalApiContractApp({ contract, scenario = 'normal', delayMs = 0 } = {}) { + if (!contract) throw new Error('A canonical external API contract is required'); + + const app = express(); + const initialRequests = () => { + const unknownStatusVideoRequest = { + ...clone(contract.unknownEnumSamples.request), + id: '00000000-0000-4000-8000-000000000097', + type: 'video', + target: clone(contract.requests[0].target), + }; + const completedVideoRequest = { + ...clone(contract.requests[0]), + id: '00000000-0000-4000-8000-000000000017', + status: 'completed', + createdAt: '2026-07-30T12:00:00.000Z', + updatedAt: '2026-07-30T12:05:00.000Z', + completedAt: '2026-07-30T12:05:00.000Z', + }; + return [ + clone(contract.unknownEnumSamples.request), + unknownStatusVideoRequest, + completedVideoRequest, + ...clone(contract.requests), + ]; + }; + const state = { + lastVideoRequest: null, + lastVideoQuery: null, + artworkRequests: [], + requests: initialRequests(), + scenario, + delayMs, + }; + const assetDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'youtarr-contract-assets-')); + const channelArtworkPath = path.join(assetDirectory, 'channel.png'); + fs.writeFileSync(channelArtworkPath, SQUARE_PNG); + + app.use((req, _res, next) => { + req.id = 'synthetic-contract-request'; + next(); + }); + app.use('/__contract', express.json({ limit: '4kb' })); + app.get('/__contract/ready', (_req, res) => res.json({ ready: true, scenario: state.scenario })); + app.get('/__contract/state', (req, res) => { + if (req.get('x-api-key') !== SYNTHETIC_API_KEY) return res.sendStatus(401); + return res.json(state); + }); + app.post('/__contract/scenario', (req, res) => { + if (req.get('x-api-key') !== SYNTHETIC_API_KEY) return res.sendStatus(401); + state.scenario = String(req.body.scenario || 'normal'); + state.delayMs = Math.max(0, Number(req.body.delayMs) || 0); + state.lastVideoRequest = null; + state.lastVideoQuery = null; + state.artworkRequests = []; + state.requests = initialRequests(); + return res.json({ scenario: state.scenario, delayMs: state.delayMs }); + }); + + const pause = async () => { + if (state.delayMs > 0) await new Promise((resolve) => setTimeout(resolve, state.delayMs)); + }; + const failIfRequested = () => { + if (state.scenario === 'server-error') throw new Error('synthetic server failure'); + }; + const terminalPage = (data) => ({ + ...clone(contract.catalogPage), + data: clone(data), + pagination: { + ...clone(contract.catalogPage.pagination), + total: data.length, + totalPages: data.length === 0 ? 0 : 1, + nextCursor: null, + }, + }); + const visibleCatalog = () => { + if (state.scenario === 'empty') return terminalPage([]); + if (state.scenario === 'filtered') { + return terminalPage([{ + ...clone(contract.catalogPage.data.find((video) => video.youtubeId === 'abcdefghijk')), + youtubeId: 'unsafe00001', + title: 'Locally filtered mature fixture', + rating: 'TV-MA', + }]); + } + if (state.scenario === 'malformed') return { malformed: true }; + const response = clone(contract.catalogPage); + response.data.push({ + ...clone(response.data[0]), + youtubeId: 'unsafe00001', + title: 'Locally filtered mature fixture', + rating: 'TV-MA', + }); + return response; + }; + + const catalogService = { + async listChannels() { + await pause(); + failIfRequested(); + return clone(contract.channelsPage); + }, + async listChannelVideos() { + await pause(); + failIfRequested(); + return visibleCatalog(); + }, + async listVideos(_key, query) { + await pause(); + failIfRequested(); + validatePagination(query, 100); + if (query.search && String(query.search).length > 200) { + throw new CatalogError('search must be at most 200 characters'); + } + state.lastVideoQuery = clone(query); + if (query.cursor) return clone(contract.catalogNextPage); + const response = visibleCatalog(); + if (!Array.isArray(response.data)) return response; + if (query.search) { + response.data = response.data.filter((video) => + video.title.toLowerCase().includes(String(query.search).toLowerCase()) + ); + response.pagination.total = response.data.length; + } + return response; + }, + async getVideoDetail(_key, youtubeId) { + await pause(); + failIfRequested(); + const catalogVideo = contract.catalogPage.data.find((video) => video.youtubeId === youtubeId); + if (!catalogVideo) { + const error = new Error('Video not found'); + error.name = 'CatalogError'; + error.status = 404; + throw error; + } + return { + ...clone(contract.videoDetail), + ...clone(catalogVideo), + metadata: clone(contract.videoDetail.metadata), + }; + }, + async getChannelThumbnail(_key, channelId) { + state.artworkRequests.push(`channel:${channelId}`); + return channelArtworkPath; + }, + async getVideoThumbnail(_key, youtubeId) { + state.artworkRequests.push(`video:${youtubeId}`); + return { source: 'upstream', url: 'https://synthetic.invalid/video.png' }; + }, + }; + + const requestService = { + async createVideoRequest(_key, body) { + state.lastVideoRequest = clone(body); + const request = { + id: '00000000-0000-4000-8000-000000000098', + type: 'video', + status: 'pending', + target: { youtubeId: body.youtubeId, channelId: body.channelId, channelUrl: null }, + createdAt: '2026-07-31T12:30:00.000Z', + updatedAt: '2026-07-31T12:30:00.000Z', + decidedAt: null, + completedAt: null, + message: null, + }; + state.requests.unshift(request); + return { outcome: 'created', request }; + }, + async createChannelRequest() { + return clone(contract.videoRequestResponses[1]); + }, + async createDeleteVideoRequest() { + return clone(contract.videoRequestResponses[2]); + }, + async listRequests(_key, query) { + await pause(); + failIfRequested(); + return requestPage(state.requests, query); + }, + async getRequest(_key, id) { + const request = state.requests.find((candidate) => candidate.id === id); + if (request) return clone(request); + const error = new Error('Request not found'); + error.name = 'RequestError'; + error.status = 404; + throw error; + }, + }; + + app.use('/external-api/v1', (req, res, next) => { + if (state.scenario === 'transport') { + req.socket.destroy(); + return; + } + if (state.scenario === 'unsupported' && req.path === '/capabilities') { + const sendJSON = res.json.bind(res); + res.json = (body) => sendJSON({ ...body, apiVersion: '2' }); + } + next(); + }); + + app.use('/external-api/v1', createExternalApiRoutes({ + externalApiAuth: createExternalApiAuth({ + validateApiKey: async (value) => + state.scenario !== 'unauthorized' && value === SYNTHETIC_API_KEY ? rawSyntheticKey() : null, + }), + externalApiLimiter: (_req, _res, next) => next(), + serverVersion: contract.capabilities.serverVersion, + catalogService, + thumbnailProxy: { + fetchExternalThumbnail: async () => ({ body: LANDSCAPE_PNG, contentType: 'image/png' }), + }, + externalWorkLimiter: { run: (operation) => operation() }, + requestService, + quotaService: { + status: async () => ({ + limits: { maxActiveJobs: 5, hourlyWriteLimit: 30, dailyWriteLimit: 200 }, + remaining: { activeJobs: 5, hourlyWrites: 30, dailyWrites: 200 }, + }), + }, + })); + + return { app, state }; +} + +module.exports = { + SYNTHETIC_API_KEY, + createExternalApiContractApp, +}; From 0082558098809aa581fbed4c6ae1a0f0a8377eeb Mon Sep 17 00:00:00 2001 From: bballdavis Date: Tue, 8 Sep 2026 16:59:09 -0500 Subject: [PATCH 02/27] fix: make external API opt-in and verify MariaDB migrations --- .env.example | 4 +- docker-compose.dev.yml | 2 +- docker-compose.external-db.yml | 4 +- docker-compose.yml | 4 +- docs/ENVIRONMENT_VARIABLES.md | 7 +- docs/EXTERNAL_API.md | 15 +- docs/PRD_EXTERNAL_REQUEST_API.md | 766 ------------------ .../__tests__/externalApiConfig.test.js | 37 + server/modules/externalApiConfig.js | 9 + server/routes/index.js | 8 +- 10 files changed, 70 insertions(+), 786 deletions(-) delete mode 100644 docs/PRD_EXTERNAL_REQUEST_API.md create mode 100644 server/modules/__tests__/externalApiConfig.test.js create mode 100644 server/modules/externalApiConfig.js diff --git a/.env.example b/.env.example index b9b78d59e..103e2725b 100644 --- a/.env.example +++ b/.env.example @@ -79,8 +79,8 @@ YOUTUBE_OUTPUT_DIR=./downloads AUTH_ENABLED=true # Versioned external API. This API always requires an external API key even -# when AUTH_ENABLED=false. Set to false to disable the API namespace. -EXTERNAL_API_ENABLED=true +# when AUTH_ENABLED=false. It is disabled by default; set to true to enable it. +EXTERNAL_API_ENABLED=false # Initial Admin Credentials (for headless/automated deployments) # If not set, you'll create credentials through the web UI on first access diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 0c37179f3..18f6d855f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -49,7 +49,7 @@ services: - DB_PASSWORD=${DB_PASSWORD:-youtarrpassword} - YOUTUBE_OUTPUT_DIR=/usr/src/app/data - AUTH_ENABLED=${AUTH_ENABLED:-true} - - EXTERNAL_API_ENABLED=${EXTERNAL_API_ENABLED:-true} + - EXTERNAL_API_ENABLED=${EXTERNAL_API_ENABLED:-false} - AUTH_PRESET_USERNAME=${AUTH_PRESET_USERNAME:-} - AUTH_PRESET_PASSWORD=${AUTH_PRESET_PASSWORD:-} - TRUST_PROXY=${TRUST_PROXY:-} diff --git a/docker-compose.external-db.yml b/docker-compose.external-db.yml index 497aae2eb..c179bea1a 100644 --- a/docker-compose.external-db.yml +++ b/docker-compose.external-db.yml @@ -39,8 +39,8 @@ services: DB_NAME: ${DB_NAME:-youtarr} # Optional: Disable authentication (for platforms with external auth like Elfhosted) AUTH_ENABLED: ${AUTH_ENABLED:-} - # Optional: Disable the separately authenticated versioned external API - EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-true} + # Optional: Enable the separately authenticated versioned external API + EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-false} # Optional: Seed initial admin credentials for headless deployments AUTH_PRESET_USERNAME: ${AUTH_PRESET_USERNAME:-} AUTH_PRESET_PASSWORD: ${AUTH_PRESET_PASSWORD:-} diff --git a/docker-compose.yml b/docker-compose.yml index ed8e10910..e330a2138 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -67,8 +67,8 @@ services: DB_NAME: ${DB_NAME:-youtarr} # Optional: Disable authentication (for platforms with external auth like Elfhosted) AUTH_ENABLED: ${AUTH_ENABLED:-} - # Optional: Disable the separately authenticated versioned external API - EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-true} + # Optional: Enable the separately authenticated versioned external API + EXTERNAL_API_ENABLED: ${EXTERNAL_API_ENABLED:-false} # Optional: Seed initial admin credentials for headless deployments AUTH_PRESET_USERNAME: ${AUTH_PRESET_USERNAME:-} AUTH_PRESET_PASSWORD: ${AUTH_PRESET_PASSWORD:-} diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 0d329f600..640c8910b 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -101,12 +101,13 @@ To use an external database: ### EXTERNAL_API_ENABLED **Required**: No -**Default**: `true` +**Default**: `false` **Options**: `true`, `false` -**Description**: Enables the versioned `/external-api/v1` integration API by default +**Description**: Enables the versioned `/external-api/v1` integration API when set to `true` **Security**: External API routes always require a non-legacy `x-api-key`, even when `AUTH_ENABLED=false` -Set this to `false` only when the versioned API namespace should be disabled. +Set this to `true` only after the external trust boundary, key policy, and +reverse-proxy rules have been reviewed. The namespace is disabled by default. The routes remain protected until an administrator has assigned an API key an external role and policy. Existing API keys migrate as `legacy_download` and cannot use the versioned API. For endpoint-only public access, keep the Youtarr diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md index 995e06312..272a4c815 100644 --- a/docs/EXTERNAL_API.md +++ b/docs/EXTERNAL_API.md @@ -1,16 +1,16 @@ # Youtarr External API `/external-api/v1` is Youtarr's constrained integration boundary for approved -clients. It is enabled by default and never accepts a Youtarr browser session -or a legacy download key. Set `EXTERNAL_API_ENABLED=false` to disable the -namespace for a deployment. +clients. It is disabled by default and never accepts a Youtarr browser session +or a legacy download key. Set `EXTERNAL_API_ENABLED=true` only after the +external trust boundary and proxy rules have been reviewed. ## Enablement and authentication Create an external-role key in **Settings → API Keys → API Keys & External Access**. The key is revealed once. Send it only in the `x-api-key` header over -HTTPS. Set `EXTERNAL_API_ENABLED=false` and restart Youtarr if the namespace -should be disabled. +HTTPS. Set `EXTERNAL_API_ENABLED=true` and restart Youtarr to enable the +namespace; leave it unset or set to `false` to keep it disabled. Existing keys migrate as `legacy_download`. They continue to work only with `POST /api/videos/download`. Conversely, external-role keys cannot use that @@ -363,8 +363,9 @@ The proxy must: Start with auto-approval disabled, a database and configuration backup, and a view-only key with the minimum channel grants. Confirm the private Youtarr application and proxy rules before exposing the service. Revoke a compromised -key immediately; to shut down the whole namespace, set -`EXTERNAL_API_ENABLED=false` and restart Youtarr. +key immediately. Keep the namespace disabled unless the external trust boundary +and proxy rules have been reviewed. To enable it, set +`EXTERNAL_API_ENABLED=true` and restart Youtarr. Each key defaults to at most 5 active jobs, 30 accepted writes per UTC hour, and 200 per UTC day. Administrators may select lower limits. A shared diff --git a/docs/PRD_EXTERNAL_REQUEST_API.md b/docs/PRD_EXTERNAL_REQUEST_API.md deleted file mode 100644 index 54a4ced22..000000000 --- a/docs/PRD_EXTERNAL_REQUEST_API.md +++ /dev/null @@ -1,766 +0,0 @@ -# PRD: Youtarr External API and Request Management - -## Document Status - -- Status: Draft for maintainer review -- Target project: Youtarr -- Last updated: 2026-07-29 -- Audience: Any external client integrating with a Youtarr instance -- Primary use case: Provide a stable, documented, API-key-protected Youtarr API for browsing eligible content and submitting approval-backed requests. - -This API is a Youtarr product capability. Its contract, authorization model, documentation, and delivery plan must remain client-agnostic and must not depend on any one downstream application. - -## Background - -Youtarr already supports: - -- session-authenticated web UI access -- API key creation in the **API Keys & External Access** settings section -- a REST API with Swagger/OpenAPI docs -- a limited API-key workflow for `POST /api/videos/download` - -Today, API keys are intentionally restricted to a single endpoint that queues individual video downloads. That is a good security baseline, but it is too narrow for integrations that need to: - -- show available channels -- show videos within those channels, including thumbnails and download state -- request videos to be downloaded -- optionally request new channels -- operate without exposing the full Youtarr web UI - -This PRD defines a safety-first expansion of Youtarr so it can support general-purpose external integrations while keeping the main app protected behind separate middleware if desired. - -## Current Implementation Status - -As of 2026-07-29: - -- The upstream API enhancement is tracked in issue `#568`; no implementation PR is open. -- The current `api-expansion` branch is a design branch. Relative to upstream `dev`, it adds this PRD and clarifies the authenticated thumbnail/asset contract; it does not contain the broader API implementation. -- Existing API keys, their management UI, and the legacy `POST /api/videos/download` flow provide the baseline to evolve. -- The dedicated `/external-api/v1` namespace, expanded roles and policies, channel grants, catalog endpoints, durable request lifecycle, administrator queue, and operational hardening remain to be implemented. - -Implementation therefore begins with the foundation phase below. Later phases must not be described as complete until their individual PRs are merged and their acceptance checks pass. - -## Problem Statement - -Instance owners and integration authors need a limited, API-key-protected surface of Youtarr without exposing the full admin UI. External clients need read access to eligible content and limited request access, while instance owners need: - -- strict API-key enforcement -- simple reverse-proxy isolation -- approval controls for risky actions -- auditing and revocation -- predictable paging and filtering for very large channels - -## Goals - -1. Add a dedicated external API namespace that can be exposed separately from the main app. -2. Require API-key authentication for every external API request. -3. Support read access to channels and channel videos for external clients. -4. Support request-based workflows for downloading videos and optionally adding channels. -5. Add role/scoped API keys with approval-related behavior. -6. Add an in-app request queue for approving or rejecting outstanding requests. -7. Document safe reverse-proxy deployment patterns clearly. -8. Break implementation into manageable PR-sized phases. -9. Keep the public contract generic so browser extensions, native apps, automations, dashboards, and other third-party clients can integrate without consumer-specific behavior. - -## Non-Goals - -- Replacing session auth for the main Youtarr web UI in this implementation program -- Building multi-user household accounts -- Exposing every existing Youtarr endpoint to API keys -- Full external CRUD over all settings and channel management -- Solving public-internet hardening beyond Youtarr's documented safe deployment patterns -- Adding behavior, fields, or terminology that exists only for one named downstream client - -## Users and Personas - -### Instance Admin - -The Youtarr owner who manages channels, downloads, settings, and approvals. - -### External Viewer - -A client/device allowed to browse approved channels and videos only. - -### External Requester - -A client/device allowed to browse approved content and submit requests for download or new channels/videos, subject to auto-approval or admin review. - -### External Admin - -A future phase role that can use a constrained web UI backed by API keys instead of session auth. - -## Product Principles - -### Safety First - -External access must default to least privilege. The new API should be easy to place behind a separate reverse-proxy path rule such as `/external-api/*`, while the rest of Youtarr remains behind stronger middleware or private-only access. - -### Explicit Separation - -External API routes should live under their own prefix instead of reusing the mixed existing route shape. This simplifies: - -- reverse-proxy rules -- auth policy -- logging -- documentation -- future rate-limiting and abuse controls - -### Requests, Not Direct Power - -Risky actions should be modeled as requests first unless a key's policy explicitly allows auto-approval. - -### Client-Agnostic Contract - -The external API exposes Youtarr resources and workflows, not consumer-specific screens or recommendation logic. Examples may illustrate how a client could use the API, but no named integration defines the contract. - -## Proposed Solution Overview - -Add a new external API namespace: - -- Proposed prefix: `/external-api/v1` - -All routes under this prefix: - -- require `x-api-key` -- reject session-token-only access -- are governed by API key scopes/role policies -- are documented separately in Swagger/OpenAPI - -Program capabilities: - -1. Read channel list -2. Read channel video lists with paging and filters -3. Submit video download requests -4. Submit new channel requests -5. Submit approval-backed downloaded-video deletion requests -6. Read a bounded, policy-filtered cross-channel video feed -7. Review and process pending requests in a new Requests section in the main web UI - -## Authentication and Authorization - -### External API Authentication - -- All `/external-api/v1/*` endpoints require `x-api-key`. -- Requests without a valid API key return `401`. -- Session tokens (`x-access-token`) do not grant access to this namespace. -- No external endpoint is anonymous. - -### API Key Model - -Expand the existing API key model to support role/scope metadata. - -#### Access levels - -- `view`: read-only access to allowed external browse endpoints -- `request`: includes `view` plus ability to create requests -- `delete`: includes `request` plus delete requests where allowed by policy -- `admin`: the broadest external API policy scope; it does not grant session administration or request-review authority - -#### Key policy flags - -- `autoApproveVideoRequests` -- `autoApproveDeleteRequests` -- `autoApproveChannelRequests` - -These flags should only be available where they make sense for the selected access level. - -#### Notes - -- Existing keys should migrate safely with a legacy-compatible default, likely equivalent to a narrowly scoped request key for the current single-video endpoint. -- Keys remain hashed at rest. -- Keys remain one-time-view on creation. -- Keys should display metadata such as scope, last used, usage count, and created date. - -## External API Surface - -The exact path names can change during tech design, but the namespace and capabilities should remain stable. - -### 1. Channels List - -Purpose: allow external clients to display approved channels. - -Example: - -- `GET /external-api/v1/channels` - -Suggested query support: - -- `page` -- `pageSize` -- `search` -- `subfolder` -- `sortBy` -- `sortOrder` - -Suggested response fields: - -- `id` -- `channelId` -- `title` -- `thumbnailUrl` -- `description` or truncated summary -- `subfolder` -- `videoCount` -- `downloadedCount` -- `lastFetchedAt` - -Notes: - -- Reuse existing internal channel pagination semantics where possible. -- Return enough metadata for a client to render a browsable channel list without needing extra follow-up calls for basic display. -- Thumbnail references must be usable by authenticated external clients without requiring a Youtarr session cookie. - -### 2. Channel Videos - -Purpose: allow external clients to browse a channel's videos, thumbnails, metadata, and download state. - -Example: - -- `GET /external-api/v1/channels/:channelId/videos` - -Suggested query support: - -- `page` -- `pageSize` -- `offset` -- `limit` -- `status=all|downloaded|not_downloaded|requested|pending` -- `search` -- `sortBy` -- `sortOrder` -- `tabType=videos|shorts|streams` -- `dateFrom` -- `dateTo` -- `minDuration` -- `maxDuration` -- `maxRating` - -Suggested response fields: - -- `youtubeId` -- `title` -- `thumbnailUrl` -- `publishedAt` -- `duration` -- `description` -- `isDownloaded` -- `isRequested` -- `requestStatus` -- `rating` -- `channelId` -- `channelTitle` - -#### Thumbnail and media asset delivery - -This needs to be explicit because external clients may not be logged into the main Youtarr web UI. - -Requirements: - -- Thumbnail and related image assets exposed to external clients must be retrievable with API-key-based access, not only browser session auth. -- The external API should not return asset URLs that silently depend on existing Youtarr login cookies. -- The asset delivery contract must work for native clients that can attach an API key to each request. - -Recommended approach: - -- Return external-API-scoped asset URLs such as `/external-api/v1/assets/...` or `/external-api/v1/thumbnails/...`. -- Protect those asset endpoints with the same `x-api-key` requirement as the rest of the external namespace. -- Keep those asset URLs stable enough for clients to render list views without custom scraping or cookie-sharing. - -Compatibility note: - -- If some thumbnail values currently point directly to YouTube-hosted images, that is acceptable as a fallback for non-sensitive public imagery. -- If Youtarr serves locally cached thumbnails, posters, or protected assets, the PRD should assume those must be reachable through authenticated external asset endpoints. - -#### Paging requirements - -This area is critical because some channels have thousands of videos. - -Requirements: - -- Support deterministic paging over stable, explicit sort orders. -- Prefer `page + pageSize` as the public contract for consistency with current Youtarr behavior. -- Also consider `offset + limit` support if that improves client UX for "videos 30-40" style access. -- Response must include paging metadata: -- `page` -- `pageSize` -- `total` -- `totalPages` -- optionally `offset` and `limit` if supported - -#### Data sourcing behavior - -The implementation should not require a full fresh YouTube fetch for every external browse request. - -Expected behavior: - -- External browse endpoints primarily serve from Youtarr's cached/database-backed channel video records. -- Existing channel refresh behavior remains separate from external browsing. -- If the channel has not yet been fully indexed, the API should return a truthful partial-state indicator rather than blocking for a long-running scrape. - -Potential response metadata: - -- `dataSource=cache|partial_cache` -- `isFullyIndexed` -- `lastIndexedAt` -- `indexingHint` - -This keeps response latency predictable and avoids coupling player UX to yt-dlp fetch timing. - -### 3. Video Download Request - -Purpose: allow external clients to request a video download. - -Example: - -- `POST /external-api/v1/requests/videos` - -Request body: - -- `youtubeId` or canonical YouTube URL -- optional `channelId` -- optional client metadata - -Behavior: - -- If the key policy allows auto-approval, create and execute the download immediately. -- Otherwise, create a pending request record for approval. -- Duplicate requests should be idempotent or return a clear "already requested/downloaded" response. - -Response should indicate: - -- `status=approved|pending|duplicate|already_downloaded|rejected` -- `requestId` -- `message` - -### 4. Channel Request - -Purpose: allow external clients to request that a new channel be added. - -Example: - -- `POST /external-api/v1/requests/channels` - -Request body: - -- channel URL - -Behavior: - -- validate URL -- resolve channel metadata when practical -- create pending or auto-approved request based on key policy - -This supports the future simplified requester UI and direct client integrations. - -### 5. Request Status - -Purpose: allow external clients to check the status of their requests. - -Examples: - -- `GET /external-api/v1/requests` -- `GET /external-api/v1/requests/:requestId` - -Suggested filters: - -- `status=pending|approved|rejected|completed` -- `type=video|channel|delete` - -This is especially useful for mobile apps that need to reflect whether a request is still waiting on admin approval. - -### 6. Downloaded-Video Deletion Request - -Purpose: allow an authorized external client to request deletion of an already-downloaded video asset. - -Example: - -- `POST /external-api/v1/requests/delete-videos` - -Behavior: - -- always use the durable request lifecycle -- never directly delete through the external API -- revalidate role, grants, content policy, and downloaded state before approval -- never remove a channel subscription -- complete idempotently when the asset is already absent - -### 7. Cross-Channel Video Feed - -Purpose: allow external clients to discover eligible videos across their granted channels without defining recommendation behavior inside Youtarr. - -Example: - -- `GET /external-api/v1/videos` - -Behavior: - -- use the same grant, channel-state, rating, and media-type policy as all other catalog endpoints -- return stable, bounded pages from cached Youtarr data -- expose generic candidate records only; do not accept client profiles, playback history, or ranking signals - -## Request Approval Workflow - -Add a new Requests area to the main Youtarr web UI for session-authenticated admins. - -### UI requirements - -- show pending requests -- show approved/rejected/completed history as useful follow-up scope -- filter by request type and status -- display requester key name/prefix for auditing -- approve or reject individual requests -- bulk approve/reject may be added later, but not required for initial delivery - -### Initial request types - -- video download request -- channel add request - -### Later request type in this implementation program - -- delete downloaded video - -### Potential future request types - -- delete channel -- refresh channel - -## Configuration UX Changes - -Extend the existing **API Keys & External Access** section instead of creating a parallel configuration area. - -### Additions - -- access level selector: `view`, `request`, `delete`, `admin` -- auto-approve toggles appropriate to the role -- clearer description of what each access level can do -- visibility into rate limits, last used, and request policy -- migration/labeling for legacy keys - -### Recommended UX constraints - -- `view` cannot create requests -- `request` can create requests but cannot directly delete content -- `delete` only enables delete request flows, not unrestricted deletion -- `admin` should be clearly labeled as high-risk - -## Security Requirements - -### Namespace Separation - -The external API must have its own top-level prefix, recommended as `/external-api/v1`, to enable safe reverse-proxy handling. - -### Mandatory API-Key Protection - -- Every endpoint under the external prefix requires a valid API key. -- No endpoint under the external prefix is accessible with no auth. -- No fallback to session auth on the external prefix. - -### Reverse Proxy Guidance - -Documentation must include safe examples for: - -- Traefik -- Caddy -- Nginx - -Recommended pattern: - -- protect the main Youtarr UI with stronger middleware, private network access, or SSO -- optionally expose only `/external-api/*` -- keep Swagger/docs for the external API behind admin access unless intentionally exposed - -### Logging and Auditing - -Log for each external request: - -- API key id/prefix/name -- path -- method -- result -- request id where applicable - -### Rate Limiting - -The foundation PR should keep per-key rate limiting and extend it to the new external namespace. - -Consider: - -- different limits for browse vs write endpoints -- lower limits for request-creation endpoints - -### Error Handling - -Use clear API responses for: - -- invalid key -- insufficient scope -- duplicate request -- already downloaded -- resource not found -- approval required -- rate limited - -## Future: API-Key-Backed Simplified Web UI - -This is intentionally outside the initial implementation program and must not block the external API. - -Goal: allow API keys to authenticate into a simplified Youtarr web experience. - -### Role-based UI behavior - -- `view`: videos-only UI, browse existing videos and metadata -- `request`: channel list + channel detail pages + ability to request videos and request new channels -- `admin`: near-full UI access - -### Restrictions - -- request users should not gain access to settings -- request users should not gain ignore/delete/admin controls unless explicitly authorized -- UI should be capability-driven from server-provided key scope, not hardcoded only in the client - -This work should be treated as a separate PRD or technical design. - -## Functional Requirements Summary - -1. Youtarr must expose a dedicated external API prefix. -2. All external API endpoints must require API keys. -3. The API key model must support access levels and approval-related policy flags. -4. External clients must be able to list channels. -5. External clients must be able to list channel videos with paging, filters, download-state metadata, and usable authenticated thumbnail references. -6. External clients must be able to create video download requests. -7. External clients must be able to create channel-add requests. -8. Authorized external clients must be able to create approval-backed downloaded-video deletion requests. -9. External clients must be able to read a bounded, policy-filtered cross-channel video feed without consumer-specific ranking behavior. -10. Admins must be able to approve or reject pending requests in the web UI. -11. Documentation must explain safe reverse-proxy deployment. -12. Existing API key behavior must remain backward compatible during migration. - -## Success Metrics - -- Admin can safely expose only the external API via reverse proxy. -- External client can render channels and videos without session auth. -- External client can fetch thumbnail/image assets needed for those views using API-key-protected routes. -- External requester can submit a video request that appears in the approval queue. -- Admin can approve the request and the download proceeds normally. -- No external endpoint is accessible without an API key. -- Existing bookmarklet/single-video flows continue to work. - -## Sequential Implementation Strategy - -Implementation will use three sequential, independently reviewable PRs. PRs will not be stacked: after each merge, the next PR will be rebased onto the updated upstream branch and its focused and regression checks will be rerun. - -Documentation and tests ship with the behavior they describe. Each PR must be independently testable at its own boundary, and later PRs must not be required to prove an earlier PR safe. - -### PR 1: Preparation, Data Model, and Migrations - -**Purpose** - -Create the persistence and configuration foundation required by the API without exposing new runtime routes. - -**Includes** - -- extend `ApiKeys` with role, policy, media, auto-approval, revocation, and legacy-compatibility fields -- migrate existing keys to a `legacy_download` scope that preserves the current `POST /api/videos/download` behavior -- add `ApiKeyChannelGrants` with unique key/channel membership and indexed foreign keys -- add `ExternalRequests` with opaque ID, owner, type, status, idempotency/deduplication data, nullable video/channel targets, persisted grant decision, job linkage, messages, and timestamps -- add indexes for owner/status reads, management filters, active-target deduplication, and bounded catalog queries -- add Sequelize models, associations, shared enums/constants, and validation primitives -- add `EXTERNAL_API_ENABLED=false` configuration and environment documentation -- document upgrade, rollback, re-application, data-retention, and destructive rollback implications - -**Excludes** - -- `/external-api/v1` routes -- external authentication and authorization middleware -- request execution or approval behavior -- API-key policy management endpoints -- administrator GUI - -**Required test coverage** - -- migration unit tests for every `up` and `down` path, including missing/existing table and column behavior -- isolated MariaDB integration tests for fresh install, pre-feature upgrade, current-schema restart, idempotent re-run, rollback, and re-application -- model and association Jest tests for defaults, enum validation, foreign keys, uniqueness, ownership, and timestamps -- API-key module regression tests proving existing key creation, hashing, one-time reveal, usage tracking, and legacy downloads remain unchanged -- negative tests for invalid roles, request types/statuses, malformed policy data, duplicate grants, duplicate active requests, and orphaned references -- rollback tests proving request/grant rows are removed only as documented and that database rollback does not claim to restore deleted media - -**Acceptance** - -- all migration paths pass against a real supported database, not only mocked Sequelize calls -- existing installations and existing API keys upgrade without losing access to the legacy endpoint -- the application can start safely with the new schema while the external namespace remains disabled -- no new public or management API behavior is reachable - -### PR 2: External API Surface and Backend Control Plane - -**Purpose** - -Deliver the complete headless integration contract and all session-authenticated backend controls. After this PR, the feature must be fully usable and testable through an ordinary API client without the new GUI. - -**Includes** - -- disabled-by-default `/external-api/v1` namespace and normalized error envelope -- API-key-only authentication that rejects missing, invalid, revoked, legacy-only, and session-only credentials -- cumulative role checks, per-key channel grants, rating/media policy, auto-approval policy, and separate read/write rate limits -- capabilities, granted-channel, channel-video, cross-channel-video, and authenticated asset endpoints -- owner-scoped video, channel, and downloaded-video deletion request creation/list/detail endpoints -- idempotency, concurrent deduplication, reconciliation, bounded stale-work recovery, and terminal-state handling -- approval-time revalidation and integration with existing downloader, channel provisioning, and downloaded-video deletion operations -- session-authenticated management endpoints for listing/detailing/approving/rejecting requests -- session-authenticated API-key endpoints for transactional role, policy, auto-approval, and grant management -- immediate key revocation and namespace-wide feature shutdown -- OpenAPI contract, operator documentation, proxy examples, migration/rollback runbook, legacy deprecation guidance, correlation IDs, and redacted structured logging - -**Excludes** - -- Requests page -- API-key policy/grant editor UI -- external-client-specific recommendation behavior -- API-key access to session administration or request-review authority - -**Required test coverage** - -- route-level Jest/Supertest coverage for every external and management endpoint, success response, validation error, and normalized error envelope -- authentication matrix covering missing, malformed, invalid, revoked, legacy, insufficient-role, ungranted, session-only, and mixed session/API-key credentials -- authorization matrix covering roles, grants, enabled/terminated channels, rating ceilings, unrated behavior, media types, direct IDs, counts, lists, assets, candidates, request creation, and approval revalidation -- pagination/filter tests for stable ordering, tie-breakers, maximum page size, empty pages, invalid parameters, and large channels -- asset tests covering public thumbnails, authenticated local assets, traversal, symlinks, redirects, missing files, unauthorized resources, and secret-free URLs -- request lifecycle Jest tests for pending, auto-approved, processing, completed, rejected, failed, cancelled, already-downloaded, missing-output, and already-deleted behavior -- idempotency and concurrency tests proving duplicate submissions cannot create multiple active requests -- recovery tests for interrupted channel and delete work, stale claims, repeated approval, and terminal-state reconciliation -- integration tests proving video requests reach the downloader, channel approval provisions/enables the canonical channel, and delete approval removes only the downloaded asset -- management API tests proving API keys cannot review requests or manage keys even when a browser session is also present -- rate-limit, feature-off `404`, immediate-revocation, log-redaction, correlation-ID, proxy allow/deny, and legacy-endpoint regression tests -- performance tests with thousands of cached videos covering query plans, bounded pages, timing, and memory -- OpenAPI validation proving every published endpoint, schema, status, security requirement, and example matches runtime behavior - -**Acceptance** - -- the complete contract works without the GUI through documented API calls -- every externally visible resource applies one shared authorization and eligibility policy -- all request types deterministically reach a correct terminal or recoverable state -- the external namespace can be shut down without exposing the main UI or other private routes -- existing session routes and the legacy download endpoint retain regression coverage - -### PR 3: Administrator GUI - -**Purpose** - -Add the Youtarr administrator experience on top of the stable management API from PR 2 without introducing new backend contract or authorization behavior. - -**Includes** - -- responsive, session-only Requests page with filters, history, details, current-state warnings, approve/reject actions, bounded rejection reasons, requester identity, target/job state, and audit information -- desktop table and tablet/mobile card presentations without compressed columns or horizontal overflow -- video/delete thumbnails and established video-chip treatments -- channel identity, thumbnail/handle, and concise canonical target presentation -- empty, loading, error, retry, long-content, stale-state, media-failure, and action-in-progress states -- role, human-readable rating, media-type, auto-approval, and channel-grant controls in the existing API Keys & External Access section -- transactional create/edit flows, privilege-increase confirmation, one-time secret reveal, and immediate-revocation guidance -- Requests routing/navigation and direct links between request review and API-key management -- administrator/user documentation and responsive screenshots - -**Excludes** - -- migrations -- new backend endpoints -- new authorization decisions or request-state transitions -- external-client UI - -**Required Jest coverage** - -- request-list rendering, paging, filters, sorting, empty/loading/error/retry states, and response-shape validation -- request-detail rendering for video, channel, and delete requests -- approve/reject flows, confirmation, bounded rejection reason, optimistic/in-progress behavior, API failures, stale-state conflicts, and post-action refresh -- role/policy/grant creation and editing, validation, privilege-increase confirmation, transactional failures, secret reveal/copy guidance, and revocation -- route and navigation coverage for `/requests`, direct links, browser back/forward behavior, and protected session-only rendering -- responsive rendering assertions for desktop table and mobile card selection -- keyboard navigation, focus placement/restoration, dialog behavior, screen-reader names, reduced-motion behavior, and touch-target semantics -- regression coverage for the existing bookmarklet, legacy key display, Configuration/Settings routes, navigation shell, and supported themes - -**Required Storybook coverage** - -- stories for the Requests page and its independently meaningful table, card, detail, and action-dialog components -- desktop, tablet, and mobile viewports -- every request type and lifecycle status -- empty, loading, error, retry, stale, long-content, media-failure, and action-in-progress states -- API-key policy cards/editor states, including least privilege, elevated privilege, validation errors, and revoked keys -- light/dark and supported theme-token coverage -- `play` interactions for filtering, opening details, approving, rejecting, retrying, editing policy/grants, confirmation, and revocation -- every critical story added to the existing Jest Storybook parity suite through `runStoryWithPlay` -- successful static Storybook build -- CI-enforced accessibility assertions; merely loading the a11y addon without a failing check is not sufficient - -**Acceptance** - -- backend behavior from PR 2 is consumed without contract changes -- desktop and mobile workflows complete without overflow or inaccessible controls -- Jest component/interaction tests and Storybook interaction coverage pass for every critical state -- keyboard, screen-reader, focus, contrast, reduced-motion, and responsive touch-target checks pass - -## Test Strategy and Merge Gates - -Test ownership follows the implementation: - -- PR 1 owns schema, migration, model, and legacy compatibility tests. -- PR 2 owns backend module, route, authorization, lifecycle, integration, performance, OpenAPI, operational, and proxy-isolation tests. -- PR 3 owns frontend Jest, Storybook, responsive, interaction, and accessibility tests. - -The repository currently provides backend Jest, frontend Jest, a static Storybook build, and Jest-driven Storybook `play` coverage. The implementation must extend those real harnesses rather than creating parallel one-off test systems. - -At minimum, every PR must pass the checks relevant to its diff: - -```text -npm run lint -npm run lint:ts -npm run test:backend -npm run test:frontend -npm run test:coverage -cd client && npm run build -cd client && npm run build-storybook -cd client && npm test -- src/tests/storybook_coverage.test.js --passWithNoTests -``` - -PR 1 must additionally run the isolated real-database migration matrix. PR 2 must additionally run deterministic request-worker, proxy allow/deny, large-catalog performance, and API contract validation suites. PR 3 must additionally run the blocking accessibility and responsive browser validation introduced for the feature. - -Coverage percentage alone is not sufficient. Each requirement and state transition must map to at least one named test, and every defect found during implementation or review must receive a regression test at the lowest useful layer. - -## Definition of Done for Every PR - -Each PR must: - -- state its purpose, included behavior, and explicit exclusions -- update the authoritative API contract and the relevant operator or administrator guide -- describe API, data, authorization, migration, rollback, and compatibility implications -- include focused acceptance tests plus regression coverage for existing behavior -- include responsive screenshots or Storybook states when UI changes -- report only the phase that is actually complete -- identify dependent follow-ups and known limitations -- remain small enough to review as one cohesive ownership seam - -## Open Questions - -1. Should the public external browse contract use `page/pageSize` only, or support both `page/pageSize` and `offset/limit`? -2. Should a `request` key be able to auto-approve downloads only for videos belonging to already-subscribed channels? -3. Should channel requests always require approval even when video requests can auto-approve? -4. Are there additional safeguards required beyond the approval-backed, downloaded-asset-only deletion model defined here? -5. Should external clients be allowed to browse only certain subfolders/libraries per key in a later phase? -6. Should Swagger expose external endpoints publicly, or only when the main admin UI is accessible? -7. Should the role currently named `admin` be renamed before implementation to avoid implying external request-review authority? - -## Recommendations - -- Use `/external-api/v1` as the dedicated prefix. -- Deliver the API through the sequential Youtarr PR program above; do not bundle consumer integrations into those reviews. -- Treat destructive operations as approval-backed requests, not direct actions. -- Reuse existing channel/video pagination and filtering logic where possible, but define a cleaner external contract. -- Make documentation part of the feature, not follow-up work. -- Evaluate every endpoint and response field as a reusable Youtarr capability rather than against one consuming application. - -## References - -- Existing Youtarr API key docs currently describe API keys as limited to single-video downloads. -- Existing Youtarr channel/video routes already support paging and filters for session-authenticated use. -- Existing **API Keys & External Access** settings UX is the right place to extend key capabilities. diff --git a/server/modules/__tests__/externalApiConfig.test.js b/server/modules/__tests__/externalApiConfig.test.js new file mode 100644 index 000000000..02c25caf4 --- /dev/null +++ b/server/modules/__tests__/externalApiConfig.test.js @@ -0,0 +1,37 @@ +'use strict'; + +const { isExternalApiEnabled } = require('../externalApiConfig'); + +describe('isExternalApiEnabled', () => { + const originalValue = process.env.EXTERNAL_API_ENABLED; + + afterEach(() => { + if (originalValue === undefined) { + delete process.env.EXTERNAL_API_ENABLED; + } else { + process.env.EXTERNAL_API_ENABLED = originalValue; + } + }); + + test.each([undefined, null, '', 'false', 'FALSE', '0', 'yes'])( + 'fails closed for %p', + (value) => { + expect(isExternalApiEnabled(value)).toBe(false); + } + ); + + test.each(['true', 'TRUE', ' true '])( + 'requires an explicit true value for %p', + (value) => { + expect(isExternalApiEnabled(value)).toBe(true); + } + ); + + test('reads the process environment when no value is supplied', () => { + delete process.env.EXTERNAL_API_ENABLED; + expect(isExternalApiEnabled()).toBe(false); + + process.env.EXTERNAL_API_ENABLED = 'true'; + expect(isExternalApiEnabled()).toBe(true); + }); +}); diff --git a/server/modules/externalApiConfig.js b/server/modules/externalApiConfig.js new file mode 100644 index 000000000..1b79b49f5 --- /dev/null +++ b/server/modules/externalApiConfig.js @@ -0,0 +1,9 @@ +'use strict'; + +function isExternalApiEnabled(value = process.env.EXTERNAL_API_ENABLED) { + return String(value ?? '').trim().toLowerCase() === 'true'; +} + +module.exports = { + isExternalApiEnabled, +}; diff --git a/server/routes/index.js b/server/routes/index.js index 8fc6c2e73..c70867f29 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -51,6 +51,7 @@ const externalThumbnailProxy = require('../modules/externalThumbnailProxy'); const { sharedExternalWorkLimiter } = require('../modules/externalWorkLimiter'); const { createExternalRequestService } = require('../modules/externalRequestService'); const { createExternalQuotaService } = require('../modules/externalQuotaService'); +const { isExternalApiEnabled } = require('../modules/externalApiConfig'); /** * Registers all route modules with the Express app @@ -171,9 +172,10 @@ function registerRoutes(app, deps) { // Log file download app.use(createLogRoutes({ verifyToken, logFilesModule, configModule })); - // The versioned external API is available by default. Keep an explicit - // false opt-out for deployments that do not want to expose the namespace. - if (process.env.EXTERNAL_API_ENABLED !== 'false') { + + // The versioned external API stays unreachable until a deployment opts in + // explicitly. This fail-closed default is independent of AUTH_ENABLED. + if (isExternalApiEnabled()) { app.use('/external-api/v1', createExternalApiRoutes({ externalApiAuth, externalApiIngressLimiter, From 89a85643bc742873a705bb784aa74a7b6846a39c Mon Sep 17 00:00:00 2001 From: bballdavis Date: Tue, 8 Sep 2026 18:35:25 -0500 Subject: [PATCH 03/27] fix: close config watcher after fixture validation --- server/modules/__tests__/externalContractFixtures.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/modules/__tests__/externalContractFixtures.test.js b/server/modules/__tests__/externalContractFixtures.test.js index e7aa0832d..1b05cfdc7 100644 --- a/server/modules/__tests__/externalContractFixtures.test.js +++ b/server/modules/__tests__/externalContractFixtures.test.js @@ -3,6 +3,7 @@ const path = require('path'); const crypto = require('crypto'); const { externalErrorBody } = require('../externalApiResponse'); const { dto } = require('../externalRequestService'); +const configModule = require('../configModule'); const fixtureDirectory = path.join( __dirname, @@ -13,6 +14,10 @@ const canonicalFixtureBytes = Buffer.from(fixtureBytes.toString('utf8').replace( const fixture = JSON.parse(fixtureBytes.toString('utf8')); describe('external API shared contract fixture', () => { + afterAll(() => { + configModule.stopWatchingConfig(); + }); + test('covers the complete Plinx-consumed v1 surface', () => { expect(fixture.fixtureVersion).toBe(4); expect(fixture.capabilities).toEqual(expect.objectContaining({ From f5e83409086dfc66a9da9291126d9a0daa2dd5c8 Mon Sep 17 00:00:00 2001 From: bballdavis Date: Tue, 8 Sep 2026 19:14:37 -0500 Subject: [PATCH 04/27] fix: keep contract pagination tests watcher-free --- server/modules/externalCatalogService.js | 43 +++--------------- server/modules/externalPagination.js | 57 ++++++++++++++++++++++++ server/testing/externalApiContractApp.js | 2 +- 3 files changed, 64 insertions(+), 38 deletions(-) create mode 100644 server/modules/externalPagination.js diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js index b374d83e3..b0db8fa89 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalCatalogService.js @@ -5,6 +5,12 @@ const { QueryTypes } = require('sequelize'); const { sequelize } = require('../db'); const configModule = require('./configModule'); const { normalizeExternalPolicy, ratingPolicy } = require('./externalEligibility'); +const { + CatalogError, + decodePageCursor, + encodePageCursor, + pagination, +} = require('./externalPagination'); const TAB_MEDIA_TYPES = { videos: 'video', shorts: 'short', streams: 'livestream' }; const SAFE_THUMBNAIL_HOSTS = ['ytimg.com', 'ggpht.com', 'googleusercontent.com']; @@ -19,13 +25,6 @@ function boundedString(value, maximum) { return String(value).slice(0, maximum); } -class CatalogError extends Error { - constructor(message, status = 400) { - super(message); - this.name = 'CatalogError'; - this.status = status; - } -} function parseInteger(value, fallback, minimum, maximum, name) { if (value === undefined) return fallback; @@ -74,36 +73,6 @@ function ratingSql(policy, effectiveRatingSql) { }; } -function decodePageCursor(value, maximumPage) { - if (value === undefined) return null; - if (typeof value !== 'string' || value.length > 200) { - throw new CatalogError('cursor is invalid'); - } - try { - const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); - if (parsed?.v !== 1 || !Number.isSafeInteger(parsed.page) || - parsed.page < 1 || parsed.page > maximumPage) { - throw new Error('invalid cursor'); - } - return parsed.page; - } catch (_error) { - throw new CatalogError('cursor is invalid'); - } -} - -function encodePageCursor(page) { - return Buffer.from(JSON.stringify({ v: 1, page }), 'utf8').toString('base64url'); -} - -function pagination(query, maximumPage = MAX_PAGE) { - if (query.cursor !== undefined && query.page !== undefined) { - throw new CatalogError('cursor and page cannot be used together'); - } - const cursorPage = decodePageCursor(query.cursor, maximumPage); - const page = cursorPage || parseInteger(query.page, 1, 1, maximumPage, 'page'); - const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); - return { page, pageSize, offset: (page - 1) * pageSize }; -} function paginationDto(page, pageSize, total, maximumPage = MAX_PAGE) { const totalPages = total === 0 ? 0 : Math.min(maximumPage, Math.ceil(total / pageSize)); diff --git a/server/modules/externalPagination.js b/server/modules/externalPagination.js new file mode 100644 index 000000000..a846b3e1d --- /dev/null +++ b/server/modules/externalPagination.js @@ -0,0 +1,57 @@ +'use strict'; + +class CatalogError extends Error { + constructor(message, status = 400) { + super(message); + this.name = 'CatalogError'; + this.status = status; + } +} + +function parseInteger(value, fallback, minimum, maximum, name) { + if (value === undefined) return fallback; + if (!/^\d+$/.test(String(value))) throw new CatalogError(`${name} must be an integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { + throw new CatalogError(`${name} must be between ${minimum} and ${maximum}`); + } + return parsed; +} + +function decodePageCursor(value, maximumPage) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length > 200) { + throw new CatalogError('cursor is invalid'); + } + try { + const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); + if (parsed?.v !== 1 || !Number.isSafeInteger(parsed.page) || + parsed.page < 1 || parsed.page > maximumPage) { + throw new Error('invalid cursor'); + } + return parsed.page; + } catch (_error) { + throw new CatalogError('cursor is invalid'); + } +} + +function encodePageCursor(page) { + return Buffer.from(JSON.stringify({ v: 1, page }), 'utf8').toString('base64url'); +} + +function pagination(query, maximumPage = 100) { + if (query.cursor !== undefined && query.page !== undefined) { + throw new CatalogError('cursor and page cannot be used together'); + } + const cursorPage = decodePageCursor(query.cursor, maximumPage); + const page = cursorPage || parseInteger(query.page, 1, 1, maximumPage, 'page'); + const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); + return { page, pageSize, offset: (page - 1) * pageSize }; +} + +module.exports = { + CatalogError, + decodePageCursor, + encodePageCursor, + pagination, +}; \ No newline at end of file diff --git a/server/testing/externalApiContractApp.js b/server/testing/externalApiContractApp.js index 6389ce8a1..e76635078 100644 --- a/server/testing/externalApiContractApp.js +++ b/server/testing/externalApiContractApp.js @@ -6,7 +6,7 @@ const os = require('os'); const path = require('path'); const { createExternalApiAuth } = require('../middleware/externalApiAuth'); const { createExternalApiRoutes } = require('../routes/externalApi'); -const { CatalogError, pagination: validatePagination } = require('../modules/externalCatalogService'); +const { CatalogError, pagination: validatePagination } = require('../modules/externalPagination'); const SYNTHETIC_API_KEY = 'plinx-synthetic-key'; const LANDSCAPE_PNG = Buffer.from( From 36f5f5df6a73109e8d8b65c896c943f56814adcc Mon Sep 17 00:00:00 2001 From: bballdavis Date: Wed, 9 Sep 2026 08:15:44 -0500 Subject: [PATCH 05/27] fix: keep external request tests watcher-free --- server/modules/externalCatalogService.js | 6 +----- server/modules/externalEligibility.js | 6 ++++++ server/modules/externalRequestService.js | 3 +-- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js index b0db8fa89..69dbb280c 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalCatalogService.js @@ -4,7 +4,7 @@ const crypto = require('crypto'); const { QueryTypes } = require('sequelize'); const { sequelize } = require('../db'); const configModule = require('./configModule'); -const { normalizeExternalPolicy, ratingPolicy } = require('./externalEligibility'); +const { normalizePolicy, ratingPolicy } = require('./externalEligibility'); const { CatalogError, decodePageCursor, @@ -57,10 +57,6 @@ function normalizeDate(value, name, endOfDay = false) { return parsed.toISOString(); } -function normalizePolicy(policy) { - return normalizeExternalPolicy(policy, CatalogError); -} - function ratingSql(policy, effectiveRatingSql) { const { recognizedRatings, allowedRatings } = ratingPolicy(policy); const clauses = [`${effectiveRatingSql} IN (:allowedRatings)`]; diff --git a/server/modules/externalEligibility.js b/server/modules/externalEligibility.js index 19c8b3be5..b2db66e11 100644 --- a/server/modules/externalEligibility.js +++ b/server/modules/externalEligibility.js @@ -1,4 +1,5 @@ const ratingMapper = require('./ratingMapper'); +const { CatalogError } = require('./externalPagination'); const MEDIA_TYPES = ['video', 'short', 'livestream']; const RATING_LABELS = [ @@ -23,6 +24,10 @@ function normalizeExternalPolicy(policy, ErrorType = Error) { }; } +function normalizePolicy(policy) { + return normalizeExternalPolicy(policy, CatalogError); +} + function effectiveRating(videoRating, channelDefaultRating) { return videoRating == null || videoRating === '' ? (channelDefaultRating == null || channelDefaultRating === '' ? null : channelDefaultRating) @@ -56,6 +61,7 @@ module.exports = { MEDIA_TYPES, RATING_LABELS, normalizeExternalPolicy, + normalizePolicy, effectiveRating, isRatingEligible, isMediaTypeEligible, diff --git a/server/modules/externalRequestService.js b/server/modules/externalRequestService.js index f075d1906..aa03fb415 100644 --- a/server/modules/externalRequestService.js +++ b/server/modules/externalRequestService.js @@ -1,7 +1,6 @@ const crypto = require('crypto'); const { Op, UniqueConstraintError } = require('sequelize'); -const { normalizePolicy } = require('./externalCatalogService'); -const { isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); +const { normalizePolicy, isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); const { hasExternalScope, normalizeExternalPermissions } = require('./externalPermissions'); const REQUEST_STATUSES = [ From 9d78863f8bc82b6d3881b06739777da8e8e1d742 Mon Sep 17 00:00:00 2001 From: bballdavis Date: Wed, 9 Sep 2026 08:25:58 -0500 Subject: [PATCH 06/27] test: keep auto-removal route mocks isolated --- .../server.additional-routes.test.js | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/server/__tests__/server.additional-routes.test.js b/server/__tests__/server.additional-routes.test.js index 0033bb2d1..3d41d9f09 100644 --- a/server/__tests__/server.additional-routes.test.js +++ b/server/__tests__/server.additional-routes.test.js @@ -56,7 +56,8 @@ const createServerModule = ({ skipInitialize = false, configOverrides = {}, channelVideoMock = null, - archiveModuleMock = null + archiveModuleMock = null, + videoDeletionModuleMock = null } = {}) => { jest.resetModules(); jest.clearAllMocks(); @@ -265,7 +266,7 @@ const createServerModule = ({ register: jest.fn().mockResolvedValue(undefined), delete: jest.fn().mockResolvedValue(undefined), })); - jest.doMock('../modules/videoDeletionModule', () => ({ + jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock || ({ deleteVideos: jest.fn().mockResolvedValue({ deleted: [], failed: [] }), deleteVideosByYoutubeIds: jest.fn().mockResolvedValue({ deleted: [], failed: [] }) })); @@ -704,8 +705,6 @@ describe('server routes - validateToken', () => { describe('server routes - auto-removal dry run', () => { test('performs dry run with boolean autoRemovalEnabled', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -723,7 +722,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -753,8 +752,6 @@ describe('server routes - auto-removal dry run', () => { }); test('performs dry run with string "true" autoRemovalEnabled', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -764,7 +761,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -788,8 +785,6 @@ describe('server routes - auto-removal dry run', () => { }); test('performs dry run with string "false" autoRemovalEnabled', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -799,7 +794,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -823,8 +818,6 @@ describe('server routes - auto-removal dry run', () => { }); test('performs dry run with numeric autoRemovalEnabled (truthy)', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -834,7 +827,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -858,8 +851,6 @@ describe('server routes - auto-removal dry run', () => { }); test('performs dry run with only threshold values', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -869,7 +860,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -895,8 +886,6 @@ describe('server routes - auto-removal dry run', () => { }); test('performs dry run with empty body', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockResolvedValue({ success: true, @@ -906,7 +895,7 @@ describe('server routes - auto-removal dry run', () => { }) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; @@ -926,13 +915,11 @@ describe('server routes - auto-removal dry run', () => { }); test('handles error during dry run', async () => { - const { app } = await createServerModule(); - const videoDeletionModuleMock = { performAutomaticCleanup: jest.fn().mockRejectedValue(new Error('Cleanup failed')) }; - jest.doMock('../modules/videoDeletionModule', () => videoDeletionModuleMock); + const { app } = await createServerModule({ videoDeletionModuleMock }); const handlers = findRouteHandlers(app, 'post', '/api/auto-removal/dry-run'); const dryRunHandler = handlers[handlers.length - 1]; From 188243877acfedecf42ec680c7c8e33f63ffcd99 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:04:41 +0000 Subject: [PATCH 07/27] fix: preserve external request retries and completed results --- jest.config.js | 2 +- jest.external-api-database.config.js | 4 +- .../__tests__/externalRequestService.test.js | 6 +- server/modules/externalRequestService.js | 48 +++-- .../testing/externalRuntime.database.test.js | 192 ++++++++++++++++++ 5 files changed, 227 insertions(+), 25 deletions(-) create mode 100644 server/testing/externalRuntime.database.test.js diff --git a/jest.config.js b/jest.config.js index 74c7d50a3..a55715112 100644 --- a/jest.config.js +++ b/jest.config.js @@ -18,6 +18,6 @@ module.exports = { coverageReporters: ['text', 'lcov', 'html'], testTimeout: 10000, setupFilesAfterEnv: ['/jest.setup.js'], - testPathIgnorePatterns: ['/migrations/__tests__/externalApiDatabase\\.integration\\.test\\.js$'], + testPathIgnorePatterns: ['/server/testing/externalRuntime.database.test.js', '/migrations/__tests__/externalApiDatabase\\.integration\\.test\\.js$'], maxWorkers: 1 }; diff --git a/jest.external-api-database.config.js b/jest.external-api-database.config.js index 4a4c64a84..c61b30324 100644 --- a/jest.external-api-database.config.js +++ b/jest.external-api-database.config.js @@ -2,7 +2,7 @@ const backendConfig = require('./jest.config'); module.exports = { ...backendConfig, - roots: ['/migrations'], - testMatch: ['/migrations/__tests__/externalApiDatabase.integration.test.js'], + roots: ['/migrations', '/server/testing'], + testMatch: ['/migrations/__tests__/externalApiDatabase.integration.test.js', '/server/testing/externalRuntime.database.test.js'], testPathIgnorePatterns: [], }; diff --git a/server/modules/__tests__/externalRequestService.test.js b/server/modules/__tests__/externalRequestService.test.js index 563c1f54c..8d9ccdb6a 100644 --- a/server/modules/__tests__/externalRequestService.test.js +++ b/server/modules/__tests__/externalRequestService.test.js @@ -225,10 +225,8 @@ describe('external video request service', () => { status: 503, code: 'work_queue_full', }); - expect(limited.created.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - })); + expect(limited.created.status).toBe('pending'); + expect(limited.created.active_dedupe_key).toBe(`4:video:${youtubeId}`); }); test('revalidates a channel grant inside the work slot before enqueue', async () => { diff --git a/server/modules/externalRequestService.js b/server/modules/externalRequestService.js index aa03fb415..3d784bbb5 100644 --- a/server/modules/externalRequestService.js +++ b/server/modules/externalRequestService.js @@ -336,7 +336,7 @@ function createExternalRequestService({ } const [claimed] = await ExternalRequest.update( { status: 'processing', updated_at: timestamp }, - { where: { id: record.id, status: whereStatus } } + { where: { id: record.id, status: whereStatus, updated_at: record.updated_at } } ); if (claimed !== 1) return false; record.status = 'processing'; @@ -484,7 +484,7 @@ function createExternalRequestService({ if (!existing) return null; if (existing.request_type !== requestType || existing.youtube_id !== youtubeId || - existing.channel_id !== channelId || + (requestType !== 'channel' && existing.channel_id !== channelId) || existing.channel_url !== channelUrl) { throw new RequestError('Idempotency key was already used for another target', 409); } @@ -530,6 +530,10 @@ function createExternalRequestService({ updated_at: acceptedAt, }); } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: 'pending', updated_at: now() }); + rethrowWorkLimit(error); + } const failedAt = now(); await record.update({ status: 'failed', @@ -581,7 +585,8 @@ function createExternalRequestService({ channelId ); await dispatchAutoApproved(existing, currentKey, currentTarget.channel); - } catch (_error) { + } catch (error) { + if (error?.status === 503) throw error; await failAuthorityChange(existing); } } @@ -639,6 +644,7 @@ function createExternalRequestService({ key, { grantToRequestingKey, requireAutoApproval = false } = {} ) { + const retryStatus = record.status === 'processing' ? 'approved' : record.status; if (!(await claimAuxiliaryRequest(record))) return record; const shouldGrant = grantToRequestingKey ?? (record.grant_to_requesting_key !== false); @@ -691,6 +697,10 @@ function createExternalRequestService({ }, { transaction }); })); } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: retryStatus, updated_at: now() }); + rethrowWorkLimit(error); + } const failedAt = now(); await record.update({ status: 'failed', @@ -734,7 +744,8 @@ function createExternalRequestService({ await provisionChannelRequest(existing, currentKey, { requireAutoApproval: existing.status !== 'approved', }); - } catch (_error) { + } catch (error) { + if (error?.status === 503) throw error; await failAuthorityChange(existing); } } @@ -789,6 +800,7 @@ function createExternalRequestService({ } async function executeDeleteRequest(record, { requireAutoApproval = false } = {}) { + const retryStatus = record.status === 'processing' ? 'approved' : record.status; if (!(await claimAuxiliaryRequest(record))) return record; try { await workLimiter.run(() => sequelize.transaction(async (transaction) => { @@ -807,13 +819,12 @@ function createExternalRequestService({ record.channel_id, transaction ); - if (!target.downloaded) { - throw new RequestError('Video not found', 404); - } - const result = await videoDeleter.deleteVideoById( - target.downloaded.id, - { transaction, video: target.downloaded } - ); + const result = target.downloaded + ? await videoDeleter.deleteVideoById( + target.downloaded.id, + { transaction, video: target.downloaded } + ) + : { success: false, error: 'Video is already removed' }; const alreadyAbsent = result?.success === false && /not found|already (?:marked as )?removed/i.test(result?.error || ''); if (result?.success === false && !alreadyAbsent) { @@ -829,6 +840,10 @@ function createExternalRequestService({ }, { transaction }); })); } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: retryStatus, updated_at: now() }); + rethrowWorkLimit(error); + } const failedAt = now(); await record.update({ status: 'failed', @@ -863,12 +878,6 @@ function createExternalRequestService({ 'channelId' ); const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); - const target = await validateTarget(key, input.youtubeId, channelId); - if (!target.downloaded) { - // Removed and never-downloaded targets are deliberately indistinguishable - // from every other hidden target state. - throw new RequestError('Video not found', 404); - } const activeDedupeKey = `${key.id}:delete_video:${input.youtubeId}`; const existing = await findTypedDuplicate({ keyId: key.id, @@ -889,12 +898,15 @@ function createExternalRequestService({ await executeDeleteRequest(existing, { requireAutoApproval: existing.status !== 'approved', }); - } catch (_error) { + } catch (error) { + if (error?.status === 503) throw error; await failAuthorityChange(existing); } } return { outcome: 'duplicate', request: dto(existing) }; } + const target = await validateTarget(key, input.youtubeId, channelId); + if (!target.downloaded) throw new RequestError('Video not found', 404); const timestamp = now(); let currentKey; let record; diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js new file mode 100644 index 000000000..c003737f8 --- /dev/null +++ b/server/testing/externalRuntime.database.test.js @@ -0,0 +1,192 @@ +'use strict'; + +const path = require('path'); +const { Sequelize } = require('sequelize'); +const Umzug = require('umzug'); + +process.env.DB_NAME = `youtarr_runtime_test_${process.pid}`; +const { sequelize } = require('../db'); +const models = require('../models'); +const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); +const { createExternalRequestService } = require('../modules/externalRequestService'); +const { createExternalQuotaService } = require('../modules/externalQuotaService'); +const { createExternalWorkLimiter } = require('../modules/externalWorkLimiter'); +const catalog = require('../modules/externalCatalogService'); +const configModule = require('../modules/configModule'); + +const youtubeId = 'abcdefghijk'; +const youtubeChannelId = 'UC1234567890123456789012'; +const timestamp = new Date('2026-10-07T12:00:00Z'); +let admin; +let channel; +let keyRecord; +let key; + +function service(options = {}) { + return createExternalRequestService({ + models, sequelize, now: () => timestamp, + executor: async ({ body }) => { + await models.Job.findOrCreate({ + where: { id: body.externalRequestId }, + defaults: { status: 'Pending', timeCreated: timestamp, timeInitiated: timestamp, jobType: 'Manual Download' }, + }); + return body.externalRequestId; + }, + channelProvisioner: { getChannelInfo: async () => ({ channel_id: youtubeChannelId }) }, + videoDeleter: { + deleteVideoById: async (_id, { video, transaction }) => { + await video.update({ removed: true }, { transaction }); + return { success: true }; + }, + }, + ...options, + }); +} + +async function policy(values) { + await keyRecord.update(values); + key = normalizeExternalApiKey(keyRecord); +} + +async function downloadedVideo() { + return models.Video.create({ youtubeId, youTubeChannelName: 'Synthetic channel', + youTubeVideoName: 'Synthetic video', channel_id: youtubeChannelId, normalized_rating: 'TV-Y' }); +} + +beforeAll(async () => { + admin = new Sequelize('mysql', process.env.DB_ADMIN_USER || process.env.DB_USER || 'root', + process.env.DB_ADMIN_PASSWORD || process.env.DB_PASSWORD, { + dialect: 'mysql', host: process.env.DB_HOST || '127.0.0.1', + port: Number(process.env.DB_PORT || 3321), logging: false, + }); + await admin.query(`CREATE DATABASE \`${process.env.DB_NAME}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`); + const migrator = new Umzug({ + migrations: { path: path.join(__dirname, '../../migrations'), params: [sequelize.getQueryInterface(), Sequelize] }, + storage: 'sequelize', storageOptions: { sequelize }, logging: false, + }); + await migrator.storage.logMigration('20250907000000-upgrade-to-utf8mb4-if-needed.js'); + await migrator.up(); +}, 120000); + +beforeEach(async () => { + for (const model of [models.ExternalRequest, models.ExternalApiUsageBucket, + models.ApiKeyChannelGrant, models.ApiKey, models.ChannelVideo, models.Video, models.Channel, models.Job]) { + await model.destroy({ where: {} }); + } + channel = await models.Channel.create({ channel_id: youtubeChannelId, + title: 'Synthetic channel', enabled: true, default_rating: 'TV-Y' }); + await models.ChannelVideo.create({ youtube_id: youtubeId, channel_id: youtubeChannelId, + title: 'Synthetic video', publishedAt: '20261007' }); + keyRecord = await models.ApiKey.create({ name: 'Synthetic test key', key_hash: 'test-only-hash', + key_prefix: 'testonly', created_at: timestamp, role: 'admin', allow_video_requests: true, + allow_channel_requests: true, allow_delete_video_requests: true, allow_unrated: true }); + await models.ApiKeyChannelGrant.create({ api_key_id: keyRecord.id, channel_id: channel.id }); + key = normalizeExternalApiKey(keyRecord); +}); + +afterAll(async () => { + configModule.stopWatchingConfig(); + await sequelize.close(); + if (admin) { + await admin.query(`DROP DATABASE IF EXISTS \`${process.env.DB_NAME}\``); + await admin.close(); + } +}); + +test('catalog IDs feed real request models and management DTOs', async () => { + const channels = await catalog.listChannels(key); + expect(channels.data[0]).toMatchObject({ id: channel.id, channelId: youtubeChannelId }); + const api = service(); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channels.data[0].id }); + const detail = await api.getAdminRequest(created.request.id); + expect(detail.target).toMatchObject({ channelId: channel.id, youtubeChannelId, title: 'Synthetic video' }); +}); + +test('concurrent duplicates reserve one accepted write', async () => { + const api = service(); + const results = await Promise.all(Array.from({ length: 5 }, () => api.createVideoRequest(key, + { youtubeId, channelId: channel.id, idempotencyKey: 'same-operation' }))); + expect(new Set(results.map(result => result.request.id)).size).toBe(1); + expect((await models.ExternalApiUsageBucket.findAll()).map(bucket => bucket.accepted_writes)).toEqual([1, 1]); +}); + +test('completed channel requests replay the original idempotency key', async () => { + await policy({ auto_approve_channel_requests: true }); + const api = service(); + const input = { channelUrl: 'https://www.youtube.com/@synthetic', idempotencyKey: 'channel-replay' }; + const first = await api.createChannelRequest(key, input); + expect(first.request.status).toBe('completed'); + const replay = await api.createChannelRequest(key, input); + expect(replay.request.id).toBe(first.request.id); +}); + +test('completed deletion requests replay after the video is removed', async () => { + await policy({ auto_approve_delete_requests: true }); + await downloadedVideo(); + const api = service(); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'delete-replay' }; + const first = await api.createDeleteVideoRequest(key, input); + expect(first.request.status).toBe('completed'); + const replay = await api.createDeleteVideoRequest(key, input); + expect(replay.request.id).toBe(first.request.id); +}); + +test('a saturated queue can retry the same accepted request without spending another write', async () => { + await policy({ auto_approve_video_requests: true }); + const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 0 }); + let release; + const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); + const api = service({ workLimiter: limiter }); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'retry-queue' }; + try { + await expect(api.createVideoRequest(key, input)).rejects.toMatchObject({ status: 503 }); + } finally { release(); await blocker; } + const retried = await api.createVideoRequest(key, input); + expect(retried.request.status).toBe('processing'); + expect(await models.Job.count()).toBe(1); + expect((await models.ExternalApiUsageBucket.findAll()).map(bucket => bucket.accepted_writes)).toEqual([1, 1]); +}); + +test('quota windows roll over at UTC midnight without carrying previous writes', async () => { + const before = createExternalQuotaService({ models, sequelize, now: () => new Date('2026-10-07T23:59:59Z') }); + const after = createExternalQuotaService({ models, sequelize, now: () => new Date('2026-10-08T00:00:00Z') }); + await before.reserveWrite(key.id, 'video:request'); + expect((await after.status(key)).remaining).toMatchObject({ hourlyWrites: 30, dailyWrites: 200 }); +}); + +test('two stale recovery readers cannot both reclaim a channel request', async () => { + await policy({ auto_approve_channel_requests: true }); + const api = service(); + await policy({ auto_approve_channel_requests: false }); + const input = { channelUrl: 'https://www.youtube.com/@synthetic', idempotencyKey: 'stale-recovery' }; + const created = await api.createChannelRequest(key, input); + await models.ExternalRequest.update({ status: 'processing', decided_at: timestamp, + updated_at: new Date(timestamp.getTime() - 600000) }, { where: { id: created.request.id } }); + await policy({ auto_approve_channel_requests: true }); + let readers = 0; + let release; + const bothRead = new Promise(resolve => { release = resolve; }); + models.ExternalRequest.addHook('afterFind', 'coordinateRecovery', async record => { + if (record?.id !== created.request.id || record.updated_at >= timestamp) return; + readers += 1; + if (readers === 2) release(); + await bothRead; + }); + const provisioner = { getChannelInfo: jest.fn(async () => ({ channel_id: youtubeChannelId })) }; + try { + await Promise.all([service({ channelProvisioner: provisioner }).createChannelRequest(key, input), + service({ channelProvisioner: provisioner }).createChannelRequest(key, input)]); + expect(provisioner.getChannelInfo).toHaveBeenCalledTimes(1); + } finally { + models.ExternalRequest.removeHook('afterFind', 'coordinateRecovery'); + } +}); + +test('reconciliation keeps a downloaded video completed when its job is Complete', async () => { + await policy({ auto_approve_video_requests: true }); + const api = service(); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + await downloadedVideo(); + await models.Job.update({ status: 'Complete' }, { where: { id: created.request.id } }); + expect((await api.getRequest(key, created.request.id)).status).toBe('completed'); +}); From 10a9b86e976fb134c1c3a59bda5c7bbaf74e6be5 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:07:58 +0000 Subject: [PATCH 08/27] fix: serialize external approvals and recover durable download jobs --- .../apiKeyChannelGrantModule.test.js | 6 +- .../__tests__/externalRequestService.test.js | 9 +-- server/modules/apiKeyChannelGrantModule.js | 2 +- server/modules/apiKeyModule.js | 2 +- server/modules/externalRequestService.js | 59 ++++++++++++------- .../testing/externalRuntime.database.test.js | 56 +++++++++++++++++- 6 files changed, 104 insertions(+), 30 deletions(-) diff --git a/server/modules/__tests__/apiKeyChannelGrantModule.test.js b/server/modules/__tests__/apiKeyChannelGrantModule.test.js index b5c65d477..985090d9d 100644 --- a/server/modules/__tests__/apiKeyChannelGrantModule.test.js +++ b/server/modules/__tests__/apiKeyChannelGrantModule.test.js @@ -1,6 +1,6 @@ jest.mock('../../db', () => ({ sequelize: { - transaction: jest.fn(async (callback) => callback({ id: 'transaction' })), + transaction: jest.fn(async (callback) => callback({ id: 'transaction', LOCK: { UPDATE: 'UPDATE' } })), }, })); const { Op } = require('sequelize'); @@ -26,11 +26,11 @@ describe('API key channel grants', () => { const result = await grants.replaceChannelGrants(7, [9, 3, 9]); expect(result).toEqual({ keyId: 7, channelIds: [3, 9] }); expect(ApiKeyChannelGrant.destroy).toHaveBeenCalledWith({ - where: { api_key_id: 7 }, transaction: { id: 'transaction' }, + where: { api_key_id: 7 }, transaction: { id: 'transaction', LOCK: { UPDATE: 'UPDATE' } }, }); expect(ApiKeyChannelGrant.bulkCreate).toHaveBeenCalledWith( [{ api_key_id: 7, channel_id: 3 }, { api_key_id: 7, channel_id: 9 }], - { transaction: { id: 'transaction' } } + { transaction: { id: 'transaction', LOCK: { UPDATE: 'UPDATE' } } } ); }); diff --git a/server/modules/__tests__/externalRequestService.test.js b/server/modules/__tests__/externalRequestService.test.js index 8d9ccdb6a..4c6c6ca2e 100644 --- a/server/modules/__tests__/externalRequestService.test.js +++ b/server/modules/__tests__/externalRequestService.test.js @@ -31,7 +31,7 @@ function fixture(overrides = {}) { create: jest.fn(async (values) => Object.assign(created, values)), update: jest.fn().mockResolvedValue([1]), findOne: jest.fn().mockResolvedValue(null), - findByPk: jest.fn().mockResolvedValue(null), + findByPk: jest.fn().mockResolvedValue(created), findAndCountAll: jest.fn().mockResolvedValue({ rows: [], count: 0 }), }, ApiKeyChannelGrant: { @@ -54,7 +54,7 @@ function fixture(overrides = {}) { findAll: jest.fn().mockResolvedValue([]), }, Video: { findOne: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, - Job: { findAll: jest.fn().mockResolvedValue([]) }, + Job: { findByPk: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, ApiKey: { findByPk: jest.fn().mockResolvedValue({ id: 4, @@ -202,7 +202,7 @@ describe('external video request service', () => { channelId: 'UC1234567890123456789012', ownerChannelMap: { [youtubeId]: 'UC1234567890123456789012' }, initiatedBy: { type: 'api_key', name: 'External Client' }, - jobLabel: 'External video request', + jobLabel: 'Manually Added Urls (external request)', externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', }, }); @@ -276,6 +276,7 @@ describe('external video request service', () => { executor: retryExecutor, }); retry.models.ExternalRequest.findOne.mockResolvedValue(interrupted); + retry.models.ExternalRequest.findByPk.mockResolvedValue(interrupted); const result = await retry.service.createVideoRequest( retry.key, { youtubeId, channelId: 8, idempotencyKey: 'same-request' } @@ -287,7 +288,7 @@ describe('external video request service', () => { expect(interrupted.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'processing', job_id: interrupted.id, - })); + }), { transaction: retry.transaction }); expect(result).toMatchObject({ outcome: 'duplicate', request: { status: 'processing' }, diff --git a/server/modules/apiKeyChannelGrantModule.js b/server/modules/apiKeyChannelGrantModule.js index 29bbf1491..a197ee232 100644 --- a/server/modules/apiKeyChannelGrantModule.js +++ b/server/modules/apiKeyChannelGrantModule.js @@ -12,7 +12,7 @@ function normalizeChannelIds(channelIds) { } async function requireExternalKey(keyId, transaction) { - const key = await ApiKey.findByPk(keyId, { transaction }); + const key = await ApiKey.findByPk(keyId, { transaction, ...(transaction ? { lock: transaction.LOCK.UPDATE } : {}) }); if (!key) return null; if (!key.is_active || key.revoked_at || key.role === 'legacy_download') { throw new Error('Only active external API keys can receive channel grants'); diff --git a/server/modules/apiKeyModule.js b/server/modules/apiKeyModule.js index c9bd3f4b4..83cef5bbe 100644 --- a/server/modules/apiKeyModule.js +++ b/server/modules/apiKeyModule.js @@ -231,7 +231,7 @@ class ApiKeyModule { async updateApiKey(id, policy, { transaction } = {}) { const apiKey = transaction - ? await ApiKey.findByPk(id, { transaction }) + ? await ApiKey.findByPk(id, { transaction, lock: transaction.LOCK.UPDATE }) : await ApiKey.findByPk(id); if (!apiKey || !apiKey.is_active || apiKey.revoked_at) return null; if (apiKey.role === 'legacy_download' || policy?.role === 'legacy_download') { diff --git a/server/modules/externalRequestService.js b/server/modules/externalRequestService.js index 3d784bbb5..2d163ba8a 100644 --- a/server/modules/externalRequestService.js +++ b/server/modules/externalRequestService.js @@ -1,4 +1,5 @@ const crypto = require('crypto'); +const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); const { Op, UniqueConstraintError } = require('sequelize'); const { normalizePolicy, isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); const { hasExternalScope, normalizeExternalPermissions } = require('./externalPermissions'); @@ -467,7 +468,7 @@ function createExternalRequestService({ if (idempotencyHash) clauses.push({ api_key_id: keyId, idempotency_hash: idempotencyHash }); const existing = await ExternalRequest.findOne({ where: { [Op.or]: clauses } }); if (!existing) return null; - if (existing.youtube_id !== youtubeId || existing.channel_id !== channelId) { + if (existing.request_type !== 'video' || existing.youtube_id !== youtubeId || existing.channel_id !== channelId) { throw new RequestError('Idempotency key was already used for another target', 409); } await reconcile([existing]); @@ -492,8 +493,9 @@ function createExternalRequestService({ } async function dispatchAutoApproved(record, key) { + let accepted = false; try { - const jobId = await workLimiter.run(() => + await workLimiter.run(() => sequelize.transaction(async (transaction) => { const currentKey = await quotas.assertExecutionCapacity( key.id, @@ -501,6 +503,8 @@ function createExternalRequestService({ record.id, transaction ); + const locked = await ExternalRequest.findByPk(record.id, { transaction, lock: transaction.LOCK.UPDATE }); + if (!locked || locked.status !== 'pending' || locked.job_id) return; requireCurrentAutoApproval(currentKey, 'video'); const target = await validateTarget( currentKey, @@ -508,28 +512,31 @@ function createExternalRequestService({ record.channel_id, transaction ); - return executor({ + const existingJob = await Job.findByPk(record.id, { transaction }); + const jobId = existingJob?.id || await executor({ body: { urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], channelId: target.channel.channel_id, ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, initiatedBy: { type: 'api_key', name: currentKey.name }, - jobLabel: 'External video request', + jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, // The downloader uses this UUID as the job identity. A retry after a // crash can therefore observe/reuse the accepted job instead of // starting the same download twice. externalRequestId: record.id, }, }); + if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); + accepted = true; + await record.update({ + status: 'processing', job_id: jobId || record.id, updated_at: now(), + }, { transaction }); }) ); - const acceptedAt = now(); - await record.update({ - status: 'processing', - job_id: jobId || record.id, - updated_at: acceptedAt, - }); } catch (error) { + // The downloader persists separately. Leave the request retryable if the + // process loses its database connection after enqueueing the stable job. + if (accepted) throw error; if (error?.name === 'ExternalWorkLimitError') { await record.update({ status: 'pending', updated_at: now() }); rethrowWorkLimit(error); @@ -586,7 +593,7 @@ function createExternalRequestService({ ); await dispatchAutoApproved(existing, currentKey, currentTarget.channel); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(existing); } } @@ -632,7 +639,7 @@ function createExternalRequestService({ const finalTarget = await validateTarget(finalKey, input.youtubeId, channelId); await dispatchAutoApproved(record, finalKey, finalTarget.channel); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(record); } } @@ -745,7 +752,7 @@ function createExternalRequestService({ requireAutoApproval: existing.status !== 'approved', }); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(existing); } } @@ -792,7 +799,7 @@ function createExternalRequestService({ ); await provisionChannelRequest(record, finalKey, { requireAutoApproval: true }); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(record); } } @@ -899,7 +906,7 @@ function createExternalRequestService({ requireAutoApproval: existing.status !== 'approved', }); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(existing); } } @@ -950,7 +957,7 @@ function createExternalRequestService({ await validateTarget(finalKey, input.youtubeId, channelId); await executeDeleteRequest(record, { requireAutoApproval: true }); } catch (error) { - if (error?.status === 503) throw error; + if (![403, 404].includes(error?.status)) throw error; await failAuthorityChange(record); } } @@ -1087,7 +1094,10 @@ function createExternalRequestService({ } const reason = action === 'reject' ? sanitizeReason(input.reason) : null; - await sequelize.transaction(async (transaction) => { + const current = await ExternalRequest.findByPk(id); + if (!current || current.request_type !== 'video') throw new RequestError('Request not found', 404); + const review = () => sequelize.transaction(async (transaction) => { + await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); const record = await ExternalRequest.findOne({ where: { id, request_type: 'video' }, transaction, @@ -1200,16 +1210,18 @@ function createExternalRequestService({ }, { transaction }); } try { - const jobId = await workLimiter.run(() => executor({ + const existingJob = await Job.findByPk(record.id, { transaction }); + const jobId = existingJob?.id || await executor({ body: { urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], channelId: target.channel.channel_id, ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, initiatedBy: { type: 'api_key', name: key.name }, - jobLabel: 'External video request', + jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, externalRequestId: record.id, }, - })); + }); + if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); const acceptedAt = now(); await record.update({ status: 'processing', @@ -1220,6 +1232,12 @@ function createExternalRequestService({ await failApproval('Download could not be queued'); } }); + try { + await (action === 'approve' ? workLimiter.run(review) : review()); + } catch (error) { + rethrowWorkLimit(error); + throw error; + } return getAdminRequest(id); } @@ -1253,6 +1271,7 @@ function createExternalRequestService({ let key; let claimed; await sequelize.transaction(async (transaction) => { + await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); const record = await ExternalRequest.findByPk(id, { transaction, lock: transaction.LOCK.UPDATE, diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index c003737f8..1ce988b6f 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -11,6 +11,7 @@ const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); const { createExternalRequestService } = require('../modules/externalRequestService'); const { createExternalQuotaService } = require('../modules/externalQuotaService'); const { createExternalWorkLimiter } = require('../modules/externalWorkLimiter'); +const { isSpecificUrlDownloadJob } = require('../modules/download/jobTypes'); const catalog = require('../modules/externalCatalogService'); const configModule = require('../modules/configModule'); @@ -26,6 +27,7 @@ function service(options = {}) { return createExternalRequestService({ models, sequelize, now: () => timestamp, executor: async ({ body }) => { + expect(isSpecificUrlDownloadJob(body.jobLabel)).toBe(true); await models.Job.findOrCreate({ where: { id: body.externalRequestId }, defaults: { status: 'Pending', timeCreated: timestamp, timeInitiated: timestamp, jobType: 'Manual Download' }, @@ -76,7 +78,7 @@ beforeEach(async () => { channel = await models.Channel.create({ channel_id: youtubeChannelId, title: 'Synthetic channel', enabled: true, default_rating: 'TV-Y' }); await models.ChannelVideo.create({ youtube_id: youtubeId, channel_id: youtubeChannelId, - title: 'Synthetic video', publishedAt: '20261007' }); + title: 'Synthetic video', publishedAt: '2026-10-07T00:00:00.000Z' }); keyRecord = await models.ApiKey.create({ name: 'Synthetic test key', key_hash: 'test-only-hash', key_prefix: 'testonly', created_at: timestamp, role: 'admin', allow_video_requests: true, allow_channel_requests: true, allow_delete_video_requests: true, allow_unrated: true }); @@ -190,3 +192,55 @@ test('reconciliation keeps a downloaded video completed when its job is Complete await models.Job.update({ status: 'Complete' }, { where: { id: created.request.id } }); expect((await api.getRequest(key, created.request.id)).status).toBe('completed'); }); + +test('two administrators cannot enqueue the same approval twice', async () => { + const api = service(); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + const results = await Promise.allSettled([api.reviewRequest(created.request.id, 'approve'), + api.reviewRequest(created.request.id, 'approve')]); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); + expect(results.find(result => result.status === 'rejected').reason.status).toBe(409); + expect(await models.Job.count()).toBe(1); +}); + +test('grant revocation while an approval waits for a work slot prevents enqueue', async () => { + const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 1 }); + let release; + const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); + const api = service({ workLimiter: limiter }); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + const approval = api.reviewRequest(created.request.id, 'approve'); + await models.ApiKeyChannelGrant.destroy({ where: { api_key_id: key.id } }); + release(); + await blocker; + expect((await approval).status).toBe('failed'); + expect(await models.Job.count()).toBe(0); +}); + +test('retry after enqueue but before request persistence reuses the durable job', async () => { + await policy({ auto_approve_video_requests: true }); + const api = service(); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'enqueue-crash' }; + models.ExternalRequest.addHook('beforeUpdate', 'simulateConnectionLoss', row => { + if (row.status === 'processing') throw new Error('Synthetic connection loss after enqueue'); + }); + try { + await expect(api.createVideoRequest(key, input)).rejects.toThrow('Synthetic connection loss'); + } finally { + models.ExternalRequest.removeHook('beforeUpdate', 'simulateConnectionLoss'); + } + const executor = jest.fn(); + const result = await service({ executor }).createVideoRequest(key, input); + expect(result.request.status).toBe('processing'); + expect(executor).not.toHaveBeenCalled(); + expect(await models.Job.count()).toBe(1); +}); + +test('idempotency keys cannot replay a deletion as a video request', async () => { + const video = await downloadedVideo(); + const api = service(); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'cross-operation' }; + await api.createDeleteVideoRequest(key, input); + await video.update({ removed: true }); + await expect(api.createVideoRequest(key, input)).rejects.toMatchObject({ status: 409 }); +}); From 5595104c011d2e8484321be253961397339059aa Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:13:13 +0000 Subject: [PATCH 09/27] fix: harden external API inputs and expose effective availability --- .../__tests__/externalThumbnailProxy.test.js | 10 +++++++ server/modules/channel/channelMappers.js | 1 + server/modules/externalCatalogService.js | 28 +++++-------------- server/modules/externalThumbnailProxy.js | 28 +++++++++++++++++-- server/routes/__tests__/config.test.js | 16 +++++++++++ server/routes/__tests__/externalApi.test.js | 7 +++++ server/routes/config.js | 9 +++++- server/routes/externalApi.js | 6 ++++ server/routes/externalRequests.js | 2 +- .../testing/externalRuntime.database.test.js | 13 +++++++++ 10 files changed, 94 insertions(+), 26 deletions(-) diff --git a/server/modules/__tests__/externalThumbnailProxy.test.js b/server/modules/__tests__/externalThumbnailProxy.test.js index 9cf99b88c..1ed327be3 100644 --- a/server/modules/__tests__/externalThumbnailProxy.test.js +++ b/server/modules/__tests__/externalThumbnailProxy.test.js @@ -15,6 +15,16 @@ function response({ body = Buffer.from('jpeg'), contentType = 'image/jpeg', leng } describe('external thumbnail proxy', () => { + test.each([ + 'http://i.ytimg.com/image.jpg', 'https://i.ytimg.com.evil.example/image.jpg', + 'https://127.0.0.1/image.jpg', 'https://secret@i.ytimg.com/image.jpg', + 'https://i.ytimg.com:8443/image.jpg', 'file:///tmp/image.jpg', + ])('rejects unsafe destinations before fetching: %s', async url => { + const fetchImpl = jest.fn(); + await expect(fetchExternalThumbnail(url, fetchImpl)).rejects.toBeInstanceOf(ThumbnailProxyError); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + test('returns bounded image bytes without following redirects', async () => { const fetchImpl = jest.fn().mockResolvedValue(response()); diff --git a/server/modules/channel/channelMappers.js b/server/modules/channel/channelMappers.js index 45ebb6aae..3c17d1c79 100644 --- a/server/modules/channel/channelMappers.js +++ b/server/modules/channel/channelMappers.js @@ -43,6 +43,7 @@ class ChannelMappers { mapChannelListEntry(channel, tabStats = null) { const effectiveTabs = tabState.computeEffectiveTabs(channel.available_tabs, channel.hidden_tabs); const out = { + database_id: channel.id, url: channel.url, uploader: channel.uploader || '', channel_id: channel.channel_id || '', diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js index 69dbb280c..aac4f7feb 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalCatalogService.js @@ -5,6 +5,7 @@ const { QueryTypes } = require('sequelize'); const { sequelize } = require('../db'); const configModule = require('./configModule'); const { normalizePolicy, ratingPolicy } = require('./externalEligibility'); +const { publicVideoThumbnail } = require('./externalThumbnailProxy'); const { CatalogError, decodePageCursor, @@ -13,12 +14,10 @@ const { } = require('./externalPagination'); const TAB_MEDIA_TYPES = { videos: 'video', shorts: 'short', streams: 'livestream' }; -const SAFE_THUMBNAIL_HOSTS = ['ytimg.com', 'ggpht.com', 'googleusercontent.com']; const ACTIVE_REQUEST_STATUSES = ['pending', 'approved', 'processing']; const MAX_PAGE = 100; const MAX_DESCRIPTION_LENGTH = 2000; const MAX_TITLE_LENGTH = 500; -const MAX_PUBLIC_URL_LENGTH = 2048; function boundedString(value, maximum) { if (value === null || value === undefined) return null; @@ -90,7 +89,7 @@ function catalogCursorFingerprint(endpoint, filters) { function decodeCatalogCursor(value, expected) { if (value === undefined) return null; - if (typeof value !== 'string' || value.length > 500) { + if (typeof value !== 'string' || value.length > 4096) { throw new CatalogError('cursor is invalid'); } try { @@ -104,7 +103,9 @@ function decodeCatalogCursor(value, expected) { !Number.isSafeInteger(parsed.channelDatabaseId) || parsed.channelDatabaseId < 1 || typeof parsed.youtubeId !== 'string' || parsed.youtubeId.length === 0 || parsed.youtubeId.length > 32 || - !Object.prototype.hasOwnProperty.call(parsed, 'sortValue')) { + (expected.sortBy === 'duration' + ? !Number.isFinite(parsed.sortValue) || parsed.sortValue < -1 + : typeof parsed.sortValue !== 'string' || parsed.sortValue.length > MAX_TITLE_LENGTH)) { throw new Error('invalid cursor'); } return parsed; @@ -190,21 +191,6 @@ function lastFetched(channel, mediaType = null) { } } -function publicVideoThumbnail(value) { - if (typeof value !== 'string' || value.length > MAX_PUBLIC_URL_LENGTH) return null; - try { - const url = new URL(value); - const hostname = url.hostname.toLowerCase(); - if (url.protocol !== 'https:' || - !SAFE_THUMBNAIL_HOSTS.some((host) => hostname === host || hostname.endsWith(`.${host}`))) { - return null; - } - return url.toString(); - } catch (_error) { - return null; - } -} - function videoThumbnailUrl(youtubeId) { return `/external-api/v1/assets/videos/${youtubeId}/thumbnail`; } @@ -307,7 +293,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { duration: 'COALESCE(cv.duration, -1)', }; const sortBy = query.sortBy || 'date'; - if (!sortColumns[sortBy]) throw new CatalogError('sortBy must be date, title, or duration'); + if (!Object.hasOwn(sortColumns, sortBy)) throw new CatalogError('sortBy must be date, title, or duration'); const sortOrder = (query.sortOrder || 'desc').toLowerCase(); if (!['asc', 'desc'].includes(sortOrder)) throw new CatalogError('sortOrder must be asc or desc'); const minDuration = parseInteger(query.minDuration, null, 0, 604800, 'minDuration'); @@ -467,7 +453,7 @@ async function listVideos(key, query = {}) { duration: 'COALESCE(cv.duration, -1)', }; const sortBy = query.sortBy || 'date'; - if (!sortColumns[sortBy]) throw new CatalogError('sortBy must be date, title, or duration'); + if (!Object.hasOwn(sortColumns, sortBy)) throw new CatalogError('sortBy must be date, title, or duration'); const sortOrder = (query.sortOrder || 'desc').toLowerCase(); if (!['asc', 'desc'].includes(sortOrder)) throw new CatalogError('sortOrder must be asc or desc'); const minDuration = parseInteger(query.minDuration, null, 0, 604800, 'minDuration'); diff --git a/server/modules/externalThumbnailProxy.js b/server/modules/externalThumbnailProxy.js index 8c696df05..ca16a4ffc 100644 --- a/server/modules/externalThumbnailProxy.js +++ b/server/modules/externalThumbnailProxy.js @@ -1,6 +1,22 @@ const THUMBNAIL_FETCH_TIMEOUT_MS = 10000; const THUMBNAIL_MAX_BYTES = 5 * 1024 * 1024; const ALLOWED_IMAGE_TYPES = new Set(['image/jpeg', 'image/png', 'image/webp']); +const SAFE_THUMBNAIL_HOSTS = ['ytimg.com', 'ggpht.com', 'googleusercontent.com']; + +function publicVideoThumbnail(value) { + if (typeof value !== 'string' || value.length > 2048) return null; + try { + const url = new URL(value); + const hostname = url.hostname.toLowerCase(); + if (url.protocol !== 'https:' || url.username || url.password || url.port || + !SAFE_THUMBNAIL_HOSTS.some(host => hostname === host || hostname.endsWith(`.${host}`))) { + return null; + } + return url.toString(); + } catch (_error) { + return null; + } +} class ThumbnailProxyError extends Error { constructor(message) { @@ -10,6 +26,7 @@ class ThumbnailProxyError extends Error { } async function fetchExternalThumbnail(url, fetchImpl = global.fetch) { + if (!publicVideoThumbnail(url)) throw new ThumbnailProxyError('Thumbnail URL is not allowed'); let response; try { response = await fetchImpl(url, { @@ -20,18 +37,22 @@ async function fetchExternalThumbnail(url, fetchImpl = global.fetch) { } catch (_error) { throw new ThumbnailProxyError('Thumbnail fetch failed'); } - if (!response.ok) throw new ThumbnailProxyError('Thumbnail fetch failed'); + const rejectResponse = async message => { + await response.body?.cancel(); + throw new ThumbnailProxyError(message); + }; + if (!response.ok) return rejectResponse('Thumbnail fetch failed'); const contentType = String(response.headers.get('content-type') || '') .split(';', 1)[0] .trim() .toLowerCase(); if (!ALLOWED_IMAGE_TYPES.has(contentType)) { - throw new ThumbnailProxyError('Thumbnail response was not an image'); + return rejectResponse('Thumbnail response was not an image'); } const declaredLength = Number(response.headers.get('content-length')); if (Number.isFinite(declaredLength) && declaredLength > THUMBNAIL_MAX_BYTES) { - throw new ThumbnailProxyError('Thumbnail response was too large'); + return rejectResponse('Thumbnail response was too large'); } const reader = response.body?.getReader(); @@ -65,4 +86,5 @@ module.exports = { ThumbnailProxyError, THUMBNAIL_FETCH_TIMEOUT_MS, THUMBNAIL_MAX_BYTES, + publicVideoThumbnail, }; diff --git a/server/routes/__tests__/config.test.js b/server/routes/__tests__/config.test.js index c153f1c64..5d9a9b382 100644 --- a/server/routes/__tests__/config.test.js +++ b/server/routes/__tests__/config.test.js @@ -738,3 +738,19 @@ describe('schedule configuration', () => { expect(res.body.logging).toEqual(LOGGING_STATUS); }); }); + +test.each([[undefined, false], ['false', false], ['invalid', false], ['true', true]])( + 'reports effective external API availability for %s', async (value, enabled) => { + const original = process.env.EXTERNAL_API_ENABLED; + try { + if (value === undefined) delete process.env.EXTERNAL_API_ENABLED; + else process.env.EXTERNAL_API_ENABLED = value; + const { app } = makeApp(); + const response = await supertest(app).get('/getconfig').expect(200); + expect(response.body.isPlatformManaged.externalApiEnabled).toBe(enabled); + } finally { + if (original === undefined) delete process.env.EXTERNAL_API_ENABLED; + else process.env.EXTERNAL_API_ENABLED = original; + } + } +); diff --git a/server/routes/__tests__/externalApi.test.js b/server/routes/__tests__/externalApi.test.js index f6babe5d4..39408b91a 100644 --- a/server/routes/__tests__/externalApi.test.js +++ b/server/routes/__tests__/externalApi.test.js @@ -541,3 +541,10 @@ describe('external API capabilities', () => { expect(response.headers['cache-control']).toBe('private, no-store'); }); }); + +test.each(['sortOrder[]=desc', 'pageSize[x]=2', 'search=a&search=b'])( + 'rejects non-scalar query parameters: %s', async query => { + const { app } = makeApp({ key: externalKey() }); + await request(app).get(`/external-api/v1/videos?${query}`).set('x-api-key', 'valid').expect(400); + } +); diff --git a/server/routes/config.js b/server/routes/config.js index 1a9ce5ade..5713a171e 100644 --- a/server/routes/config.js +++ b/server/routes/config.js @@ -4,6 +4,7 @@ const { SCHEDULES, getScheduleError, getSchedule } = require('../modules/schedul const customArgsParser = require('../modules/download/customArgsParser'); const filenamePreview = require('../modules/filenamePreview'); const { getExternalCookiesPath } = require('../modules/externalCookies'); +const { isExternalApiEnabled } = require('../modules/externalApiConfig'); const { isValidLevelSetting } = require('../logging/logLevel'); // Mirror of the frontend RATE_LIMIT_REGEX. Matches yt-dlp's --limit-rate @@ -137,6 +138,11 @@ module.exports = function createConfigRoutes({ * type: string * ytdlpUpdateFrequency: * type: string + * isPlatformManaged: + * type: object + * properties: + * externalApiEnabled: + * type: boolean * channelVideoCountsFrequency: * type: string * logLevel: @@ -158,7 +164,8 @@ module.exports = function createConfigRoutes({ plexUrl: !!process.env.PLEX_URL, authEnabled: process.env.AUTH_ENABLED === 'false' ? false : true, useTmpForDownloads: configModule.isElfhostedPlatform(), - ytdlpUpdates: configModule.isElfhostedPlatform() + ytdlpUpdates: configModule.isElfhostedPlatform(), + externalApiEnabled: isExternalApiEnabled() }; safeConfig.deploymentEnvironment = { diff --git a/server/routes/externalApi.js b/server/routes/externalApi.js index 8be174ee4..2db2ca8b4 100644 --- a/server/routes/externalApi.js +++ b/server/routes/externalApi.js @@ -37,6 +37,12 @@ function createExternalApiRoutes({ return next(); }); router.use(externalApiAuth, externalApiLimiter, recordExternalApiUse); + router.use((req, res, next) => { + if (Object.values(req.query).some(value => typeof value !== 'string')) { + return sendExternalError(res, 400, 'Query parameters must be single string values', { requestId: req.id }); + } + return next(); + }); router.use((req, res, next) => { if (req.method !== 'POST') return next(); const contentEncoding = String(req.get('content-encoding') || 'identity').toLowerCase(); diff --git a/server/routes/externalRequests.js b/server/routes/externalRequests.js index 17f8b3ebe..f69a8e93a 100644 --- a/server/routes/externalRequests.js +++ b/server/routes/externalRequests.js @@ -16,7 +16,7 @@ function createExternalRequestReviewRoutes({ }; const sendError = (req, res, error) => { - if (error.name === 'RequestError' && error.status >= 400 && error.status < 500) { + if (error.name === 'RequestError' && error.status >= 400 && error.status <= 503) { return res.status(error.status).json({ error: error.message }); } req.log?.error({ err: error }, 'External request review operation failed'); diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index 1ce988b6f..980246c4b 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -96,6 +96,8 @@ afterAll(async () => { }); test('catalog IDs feed real request models and management DTOs', async () => { + const mapper = require('../modules/channel/channelMappers'); + expect(mapper.mapChannelListEntry(channel).database_id).toBe(channel.id); const channels = await catalog.listChannels(key); expect(channels.data[0]).toMatchObject({ id: channel.id, channelId: youtubeChannelId }); const api = service(); @@ -104,6 +106,17 @@ test('catalog IDs feed real request models and management DTOs', async () => { expect(detail.target).toMatchObject({ channelId: channel.id, youtubeChannelId, title: 'Synthetic video' }); }); +test('title cursors round trip long Unicode titles from real catalog rows', async () => { + await models.ChannelVideo.update({ title: '長'.repeat(240) }, { where: { youtube_id: youtubeId } }); + await models.ChannelVideo.create({ youtube_id: 'lmnopqrstuv', channel_id: youtubeChannelId, + title: '長'.repeat(240), publishedAt: timestamp.toISOString() }); + const first = await catalog.listVideos(key, { sortBy: 'title', pageSize: '1' }); + const second = await catalog.listVideos(key, { sortBy: 'title', pageSize: '1', cursor: first.pagination.nextCursor }); + expect(second.data).toHaveLength(1); + expect(second.data[0].youtubeId).not.toBe(first.data[0].youtubeId); +}); + + test('concurrent duplicates reserve one accepted write', async () => { const api = service(); const results = await Promise.all(Array.from({ length: 5 }, () => api.createVideoRequest(key, From b2213f5d899cd0dc38814164938177788f1483e3 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:19:49 +0000 Subject: [PATCH 10/27] perf: batch external request history and verify runtime indexes --- .github/workflows/ci.yml | 17 +++- docs/DATABASE.md | 2 +- docs/EXTERNAL_API.md | 89 +++++++++++++++-- ...041058-add-external-api-runtime-indexes.js | 39 ++++++++ .../externalApiRuntimeIndexes.test.js | 34 +++++++ package.json | 1 + scripts/external-api-index-plans.js | 98 +++++++++++++++++++ server/models/externalrequest.js | 2 + .../__tests__/externalCatalogRequests.test.js | 20 ++++ .../__tests__/externalCatalogService.test.js | 21 ++-- server/modules/externalCatalogRequests.js | 28 ++++++ server/modules/externalCatalogService.js | 31 ++---- .../testing/externalRuntime.database.test.js | 22 +++++ 13 files changed, 358 insertions(+), 46 deletions(-) create mode 100644 migrations/20261007041058-add-external-api-runtime-indexes.js create mode 100644 migrations/__tests__/externalApiRuntimeIndexes.test.js create mode 100644 scripts/external-api-index-plans.js create mode 100644 server/modules/__tests__/externalCatalogRequests.test.js create mode 100644 server/modules/externalCatalogRequests.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5053f9573..9066ff095 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,7 +164,7 @@ jobs: coverage/lcov.info test-external-api-database: - name: External API DB migrations (${{ matrix.engine }}) + name: External API DB runtime and migrations (${{ matrix.engine }}) runs-on: ubuntu-latest timeout-minutes: 10 strategy: @@ -200,7 +200,7 @@ jobs: run: npm install -g npm@11.15.0 --ignore-scripts - name: Install dependencies run: npm ci --ignore-scripts - - name: Run external API migration lifecycle + - name: Run external API runtime and migration lifecycle run: | npm run test:external-api-database -- --json --outputFile=external-api-database-results.json node -e "const result = require('./external-api-database-results.json'); if (result.numPassedTests < 1 || result.numPendingTestSuites !== 0 || result.numPendingTests !== 0 || result.numTodoTests !== 0) process.exit(1);" @@ -211,6 +211,19 @@ jobs: DB_PASSWORD: rootpass DB_ADMIN_USER: root DB_ADMIN_PASSWORD: rootpass + - name: Verify runtime query plans on synthetic data + run: npm run test:external-index-plans + env: + DB_HOST: 127.0.0.1 + DB_PORT: 3321 + DB_USER: root + DB_PASSWORD: rootpass + EXTERNAL_API_INDEX_REPORT: external-api-index-plans.json + - name: Upload runtime query plans + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: external-api-index-plans-${{ matrix.engine }} + path: external-api-index-plans.json test-frontend: name: Frontend Tests diff --git a/docs/DATABASE.md b/docs/DATABASE.md index 5c99e5dab..5234e689b 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -45,7 +45,7 @@ Youtarr uses MariaDB/MySQL for storing: ### External API Policy Foundation -The external API policy fields and supporting tables are dormant until the runtime/control plane is enabled. Existing keys retain their single-video download behavior and are backfilled with the `legacy_download` role. API-key values remain hashed and cannot be recovered from the database. Revocation is represented by `revoked_at` together with the key's active status. +The external API policy fields and supporting tables are enforced when `EXTERNAL_API_ENABLED=true`; the runtime is disabled by default. Existing keys retain their single-video download behavior and are backfilled with the `legacy_download` role. API-key values remain hashed and cannot be recovered from the database. Revocation is represented by `revoked_at` together with the key's active status. The runtime adds two request-history indexes for catalog status and administrator queue queries; see [External API query plans](EXTERNAL_API.md#database-integration-and-query-plans) for measured plans and rollback behavior. ## Internal Database (Default) diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md index 272a4c815..4b31f434f 100644 --- a/docs/EXTERNAL_API.md +++ b/docs/EXTERNAL_API.md @@ -7,8 +7,8 @@ external trust boundary and proxy rules have been reviewed. ## Enablement and authentication -Create an external-role key in **Settings → API Keys → API Keys & External -Access**. The key is revealed once. Send it only in the `x-api-key` header over +Create an external-role key through the session-authenticated management API +described below. The key is revealed once. Send it only in the `x-api-key` header over HTTPS. Set `EXTERNAL_API_ENABLED=true` and restart Youtarr to enable the namespace; leave it unset or set to `false` to keep it disabled. @@ -47,8 +47,7 @@ Every external key has: per UTC hour, and 200 per UTC day (administrators may select lower values); - an explicit set of granted Youtarr channel database IDs. -The HTTP contract retains a compact `maxRatingLevel` value, but the -administrator UI presents the actual rating ceilings: +The HTTP contract uses a compact `maxRatingLevel` value with these rating ceilings: | Level | Administrator label | Movie ratings | TV ratings | | --- | --- | --- | --- | @@ -315,14 +314,14 @@ its idempotent operation. ## Administrator review -The web queue uses session-only endpoints under `/api/external-requests`: +Administrator clients use session-only endpoints under `/api/external-requests`: - list with `status`, `requestType`, and `apiKeyId` filters; - request detail; - approve; - reject with a 1–300 character reason. -Approval locks the current request, reloads the current key policy, and +Approval locks the key before the request, reloads the current key policy, and revalidates authorization and target state immediately before execution. Revocation or a policy/grant change therefore takes effect immediately. @@ -380,15 +379,38 @@ may contain secrets. ## Configuration and administration -1. In **Settings → API Keys → API Keys & External Access**, create an external - key and save the raw secret; it is shown only once. +This runtime includes the management APIs. The external-access editor and +request-review UI are a separate follow-up in [PR #809](https://github.com/DialmasterOrg/Youtarr/pull/809). + +1. Create an external key with session-authenticated `POST /api/keys` and save + the returned raw secret; it is shown only once. Omitting `policy` creates a + legacy download key. For example, a body for a read-only external key is: + + ```json + {"name":"Catalog reader","policy":{"role":"view","maxRatingLevel":2,"allowUnrated":false,"allowedMediaTypes":["video"]},"channelIds":[12]} + ``` + + Use `x-access-token` with an administrator session for management calls. + Obtain channel database IDs from `GET /getchannels` (`database_id`), not its + YouTube `channel_id`. Do not copy the example ID without checking your channels. 2. The session-authenticated `GET /api/keys` management response includes `channel_grant_count`: the number of grants whose channels are currently enabled and non-terminated. It is not part of the `/external-api/v1` contract. 3. Choose the smallest request permissions, rating ceiling, media types, quotas, and enabled channel grants. -4. Use the administrator **External Requests** queue to approve or reject - pending requests when auto-approval is not enabled. +4. Review pending requests using `GET /api/external-requests` and + `POST /api/external-requests/{id}/approve` or `/reject` (a rejection requires + a JSON `reason`). Policy and grants can be replaced atomically with + `PUT /api/keys/{id}/external-access` and a `{ "policy": ..., "channelIds": [...] }` body. + +`GET /getconfig` reports the effective feature flag in +`isPlatformManaged.externalApiEnabled`; clients should use this value when +deciding whether to offer external API controls. + +When the shared work queue is full, writes return 503 while retaining the +accepted request and its idempotency key. Retry the same body to resume it +without charging another accepted write. An already completed channel or +deletion request returns its original result on an idempotent retry. Saving an external key with zero approved channels is intentional and does not backfill or grant access. That key fails closed: catalog reads and requests @@ -401,6 +423,53 @@ keys cannot use the legacy direct-download endpoint. ## Synthetic consumer contract +### Database integration and query plans + +`npm run test:external-api-database` runs the migration lifecycle and runtime +regressions against a real local database. CI runs both MariaDB 10.3 and +MySQL 8.0; the default backend suite excludes these files instead of skipping +tests. The tests create and remove disposable databases and use synthetic data. +Configure `DB_HOST`, `DB_PORT`, `DB_USER`, `DB_PASSWORD` and, if different, +`DB_ADMIN_USER` / `DB_ADMIN_PASSWORD`. The administrator needs database +creation and removal privileges. + +`npm run test:external-index-plans` creates 300,000 cached videos, 1,000 channels, +30,000 request records, and 10 channel grants in a disposable local database. +It executes the actual catalog and management service methods, captures their +SQL, compares results before and after the runtime migration, and records +`EXPLAIN` plans in `/tmp/youtarr-external-index-plans.json`. Set +`EXTERNAL_API_INDEX_REPORT` to choose another report path. CI uploads this report +for each engine. No YouTube or media-server calls are made. + +Only two indexes are added, both on `external_requests`: + +- `external_requests_catalog_status_idx`: `(api_key_id, request_type, youtube_id, created_at, id, status)`. + Latest request status is fetched in a batch for the displayed page using + `ROW_NUMBER()`, avoiding a correlated history scan for every candidate video. +- `external_requests_management_idx`: `(request_type, status, created_at, id)` + supports the filtered administrator queue. + +Measured plans on the synthetic fixture (estimated rows for the request-table +access, not actual rows returned): + +| Engine | Query | Before index | After index | +| --- | --- | --- | --- | +| MariaDB 10.3.39 | Latest status for displayed videos | 14,935 | 500, covering catalog-status index | +| MySQL 8.0.46 | Latest status for displayed videos | 19,923 | 50, covering catalog-status index | +| MariaDB 10.3.39 | Video requests, pending | 29,870, table scan | 1,500, management index | +| MySQL 8.0.46 | Video requests, pending | 39,847, table scan | 1,500, management index | + +The four service reads took 119 → 25 ms on MariaDB and 180 → 64 ms on MySQL +with the batched lookup; building both indexes took 168 ms and 255 ms respectively +in this local run. These are fixture measurements, not deployment guarantees. +MySQL still sorts the joined management result. Existing channel lookup indexes +already cover granted-channel joins; this migration adds no catalog-table indexes +and does not consolidate duplicate channels. Equivalent indexes under other names +are retained, and rollback removes only matching definitions under the migration's +own index names. + +### HTTP fixture + The canonical, production-free consumer dataset lives in `fixtures/external-api-v1/contract.json`. Validate its published checksum with `npm run test:external-fixture`. diff --git a/migrations/20261007041058-add-external-api-runtime-indexes.js b/migrations/20261007041058-add-external-api-runtime-indexes.js new file mode 100644 index 000000000..1794445e8 --- /dev/null +++ b/migrations/20261007041058-add-external-api-runtime-indexes.js @@ -0,0 +1,39 @@ +'use strict'; + +const { indexExists } = require('./helpers'); + +// EXPLAIN evidence for the actual runtime queries is in docs/EXTERNAL_API.md. +// Existing channel/catalog lookup indexes already cover the granted-channel +// joins; no new indexes or uniqueness constraints are needed on those tables. +const INDEXES = [ + { + fields: ['api_key_id', 'request_type', 'youtube_id', 'created_at', 'id', 'status'], + name: 'external_requests_catalog_status_idx', + }, + { + fields: ['request_type', 'status', 'created_at', 'id'], + name: 'external_requests_management_idx', + }, +]; + +module.exports = { + async up(queryInterface) { + for (const { fields, name } of INDEXES) { + if (await indexExists(queryInterface, 'external_requests', { fields }) || + await indexExists(queryInterface, 'external_requests', { name })) continue; + await queryInterface.addIndex('external_requests', fields, { name }); + } + }, + + async down(queryInterface) { + for (const { fields, name } of [...INDEXES].reverse()) { + // Preserve equivalent indexes under other names and unrelated same-name + // indexes. Only remove this migration's named, matching definitions. + if (await indexExists(queryInterface, 'external_requests', { name, fields })) { + await queryInterface.removeIndex('external_requests', name); + } + } + }, + + INDEXES, +}; diff --git a/migrations/__tests__/externalApiRuntimeIndexes.test.js b/migrations/__tests__/externalApiRuntimeIndexes.test.js new file mode 100644 index 000000000..b506b83a8 --- /dev/null +++ b/migrations/__tests__/externalApiRuntimeIndexes.test.js @@ -0,0 +1,34 @@ +const migration = require('../20261007041058-add-external-api-runtime-indexes'); + +function queryInterface(initial = []) { + const indexes = [...initial]; + return { + indexes, + showIndex: async () => indexes, + addIndex: jest.fn(async (_table, fields, { name }) => { + indexes.push({ name, fields: fields.map(attribute => ({ attribute })) }); + }), + removeIndex: jest.fn(async (_table, name) => { + indexes.splice(indexes.findIndex(index => index.name === name), 1); + }), + }; +} + +test('reapply repairs missing runtime indexes without duplicating an equivalent index', async () => { + const custom = { name: 'operator_catalog_index', + fields: migration.INDEXES[0].fields.map(attribute => ({ attribute })) }; + const query = queryInterface([custom]); + await migration.up(query); + await migration.up(query); + expect(query.addIndex).toHaveBeenCalledTimes(1); + await migration.down(query); + expect(query.indexes).toEqual([custom]); +}); + +test('rollback preserves unrelated indexes even when their names collide', async () => { + const unrelated = { name: migration.INDEXES[0].name, fields: [{ attribute: 'message' }] }; + const query = queryInterface([unrelated]); + await migration.up(query); + await migration.down(query); + expect(query.indexes).toEqual([unrelated]); +}); diff --git a/package.json b/package.json index 9399f72eb..872693b54 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "test": "npm run lint && npm run test:unit", "test:unit": "npm run test:backend -- --passWithNoTests && npm run test:frontend -- --passWithNoTests", "test:backend": "jest --config jest.config.js", + "test:external-index-plans": "node scripts/external-api-index-plans.js", "test:external-api-database": "jest --config jest.external-api-database.config.js --runInBand", "test:frontend": "cd client && npm test -- --watchAll=false", "test:coverage": "npm run lint && jest --config jest.config.js --coverage && cd client && npm test -- --coverage --watchAll=false", diff --git a/scripts/external-api-index-plans.js b/scripts/external-api-index-plans.js new file mode 100644 index 000000000..a7d912a6e --- /dev/null +++ b/scripts/external-api-index-plans.js @@ -0,0 +1,98 @@ +#!/usr/bin/env node +'use strict'; + +// Runs only against a local disposable database with entirely synthetic rows. +const path = require('path'); +const fs = require('fs'); +const { Sequelize, QueryTypes } = require('sequelize'); +const Umzug = require('umzug'); +const root = path.resolve(__dirname, '..'); +const databaseName = `youtarr_external_index_${process.pid}`; +const host = process.env.DB_HOST || '127.0.0.1'; +if (!['127.0.0.1', 'localhost', '::1'].includes(host)) { + throw new Error('Index-plan checks require a local disposable database'); +} +process.env.DB_NAME = databaseName; +process.env.DB_HOST = host; +const { sequelize } = require('../server/db'); +const catalog = require('../server/modules/externalCatalogService'); +const config = require('../server/modules/configModule'); +const logger = require('../server/logger'); +const { createExternalRequestService } = require('../server/modules/externalRequestService'); +const migration = require('../migrations/20261007041058-add-external-api-runtime-indexes'); + +async function run() { + const admin = new Sequelize('mysql', process.env.DB_ADMIN_USER || process.env.DB_USER || 'root', + process.env.DB_ADMIN_PASSWORD || process.env.DB_PASSWORD || '123qweasd', { + dialect: 'mysql', host, port: Number(process.env.DB_PORT || 3321), logging: false, + }); + let created = false; + try { + await admin.query(`CREATE DATABASE \`${databaseName}\` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci`); + created = true; + const queryInterface = sequelize.getQueryInterface(); + const migrator = new Umzug({ + migrations: { path: path.join(root, 'migrations'), params: [queryInterface, Sequelize] }, + storage: 'sequelize', storageOptions: { sequelize }, logging: false, + }); + await migrator.storage.logMigration('20250907000000-upgrade-to-utf8mb4-if-needed.js'); + await migrator.up(); + await migration.down(queryInterface); + const digit='(SELECT 0 n UNION ALL SELECT 1 UNION ALL SELECT 2 UNION ALL SELECT 3 UNION ALL SELECT 4 UNION ALL SELECT 5 UNION ALL SELECT 6 UNION ALL SELECT 7 UNION ALL SELECT 8 UNION ALL SELECT 9)'; + await sequelize.query(`CREATE TABLE fixture_numbers AS SELECT a.n + b.n*10+c.n*100+d.n*1000+e.n*10000+f.n*100000 n FROM ${digit} a CROSS JOIN ${digit} b CROSS JOIN ${digit} c CROSS JOIN ${digit} d CROSS JOIN ${digit} e CROSS JOIN ${digit} f WHERE a.n+b.n*10+c.n*100+d.n*1000+e.n*10000+f.n*100000 < 300000`); + await sequelize.query("INSERT INTO channels (channel_id,title,enabled,default_rating,auto_download_enabled_tabs) SELECT CONCAT('UC',LPAD(n,22,'0')),CONCAT('Synthetic ',n),1,'TV-Y','video' FROM fixture_numbers WHERE n<1000"); + await sequelize.query("INSERT INTO channelvideos (youtube_id,channel_id,title,media_type,youtube_removed,ignored,published_at) SELECT LPAD(n,11,'0'),CONCAT('UC',LPAD(FLOOR(n/300),22,'0')),CONCAT('Synthetic video ',n),IF(MOD(n,10)=0,'short','video'),MOD(n,20)=0,MOD(n,30)=0,'2026-10-07T00:00:00.000Z' FROM fixture_numbers"); + await sequelize.query("INSERT INTO apikeys (id,name,key_hash,key_prefix,created_at,role,allowed_media_types) VALUES (1,'Synthetic','fake','fake',NOW(),'admin','[\"video\"]')"); + await sequelize.query('INSERT INTO api_key_channel_grants (api_key_id,channel_id,created_at) SELECT 1,id,NOW() FROM channels WHERE id<=10'); + await sequelize.query("INSERT INTO external_requests (id,api_key_id,channel_id,youtube_id,request_type,status,created_at,updated_at) SELECT CONCAT('00000000-0000-4000-8000-',LPAD(n,12,'0')),1,MOD(n,1000)+1,LPAD(MOD(n,3000),11,'0'),IF(MOD(n,10)=0,'channel','video'),IF(MOD(n,20)=1,'pending','completed'),DATE_ADD('2026-01-01',INTERVAL n SECOND),NOW() FROM fixture_numbers WHERE n<30000"); + await sequelize.query('ANALYZE TABLE channels,channelvideos,external_requests'); + + const key = { id: 1, role: 'admin', maxRatingLevel: 4, allowUnrated: true, allowedMediaTypes: ['video'] }; + const requests = createExternalRequestService({ executor: () => {}, videoDeleter: {} }); + const reads = async () => [ + await catalog.listChannels(key), + await catalog.listVideos(key, { pageSize: '50' }), + await catalog.listChannelVideos(key, 1, { pageSize: '50' }), + await requests.listAdminRequests({ requestType: 'video', status: 'pending' }), + ]; + const statements = []; + sequelize.options.logging = sql => statements.push(sql.replace(/^Executing \([^)]+\): /, '')); + const beforeStart = Date.now(); + const beforeResults = await reads(); + const beforeMs = Date.now() - beforeStart; + sequelize.options.logging = false; + const queries = statements.filter(sql => /^SELECT/i.test(sql)); + const explain = async () => { + const plans = []; + for (const sql of queries) plans.push(await sequelize.query(`EXPLAIN ${sql}`, { type: QueryTypes.SELECT })); + return plans; + }; + const before = await explain(); + const buildStart = Date.now(); + await migration.up(queryInterface); + const indexBuildMs = Date.now() - buildStart; + await sequelize.query('ANALYZE TABLE external_requests'); + const after = await explain(); + const afterStart = Date.now(); + const afterResults = await reads(); + const afterMs = Date.now() - afterStart; + if (JSON.stringify(beforeResults) !== JSON.stringify(afterResults)) { + throw new Error('Runtime index migration changed query results'); + } + const [version] = await sequelize.query('SELECT VERSION() version', { type: QueryTypes.SELECT }); + const report = { version: version.version, + seed: { channelvideos: 300000, channels: 1000, requests: 30000, grants: 10 }, + beforeMs, afterMs, indexBuildMs, queries, before, after }; + const destination = process.env.EXTERNAL_API_INDEX_REPORT || '/tmp/youtarr-external-index-plans.json'; + fs.writeFileSync(destination, JSON.stringify(report, null, 2) + '\n'); + logger.info({ destination, version: report.version, beforeMs, afterMs, indexBuildMs }, 'External API index plans verified'); + } finally { + sequelize.options.logging = false; + await sequelize.close(); + if (created) await admin.query(`DROP DATABASE \`${databaseName}\``); + await admin.close(); + config.stopWatchingConfig(); + } +} + +run().catch(error => { logger.error({ err: error }, 'External API index-plan checks failed'); process.exitCode = 1; }); diff --git a/server/models/externalrequest.js b/server/models/externalrequest.js index da0339cb0..d1b6070e2 100644 --- a/server/models/externalrequest.js +++ b/server/models/externalrequest.js @@ -25,6 +25,8 @@ ExternalRequest.init({ { unique: true, fields: ['api_key_id', 'idempotency_hash'], name: 'external_requests_key_idempotency_uq' }, { fields: ['api_key_id', 'created_at'], name: 'external_requests_key_created_idx' }, { fields: ['api_key_id', 'status'], name: 'external_requests_key_status_idx' }, + { fields: ['api_key_id', 'request_type', 'youtube_id', 'created_at', 'id', 'status'], name: 'external_requests_catalog_status_idx' }, + { fields: ['request_type', 'status', 'created_at', 'id'], name: 'external_requests_management_idx' }, ] }); ExternalRequest.REQUEST_STATUSES = REQUEST_STATUSES; module.exports = ExternalRequest; diff --git a/server/modules/__tests__/externalCatalogRequests.test.js b/server/modules/__tests__/externalCatalogRequests.test.js new file mode 100644 index 000000000..a406429f7 --- /dev/null +++ b/server/modules/__tests__/externalCatalogRequests.test.js @@ -0,0 +1,20 @@ +jest.mock('../../db', () => ({ sequelize: { query: jest.fn() } })); +const { sequelize } = require('../../db'); +const { attachRequestStatuses } = require('../externalCatalogRequests'); + +beforeEach(() => jest.clearAllMocks()); + +test('empty catalog pages do not query request history', async () => { + await attachRequestStatuses({ id: 7 }, []); + expect(sequelize.query).not.toHaveBeenCalled(); +}); + +test('attaches latest statuses only to the displayed videos for the calling key', async () => { + sequelize.query.mockResolvedValue([{ youtube_id: 'abcdefghijk', status: 'pending' }]); + const rows = [{ youtube_id: 'abcdefghijk' }, { youtube_id: 'lmnopqrstuv' }]; + await attachRequestStatuses({ id: 7 }, rows); + expect(rows.map(row => row.request_status)).toEqual(['pending', null]); + expect(sequelize.query).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ + replacements: { keyId: 7, youtubeIds: ['abcdefghijk', 'lmnopqrstuv'] }, + })); +}); diff --git a/server/modules/__tests__/externalCatalogService.test.js b/server/modules/__tests__/externalCatalogService.test.js index 0d610d5f1..a2620b6dc 100644 --- a/server/modules/__tests__/externalCatalogService.test.js +++ b/server/modules/__tests__/externalCatalogService.test.js @@ -17,7 +17,10 @@ const key = (overrides = {}) => ({ }); describe('external cached catalog', () => { - beforeEach(() => jest.clearAllMocks()); + beforeEach(() => { + jest.clearAllMocks(); + sequelize.query.mockResolvedValue([]); + }); test('lists only granted enabled channels with SQL paging and policy-filtered counts', async () => { sequelize.query @@ -71,8 +74,9 @@ describe('external cached catalog', () => { youtube_id: 'abc', title: 'Allowed', thumbnail: 'https://i.ytimg.com/vi/abc/hqdefault.jpg', publishedAt: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', duration: 90, media_type: 'short', description: null, downloaded_id: null, - downloaded_removed: null, rating: 'TV-Y', request_status: 'processing', - }]); + downloaded_removed: null, rating: 'TV-Y', + }]) + .mockResolvedValueOnce([{ youtube_id: 'abc', status: 'processing' }]); const result = await catalog.listChannelVideos( key({ allowedMediaTypes: ['video', 'short'] }), '8', @@ -101,8 +105,7 @@ describe('external cached catalog', () => { 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC' ); expect(listSql).toContain('LIMIT :fetchLimit OFFSET :offset'); - expect(listSql).toContain('FROM external_requests er'); - expect(listSql).toContain('er.api_key_id = :keyId'); + expect(sequelize.query.mock.calls[3][1].replacements).toMatchObject({ keyId: 4, youtubeIds: ['abc'] }); }); test('returns the complete deterministic cross-channel catalog beyond the former three-page cap', async () => { @@ -123,8 +126,8 @@ describe('external cached catalog', () => { channel_database_id: 8, channel_id: 'UCsafe', channel_title: 'Safe Channel', - request_status: 'failed', - }]); + }]) + .mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', status: 'failed' }]); const result = await catalog.listVideos(key(), { page: '3', @@ -320,8 +323,8 @@ describe('external cached catalog', () => { channel_database_id: 8, channel_id: 'UCsafe', channel_title: 'Safe Channel', - request_status: 'approved', - }]); + }]) + .mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', status: 'approved' }]); const metadataService = { getVideoMetadata: jest.fn().mockResolvedValue({ description: 'full description', diff --git a/server/modules/externalCatalogRequests.js b/server/modules/externalCatalogRequests.js new file mode 100644 index 000000000..342e06551 --- /dev/null +++ b/server/modules/externalCatalogRequests.js @@ -0,0 +1,28 @@ +const { QueryTypes } = require('sequelize'); +const { sequelize } = require('../db'); + +async function attachRequestStatuses(key, rows) { + if (rows.length === 0) return; + // Limit the history lookup to the displayed page. A correlated ORDER BY / + // LIMIT subquery can scan a key's entire history for every catalog candidate + // on MySQL, even when a matching composite index exists. + const requests = await sequelize.query( + `SELECT youtube_id, status FROM ( + SELECT youtube_id, status, + ROW_NUMBER() OVER ( + PARTITION BY youtube_id ORDER BY created_at DESC, id DESC + ) AS request_rank + FROM external_requests + WHERE api_key_id = :keyId AND request_type = 'video' + AND youtube_id IN (:youtubeIds) + ) ranked WHERE request_rank = 1`, + { + replacements: { keyId: key.id, youtubeIds: [...new Set(rows.map(row => row.youtube_id))] }, + type: QueryTypes.SELECT, + } + ); + const statuses = new Map(requests.map(request => [request.youtube_id, request.status])); + for (const row of rows) row.request_status = statuses.get(row.youtube_id) || null; +} + +module.exports = { attachRequestStatuses }; diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js index aac4f7feb..af09ed518 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalCatalogService.js @@ -1,6 +1,7 @@ const fs = require('fs').promises; const path = require('path'); const crypto = require('crypto'); +const { attachRequestStatuses } = require('./externalCatalogRequests'); const { QueryTypes } = require('sequelize'); const { sequelize } = require('../db'); const configModule = require('./configModule'); @@ -388,14 +389,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { cv.duration, cv.media_type, v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, ${effectiveRating} AS rating, c.id AS channel_database_id, - ${sortExpressions[sortBy]} AS cursor_sort_value, - (SELECT er.status - FROM external_requests er - WHERE er.api_key_id = :keyId - AND er.request_type = 'video' - AND er.youtube_id = cv.youtube_id - ORDER BY er.created_at DESC, er.id DESC - LIMIT 1) AS request_status + ${sortExpressions[sortBy]} AS cursor_sort_value ${from} ${seekSql} ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC @@ -404,6 +398,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { ); const hasMore = fetchedRows.length > pageSize; const rows = fetchedRows.slice(0, pageSize); + await attachRequestStatuses(key, rows); const total = Number(countRows[0]?.total || 0); const lastIndexedAt = lastFetched(channel, mediaType); return { @@ -548,14 +543,7 @@ async function listVideos(key, query = {}) { cv.duration, cv.media_type, v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, ${effectiveRating} AS rating, c.id AS channel_database_id, c.channel_id, COALESCE(c.title, c.uploader, '') AS channel_title, - ${sortExpressions[sortBy]} AS cursor_sort_value, - (SELECT er.status - FROM external_requests er - WHERE er.api_key_id = :keyId - AND er.request_type = 'video' - AND er.youtube_id = cv.youtube_id - ORDER BY er.created_at DESC, er.id DESC - LIMIT 1) AS request_status + ${sortExpressions[sortBy]} AS cursor_sort_value ${from} ${seekSql} ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC @@ -564,6 +552,7 @@ async function listVideos(key, query = {}) { ); const hasMore = fetchedRows.length > pageSize; const rows = fetchedRows.slice(0, pageSize); + await attachRequestStatuses(key, rows); const total = Number(countRows[0]?.total || 0); return { data: rows.map((row) => ({ @@ -612,14 +601,7 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { v.last_downloaded_at, v.file_size AS fileSize, v.audio_file_size AS audioFileSize, v.protected, v.rating_source, v.video_resolution, ${effectiveRating} AS rating, c.id AS channel_database_id, c.channel_id, - COALESCE(c.title, c.uploader, '') AS channel_title, - (SELECT er.status - FROM external_requests er - WHERE er.api_key_id = :keyId - AND er.request_type = 'video' - AND er.youtube_id = cv.youtube_id - ORDER BY er.created_at DESC, er.id DESC - LIMIT 1) AS request_status + COALESCE(c.title, c.uploader, '') AS channel_title FROM channelvideos cv INNER JOIN channels c ON c.channel_id = cv.channel_id INNER JOIN api_key_channel_grants g @@ -644,6 +626,7 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { ); const row = rows[0]; if (!row) throw new CatalogError('Video not found', 404); + await attachRequestStatuses(key, [row]); const metadataProvider = metadataService || require('./videoMetadataModule'); const metadata = await metadataProvider.getVideoMetadata(youtubeId); diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index 980246c4b..a2cbac0cb 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -257,3 +257,25 @@ test('idempotency keys cannot replay a deletion as a video request', async () => await video.update({ removed: true }); await expect(api.createVideoRequest(key, input)).rejects.toMatchObject({ status: 409 }); }); +test('runtime indexes preserve equivalent indexes and survive rollback and partial reapply', async () => { + const migration = require('../../migrations/20261007041058-add-external-api-runtime-indexes'); + const queryInterface = sequelize.getQueryInterface(); + await migration.down(queryInterface); + const customName = 'custom_external_catalog_lookup'; + await queryInterface.addIndex('external_requests', migration.INDEXES[0].fields, { name: customName }); + try { + await migration.up(queryInterface); + await migration.up(queryInterface); + let names = (await queryInterface.showIndex('external_requests')).map(index => index.name); + expect(names).toContain(customName); + expect(names).not.toContain(migration.INDEXES[0].name); + expect(names).toContain(migration.INDEXES[1].name); + await migration.down(queryInterface); + names = (await queryInterface.showIndex('external_requests')).map(index => index.name); + expect(names).toContain(customName); + expect(names).not.toContain(migration.INDEXES[1].name); + } finally { + await queryInterface.removeIndex('external_requests', customName); + await migration.up(queryInterface); + } +}); From 62eaf055e9ef50ee7dca89854827bb23b05ef0a0 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:27:16 +0000 Subject: [PATCH 11/27] refactor: separate external request lifecycle and preserve approval retries --- CLAUDE.md | 5 + docs/DEVELOPMENT.md | 11 +- .../__tests__/externalPagination.test.js | 11 + .../__tests__/externalQuotaService.test.js | 3 +- .../externalRequestExecution.test.js | 321 ++++++ .../externalRequestPrimitives.test.js | 28 + .../externalRequestReconciliation.test.js | 72 ++ .../externalRequestReviewService.test.js | 201 ++++ .../__tests__/externalRequestService.test.js | 747 +------------ .../__tests__/fixtures/externalRequests.js | 166 +++ server/modules/externalCatalogService.js | 23 +- server/modules/externalPagination.js | 37 +- server/modules/externalQuotaService.js | 52 +- server/modules/externalRequestExecution.js | 232 ++++ server/modules/externalRequestPrimitives.js | 194 ++++ .../modules/externalRequestReconciliation.js | 61 ++ .../modules/externalRequestReviewService.js | 475 +++++++++ server/modules/externalRequestService.js | 996 +----------------- .../testing/externalRuntime.database.test.js | 68 ++ 19 files changed, 1914 insertions(+), 1789 deletions(-) create mode 100644 server/modules/__tests__/externalPagination.test.js create mode 100644 server/modules/__tests__/externalRequestExecution.test.js create mode 100644 server/modules/__tests__/externalRequestPrimitives.test.js create mode 100644 server/modules/__tests__/externalRequestReconciliation.test.js create mode 100644 server/modules/__tests__/externalRequestReviewService.test.js create mode 100644 server/modules/__tests__/fixtures/externalRequests.js create mode 100644 server/modules/externalRequestExecution.js create mode 100644 server/modules/externalRequestPrimitives.js create mode 100644 server/modules/externalRequestReconciliation.js create mode 100644 server/modules/externalRequestReviewService.js diff --git a/CLAUDE.md b/CLAUDE.md index 3910fcb32..0f21aef6d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,6 +29,9 @@ For multi-part requests (e.g., "review this PR AND explain WebSocket handling"), ## Architecture ### Backend (server/) +- `routes/externalApi.js` and `middleware/externalApiAuth.js`: opt-in `/external-api/v1` routes with mandatory external-key authentication, bounded JSON bodies, private responses, and per-key limits. `modules/externalApiConfig.js`, `externalApiResponse.js`, `externalPermissions.js`, and `externalEligibility.js` centralize feature availability, errors, scopes, and media/rating policy. +- `modules/externalCatalogService.js`: cached, grant-filtered catalog and asset lookup; `externalCatalogRequests.js` batches current-page request status, `externalPagination.js` shares bounded page cursors, and `externalThumbnailProxy.js` enforces upstream image URL, size, and timeout limits. +- `modules/externalRequestService.js`: request creation and owner history; `externalRequestExecution.js` owns dispatch/recovery, `externalRequestReviewService.js` owns administrator decisions, `externalRequestPrimitives.js` shares validation/DTOs, and `externalRequestReconciliation.js` reconciles durable jobs for both request reads and `externalQuotaService.js`. Key-before-request locks serialize decisions; `externalWorkLimiter.js` bounds expensive work. `routes/externalRequests.js` exposes session-authenticated management endpoints; their UI is a separate follow-up. - `modules/externalCookies.js`: validates external cookie snapshots through the installed yt-dlp parser in `utils/validate-cookies.py`, memoizes unchanged results, and reports unusable files for omission; `modules/uploadedCookies.js` gives the uploaded `config/cookies.user.txt` a disposable private copy per run (yt-dlp rewrites its cookie file in place on exit, so concurrent runs sharing it could read a half-written jar or undo a Settings upload; nothing is written back, and Settings uploads are written to a temp file and renamed into place; if the copy fails, the run goes without cookies (error-level log; info when the upload was just deleted) and yt-dlp is never handed the uploaded file itself, whose exit-time jar save could overwrite a newer upload or recreate deleted cookies); `modules/ytdlpProcess.js` owns both kinds of copy's lifetime around yt-dlp processes. No persistent cookie fallback. - `modules/rescanRowUpdate.js` + `modules/videoRowGuard.js`: the filesystem rescan (`videosModule.backfillVideoMetadata`) walks the disk once and reads rows in chunks, so downloads and deletions in between make the walk stale. The walk only nominates rows (`isRescanCandidate`); `resolveRescanUpdate` recomputes each nominated row's write from a fresh per-format stat (present only for a regular file / missing on `ENOENT` or `ENOTDIR` or a non-file / unknown on any other error; both formats missing marks the row missing and keeps its stored paths; a missing format is cleared only when the other is present; any present format means `removed: false` even if the other is unknown; an unknown format is never cleared, and with nothing present an unknown writes nothing), and the write only lands if the row is unchanged since read (`videoRowGuard.unchangedSinceRead`: removed, both paths and sizes, `video_resolution`, `last_downloaded_at`; zero affected rows means a conflict only when the write includes a column that really differs, because Sequelize's MySQL dialect connects with -FOUND_ROWS and mysql2 then counts changed rows, not matched rows). Nominated rows left as recorded (files still at a stored path, e.g. outside the downloads folder, or files that could not be checked; re-checked every run) are counted as `kept` (`videosKept` in the run details, not the message). The walk's count of folders and files it could not read is `unreadableOnDisk` in the run details and log (not the message: those rows are re-checked by path). Rows another writer changed are counted as `skippedChanged` (`videosSkipped` in the run record) and failed writes as `failed` (`videosFailed`); both appear in the run message, and a finished scan with failed writes records as `error`/`partial` (a timed-out scan stays `timed-out` so Maintenance can continue it); the completion `rescanStatus` broadcast is built through the same `rescanRunSummary` record so the live Maintenance view matches the run history. Library repair (`jobModule.backfillFromCompleteList`) uses the same guard: it reads the row before searching the disk, writes file details and `removed: false` only for a file it actually found, and looks up a lost row's files by YouTube id with one `videosModule.scanForVideoFiles` walk per run (only when a lost row exists; video and audio, any folder layout), recreating it as missing when the walk finds nothing. The walk skips and counts (`unreadable`) folders and files it cannot read rather than stopping, logging unreadable files once per folder; repair still recreates unmatched rows as missing after a partial walk, because deferring would block recovery wherever a folder is never readable (lost+found, NAS recycle bins), and the next rescan restores a row whose file it finds. Such a run records as `error`/`partial`, naming the count in its message (`recreatedUnverified` and `unreadableOnDisk` in the details); a partial walk that affected no row stays `completed`. - `modules/cookieDetails.js`: summarizes the YouTube login cookies (names, counts, earliest expiry; never values) in the active cookie file as `details` on `GET /api/cookies/status` and the upload/delete responses, reading external files through `externalCookies.readExternalCookies`. `modules/cookieTest.js`: `POST /api/cookies/test`, one yt-dlp request to the subscriptions feed with `buildCommonArgs` (same proxy/IP family/cookies/cache as downloads) through `ytDlpRunner`, against a private temp copy of uploaded cookies (yt-dlp writes its jar back on exit, which would otherwise undo a replace/delete made mid-test; external files get spawnYtDlp's snapshot) and never logs stderr (yt-dlp echoes malformed cookie lines with their values); any exit 0 with JSON is signed in (even zero subscriptions), and the feed's "Incomplete yt initial data" warnings mean signed out (EXPIRED_COOKIES), a meaning specific to this feed, so it is not in the shared subscription-import classifier. One test at a time (409), rate-limited by `cookieTestRateLimiter`. @@ -226,6 +229,8 @@ npm run test:frontend # all frontend test npm run test:frontend -- client/src/hooks/__tests__/foo.ts # one frontend test npm run test:backend # all backend tests npm run test:backend -- server/modules/__tests__/foo.test.js # one backend test +npm run test:external-api-database # real database migration/runtime regressions +npm run test:external-index-plans # local synthetic query/EXPLAIN report npm run lint # lint front + back npm run lint:ts # TypeScript typecheck ``` diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 722c052ee..3d5119fd5 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -445,7 +445,7 @@ git commit --no-verify ### Running Tests -All tests run on your host machine (not in Docker) since they're isolated unit tests: +The unit suites run on your host machine without a database: ```bash # Run all tests (backend + frontend) @@ -464,6 +464,15 @@ npm run test:coverage npm run test:watch ``` +### External API Database Tests + +The separate external API integration suite requires a disposable MariaDB 10.3 +or MySQL 8.0 server. It runs the real migrations and model-backed request, +approval, quota, and catalog flows without contacting YouTube. Follow the +[database setup and query-plan commands](EXTERNAL_API.md#database-integration-and-query-plans) +to run both engines, including rollback/reapply and the synthetic index report. +The ordinary backend Jest command does not run these database tests. + ### External Cookie Validation Tests The backend Jest suites cover external-file handling and process cleanup. A diff --git a/server/modules/__tests__/externalPagination.test.js b/server/modules/__tests__/externalPagination.test.js new file mode 100644 index 000000000..dc0cfdfe8 --- /dev/null +++ b/server/modules/__tests__/externalPagination.test.js @@ -0,0 +1,11 @@ +const { pagination, paginationDto, CatalogError } = require('../externalPagination'); + +test('catalog pagination rejects unbounded or conflicting parameters', () => { + for (const query of [{ page: '101' }, { pageSize: '101' }, { page: '-1' }, + { cursor: 'x'.repeat(201) }, { page: '1', cursor: 'anything' }]) { + expect(() => pagination(query)).toThrow(CatalogError); + } + const result = paginationDto(1, 50, 60); + expect(pagination({ cursor: result.nextCursor })).toEqual({ page: 2, pageSize: 50, offset: 50 }); + expect(paginationDto(1, 50, 0)).toMatchObject({ totalPages: 0, nextCursor: null }); +}); diff --git a/server/modules/__tests__/externalQuotaService.test.js b/server/modules/__tests__/externalQuotaService.test.js index a9c1660d5..6c16410c6 100644 --- a/server/modules/__tests__/externalQuotaService.test.js +++ b/server/modules/__tests__/externalQuotaService.test.js @@ -145,8 +145,7 @@ describe('external API durable quotas', () => { }); expect(completedRequest.update).toHaveBeenCalledWith( - expect.objectContaining({ status: 'completed', active_dedupe_key: null }), - {} + expect.objectContaining({ status: 'completed', active_dedupe_key: null }) ); expect(result.remaining.activeJobs).toBe(5); }); diff --git a/server/modules/__tests__/externalRequestExecution.test.js b/server/modules/__tests__/externalRequestExecution.test.js new file mode 100644 index 000000000..5d1f9dd49 --- /dev/null +++ b/server/modules/__tests__/externalRequestExecution.test.js @@ -0,0 +1,321 @@ +const { fixture, record, timestamp, youtubeId } = require('./fixtures/externalRequests'); + +describe('external request execution', () => { + test('auto-approval queues only the canonical URL and server-owned channel mapping', async () => { + const { service, models, executor, key, transaction } = fixture({ + key: { autoApproveVideoRequests: true }, + }); + const result = await service.createVideoRequest(key, { + youtubeId, channelId: 8, idempotencyKey: 'external-client-1', + }); + expect(models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ + status: 'pending', + decided_at: timestamp, + idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), + }), { transaction }); + expect(executor).toHaveBeenCalledWith({ + body: { + urls: [`https://www.youtube.com/watch?v=${youtubeId}`], + channelId: 'UC1234567890123456789012', + ownerChannelMap: { [youtubeId]: 'UC1234567890123456789012' }, + initiatedBy: { type: 'api_key', name: 'External Client' }, + jobLabel: 'Manually Added Urls (external request)', + externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + }, + }); + expect(result.request.status).toBe('processing'); + }); + + test('returns service unavailable when the shared work queue is full', async () => { + const workError = Object.assign(new Error('full'), { + name: 'ExternalWorkLimitError', + }); + const limited = fixture({ + key: { autoApproveVideoRequests: true }, + workLimiter: { run: jest.fn().mockRejectedValue(workError) }, + }); + await expect(limited.service.createVideoRequest(limited.key, { + youtubeId, + channelId: 8, + })).rejects.toMatchObject({ + name: 'RequestError', + status: 503, + code: 'work_queue_full', + }); + expect(limited.created.status).toBe('pending'); + expect(limited.created.active_dedupe_key).toBe(`4:video:${youtubeId}`); + }); + + test('revalidates a channel grant inside the work slot before enqueue', async () => { + const raced = fixture({ + key: { autoApproveVideoRequests: true }, + }); + raced.models.ApiKeyChannelGrant.findOne + .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) + .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) + .mockResolvedValueOnce(null); + + const result = await raced.service.createVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.executor).not.toHaveBeenCalled(); + expect(raced.created.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + })); + expect(result.request.status).toBe('failed'); + }); + + test('does not enqueue when video auto-approval is disabled before execution', async () => { + const raced = fixture({ key: { autoApproveVideoRequests: true } }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + allowVideoRequests: true, + autoApproveVideoRequests: false, + }); + + const result = await raced.service.createVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.executor).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('resumes an interrupted auto-approved dispatch with the stable request job id', async () => { + const interrupted = record({ status: 'pending', decided_at: timestamp, job_id: null }); + const retryExecutor = jest.fn().mockResolvedValue(interrupted.id); + const retry = fixture({ + key: { autoApproveVideoRequests: true }, + executor: retryExecutor, + }); + retry.models.ExternalRequest.findOne.mockResolvedValue(interrupted); + retry.models.ExternalRequest.findByPk.mockResolvedValue(interrupted); + const result = await retry.service.createVideoRequest( + retry.key, + { youtubeId, channelId: 8, idempotencyKey: 'same-request' } + ); + expect(retry.models.ExternalRequest.create).not.toHaveBeenCalled(); + expect(retryExecutor).toHaveBeenCalledWith(expect.objectContaining({ + body: expect.objectContaining({ externalRequestId: interrupted.id }), + })); + expect(interrupted.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'processing', + job_id: interrupted.id, + }), { transaction: retry.transaction }); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { status: 'processing' }, + }); + }); + + test('auto-approved channel requests provision and grant the resulting channel', async () => { + const approved = fixture({ key: { autoApproveChannelRequests: true } }); + const result = await approved.service.createChannelRequest(approved.key, { + channelUrl: 'https://www.youtube.com/channel/UC1234567890123456789012', + }); + + expect(approved.channelProvisioner.getChannelInfo).toHaveBeenCalledWith( + 'https://www.youtube.com/channel/UC1234567890123456789012', + false, + true, + {}, + { skipTabDetection: true } + ); + expect(approved.models.ApiKeyChannelGrant.findOrCreate).toHaveBeenCalledWith({ + where: { api_key_id: 4, channel_id: 8 }, + defaults: { api_key_id: 4, channel_id: 8 }, + transaction: approved.transaction, + }); + expect(result.request).toMatchObject({ + type: 'channel', + status: 'completed', + target: { channelId: 8 }, + }); + }); + + test('does not provision when channel auto-approval is disabled before execution', async () => { + const raced = fixture({ key: { autoApproveChannelRequests: true } }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + allowChannelRequests: true, + autoApproveChannelRequests: false, + }); + + const result = await raced.service.createChannelRequest(raced.key, { + channelUrl: 'https://www.youtube.com/@safe', + }); + + expect(raced.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('resumes an interrupted auto-approved channel request on idempotent retry', async () => { + const interrupted = record({ + request_type: 'channel', + channel_id: null, + youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', + status: 'pending', + decided_at: timestamp, + }); + const recovering = fixture({ + key: { autoApproveChannelRequests: true }, + }); + recovering.models.ExternalRequest.findOne.mockResolvedValue(interrupted); + + const result = await recovering.service.createChannelRequest(recovering.key, { + channelUrl: 'https://www.youtube.com/@safe', + }); + + expect(recovering.channelProvisioner.getChannelInfo).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { type: 'channel', status: 'completed' }, + }); + }); + + test('auto-approved deletion reaches a completed terminal state', async () => { + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + idempotencyKey: 'delete-1', + }); + + expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( + 91, + expect.objectContaining({ + transaction: deleting.transaction, + video: expect.objectContaining({ id: 91, youtubeId }), + }) + ); + expect(result).toMatchObject({ + outcome: 'created', + request: { type: 'delete_video', status: 'completed' }, + }); + }); + + test('does not delete when delete auto-approval is disabled before execution', async () => { + const raced = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + raced.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + raced.quotaService.assertExecutionCapacity.mockResolvedValue({ + ...raced.key, + role: 'delete', + allowDeleteVideoRequests: true, + autoApproveDeleteRequests: false, + }); + + const result = await raced.service.createDeleteVideoRequest(raced.key, { + youtubeId, + channelId: 8, + }); + + expect(raced.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); + expect(result.request.status).toBe('failed'); + }); + + test('deletion completes idempotently when the target disappears during execution', async () => { + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + videoDeleter: { + deleteVideoById: jest.fn().mockResolvedValue({ + success: false, + error: 'Video not found in database', + }), + }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + }); + + expect(result.request).toMatchObject({ + status: 'completed', + message: 'Video is already deleted', + }); + }); + + test('recovers a stale processing deletion idempotently', async () => { + const stale = record({ + request_type: 'delete_video', + status: 'processing', + channel_url: null, + decided_at: new Date('2026-07-26T11:00:00.000Z'), + updated_at: new Date('2026-07-26T11:00:00.000Z'), + }); + const deleting = fixture({ + key: { role: 'delete', autoApproveDeleteRequests: true }, + }); + deleting.models.Video.findOne.mockResolvedValue({ + id: 91, + youtubeId, + normalized_rating: 'TV-Y', + removed: false, + }); + deleting.models.ExternalRequest.findOne.mockResolvedValue(stale); + + const result = await deleting.service.createDeleteVideoRequest(deleting.key, { + youtubeId, + channelId: 8, + }); + + expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( + 91, + expect.objectContaining({ transaction: deleting.transaction }) + ); + expect(result).toMatchObject({ + outcome: 'duplicate', + request: { type: 'delete_video', status: 'completed' }, + }); + }); + + test('marks queue failures terminal so a retry is possible', async () => { + const queueError = new Error('secret executor failure'); + const failed = fixture({ + key: { autoApproveVideoRequests: true }, + executor: jest.fn().mockRejectedValue(queueError), + }); + const result = await failed.service.createVideoRequest( + failed.key, { youtubeId, channelId: 8 } + ); + expect(failed.created.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + message: 'Download could not be queued', + })); + expect(result.request).toMatchObject({ + status: 'failed', + message: 'Download could not be queued', + }); + expect(result.request.message).not.toContain('secret'); + }); + + +}); diff --git a/server/modules/__tests__/externalRequestPrimitives.test.js b/server/modules/__tests__/externalRequestPrimitives.test.js new file mode 100644 index 000000000..6ee8879e9 --- /dev/null +++ b/server/modules/__tests__/externalRequestPrimitives.test.js @@ -0,0 +1,28 @@ +const { RequestError, requestPagination, requestPaginationDto, dto, + normalizeChannelUrl, normalizeIdempotencyKey } = require('../externalRequestPrimitives'); + +test('request pages share bounded cursors and retain request-specific errors', () => { + const page = requestPaginationDto(1, 10, 25); + expect(requestPagination({ cursor: page.nextCursor, pageSize: '10' })) + .toEqual({ page: 2, pageSize: 10, offset: 10 }); + expect(() => requestPagination({ cursor: page.nextCursor, page: '2' })).toThrow(RequestError); + expect(() => requestPagination({ page: '101' })).toThrow(RequestError); + expect(() => requestPagination({ cursor: 'invalid' })).toThrow(RequestError); + expect(requestPaginationDto(100, 10, 10000).nextCursor).toBeNull(); +}); + +test('public request DTOs omit key hashes, jobs and deduplication identifiers', () => { + const result = dto({ id: 'request', request_type: 'channel', status: 'completed', + grant_to_requesting_key: false, key_hash: 'private-hash', job_id: 'internal-job', + active_dedupe_key: 'internal-dedupe', idempotency_hash: 'private-idempotency' }); + expect(result.grantToRequestingKey).toBe(false); + expect(JSON.stringify(result)).not.toMatch(/private|internal/); +}); + +test('channel targets remain canonical and idempotency keys are bounded hashes', () => { + expect(normalizeChannelUrl('youtube.com/@Family')).toBe('https://www.youtube.com/@Family'); + expect(() => normalizeChannelUrl('https://youtube.com.attacker.test/@Family')).toThrow(RequestError); + expect(() => normalizeChannelUrl('https://youtube.com/@Family/%zz')).toThrow(RequestError); + expect(normalizeIdempotencyKey('local-test')).toMatch(/^[a-f0-9]{64}$/); + expect(() => normalizeIdempotencyKey('x'.repeat(201))).toThrow(RequestError); +}); diff --git a/server/modules/__tests__/externalRequestReconciliation.test.js b/server/modules/__tests__/externalRequestReconciliation.test.js new file mode 100644 index 000000000..4df2d8ba5 --- /dev/null +++ b/server/modules/__tests__/externalRequestReconciliation.test.js @@ -0,0 +1,72 @@ +const { fixture, record, timestamp, youtubeId } = require('./fixtures/externalRequests'); + +describe('external request reconciliation', () => { + test('reconciles a processing deletion after its video is absent', async () => { + const deleting = fixture({ key: { role: 'delete' } }); + const processing = record({ + request_type: 'delete_video', + status: 'processing', + decided_at: timestamp, + }); + deleting.models.ExternalRequest.findAndCountAll.mockResolvedValue({ + rows: [processing], + count: 1, + }); + deleting.models.Video.findAll.mockResolvedValue([]); + + const result = await deleting.service.listRequests(deleting.key); + + expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', + active_dedupe_key: null, + completed_at: timestamp, + })); + expect(result.data[0]).toMatchObject({ + type: 'delete_video', + status: 'completed', + }); + }); + + test('lists only the calling key and lazily reconciles completed downloads', async () => { + const processing = record({ status: 'processing' }); + const own = fixture(); + own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); + own.models.Video.findAll.mockResolvedValue([{ youtubeId }]); + const result = await own.service.listRequests(own.key, { + page: '1', pageSize: '10', + }); + expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith(expect.objectContaining({ + where: { api_key_id: 4 }, + limit: 10, + offset: 0, + })); + expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', + active_dedupe_key: null, + completed_at: timestamp, + })); + expect(result.data[0].status).toBe('completed'); + }); + + test('reconciles a status-filtered page and requeries its count', async () => { + const stale = record({ status: 'processing', job_id: 'job-123' }); + const own = fixture(); + own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: 'Error' }]); + own.models.ExternalRequest.findAndCountAll + .mockResolvedValueOnce({ rows: [stale], count: 1 }) + .mockResolvedValueOnce({ rows: [], count: 0 }); + + const result = await own.service.listRequests(own.key, { status: 'processing' }); + + expect(stale.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); + expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( + expect.objectContaining({ where: { api_key_id: 4, status: 'processing' } }) + ); + expect(result).toMatchObject({ + data: [], + pagination: { total: 0, totalPages: 0 }, + }); + }); + + +}); diff --git a/server/modules/__tests__/externalRequestReviewService.test.js b/server/modules/__tests__/externalRequestReviewService.test.js new file mode 100644 index 000000000..0dfc91404 --- /dev/null +++ b/server/modules/__tests__/externalRequestReviewService.test.js @@ -0,0 +1,201 @@ +const { fixture, record, timestamp, youtubeId } = require('./fixtures/externalRequests'); + +describe('external request reviewservice', () => { + test('lists administrator requests with safe joined metadata and filters', async () => { + const listed = record({ + status: 'pending', + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', key_hash: 'must-not-leak', + role: 'request', is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + default_rating: 'TV-PG', + }, + job: null, + }); + const admin = fixture(); + admin.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [listed], count: 1 }); + admin.models.ApiKey.findAll.mockResolvedValue([listed.apiKey]); + admin.models.ChannelVideo.findAll.mockResolvedValue([{ + youtube_id: youtubeId, + channel_id: 'UC1234567890123456789012', + title: 'Safe video', + media_type: 'video', + }]); + admin.models.Video.findAll.mockResolvedValue([{ + youtubeId, + normalized_rating: 'TV-Y', + }]); + + const result = await admin.service.listAdminRequests({ + page: '2', pageSize: '10', status: 'pending', apiKeyId: '4', + }); + + expect(admin.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( + expect.objectContaining({ + where: { status: 'pending', api_key_id: 4 }, + limit: 10, + offset: 10, + distinct: true, + }) + ); + const keyInclude = admin.models.ExternalRequest.findAndCountAll.mock.calls[0][0] + .include.find((item) => item.as === 'apiKey'); + expect(keyInclude.attributes).not.toContain('key_hash'); + expect(result.data[0]).toMatchObject({ + requester: { id: 4, name: 'External Client', keyPrefix: 'abcd1234' }, + target: { + youtubeId, + channelId: 8, + youtubeChannelId: 'UC1234567890123456789012', + channelTitle: 'Safe Channel', + title: 'Safe video', + mediaType: 'video', + rating: 'TV-PG', + contentRating: 'TV-Y', + }, + job: null, + }); + expect(JSON.stringify(result)).not.toContain('must-not-leak'); + expect(admin.models.Video.findAll).toHaveBeenCalledWith({ + where: { youtubeId: [youtubeId] }, + attributes: ['youtubeId', 'normalized_rating'], + }); + }); + + test('approves a pending request only after current policy revalidation and queue acceptance', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: { + id: pending.id, status: 'In Progress', jobType: 'External video request', + timeCreated: timestamp, timeInitiated: timestamp, + }, + }); + const approved = fixture(); + approved.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(hydrated); + approved.models.ChannelVideo.findAll.mockResolvedValue([{ + youtube_id: youtubeId, + channel_id: 'UC1234567890123456789012', + title: 'Safe video', + media_type: 'video', + }]); + + const result = await approved.service.reviewVideoRequest(pending.id, 'approve', {}); + + expect(approved.models.ApiKey.findByPk).toHaveBeenCalledWith(4, expect.objectContaining({ + transaction: approved.transaction, + lock: 'UPDATE', + attributes: expect.not.arrayContaining(['key_hash']), + })); + expect(approved.executor).toHaveBeenCalledWith({ + body: expect.objectContaining({ + urls: [`https://www.youtube.com/watch?v=${youtubeId}`], + channelId: 'UC1234567890123456789012', + externalRequestId: pending.id, + }), + }); + expect(pending.update.mock.calls.map(([values]) => values.status)).toEqual([ + 'approved', 'processing', + ]); + expect(result.status).toBe('processing'); + expect(result.job.id).toBe(pending.id); + }); + + test('fails approval closed when the requester key is revoked', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: false, revoked_at: timestamp, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: null, + }); + const revoked = fixture(); + revoked.models.ApiKey.findByPk.mockResolvedValue({ + id: 4, + name: 'External Client', + role: 'request', + is_active: false, + revoked_at: timestamp, + max_rating_level: 2, + allow_unrated: false, + allowed_media_types: ['video'], + }); + revoked.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(hydrated); + revoked.models.ChannelVideo.findAll.mockResolvedValue([]); + + const result = await revoked.service.reviewVideoRequest(pending.id, 'approve', {}); + + expect(revoked.executor).not.toHaveBeenCalled(); + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + }), { transaction: revoked.transaction }); + expect(result.status).toBe('failed'); + }); + + test('rejects pending requests with a bounded sanitized reason and clears dedupe', async () => { + const pending = record(); + const hydrated = Object.assign(pending, { + apiKey: { + id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', + is_active: true, revoked_at: null, + }, + channel: { + id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', + }, + job: null, + }); + const rejected = fixture(); + rejected.models.ExternalRequest.findOne + .mockResolvedValueOnce(pending) + .mockResolvedValueOnce(hydrated); + rejected.models.ChannelVideo.findAll.mockResolvedValue([]); + + const result = await rejected.service.reviewVideoRequest( + pending.id, + 'reject', + { reason: ' Not\u0000 eligible \n right now. ' } + ); + + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'rejected', + active_dedupe_key: null, + message: 'Not eligible right now.', + decided_at: timestamp, + }), { transaction: rejected.transaction }); + expect(result.status).toBe('rejected'); + }); + + test('rejects non-monotonic review transitions and malformed reasons', async () => { + const completed = fixture(); + completed.models.ExternalRequest.findOne.mockResolvedValue(record({ status: 'completed' })); + await expect(completed.service.reviewVideoRequest( + '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + 'approve', + {} + )).rejects.toMatchObject({ status: 409 }); + await expect(completed.service.reviewVideoRequest( + '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + 'reject', + { reason: 'x'.repeat(301) } + )).rejects.toThrow('reason'); + }); +}); diff --git a/server/modules/__tests__/externalRequestService.test.js b/server/modules/__tests__/externalRequestService.test.js index 4c6c6ca2e..a24020110 100644 --- a/server/modules/__tests__/externalRequestService.test.js +++ b/server/modules/__tests__/externalRequestService.test.js @@ -1,169 +1,7 @@ -const { createExternalRequestService, RequestError } = require('../externalRequestService'); +const { RequestError } = require('../externalRequestService'); +const { fixture, record, youtubeId } = require('./fixtures/externalRequests'); -const timestamp = new Date('2026-07-26T12:00:00.000Z'); -const youtubeId = 'abcdefghijk'; - -function record(overrides = {}) { - return { - id: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', - api_key_id: 4, - channel_id: 8, - youtube_id: youtubeId, - request_type: 'video', - status: 'pending', - created_at: timestamp, - updated_at: timestamp, - decided_at: null, - completed_at: null, - message: null, - update: jest.fn(async function update(values) { - Object.assign(this, values); - }), - ...overrides, - }; -} - -function fixture(overrides = {}) { - const created = record(); - const transaction = { LOCK: { UPDATE: 'UPDATE' } }; - const models = { - ExternalRequest: { - create: jest.fn(async (values) => Object.assign(created, values)), - update: jest.fn().mockResolvedValue([1]), - findOne: jest.fn().mockResolvedValue(null), - findByPk: jest.fn().mockResolvedValue(created), - findAndCountAll: jest.fn().mockResolvedValue({ rows: [], count: 0 }), - }, - ApiKeyChannelGrant: { - findOne: jest.fn().mockResolvedValue({ id: 1 }), - findOrCreate: jest.fn().mockResolvedValue([{ id: 1 }, true]), - }, - Channel: { - findByPk: jest.fn().mockResolvedValue({ - id: 8, channel_id: 'UC1234567890123456789012', enabled: true, default_rating: 'TV-Y', - }), - findOne: jest.fn().mockResolvedValue({ - id: 8, channel_id: 'UC1234567890123456789012', enabled: true, - terminated_at: null, default_rating: 'TV-Y', - }), - }, - ChannelVideo: { - findOne: jest.fn().mockResolvedValue({ - youtube_id: youtubeId, media_type: 'video', youtube_removed: false, ignored: false, - }), - findAll: jest.fn().mockResolvedValue([]), - }, - Video: { findOne: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, - Job: { findByPk: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, - ApiKey: { - findByPk: jest.fn().mockResolvedValue({ - id: 4, - name: 'External Client', - role: 'request', - is_active: true, - revoked_at: null, - allow_video_requests: true, - allow_channel_requests: true, - allow_delete_video_requests: false, - auto_approve_video_requests: false, - auto_approve_channel_requests: false, - auto_approve_delete_requests: false, - max_rating_level: 2, - allow_unrated: false, - allowed_media_types: ['video'], - max_active_jobs: 5, - hourly_write_limit: 30, - daily_write_limit: 200, - }), - findAll: jest.fn().mockResolvedValue([]), - }, - ...overrides.models, - }; - const sequelize = { - transaction: jest.fn(async (callback) => callback(transaction)), - }; - const executor = jest.fn().mockResolvedValue(undefined); - const channelProvisioner = overrides.channelProvisioner || { - getChannelInfo: jest.fn().mockResolvedValue({ channel_id: 'UC1234567890123456789012' }), - }; - const videoDeleter = overrides.videoDeleter || { - deleteVideosByYoutubeIds: jest.fn().mockResolvedValue({ success: true, failed: [] }), - deleteVideoById: jest.fn().mockResolvedValue({ success: true }), - }; - const quotaService = overrides.quotaService || { - reserveWrite: jest.fn(async () => ({ - id: 4, - name: 'External Client', - role: 'request', - isActive: true, - autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, - autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, - autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, - allowVideoRequests: overrides.key?.allowVideoRequests ?? true, - allowChannelRequests: overrides.key?.allowChannelRequests ?? true, - allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, - maxRatingLevel: 2, - allowUnrated: false, - allowedMediaTypes: ['video'], - })), - reloadAuthorizedKey: jest.fn(async () => ({ - id: 4, - name: 'External Client', - role: 'request', - autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, - autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, - autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, - allowVideoRequests: overrides.key?.allowVideoRequests ?? true, - allowChannelRequests: overrides.key?.allowChannelRequests ?? true, - allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, - maxRatingLevel: 2, - allowUnrated: false, - allowedMediaTypes: ['video'], - })), - assertExecutionCapacity: jest.fn(async () => ({ - id: 4, - name: 'External Client', - role: 'request', - autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, - autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, - autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, - allowVideoRequests: overrides.key?.allowVideoRequests ?? true, - allowChannelRequests: overrides.key?.allowChannelRequests ?? true, - allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, - maxRatingLevel: 2, - allowUnrated: false, - allowedMediaTypes: ['video'], - })), - }; - const service = createExternalRequestService({ - models, - executor: overrides.executor || executor, - channelProvisioner, - videoDeleter, - now: overrides.now || (() => timestamp), - sequelize: overrides.sequelize || sequelize, - quotaService, - workLimiter: overrides.workLimiter, - }); - const key = { - id: 4, - name: 'External Client', - role: 'request', - autoApproveVideoRequests: false, - autoApproveChannelRequests: false, - autoApproveDeleteRequests: false, - maxRatingLevel: 2, - allowUnrated: false, - allowedMediaTypes: ['video'], - ...overrides.key, - }; - return { - service, models, executor, key, created, sequelize, transaction, - channelProvisioner, videoDeleter, quotaService, - }; -} - -describe('external video request service', () => { +describe('external request service', () => { test('persists a pending request without executing a download', async () => { const { service, models, executor, key, transaction, quotaService } = fixture(); const result = await service.createVideoRequest(key, { youtubeId, channelId: 8 }); @@ -184,117 +22,6 @@ describe('external video request service', () => { expect(executor).not.toHaveBeenCalled(); }); - test('auto-approval queues only the canonical URL and server-owned channel mapping', async () => { - const { service, models, executor, key, transaction } = fixture({ - key: { autoApproveVideoRequests: true }, - }); - const result = await service.createVideoRequest(key, { - youtubeId, channelId: 8, idempotencyKey: 'external-client-1', - }); - expect(models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ - status: 'pending', - decided_at: timestamp, - idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), - }), { transaction }); - expect(executor).toHaveBeenCalledWith({ - body: { - urls: [`https://www.youtube.com/watch?v=${youtubeId}`], - channelId: 'UC1234567890123456789012', - ownerChannelMap: { [youtubeId]: 'UC1234567890123456789012' }, - initiatedBy: { type: 'api_key', name: 'External Client' }, - jobLabel: 'Manually Added Urls (external request)', - externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', - }, - }); - expect(result.request.status).toBe('processing'); - }); - - test('returns service unavailable when the shared work queue is full', async () => { - const workError = Object.assign(new Error('full'), { - name: 'ExternalWorkLimitError', - }); - const limited = fixture({ - key: { autoApproveVideoRequests: true }, - workLimiter: { run: jest.fn().mockRejectedValue(workError) }, - }); - await expect(limited.service.createVideoRequest(limited.key, { - youtubeId, - channelId: 8, - })).rejects.toMatchObject({ - name: 'RequestError', - status: 503, - code: 'work_queue_full', - }); - expect(limited.created.status).toBe('pending'); - expect(limited.created.active_dedupe_key).toBe(`4:video:${youtubeId}`); - }); - - test('revalidates a channel grant inside the work slot before enqueue', async () => { - const raced = fixture({ - key: { autoApproveVideoRequests: true }, - }); - raced.models.ApiKeyChannelGrant.findOne - .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) - .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) - .mockResolvedValueOnce(null); - - const result = await raced.service.createVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.executor).not.toHaveBeenCalled(); - expect(raced.created.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - })); - expect(result.request.status).toBe('failed'); - }); - - test('does not enqueue when video auto-approval is disabled before execution', async () => { - const raced = fixture({ key: { autoApproveVideoRequests: true } }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - allowVideoRequests: true, - autoApproveVideoRequests: false, - }); - - const result = await raced.service.createVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.executor).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - - test('resumes an interrupted auto-approved dispatch with the stable request job id', async () => { - const interrupted = record({ status: 'pending', decided_at: timestamp, job_id: null }); - const retryExecutor = jest.fn().mockResolvedValue(interrupted.id); - const retry = fixture({ - key: { autoApproveVideoRequests: true }, - executor: retryExecutor, - }); - retry.models.ExternalRequest.findOne.mockResolvedValue(interrupted); - retry.models.ExternalRequest.findByPk.mockResolvedValue(interrupted); - const result = await retry.service.createVideoRequest( - retry.key, - { youtubeId, channelId: 8, idempotencyKey: 'same-request' } - ); - expect(retry.models.ExternalRequest.create).not.toHaveBeenCalled(); - expect(retryExecutor).toHaveBeenCalledWith(expect.objectContaining({ - body: expect.objectContaining({ externalRequestId: interrupted.id }), - })); - expect(interrupted.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'processing', - job_id: interrupted.id, - }), { transaction: retry.transaction }); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { status: 'processing' }, - }); - }); - test('fails closed for missing grants, unknown media, and ratings above policy', async () => { const missingGrant = fixture(); missingGrant.models.ApiKeyChannelGrant.findOne.mockResolvedValue(null); @@ -380,47 +107,6 @@ describe('external video request service', () => { expect(pending.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); }); - test('auto-approved channel requests provision and grant the resulting channel', async () => { - const approved = fixture({ key: { autoApproveChannelRequests: true } }); - const result = await approved.service.createChannelRequest(approved.key, { - channelUrl: 'https://www.youtube.com/channel/UC1234567890123456789012', - }); - - expect(approved.channelProvisioner.getChannelInfo).toHaveBeenCalledWith( - 'https://www.youtube.com/channel/UC1234567890123456789012', - false, - true, - {}, - { skipTabDetection: true } - ); - expect(approved.models.ApiKeyChannelGrant.findOrCreate).toHaveBeenCalledWith({ - where: { api_key_id: 4, channel_id: 8 }, - defaults: { api_key_id: 4, channel_id: 8 }, - transaction: approved.transaction, - }); - expect(result.request).toMatchObject({ - type: 'channel', - status: 'completed', - target: { channelId: 8 }, - }); - }); - - test('does not provision when channel auto-approval is disabled before execution', async () => { - const raced = fixture({ key: { autoApproveChannelRequests: true } }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - allowChannelRequests: true, - autoApproveChannelRequests: false, - }); - - const result = await raced.service.createChannelRequest(raced.key, { - channelUrl: 'https://www.youtube.com/@safe', - }); - - expect(raced.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - test('returns the winning channel request when a concurrent create wins', async () => { const winner = record({ request_type: 'channel', @@ -444,31 +130,6 @@ describe('external video request service', () => { }); }); - test('resumes an interrupted auto-approved channel request on idempotent retry', async () => { - const interrupted = record({ - request_type: 'channel', - channel_id: null, - youtube_id: null, - channel_url: 'https://www.youtube.com/@safe', - status: 'pending', - decided_at: timestamp, - }); - const recovering = fixture({ - key: { autoApproveChannelRequests: true }, - }); - recovering.models.ExternalRequest.findOne.mockResolvedValue(interrupted); - - const result = await recovering.service.createChannelRequest(recovering.key, { - channelUrl: 'https://www.youtube.com/@safe', - }); - - expect(recovering.channelProvisioner.getChannelInfo).toHaveBeenCalledTimes(1); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { type: 'channel', status: 'completed' }, - }); - }); - test('delete requests are role-bound and hide absent targets', async () => { const forbidden = fixture(); await expect(forbidden.service.createDeleteVideoRequest(forbidden.key, { @@ -485,192 +146,6 @@ describe('external video request service', () => { expect(absent.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); }); - test('auto-approved deletion reaches a completed terminal state', async () => { - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - idempotencyKey: 'delete-1', - }); - - expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( - 91, - expect.objectContaining({ - transaction: deleting.transaction, - video: expect.objectContaining({ id: 91, youtubeId }), - }) - ); - expect(result).toMatchObject({ - outcome: 'created', - request: { type: 'delete_video', status: 'completed' }, - }); - }); - - test('does not delete when delete auto-approval is disabled before execution', async () => { - const raced = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - raced.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - role: 'delete', - allowDeleteVideoRequests: true, - autoApproveDeleteRequests: false, - }); - - const result = await raced.service.createDeleteVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - - test('deletion completes idempotently when the target disappears during execution', async () => { - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - videoDeleter: { - deleteVideoById: jest.fn().mockResolvedValue({ - success: false, - error: 'Video not found in database', - }), - }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - }); - - expect(result.request).toMatchObject({ - status: 'completed', - message: 'Video is already deleted', - }); - }); - - test('recovers a stale processing deletion idempotently', async () => { - const stale = record({ - request_type: 'delete_video', - status: 'processing', - channel_url: null, - decided_at: new Date('2026-07-26T11:00:00.000Z'), - updated_at: new Date('2026-07-26T11:00:00.000Z'), - }); - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - deleting.models.ExternalRequest.findOne.mockResolvedValue(stale); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - }); - - expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( - 91, - expect.objectContaining({ transaction: deleting.transaction }) - ); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { type: 'delete_video', status: 'completed' }, - }); - }); - - test('reconciles a processing deletion after its video is absent', async () => { - const deleting = fixture({ key: { role: 'delete' } }); - const processing = record({ - request_type: 'delete_video', - status: 'processing', - decided_at: timestamp, - }); - deleting.models.ExternalRequest.findAndCountAll.mockResolvedValue({ - rows: [processing], - count: 1, - }); - deleting.models.Video.findAll.mockResolvedValue([]); - - const result = await deleting.service.listRequests(deleting.key); - - expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'completed', - active_dedupe_key: null, - completed_at: timestamp, - })); - expect(result.data[0]).toMatchObject({ - type: 'delete_video', - status: 'completed', - }); - }); - - test('marks queue failures terminal so a retry is possible', async () => { - const queueError = new Error('secret executor failure'); - const failed = fixture({ - key: { autoApproveVideoRequests: true }, - executor: jest.fn().mockRejectedValue(queueError), - }); - const result = await failed.service.createVideoRequest( - failed.key, { youtubeId, channelId: 8 } - ); - expect(failed.created.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - message: 'Download could not be queued', - })); - expect(result.request).toMatchObject({ - status: 'failed', - message: 'Download could not be queued', - }); - expect(result.request.message).not.toContain('secret'); - }); - - test('lists only the calling key and lazily reconciles completed downloads', async () => { - const processing = record({ status: 'processing' }); - const own = fixture(); - own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); - own.models.Video.findAll.mockResolvedValue([{ youtubeId }]); - const result = await own.service.listRequests(own.key, { - page: '1', pageSize: '10', - }); - expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith(expect.objectContaining({ - where: { api_key_id: 4 }, - limit: 10, - offset: 0, - })); - expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'completed', - active_dedupe_key: null, - completed_at: timestamp, - })); - expect(result.data[0].status).toBe('completed'); - }); - test.each(['Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings'])( 'reconciles a %s downloader job to a retryable failed request', async (jobStatus) => { @@ -695,26 +170,6 @@ describe('external video request service', () => { } ); - test('reconciles a status-filtered page and requeries its count', async () => { - const stale = record({ status: 'processing', job_id: 'job-123' }); - const own = fixture(); - own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: 'Error' }]); - own.models.ExternalRequest.findAndCountAll - .mockResolvedValueOnce({ rows: [stale], count: 1 }) - .mockResolvedValueOnce({ rows: [], count: 0 }); - - const result = await own.service.listRequests(own.key, { status: 'processing' }); - - expect(stale.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); - expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( - expect.objectContaining({ where: { api_key_id: 4, status: 'processing' } }) - ); - expect(result).toMatchObject({ - data: [], - pagination: { total: 0, totalPages: 0 }, - }); - }); - test('validates scope, body fields, paging, and status allowlists', async () => { const { service, key } = fixture(); await expect(service.createVideoRequest( @@ -738,201 +193,5 @@ describe('external video request service', () => { await expect(service.listRequests(key, { pageSize: '101' })).rejects.toThrow('pageSize'); }); - test('lists administrator requests with safe joined metadata and filters', async () => { - const listed = record({ - status: 'pending', - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', key_hash: 'must-not-leak', - role: 'request', is_active: true, revoked_at: null, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - default_rating: 'TV-PG', - }, - job: null, - }); - const admin = fixture(); - admin.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [listed], count: 1 }); - admin.models.ApiKey.findAll.mockResolvedValue([listed.apiKey]); - admin.models.ChannelVideo.findAll.mockResolvedValue([{ - youtube_id: youtubeId, - channel_id: 'UC1234567890123456789012', - title: 'Safe video', - media_type: 'video', - }]); - admin.models.Video.findAll.mockResolvedValue([{ - youtubeId, - normalized_rating: 'TV-Y', - }]); - - const result = await admin.service.listAdminRequests({ - page: '2', pageSize: '10', status: 'pending', apiKeyId: '4', - }); - - expect(admin.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( - expect.objectContaining({ - where: { status: 'pending', api_key_id: 4 }, - limit: 10, - offset: 10, - distinct: true, - }) - ); - const keyInclude = admin.models.ExternalRequest.findAndCountAll.mock.calls[0][0] - .include.find((item) => item.as === 'apiKey'); - expect(keyInclude.attributes).not.toContain('key_hash'); - expect(result.data[0]).toMatchObject({ - requester: { id: 4, name: 'External Client', keyPrefix: 'abcd1234' }, - target: { - youtubeId, - channelId: 8, - youtubeChannelId: 'UC1234567890123456789012', - channelTitle: 'Safe Channel', - title: 'Safe video', - mediaType: 'video', - rating: 'TV-PG', - contentRating: 'TV-Y', - }, - job: null, - }); - expect(JSON.stringify(result)).not.toContain('must-not-leak'); - expect(admin.models.Video.findAll).toHaveBeenCalledWith({ - where: { youtubeId: [youtubeId] }, - attributes: ['youtubeId', 'normalized_rating'], - }); - }); - test('approves a pending request only after current policy revalidation and queue acceptance', async () => { - const pending = record(); - const hydrated = Object.assign(pending, { - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: true, revoked_at: null, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: { - id: pending.id, status: 'In Progress', jobType: 'External video request', - timeCreated: timestamp, timeInitiated: timestamp, - }, - }); - const approved = fixture(); - approved.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(null) - .mockResolvedValueOnce(hydrated); - approved.models.ChannelVideo.findAll.mockResolvedValue([{ - youtube_id: youtubeId, - channel_id: 'UC1234567890123456789012', - title: 'Safe video', - media_type: 'video', - }]); - - const result = await approved.service.reviewVideoRequest(pending.id, 'approve', {}); - - expect(approved.models.ApiKey.findByPk).toHaveBeenCalledWith(4, expect.objectContaining({ - transaction: approved.transaction, - lock: 'UPDATE', - attributes: expect.not.arrayContaining(['key_hash']), - })); - expect(approved.executor).toHaveBeenCalledWith({ - body: expect.objectContaining({ - urls: [`https://www.youtube.com/watch?v=${youtubeId}`], - channelId: 'UC1234567890123456789012', - externalRequestId: pending.id, - }), - }); - expect(pending.update.mock.calls.map(([values]) => values.status)).toEqual([ - 'approved', 'processing', - ]); - expect(result.status).toBe('processing'); - expect(result.job.id).toBe(pending.id); - }); - - test('fails approval closed when the requester key is revoked', async () => { - const pending = record(); - const hydrated = Object.assign(pending, { - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: false, revoked_at: timestamp, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: null, - }); - const revoked = fixture(); - revoked.models.ApiKey.findByPk.mockResolvedValue({ - id: 4, - name: 'External Client', - role: 'request', - is_active: false, - revoked_at: timestamp, - max_rating_level: 2, - allow_unrated: false, - allowed_media_types: ['video'], - }); - revoked.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(hydrated); - revoked.models.ChannelVideo.findAll.mockResolvedValue([]); - - const result = await revoked.service.reviewVideoRequest(pending.id, 'approve', {}); - - expect(revoked.executor).not.toHaveBeenCalled(); - expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - }), { transaction: revoked.transaction }); - expect(result.status).toBe('failed'); - }); - - test('rejects pending requests with a bounded sanitized reason and clears dedupe', async () => { - const pending = record(); - const hydrated = Object.assign(pending, { - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: true, revoked_at: null, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: null, - }); - const rejected = fixture(); - rejected.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(hydrated); - rejected.models.ChannelVideo.findAll.mockResolvedValue([]); - - const result = await rejected.service.reviewVideoRequest( - pending.id, - 'reject', - { reason: ' Not\u0000 eligible \n right now. ' } - ); - - expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'rejected', - active_dedupe_key: null, - message: 'Not eligible right now.', - decided_at: timestamp, - }), { transaction: rejected.transaction }); - expect(result.status).toBe('rejected'); - }); - - test('rejects non-monotonic review transitions and malformed reasons', async () => { - const completed = fixture(); - completed.models.ExternalRequest.findOne.mockResolvedValue(record({ status: 'completed' })); - await expect(completed.service.reviewVideoRequest( - '9b89e5bc-8c90-4e72-b245-270fed2eacc2', - 'approve', - {} - )).rejects.toMatchObject({ status: 409 }); - await expect(completed.service.reviewVideoRequest( - '9b89e5bc-8c90-4e72-b245-270fed2eacc2', - 'reject', - { reason: 'x'.repeat(301) } - )).rejects.toThrow('reason'); - }); }); diff --git a/server/modules/__tests__/fixtures/externalRequests.js b/server/modules/__tests__/fixtures/externalRequests.js new file mode 100644 index 000000000..54966022d --- /dev/null +++ b/server/modules/__tests__/fixtures/externalRequests.js @@ -0,0 +1,166 @@ +const { createExternalRequestService } = require('../../externalRequestService'); + +const timestamp = new Date('2026-07-26T12:00:00.000Z'); +const youtubeId = 'abcdefghijk'; + +function record(overrides = {}) { + return { + id: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + api_key_id: 4, + channel_id: 8, + youtube_id: youtubeId, + request_type: 'video', + status: 'pending', + created_at: timestamp, + updated_at: timestamp, + decided_at: null, + completed_at: null, + message: null, + update: jest.fn(async function update(values) { + Object.assign(this, values); + }), + ...overrides, + }; +} + +function fixture(overrides = {}) { + const created = record(); + const transaction = { LOCK: { UPDATE: 'UPDATE' } }; + const models = { + ExternalRequest: { + create: jest.fn(async (values) => Object.assign(created, values)), + update: jest.fn().mockResolvedValue([1]), + findOne: jest.fn().mockResolvedValue(null), + findByPk: jest.fn().mockResolvedValue(created), + findAndCountAll: jest.fn().mockResolvedValue({ rows: [], count: 0 }), + }, + ApiKeyChannelGrant: { + findOne: jest.fn().mockResolvedValue({ id: 1 }), + findOrCreate: jest.fn().mockResolvedValue([{ id: 1 }, true]), + }, + Channel: { + findByPk: jest.fn().mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, default_rating: 'TV-Y', + }), + findOne: jest.fn().mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, + terminated_at: null, default_rating: 'TV-Y', + }), + }, + ChannelVideo: { + findOne: jest.fn().mockResolvedValue({ + youtube_id: youtubeId, media_type: 'video', youtube_removed: false, ignored: false, + }), + findAll: jest.fn().mockResolvedValue([]), + }, + Video: { findOne: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, + Job: { findByPk: jest.fn().mockResolvedValue(null), findAll: jest.fn().mockResolvedValue([]) }, + ApiKey: { + findByPk: jest.fn().mockResolvedValue({ + id: 4, + name: 'External Client', + role: 'request', + is_active: true, + revoked_at: null, + allow_video_requests: true, + allow_channel_requests: true, + allow_delete_video_requests: false, + auto_approve_video_requests: false, + auto_approve_channel_requests: false, + auto_approve_delete_requests: false, + max_rating_level: 2, + allow_unrated: false, + allowed_media_types: ['video'], + max_active_jobs: 5, + hourly_write_limit: 30, + daily_write_limit: 200, + }), + findAll: jest.fn().mockResolvedValue([]), + }, + ...overrides.models, + }; + const sequelize = { + transaction: jest.fn(async (callback) => callback(transaction)), + }; + const executor = jest.fn().mockResolvedValue(undefined); + const channelProvisioner = overrides.channelProvisioner || { + getChannelInfo: jest.fn().mockResolvedValue({ channel_id: 'UC1234567890123456789012' }), + }; + const videoDeleter = overrides.videoDeleter || { + deleteVideosByYoutubeIds: jest.fn().mockResolvedValue({ success: true, failed: [] }), + deleteVideoById: jest.fn().mockResolvedValue({ success: true }), + }; + const quotaService = overrides.quotaService || { + reserveWrite: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + isActive: true, + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + reloadAuthorizedKey: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + assertExecutionCapacity: jest.fn(async () => ({ + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: overrides.key?.autoApproveVideoRequests ?? false, + autoApproveChannelRequests: overrides.key?.autoApproveChannelRequests ?? false, + autoApproveDeleteRequests: overrides.key?.autoApproveDeleteRequests ?? false, + allowVideoRequests: overrides.key?.allowVideoRequests ?? true, + allowChannelRequests: overrides.key?.allowChannelRequests ?? true, + allowDeleteVideoRequests: overrides.key?.allowDeleteVideoRequests ?? false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + })), + }; + const service = createExternalRequestService({ + models, + executor: overrides.executor || executor, + channelProvisioner, + videoDeleter, + now: overrides.now || (() => timestamp), + sequelize: overrides.sequelize || sequelize, + quotaService, + workLimiter: overrides.workLimiter, + }); + const key = { + id: 4, + name: 'External Client', + role: 'request', + autoApproveVideoRequests: false, + autoApproveChannelRequests: false, + autoApproveDeleteRequests: false, + maxRatingLevel: 2, + allowUnrated: false, + allowedMediaTypes: ['video'], + ...overrides.key, + }; + return { + service, models, executor, key, created, sequelize, transaction, + channelProvisioner, videoDeleter, quotaService, + }; +} + +module.exports = { fixture, record, timestamp, youtubeId }; diff --git a/server/modules/externalCatalogService.js b/server/modules/externalCatalogService.js index af09ed518..2d3927668 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalCatalogService.js @@ -9,6 +9,8 @@ const { normalizePolicy, ratingPolicy } = require('./externalEligibility'); const { publicVideoThumbnail } = require('./externalThumbnailProxy'); const { CatalogError, + parseInteger, + paginationDto, decodePageCursor, encodePageCursor, pagination, @@ -26,16 +28,6 @@ function boundedString(value, maximum) { } -function parseInteger(value, fallback, minimum, maximum, name) { - if (value === undefined) return fallback; - if (!/^\d+$/.test(String(value))) throw new CatalogError(`${name} must be an integer`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new CatalogError(`${name} must be between ${minimum} and ${maximum}`); - } - return parsed; -} - function normalizeSearch(value) { if (value === undefined) return null; if (typeof value !== 'string' || value.length > 200) { @@ -70,17 +62,6 @@ function ratingSql(policy, effectiveRatingSql) { } -function paginationDto(page, pageSize, total, maximumPage = MAX_PAGE) { - const totalPages = total === 0 ? 0 : Math.min(maximumPage, Math.ceil(total / pageSize)); - return { - page, - pageSize, - total, - totalPages, - nextCursor: page < totalPages ? encodePageCursor(page + 1) : null, - }; -} - function catalogCursorFingerprint(endpoint, filters) { return crypto.createHash('sha256') .update(JSON.stringify({ endpoint, ...filters })) diff --git a/server/modules/externalPagination.js b/server/modules/externalPagination.js index a846b3e1d..09174115b 100644 --- a/server/modules/externalPagination.js +++ b/server/modules/externalPagination.js @@ -8,20 +8,20 @@ class CatalogError extends Error { } } -function parseInteger(value, fallback, minimum, maximum, name) { +function parseInteger(value, fallback, minimum, maximum, name, ErrorType = CatalogError) { if (value === undefined) return fallback; - if (!/^\d+$/.test(String(value))) throw new CatalogError(`${name} must be an integer`); + if (!/^\d+$/.test(String(value))) throw new ErrorType(`${name} must be an integer`); const parsed = Number(value); if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new CatalogError(`${name} must be between ${minimum} and ${maximum}`); + throw new ErrorType(`${name} must be between ${minimum} and ${maximum}`); } return parsed; } -function decodePageCursor(value, maximumPage) { +function decodePageCursor(value, maximumPage, ErrorType = CatalogError) { if (value === undefined) return null; if (typeof value !== 'string' || value.length > 200) { - throw new CatalogError('cursor is invalid'); + throw new ErrorType('cursor is invalid'); } try { const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); @@ -31,7 +31,7 @@ function decodePageCursor(value, maximumPage) { } return parsed.page; } catch (_error) { - throw new CatalogError('cursor is invalid'); + throw new ErrorType('cursor is invalid'); } } @@ -39,19 +39,32 @@ function encodePageCursor(page) { return Buffer.from(JSON.stringify({ v: 1, page }), 'utf8').toString('base64url'); } -function pagination(query, maximumPage = 100) { +function pagination(query, maximumPage = 100, ErrorType = CatalogError) { if (query.cursor !== undefined && query.page !== undefined) { - throw new CatalogError('cursor and page cannot be used together'); + throw new ErrorType('cursor and page cannot be used together'); } - const cursorPage = decodePageCursor(query.cursor, maximumPage); - const page = cursorPage || parseInteger(query.page, 1, 1, maximumPage, 'page'); - const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); + const cursorPage = decodePageCursor(query.cursor, maximumPage, ErrorType); + const page = cursorPage || parseInteger(query.page, 1, 1, maximumPage, 'page', ErrorType); + const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize', ErrorType); return { page, pageSize, offset: (page - 1) * pageSize }; } +function paginationDto(page, pageSize, total, maximumPage = 100) { + const totalPages = total === 0 ? 0 : Math.min(maximumPage, Math.ceil(total / pageSize)); + return { + page, + pageSize, + total, + totalPages, + nextCursor: page < totalPages ? encodePageCursor(page + 1) : null, + }; +} + module.exports = { + parseInteger, + paginationDto, CatalogError, decodePageCursor, encodePageCursor, pagination, -}; \ No newline at end of file +}; diff --git a/server/modules/externalQuotaService.js b/server/modules/externalQuotaService.js index b442c9abe..488bbc1b7 100644 --- a/server/modules/externalQuotaService.js +++ b/server/modules/externalQuotaService.js @@ -1,4 +1,5 @@ const { Op } = require('sequelize'); +const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); const { hasExternalScope } = require('./externalPermissions'); @@ -36,7 +37,8 @@ function createExternalQuotaService({ sequelize = require('../db').sequelize, now = () => new Date(), } = {}) { - const { ApiKey, ExternalRequest, ExternalApiUsageBucket, Video, Job } = models; + const { ApiKey, ExternalRequest, ExternalApiUsageBucket } = models; + const reconcile = createExternalRequestReconciliation({ models, now }); async function reloadAuthorizedKey(keyId, requiredScope, transaction = null) { const record = await ApiKey.findByPk(keyId, { @@ -62,53 +64,7 @@ function createExternalQuotaService({ ], ...(transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}), }); - if (processing.length > 0) { - const youtubeIds = [...new Set(processing.map((record) => record.youtube_id))]; - const videos = await Video.findAll({ - where: { youtubeId: youtubeIds, removed: false }, - attributes: ['youtubeId'], - ...(transaction ? { transaction } : {}), - }); - const present = new Set(videos.map((video) => video.youtubeId)); - const terminalAt = now(); - const completed = processing.filter((record) => - (record.request_type === 'video' && present.has(record.youtube_id)) || - (record.request_type === 'delete_video' && !present.has(record.youtube_id)) - ); - await Promise.all(completed.map((record) => record.update({ - status: 'completed', - active_dedupe_key: null, - completed_at: terminalAt, - updated_at: terminalAt, - }, transaction ? { transaction } : {}))); - - const unresolved = processing.filter((record) => - record.request_type === 'video' && - !present.has(record.youtube_id) && - record.job_id && - !completed.includes(record) - ); - if (unresolved.length > 0) { - const jobs = await Job.findAll({ - where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, - attributes: ['id', 'status'], - ...(transaction ? { transaction } : {}), - }); - const terminalJobs = new Set(jobs - .filter((job) => [ - 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', - ].includes(job.status)) - .map((job) => job.id)); - await Promise.all(unresolved - .filter((record) => terminalJobs.has(record.job_id)) - .map((record) => record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Download did not complete', - updated_at: terminalAt, - }, transaction ? { transaction } : {}))); - } - } + await reconcile(processing, transaction); return ExternalRequest.count({ where: { api_key_id: keyId, diff --git a/server/modules/externalRequestExecution.js b/server/modules/externalRequestExecution.js new file mode 100644 index 000000000..238a66b74 --- /dev/null +++ b/server/modules/externalRequestExecution.js @@ -0,0 +1,232 @@ +const { Op } = require('sequelize'); +const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); +const { + AUXILIARY_RECOVERY_DELAY_MS, requireCurrentAutoApproval, rethrowWorkLimit, +} = require('./externalRequestPrimitives'); + +function createExternalRequestExecution({ + models, sequelize, now, quotas, workLimiter, executor, channelProvisioner, + videoDeleter, validateTarget, +}) { + const { ExternalRequest, ApiKeyChannelGrant, Channel, Job } = models; + async function claimAuxiliaryRequest(record) { + const timestamp = now(); + let whereStatus; + if (['pending', 'approved'].includes(record.status)) { + whereStatus = record.status; + } else if (record.status === 'processing') { + const updatedAt = new Date(record.updated_at).getTime(); + if (!Number.isFinite(updatedAt) || + timestamp.getTime() - updatedAt < AUXILIARY_RECOVERY_DELAY_MS) { + return false; + } + whereStatus = 'processing'; + } else { + return false; + } + const [claimed] = await ExternalRequest.update( + { status: 'processing', updated_at: timestamp }, + { where: { id: record.id, status: whereStatus, updated_at: record.updated_at } } + ); + if (claimed !== 1) return false; + record.status = 'processing'; + record.updated_at = timestamp; + return true; + } + + async function dispatchAutoApproved(record, key) { + let accepted = false; + try { + await workLimiter.run(() => + sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + key.id, + 'video:request', + record.id, + transaction + ); + const locked = await ExternalRequest.findByPk(record.id, { transaction, lock: transaction.LOCK.UPDATE }); + if (!locked || locked.status !== 'pending' || locked.job_id) return; + requireCurrentAutoApproval(currentKey, 'video'); + const target = await validateTarget( + currentKey, + record.youtube_id, + record.channel_id, + transaction + ); + const existingJob = await Job.findByPk(record.id, { transaction }); + const jobId = existingJob?.id || await executor({ + body: { + urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], + channelId: target.channel.channel_id, + ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, + initiatedBy: { type: 'api_key', name: currentKey.name }, + jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, + // The downloader uses this UUID as the job identity. A retry after a + // crash can therefore observe/reuse the accepted job instead of + // starting the same download twice. + externalRequestId: record.id, + }, + }); + if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); + accepted = true; + await record.update({ + status: 'processing', job_id: jobId || record.id, updated_at: now(), + }, { transaction }); + }) + ); + } catch (error) { + // The downloader persists separately. Leave the request retryable if the + // process loses its database connection after enqueueing the stable job. + if (accepted) throw error; + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: 'pending', updated_at: now() }); + rethrowWorkLimit(error); + } + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Download could not be queued', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + async function provisionChannelRequest( + record, + key, + { grantToRequestingKey, requireAutoApproval = false } = {} + ) { + const retryStatus = record.status === 'processing' ? 'approved' : record.status; + if (!(await claimAuxiliaryRequest(record))) return record; + const shouldGrant = grantToRequestingKey ?? + (record.grant_to_requesting_key !== false); + try { + await workLimiter.run(() => sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + key.id, + 'channel:request', + record.id, + transaction + ); + if (requireAutoApproval) { + requireCurrentAutoApproval(currentKey, 'channel'); + } + const provisioner = channelProvisioner || require('./channel/channelProvisioning'); + const result = await provisioner.getChannelInfo( + record.channel_url, + false, + true, + {}, + { skipTabDetection: true } + ); + const channel = await Channel.findOne({ + where: { + [Op.or]: [ + { channel_id: result.channel_id || result.id }, + { url: record.channel_url }, + ], + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!channel || channel.enabled !== true || channel.terminated_at) { + throw new Error('Provisioned channel is unavailable'); + } + if (shouldGrant) { + await ApiKeyChannelGrant.findOrCreate({ + where: { api_key_id: currentKey.id, channel_id: channel.id }, + defaults: { api_key_id: currentKey.id, channel_id: channel.id }, + transaction, + }); + } + const completedAt = now(); + await record.update({ + channel_id: channel.id, + status: 'completed', + active_dedupe_key: null, + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + })); + } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: retryStatus, updated_at: now() }); + rethrowWorkLimit(error); + } + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Channel could not be provisioned', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + async function executeDeleteRequest(record, { requireAutoApproval = false } = {}) { + const retryStatus = record.status === 'processing' ? 'approved' : record.status; + if (!(await claimAuxiliaryRequest(record))) return record; + try { + await workLimiter.run(() => sequelize.transaction(async (transaction) => { + const currentKey = await quotas.assertExecutionCapacity( + record.api_key_id, + 'video:delete', + record.id, + transaction + ); + if (requireAutoApproval) { + requireCurrentAutoApproval(currentKey, 'delete_video'); + } + const target = await validateTarget( + currentKey, + record.youtube_id, + record.channel_id, + transaction + ); + const result = target.downloaded + ? await videoDeleter.deleteVideoById( + target.downloaded.id, + { transaction, video: target.downloaded } + ) + : { success: false, error: 'Video is already removed' }; + const alreadyAbsent = result?.success === false && + /not found|already (?:marked as )?removed/i.test(result?.error || ''); + if (result?.success === false && !alreadyAbsent) { + throw new Error('Deletion failed'); + } + const completedAt = now(); + await record.update({ + status: 'completed', + active_dedupe_key: null, + ...(alreadyAbsent ? { message: 'Video is already deleted' } : {}), + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + })); + } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + await record.update({ status: retryStatus, updated_at: now() }); + rethrowWorkLimit(error); + } + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Video could not be deleted', + updated_at: failedAt, + }); + rethrowWorkLimit(error); + } + return record; + } + + return { dispatchAutoApproved, provisionChannelRequest, executeDeleteRequest }; +} + +module.exports = { createExternalRequestExecution }; diff --git a/server/modules/externalRequestPrimitives.js b/server/modules/externalRequestPrimitives.js new file mode 100644 index 000000000..f0ae67150 --- /dev/null +++ b/server/modules/externalRequestPrimitives.js @@ -0,0 +1,194 @@ +const crypto = require('crypto'); +const { UniqueConstraintError } = require('sequelize'); +const { normalizeExternalPermissions } = require('./externalPermissions'); + +const REQUEST_STATUSES = [ + 'pending', 'approved', 'processing', 'completed', + 'rejected', 'failed', 'cancelled', +]; +const ACTIVE_STATUSES = ['pending', 'approved', 'processing']; +const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; +const AUXILIARY_RECOVERY_DELAY_MS = 5 * 60 * 1000; +const paginationHelpers = require('./externalPagination'); + +class RequestError extends Error { + constructor(message, status = 400, code = null) { + super(message); + this.name = 'RequestError'; + this.status = status; + this.code = code; + } +} + +function rethrowWorkLimit(error) { + if (error?.name === 'ExternalWorkLimitError') { + throw new RequestError( + 'External API work capacity is temporarily unavailable', + 503, + 'work_queue_full' + ); + } +} + +function requireCurrentAutoApproval(key, requestType) { + const enabled = { + video: key.autoApproveVideoRequests, + channel: key.autoApproveChannelRequests, + delete_video: key.autoApproveDeleteRequests, + }[requestType]; + if (enabled !== true) { + throw new RequestError('Request is no longer eligible', 403); + } +} + +function parseInteger(value, fallback, minimum, maximum, name) { + return paginationHelpers.parseInteger(value, fallback, minimum, maximum, name, RequestError); +} + +function requestPagination(query) { + return paginationHelpers.pagination(query, 100, RequestError); +} + +const requestPaginationDto = paginationHelpers.paginationDto; + +function normalizeIdempotencyKey(value) { + if (value === undefined) return null; + if (typeof value !== 'string' || value.length < 1 || value.length > 200) { + throw new RequestError('idempotencyKey must be a string of 1 to 200 characters'); + } + return crypto.createHash('sha256').update(value).digest('hex'); +} + +function normalizeChannelUrl(value) { + if (typeof value !== 'string' || value.length < 1 || value.length > 500) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + let url; + try { + url = new URL(/^https?:\/\//i.test(value.trim()) ? value.trim() : `https://${value.trim()}`); + } catch (_error) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + const hostname = url.hostname.toLowerCase().replace(/^www\./, ''); + const segments = url.pathname.split('/').filter(Boolean); + const supportedPrefix = segments[0]?.startsWith('@') || + ['channel', 'c', 'user'].includes(segments[0]); + if (!['youtube.com', 'm.youtube.com'].includes(hostname) || + !supportedPrefix || segments.length !== (segments[0].startsWith('@') ? 1 : 2)) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } + try { + return `https://www.youtube.com/${segments.map((segment) => { + const decoded = decodeURIComponent(segment); + return decoded.startsWith('@') + ? `@${encodeURIComponent(decoded.slice(1))}` + : encodeURIComponent(decoded); + }).join('/')}`; + } catch (_error) { + throw new RequestError('channelUrl must be a supported YouTube channel URL'); + } +} + +function dto(record) { + const value = record.toJSON ? record.toJSON() : record; + return { + id: value.id, + type: value.request_type, + status: value.status, + target: { + youtubeId: value.youtube_id || null, + channelId: value.channel_id || null, + channelUrl: value.channel_url || null, + }, + createdAt: value.created_at, + updatedAt: value.updated_at, + ...(value.decided_at ? { decidedAt: value.decided_at } : {}), + ...(value.completed_at ? { completedAt: value.completed_at } : {}), + ...(value.message ? { message: value.message } : {}), + ...(value.request_type === 'channel' && value.grant_to_requesting_key !== null && + value.grant_to_requesting_key !== undefined + ? { grantToRequestingKey: value.grant_to_requesting_key === true } + : {}), + }; +} + +function normalizeStoredKey(record) { + const value = record?.toJSON ? record.toJSON() : record; + if (!value) return null; + const permissions = normalizeExternalPermissions(value); + if (!permissions) return null; + return { + id: value.id, + name: value.name, + role: value.role, + isActive: value.is_active, + revokedAt: value.revoked_at, + autoApproveVideoRequests: value.auto_approve_video_requests, + autoApproveChannelRequests: value.auto_approve_channel_requests, + autoApproveDeleteRequests: value.auto_approve_delete_requests, + ...permissions, + maxRatingLevel: value.max_rating_level, + allowUnrated: value.allow_unrated, + allowedMediaTypes: value.allowed_media_types, + }; +} + +function sanitizeReason(value) { + if (typeof value !== 'string') { + throw new RequestError('reason is required'); + } + // eslint-disable-next-line no-control-regex + const sanitized = value.replace(/[\x00-\x1F\x7F]/g, ' ').replace(/\s+/g, ' ').trim(); + if (sanitized.length < 1 || sanitized.length > 300) { + throw new RequestError('reason must be between 1 and 300 characters'); + } + return sanitized; +} + +function isUniqueConstraintError(error) { + return error instanceof UniqueConstraintError || + error?.name === 'SequelizeUniqueConstraintError'; +} + +function adminDto(record, catalogVideo = null) { + const value = record.toJSON ? record.toJSON() : record; + const key = value.apiKey || value.api_key || null; + const channel = value.channel || null; + const job = value.job || null; + return { + ...dto(record), + requester: key ? { + id: key.id, + name: key.name, + keyPrefix: key.key_prefix, + role: key.role, + isActive: key.is_active === true, + revokedAt: key.revoked_at || null, + } : null, + target: { + youtubeId: value.youtube_id || null, + channelId: value.channel_id || null, + channelUrl: value.channel_url || null, + youtubeChannelId: channel?.channel_id || null, + channelTitle: channel?.title || channel?.uploader || null, + title: catalogVideo?.title || null, + mediaType: catalogVideo?.media_type || null, + rating: channel?.default_rating || null, + contentRating: catalogVideo?.content_rating || channel?.default_rating || null, + }, + job: job ? { + id: job.id, + status: job.status, + type: job.jobType, + createdAt: job.timeCreated, + startedAt: job.timeInitiated, + } : null, + }; +} + +module.exports = { + REQUEST_STATUSES, ACTIVE_STATUSES, VIDEO_ID_PATTERN, AUXILIARY_RECOVERY_DELAY_MS, + RequestError, rethrowWorkLimit, requireCurrentAutoApproval, parseInteger, + requestPagination, requestPaginationDto, normalizeIdempotencyKey, normalizeChannelUrl, + dto, normalizeStoredKey, sanitizeReason, isUniqueConstraintError, adminDto, +}; diff --git a/server/modules/externalRequestReconciliation.js b/server/modules/externalRequestReconciliation.js new file mode 100644 index 000000000..5fe6f788c --- /dev/null +++ b/server/modules/externalRequestReconciliation.js @@ -0,0 +1,61 @@ +function createExternalRequestReconciliation({ models, now }) { + const { Video, Job } = models; + async function reconcile(records, transaction = null) { + const processing = records.filter((record) => record.status === 'processing'); + const mediaRequests = processing.filter( + (record) => ['video', 'delete_video'].includes(record.request_type) + ); + const youtubeIds = [...new Set(mediaRequests.map((record) => record.youtube_id))]; + if (youtubeIds.length === 0) return records; + const videos = await Video.findAll({ + where: { youtubeId: youtubeIds, removed: false }, + attributes: ['youtubeId'], + ...(transaction ? { transaction } : {}), + }); + const completed = new Set(videos.map((video) => video.youtubeId)); + const completedAt = now(); + const completedRecords = mediaRequests.filter((record) => + (record.request_type === 'video' && completed.has(record.youtube_id)) || + (record.request_type === 'delete_video' && !completed.has(record.youtube_id)) + ); + await Promise.all(completedRecords.map(async (record) => { + await record.update({ + status: 'completed', + active_dedupe_key: null, + completed_at: completedAt, + updated_at: completedAt, + }, ...(transaction ? [{ transaction }] : [])); + })); + const unresolved = mediaRequests.filter( + (record) => record.request_type === 'video' && + !completed.has(record.youtube_id) && record.job_id + ); + if (unresolved.length > 0) { + const jobs = await Job.findAll({ + where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, + attributes: ['id', 'status'], + ...(transaction ? { transaction } : {}), + }); + const terminalFailures = new Set( + jobs + .filter((job) => [ + 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', + ].includes(job.status)) + .map((job) => job.id) + ); + await Promise.all(unresolved + .filter((record) => terminalFailures.has(record.job_id)) + .map((record) => record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Download did not complete', + updated_at: completedAt, + }, ...(transaction ? [{ transaction }] : [])))); + } + return records; + } + + return reconcile; +} + +module.exports = { createExternalRequestReconciliation }; diff --git a/server/modules/externalRequestReviewService.js b/server/modules/externalRequestReviewService.js new file mode 100644 index 000000000..e12a54cac --- /dev/null +++ b/server/modules/externalRequestReviewService.js @@ -0,0 +1,475 @@ +const { Op } = require('sequelize'); +const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); +const { hasExternalScope } = require('./externalPermissions'); +const { + RequestError, REQUEST_STATUSES, ACTIVE_STATUSES, normalizeStoredKey, + parseInteger, requestPagination, requestPaginationDto, adminDto, + sanitizeReason, rethrowWorkLimit, +} = require('./externalRequestPrimitives'); + +function createExternalRequestReviewService({ + models, sequelize, now, quotas, workLimiter, executor, reconcile, + validateTarget, provisionChannelRequest, executeDeleteRequest, +}) { + const { ExternalRequest, ApiKey, Channel, ChannelVideo, Video, Job } = models; + const adminIncludes = () => [ + { + model: ApiKey, + as: 'apiKey', + attributes: [ + 'id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allowed_media_types', + ], + required: false, + }, + { + model: Channel, + as: 'channel', + attributes: [ + 'id', 'channel_id', 'title', 'uploader', 'url', 'default_rating', 'terminated_at', + ], + required: false, + }, + { + model: Job, + as: 'job', + attributes: ['id', 'status', 'jobType', 'timeCreated', 'timeInitiated'], + required: false, + }, + ]; + + async function catalogMetadata(records) { + const youtubeIds = [...new Set(records.map((record) => record.youtube_id).filter(Boolean))]; + if (youtubeIds.length === 0) return new Map(); + const rows = await ChannelVideo.findAll({ + where: { youtube_id: youtubeIds }, + attributes: ['youtube_id', 'channel_id', 'title', 'media_type'], + }); + const videos = await Video.findAll({ + where: { youtubeId: youtubeIds }, + attributes: ['youtubeId', 'normalized_rating'], + }); + const videoById = new Map(videos.map((row) => { + const value = row.toJSON ? row.toJSON() : row; + return [value.youtubeId, value]; + })); + return new Map(rows.map((row) => { + const value = row.toJSON ? row.toJSON() : row; + return [`${value.channel_id}:${value.youtube_id}`, value]; + }).map(([key, value]) => [key, { + ...value, + content_rating: videoById.get(value.youtube_id)?.normalized_rating || null, + }])); + } + + async function adminDtos(records) { + const metadata = await catalogMetadata(records); + return records.map((record) => { + const value = record.toJSON ? record.toJSON() : record; + const channel = value.channel; + return adminDto( + record, + metadata.get(`${channel?.channel_id || ''}:${value.youtube_id}`) || null + ); + }); + } + + async function listAdminRequests(query = {}) { + const { page, pageSize, offset } = requestPagination(query); + const status = query.status; + if (status !== undefined && !REQUEST_STATUSES.includes(status)) { + throw new RequestError(`status must be one of: ${REQUEST_STATUSES.join(', ')}`); + } + const apiKeyId = query.apiKeyId === undefined + ? null + : parseInteger(query.apiKeyId, null, 1, Number.MAX_SAFE_INTEGER, 'apiKeyId'); + const requestType = query.requestType; + if (requestType !== undefined && !['video', 'channel', 'delete_video'].includes(requestType)) { + throw new RequestError('requestType must be video, channel, or delete_video'); + } + const where = { + ...(requestType ? { request_type: requestType } : {}), + ...(status ? { status } : {}), + ...(apiKeyId ? { api_key_id: apiKeyId } : {}), + }; + const result = await ExternalRequest.findAndCountAll({ + where, + include: adminIncludes(), + distinct: true, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }); + await reconcile(result.rows); + const reconciledResult = status + ? await ExternalRequest.findAndCountAll({ + where, + include: adminIncludes(), + distinct: true, + order: [['created_at', 'DESC'], ['id', 'DESC']], + limit: pageSize, + offset, + }) + : result; + const requesters = await ApiKey.findAll({ + attributes: ['id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at'], + order: [['name', 'ASC'], ['id', 'ASC']], + }); + return { + data: await adminDtos(reconciledResult.rows), + pagination: requestPaginationDto(page, pageSize, reconciledResult.count), + filterOptions: { + requesters: requesters.map((key) => { + const value = key.toJSON ? key.toJSON() : key; + return { + id: value.id, + name: value.name, + keyPrefix: value.key_prefix, + role: value.role, + isActive: value.is_active === true, + revokedAt: value.revoked_at || null, + }; + }), + }, + }; + } + + async function getAdminRequest(id) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + const record = await ExternalRequest.findOne({ + where: { id }, + include: adminIncludes(), + }); + if (!record) throw new RequestError('Request not found', 404); + await reconcile([record]); + return (await adminDtos([record]))[0]; + } + + async function reviewVideoRequest(id, action, input = {}) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + if (!['approve', 'reject'].includes(action)) { + throw new RequestError('Unsupported review action'); + } + const allowedFields = action === 'reject' ? ['reason'] : []; + if (Object.keys(input).some((field) => !allowedFields.includes(field))) { + throw new RequestError('Request body contains unsupported fields'); + } + const reason = action === 'reject' ? sanitizeReason(input.reason) : null; + + const current = await ExternalRequest.findByPk(id); + if (!current || current.request_type !== 'video') throw new RequestError('Request not found', 404); + const review = () => sequelize.transaction(async (transaction) => { + await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); + const record = await ExternalRequest.findOne({ + where: { id, request_type: 'video' }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!record) throw new RequestError('Request not found', 404); + + if (action === 'reject') { + if (record.status !== 'pending') { + throw new RequestError('Only pending requests can be rejected', 409); + } + const rejectedAt = now(); + await record.update({ + status: 'rejected', + active_dedupe_key: null, + message: reason, + decided_at: rejectedAt, + updated_at: rejectedAt, + }, { transaction }); + return; + } + + if (record.status !== 'pending' && + !(record.status === 'approved' && !record.job_id)) { + throw new RequestError('Only pending requests can be approved', 409); + } + + const failApproval = async (message) => { + const failedAt = now(); + await record.update({ + status: 'failed', + active_dedupe_key: null, + message, + decided_at: record.decided_at || failedAt, + updated_at: failedAt, + }, { transaction }); + }; + + const storedKey = await ApiKey.findByPk(record.api_key_id, { + attributes: [ + 'id', 'name', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'allowed_media_types', + ], + transaction, + lock: transaction.LOCK.UPDATE, + }); + const key = normalizeStoredKey(storedKey); + if (!key || key.isActive !== true || key.revokedAt || + !hasExternalScope(key, 'video:request')) { + await failApproval('Request is no longer eligible'); + return; + } + try { + await quotas.assertExecutionCapacity( + record.api_key_id, + 'video:request', + record.id, + transaction + ); + } catch (error) { + if (error.name !== 'QuotaError') throw error; + await failApproval('Request exceeds the active job limit'); + return; + } + + let target; + try { + target = await validateTarget(key, record.youtube_id, record.channel_id, transaction); + } catch (error) { + if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; + await failApproval('Request is no longer eligible'); + return; + } + if (target.downloaded) { + const completedAt = now(); + await record.update({ + status: 'completed', + active_dedupe_key: null, + message: 'Video is already downloaded', + decided_at: record.decided_at || completedAt, + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + return; + } + + const duplicate = await ExternalRequest.findOne({ + where: { + id: { [Op.ne]: record.id }, + active_dedupe_key: `${record.api_key_id}:video:${record.youtube_id}`, + status: { [Op.in]: ACTIVE_STATUSES }, + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (duplicate) { + await failApproval('Another active request already exists'); + return; + } + + const approvedAt = now(); + if (record.status === 'pending') { + await record.update({ + status: 'approved', + decided_at: approvedAt, + updated_at: approvedAt, + }, { transaction }); + } + let accepted = false; + try { + const existingJob = await Job.findByPk(record.id, { transaction }); + const jobId = existingJob?.id || await executor({ + body: { + urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], + channelId: target.channel.channel_id, + ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, + initiatedBy: { type: 'api_key', name: key.name }, + jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, + externalRequestId: record.id, + }, + }); + if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); + accepted = true; + const acceptedAt = now(); + await record.update({ + status: 'processing', + job_id: jobId || record.id, + updated_at: acceptedAt, + }, { transaction }); + } catch (error) { + if (accepted) throw error; + await failApproval('Download could not be queued'); + } + }); + try { + await (action === 'approve' ? workLimiter.run(review) : review()); + } catch (error) { + rethrowWorkLimit(error); + throw error; + } + + return getAdminRequest(id); + } + + async function reviewRequest(id, action, input = {}) { + if (typeof id !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + throw new RequestError('Request not found', 404); + } + const current = await ExternalRequest.findByPk(id); + if (!current) throw new RequestError('Request not found', 404); + if (current.request_type === 'video') return reviewVideoRequest(id, action, input); + if (!input || typeof input !== 'object' || Array.isArray(input)) { + throw new RequestError('Request body must be an object'); + } + if (!['approve', 'reject'].includes(action)) { + throw new RequestError('Unsupported review action'); + } + const allowedFields = action === 'reject' + ? ['reason'] + : (current.request_type === 'channel' ? ['grantToRequestingKey'] : []); + if (Object.keys(input).some((field) => !allowedFields.includes(field))) { + throw new RequestError('Request body contains unsupported fields'); + } + if (current.request_type === 'channel' && + input.grantToRequestingKey !== undefined && + typeof input.grantToRequestingKey !== 'boolean') { + throw new RequestError('grantToRequestingKey must be a boolean'); + } + const reason = action === 'reject' ? sanitizeReason(input.reason) : null; + let key; + let claimed; + await sequelize.transaction(async (transaction) => { + await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); + const record = await ExternalRequest.findByPk(id, { + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!record) throw new RequestError('Request not found', 404); + const retryingApproval = action === 'approve' && record.status === 'approved'; + if (record.status !== 'pending' && !retryingApproval) { + throw new RequestError('Only pending requests can be reviewed', 409); + } + if (retryingApproval && input.grantToRequestingKey !== undefined && + input.grantToRequestingKey !== record.grant_to_requesting_key) { + throw new RequestError('The original grant decision cannot be changed on retry', 409); + } + const decisionAt = now(); + if (action === 'reject') { + await record.update({ + status: 'rejected', + active_dedupe_key: null, + message: reason, + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + const storedKey = await ApiKey.findByPk(record.api_key_id, { + attributes: [ + 'id', 'name', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'allowed_media_types', + ], + transaction, + lock: transaction.LOCK.UPDATE, + }); + key = normalizeStoredKey(storedKey); + const requiredScope = record.request_type === 'delete_video' + ? 'video:delete' + : 'channel:request'; + if (!key || key.isActive !== true || key.revokedAt || + !hasExternalScope(key, requiredScope)) { + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + try { + await quotas.assertExecutionCapacity( + record.api_key_id, + requiredScope, + record.id, + transaction + ); + } catch (error) { + if (error.name !== 'QuotaError') throw error; + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request exceeds the active job limit', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + if (record.request_type === 'delete_video') { + try { + const target = await validateTarget( + key, + record.youtube_id, + record.channel_id, + transaction + ); + if (!target.downloaded) { + await record.update({ + status: 'completed', + active_dedupe_key: null, + message: 'Video is already deleted', + decided_at: decisionAt, + completed_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + } catch (error) { + if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; + await record.update({ + status: 'failed', + active_dedupe_key: null, + message: 'Request is no longer eligible', + decided_at: decisionAt, + updated_at: decisionAt, + }, { transaction }); + return; + } + } + await record.update({ + status: 'approved', + ...(record.request_type === 'channel' && !retryingApproval + ? { grant_to_requesting_key: input.grantToRequestingKey !== false } + : {}), + decided_at: record.decided_at || decisionAt, + updated_at: decisionAt, + }, { transaction }); + claimed = record; + }); + if (action === 'approve' && claimed) { + if (claimed.request_type === 'channel') { + await provisionChannelRequest(claimed, key, { + grantToRequestingKey: claimed.grant_to_requesting_key !== false, + }); + } else if (claimed.request_type === 'delete_video') { + await executeDeleteRequest(claimed); + } + } + return getAdminRequest(id); + } + + return { listAdminRequests, getAdminRequest, reviewVideoRequest, reviewRequest }; +} + +module.exports = { createExternalRequestReviewService }; diff --git a/server/modules/externalRequestService.js b/server/modules/externalRequestService.js index 2d163ba8a..5d3bb0b33 100644 --- a/server/modules/externalRequestService.js +++ b/server/modules/externalRequestService.js @@ -1,232 +1,15 @@ const crypto = require('crypto'); -const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); -const { Op, UniqueConstraintError } = require('sequelize'); const { normalizePolicy, isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); -const { hasExternalScope, normalizeExternalPermissions } = require('./externalPermissions'); +const { hasExternalScope } = require('./externalPermissions'); -const REQUEST_STATUSES = [ - 'pending', 'approved', 'processing', 'completed', - 'rejected', 'failed', 'cancelled', -]; -const ACTIVE_STATUSES = ['pending', 'approved', 'processing']; -const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; -const AUXILIARY_RECOVERY_DELAY_MS = 5 * 60 * 1000; -const MAX_PAGE = 100; - -class RequestError extends Error { - constructor(message, status = 400, code = null) { - super(message); - this.name = 'RequestError'; - this.status = status; - this.code = code; - } -} - -function rethrowWorkLimit(error) { - if (error?.name === 'ExternalWorkLimitError') { - throw new RequestError( - 'External API work capacity is temporarily unavailable', - 503, - 'work_queue_full' - ); - } -} - -function requireCurrentAutoApproval(key, requestType) { - const enabled = { - video: key.autoApproveVideoRequests, - channel: key.autoApproveChannelRequests, - delete_video: key.autoApproveDeleteRequests, - }[requestType]; - if (enabled !== true) { - throw new RequestError('Request is no longer eligible', 403); - } -} - -function parseInteger(value, fallback, minimum, maximum, name) { - if (value === undefined) return fallback; - if (!/^\d+$/.test(String(value))) throw new RequestError(`${name} must be an integer`); - const parsed = Number(value); - if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) { - throw new RequestError(`${name} must be between ${minimum} and ${maximum}`); - } - return parsed; -} - -function decodeCursor(value) { - if (value === undefined) return null; - if (typeof value !== 'string' || value.length > 200) { - throw new RequestError('cursor is invalid'); - } - try { - const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); - if (parsed?.v !== 1 || !Number.isSafeInteger(parsed.page) || - parsed.page < 1 || parsed.page > MAX_PAGE) { - throw new Error('invalid cursor'); - } - return parsed.page; - } catch (_error) { - throw new RequestError('cursor is invalid'); - } -} - -function requestPagination(query) { - if (query.cursor !== undefined && query.page !== undefined) { - throw new RequestError('cursor and page cannot be used together'); - } - const page = decodeCursor(query.cursor) || - parseInteger(query.page, 1, 1, MAX_PAGE, 'page'); - const pageSize = parseInteger(query.pageSize, 50, 1, 100, 'pageSize'); - return { page, pageSize, offset: (page - 1) * pageSize }; -} - -function requestPaginationDto(page, pageSize, total) { - const totalPages = total === 0 ? 0 : Math.min(MAX_PAGE, Math.ceil(total / pageSize)); - return { - page, - pageSize, - total, - totalPages, - nextCursor: page < totalPages - ? Buffer.from(JSON.stringify({ v: 1, page: page + 1 }), 'utf8').toString('base64url') - : null, - }; -} - -function normalizeIdempotencyKey(value) { - if (value === undefined) return null; - if (typeof value !== 'string' || value.length < 1 || value.length > 200) { - throw new RequestError('idempotencyKey must be a string of 1 to 200 characters'); - } - return crypto.createHash('sha256').update(value).digest('hex'); -} - -function normalizeChannelUrl(value) { - if (typeof value !== 'string' || value.length < 1 || value.length > 500) { - throw new RequestError('channelUrl must be a supported YouTube channel URL'); - } - let url; - try { - url = new URL(/^https?:\/\//i.test(value.trim()) ? value.trim() : `https://${value.trim()}`); - } catch (_error) { - throw new RequestError('channelUrl must be a supported YouTube channel URL'); - } - const hostname = url.hostname.toLowerCase().replace(/^www\./, ''); - const segments = url.pathname.split('/').filter(Boolean); - const supportedPrefix = segments[0]?.startsWith('@') || - ['channel', 'c', 'user'].includes(segments[0]); - if (!['youtube.com', 'm.youtube.com'].includes(hostname) || - !supportedPrefix || segments.length !== (segments[0].startsWith('@') ? 1 : 2)) { - throw new RequestError('channelUrl must be a supported YouTube channel URL'); - } - try { - return `https://www.youtube.com/${segments.map((segment) => { - const decoded = decodeURIComponent(segment); - return decoded.startsWith('@') - ? `@${encodeURIComponent(decoded.slice(1))}` - : encodeURIComponent(decoded); - }).join('/')}`; - } catch (_error) { - throw new RequestError('channelUrl must be a supported YouTube channel URL'); - } -} - -function dto(record) { - const value = record.toJSON ? record.toJSON() : record; - return { - id: value.id, - type: value.request_type, - status: value.status, - target: { - youtubeId: value.youtube_id || null, - channelId: value.channel_id || null, - channelUrl: value.channel_url || null, - }, - createdAt: value.created_at, - updatedAt: value.updated_at, - ...(value.decided_at ? { decidedAt: value.decided_at } : {}), - ...(value.completed_at ? { completedAt: value.completed_at } : {}), - ...(value.message ? { message: value.message } : {}), - ...(value.request_type === 'channel' && value.grant_to_requesting_key !== null && - value.grant_to_requesting_key !== undefined - ? { grantToRequestingKey: value.grant_to_requesting_key === true } - : {}), - }; -} - -function normalizeStoredKey(record) { - const value = record?.toJSON ? record.toJSON() : record; - if (!value) return null; - const permissions = normalizeExternalPermissions(value); - if (!permissions) return null; - return { - id: value.id, - name: value.name, - role: value.role, - isActive: value.is_active, - revokedAt: value.revoked_at, - autoApproveVideoRequests: value.auto_approve_video_requests, - autoApproveChannelRequests: value.auto_approve_channel_requests, - autoApproveDeleteRequests: value.auto_approve_delete_requests, - ...permissions, - maxRatingLevel: value.max_rating_level, - allowUnrated: value.allow_unrated, - allowedMediaTypes: value.allowed_media_types, - }; -} - -function sanitizeReason(value) { - if (typeof value !== 'string') { - throw new RequestError('reason is required'); - } - // eslint-disable-next-line no-control-regex - const sanitized = value.replace(/[\x00-\x1F\x7F]/g, ' ').replace(/\s+/g, ' ').trim(); - if (sanitized.length < 1 || sanitized.length > 300) { - throw new RequestError('reason must be between 1 and 300 characters'); - } - return sanitized; -} - -function isUniqueConstraintError(error) { - return error instanceof UniqueConstraintError || - error?.name === 'SequelizeUniqueConstraintError'; -} - -function adminDto(record, catalogVideo = null) { - const value = record.toJSON ? record.toJSON() : record; - const key = value.apiKey || value.api_key || null; - const channel = value.channel || null; - const job = value.job || null; - return { - ...dto(record), - requester: key ? { - id: key.id, - name: key.name, - keyPrefix: key.key_prefix, - role: key.role, - isActive: key.is_active === true, - revokedAt: key.revoked_at || null, - } : null, - target: { - youtubeId: value.youtube_id || null, - channelId: value.channel_id || null, - channelUrl: value.channel_url || null, - youtubeChannelId: channel?.channel_id || null, - channelTitle: channel?.title || channel?.uploader || null, - title: catalogVideo?.title || null, - mediaType: catalogVideo?.media_type || null, - rating: channel?.default_rating || null, - contentRating: catalogVideo?.content_rating || channel?.default_rating || null, - }, - job: job ? { - id: job.id, - status: job.status, - type: job.jobType, - createdAt: job.timeCreated, - startedAt: job.timeInitiated, - } : null, - }; -} +const { + RequestError, REQUEST_STATUSES, ACTIVE_STATUSES, VIDEO_ID_PATTERN, + parseInteger, requestPagination, requestPaginationDto, normalizeIdempotencyKey, + normalizeChannelUrl, dto, adminDto, sanitizeReason, isUniqueConstraintError, +} = require('./externalRequestPrimitives'); +const { createExternalRequestExecution } = require('./externalRequestExecution'); +const { createExternalRequestReviewService } = require('./externalRequestReviewService'); +const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); function createExternalRequestService({ models = require('../models'), @@ -239,10 +22,20 @@ function createExternalRequestService({ workLimiter = require('./externalWorkLimiter').sharedExternalWorkLimiter, } = {}) { const { - ExternalRequest, ApiKey, ApiKeyChannelGrant, Channel, ChannelVideo, Video, Job, + ExternalRequest, ApiKeyChannelGrant, Channel, ChannelVideo, Video, } = models; const quotas = quotaService || require('./externalQuotaService').createExternalQuotaService({ models, sequelize, now }); + const reconcile = createExternalRequestReconciliation({ models, now }); + const { dispatchAutoApproved, provisionChannelRequest, executeDeleteRequest } = + createExternalRequestExecution({ + models, sequelize, now, quotas, workLimiter, executor, channelProvisioner, + videoDeleter, validateTarget, + }); + const reviewService = createExternalRequestReviewService({ + models, sequelize, now, quotas, workLimiter, executor, reconcile, + validateTarget, provisionChannelRequest, executeDeleteRequest, + }); async function createReservedRequest(keyId, scope, buildValues) { return sequelize.transaction(async (transaction) => { @@ -267,84 +60,6 @@ function createExternalRequestService({ return record; } - async function reconcile(records) { - const processing = records.filter((record) => record.status === 'processing'); - const mediaRequests = processing.filter( - (record) => ['video', 'delete_video'].includes(record.request_type) - ); - const youtubeIds = [...new Set(mediaRequests.map((record) => record.youtube_id))]; - if (youtubeIds.length === 0) return records; - const videos = await Video.findAll({ - where: { youtubeId: youtubeIds, removed: false }, - attributes: ['youtubeId'], - }); - const completed = new Set(videos.map((video) => video.youtubeId)); - const completedAt = now(); - const completedRecords = mediaRequests.filter((record) => - (record.request_type === 'video' && completed.has(record.youtube_id)) || - (record.request_type === 'delete_video' && !completed.has(record.youtube_id)) - ); - await Promise.all(completedRecords.map(async (record) => { - await record.update({ - status: 'completed', - active_dedupe_key: null, - completed_at: completedAt, - updated_at: completedAt, - }); - })); - const unresolved = mediaRequests.filter( - (record) => record.request_type === 'video' && - !completed.has(record.youtube_id) && record.job_id - ); - if (unresolved.length > 0) { - const jobs = await Job.findAll({ - where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, - attributes: ['id', 'status'], - }); - const terminalFailures = new Set( - jobs - .filter((job) => [ - 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', - ].includes(job.status)) - .map((job) => job.id) - ); - await Promise.all(unresolved - .filter((record) => terminalFailures.has(record.job_id)) - .map((record) => record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Download did not complete', - updated_at: completedAt, - }))); - } - return records; - } - - async function claimAuxiliaryRequest(record) { - const timestamp = now(); - let whereStatus; - if (['pending', 'approved'].includes(record.status)) { - whereStatus = record.status; - } else if (record.status === 'processing') { - const updatedAt = new Date(record.updated_at).getTime(); - if (!Number.isFinite(updatedAt) || - timestamp.getTime() - updatedAt < AUXILIARY_RECOVERY_DELAY_MS) { - return false; - } - whereStatus = 'processing'; - } else { - return false; - } - const [claimed] = await ExternalRequest.update( - { status: 'processing', updated_at: timestamp }, - { where: { id: record.id, status: whereStatus, updated_at: record.updated_at } } - ); - if (claimed !== 1) return false; - record.status = 'processing'; - record.updated_at = timestamp; - return true; - } - async function validateTarget( key, youtubeId, @@ -399,74 +114,8 @@ function createExternalRequestService({ return { channel, downloaded }; } - const adminIncludes = () => [ - { - model: ApiKey, - as: 'apiKey', - attributes: [ - 'id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at', - 'auto_approve_video_requests', 'auto_approve_channel_requests', - 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', - 'allowed_media_types', - ], - required: false, - }, - { - model: Channel, - as: 'channel', - attributes: [ - 'id', 'channel_id', 'title', 'uploader', 'url', 'default_rating', 'terminated_at', - ], - required: false, - }, - { - model: Job, - as: 'job', - attributes: ['id', 'status', 'jobType', 'timeCreated', 'timeInitiated'], - required: false, - }, - ]; - - async function catalogMetadata(records) { - const youtubeIds = [...new Set(records.map((record) => record.youtube_id).filter(Boolean))]; - if (youtubeIds.length === 0) return new Map(); - const rows = await ChannelVideo.findAll({ - where: { youtube_id: youtubeIds }, - attributes: ['youtube_id', 'channel_id', 'title', 'media_type'], - }); - const videos = await Video.findAll({ - where: { youtubeId: youtubeIds }, - attributes: ['youtubeId', 'normalized_rating'], - }); - const videoById = new Map(videos.map((row) => { - const value = row.toJSON ? row.toJSON() : row; - return [value.youtubeId, value]; - })); - return new Map(rows.map((row) => { - const value = row.toJSON ? row.toJSON() : row; - return [`${value.channel_id}:${value.youtube_id}`, value]; - }).map(([key, value]) => [key, { - ...value, - content_rating: videoById.get(value.youtube_id)?.normalized_rating || null, - }])); - } - - async function adminDtos(records) { - const metadata = await catalogMetadata(records); - return records.map((record) => { - const value = record.toJSON ? record.toJSON() : record; - const channel = value.channel; - return adminDto( - record, - metadata.get(`${channel?.channel_id || ''}:${value.youtube_id}`) || null - ); - }); - } - async function findDuplicate(keyId, activeDedupeKey, idempotencyHash, youtubeId, channelId) { - const clauses = [{ active_dedupe_key: activeDedupeKey }]; - if (idempotencyHash) clauses.push({ api_key_id: keyId, idempotency_hash: idempotencyHash }); - const existing = await ExternalRequest.findOne({ where: { [Op.or]: clauses } }); + const existing = await findExisting(keyId, activeDedupeKey, idempotencyHash); if (!existing) return null; if (existing.request_type !== 'video' || existing.youtube_id !== youtubeId || existing.channel_id !== channelId) { throw new RequestError('Idempotency key was already used for another target', 409); @@ -475,13 +124,19 @@ function createExternalRequestService({ return existing; } + async function findExisting(keyId, activeDedupeKey, idempotencyHash) { + // Resolve the supplied idempotency key first: a different active request + // must not hide an earlier use of that key for another operation. + return (idempotencyHash && await ExternalRequest.findOne({ + where: { api_key_id: keyId, idempotency_hash: idempotencyHash }, + })) || ExternalRequest.findOne({ where: { active_dedupe_key: activeDedupeKey } }); + } + async function findTypedDuplicate({ keyId, activeDedupeKey, idempotencyHash, requestType, youtubeId = null, channelId = null, channelUrl = null, }) { - const clauses = [{ active_dedupe_key: activeDedupeKey }]; - if (idempotencyHash) clauses.push({ api_key_id: keyId, idempotency_hash: idempotencyHash }); - const existing = await ExternalRequest.findOne({ where: { [Op.or]: clauses } }); + const existing = await findExisting(keyId, activeDedupeKey, idempotencyHash); if (!existing) return null; if (existing.request_type !== requestType || existing.youtube_id !== youtubeId || @@ -492,67 +147,6 @@ function createExternalRequestService({ return existing; } - async function dispatchAutoApproved(record, key) { - let accepted = false; - try { - await workLimiter.run(() => - sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - key.id, - 'video:request', - record.id, - transaction - ); - const locked = await ExternalRequest.findByPk(record.id, { transaction, lock: transaction.LOCK.UPDATE }); - if (!locked || locked.status !== 'pending' || locked.job_id) return; - requireCurrentAutoApproval(currentKey, 'video'); - const target = await validateTarget( - currentKey, - record.youtube_id, - record.channel_id, - transaction - ); - const existingJob = await Job.findByPk(record.id, { transaction }); - const jobId = existingJob?.id || await executor({ - body: { - urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], - channelId: target.channel.channel_id, - ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, - initiatedBy: { type: 'api_key', name: currentKey.name }, - jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, - // The downloader uses this UUID as the job identity. A retry after a - // crash can therefore observe/reuse the accepted job instead of - // starting the same download twice. - externalRequestId: record.id, - }, - }); - if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); - accepted = true; - await record.update({ - status: 'processing', job_id: jobId || record.id, updated_at: now(), - }, { transaction }); - }) - ); - } catch (error) { - // The downloader persists separately. Leave the request retryable if the - // process loses its database connection after enqueueing the stable job. - if (accepted) throw error; - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: 'pending', updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Download could not be queued', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - async function createVideoRequest(key, input) { if (!hasExternalScope(key, 'video:request')) { throw new RequestError('video:request scope is required', 403); @@ -570,9 +164,6 @@ function createExternalRequestService({ if (input.channelId === undefined) throw new RequestError('channelId is required'); const channelId = parseInteger(input.channelId, null, 1, Number.MAX_SAFE_INTEGER, 'channelId'); const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); - const { downloaded } = await validateTarget(key, input.youtubeId, channelId); - if (downloaded) return { outcome: 'already_downloaded', request: null }; - const activeDedupeKey = `${key.id}:video:${input.youtubeId}`; const existing = await findDuplicate( key.id, activeDedupeKey, idempotencyHash, input.youtubeId, channelId @@ -600,6 +191,8 @@ function createExternalRequestService({ return { outcome: 'duplicate', request: dto(existing) }; } + const { downloaded } = await validateTarget(key, input.youtubeId, channelId); + if (downloaded) return { outcome: 'already_downloaded', request: null }; const timestamp = now(); let currentKey; let record; @@ -646,80 +239,6 @@ function createExternalRequestService({ return { outcome: 'created', request: dto(record) }; } - async function provisionChannelRequest( - record, - key, - { grantToRequestingKey, requireAutoApproval = false } = {} - ) { - const retryStatus = record.status === 'processing' ? 'approved' : record.status; - if (!(await claimAuxiliaryRequest(record))) return record; - const shouldGrant = grantToRequestingKey ?? - (record.grant_to_requesting_key !== false); - try { - await workLimiter.run(() => sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - key.id, - 'channel:request', - record.id, - transaction - ); - if (requireAutoApproval) { - requireCurrentAutoApproval(currentKey, 'channel'); - } - const provisioner = channelProvisioner || require('./channel/channelProvisioning'); - const result = await provisioner.getChannelInfo( - record.channel_url, - false, - true, - {}, - { skipTabDetection: true } - ); - const channel = await Channel.findOne({ - where: { - [Op.or]: [ - { channel_id: result.channel_id || result.id }, - { url: record.channel_url }, - ], - }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (!channel || channel.enabled !== true || channel.terminated_at) { - throw new Error('Provisioned channel is unavailable'); - } - if (shouldGrant) { - await ApiKeyChannelGrant.findOrCreate({ - where: { api_key_id: currentKey.id, channel_id: channel.id }, - defaults: { api_key_id: currentKey.id, channel_id: channel.id }, - transaction, - }); - } - const completedAt = now(); - await record.update({ - channel_id: channel.id, - status: 'completed', - active_dedupe_key: null, - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - })); - } catch (error) { - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: retryStatus, updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Channel could not be provisioned', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - async function createChannelRequest(key, input) { if (!hasExternalScope(key, 'channel:request')) { throw new RequestError('channel:request scope is required', 403); @@ -749,7 +268,7 @@ function createExternalRequestService({ key.id, 'channel:request', existing.id ); await provisionChannelRequest(existing, currentKey, { - requireAutoApproval: existing.status !== 'approved', + requireAutoApproval: existing.status === 'pending', }); } catch (error) { if (![403, 404].includes(error?.status)) throw error; @@ -806,63 +325,6 @@ function createExternalRequestService({ return { outcome: 'created', request: dto(record) }; } - async function executeDeleteRequest(record, { requireAutoApproval = false } = {}) { - const retryStatus = record.status === 'processing' ? 'approved' : record.status; - if (!(await claimAuxiliaryRequest(record))) return record; - try { - await workLimiter.run(() => sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - record.api_key_id, - 'video:delete', - record.id, - transaction - ); - if (requireAutoApproval) { - requireCurrentAutoApproval(currentKey, 'delete_video'); - } - const target = await validateTarget( - currentKey, - record.youtube_id, - record.channel_id, - transaction - ); - const result = target.downloaded - ? await videoDeleter.deleteVideoById( - target.downloaded.id, - { transaction, video: target.downloaded } - ) - : { success: false, error: 'Video is already removed' }; - const alreadyAbsent = result?.success === false && - /not found|already (?:marked as )?removed/i.test(result?.error || ''); - if (result?.success === false && !alreadyAbsent) { - throw new Error('Deletion failed'); - } - const completedAt = now(); - await record.update({ - status: 'completed', - active_dedupe_key: null, - ...(alreadyAbsent ? { message: 'Video is already deleted' } : {}), - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - })); - } catch (error) { - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: retryStatus, updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Video could not be deleted', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - async function createDeleteVideoRequest(key, input) { if (!hasExternalScope(key, 'video:delete')) { throw new RequestError('video:delete scope is required', 403); @@ -903,7 +365,7 @@ function createExternalRequestService({ ); await validateTarget(currentKey, input.youtubeId, channelId); await executeDeleteRequest(existing, { - requireAutoApproval: existing.status !== 'approved', + requireAutoApproval: existing.status === 'pending', }); } catch (error) { if (![403, 404].includes(error?.status)) throw error; @@ -1003,401 +465,13 @@ function createExternalRequestService({ return dto(record); } - async function listAdminRequests(query = {}) { - const { page, pageSize, offset } = requestPagination(query); - const status = query.status; - if (status !== undefined && !REQUEST_STATUSES.includes(status)) { - throw new RequestError(`status must be one of: ${REQUEST_STATUSES.join(', ')}`); - } - const apiKeyId = query.apiKeyId === undefined - ? null - : parseInteger(query.apiKeyId, null, 1, Number.MAX_SAFE_INTEGER, 'apiKeyId'); - const requestType = query.requestType; - if (requestType !== undefined && !['video', 'channel', 'delete_video'].includes(requestType)) { - throw new RequestError('requestType must be video, channel, or delete_video'); - } - const where = { - ...(requestType ? { request_type: requestType } : {}), - ...(status ? { status } : {}), - ...(apiKeyId ? { api_key_id: apiKeyId } : {}), - }; - const result = await ExternalRequest.findAndCountAll({ - where, - include: adminIncludes(), - distinct: true, - order: [['created_at', 'DESC'], ['id', 'DESC']], - limit: pageSize, - offset, - }); - await reconcile(result.rows); - const reconciledResult = status - ? await ExternalRequest.findAndCountAll({ - where, - include: adminIncludes(), - distinct: true, - order: [['created_at', 'DESC'], ['id', 'DESC']], - limit: pageSize, - offset, - }) - : result; - const requesters = await ApiKey.findAll({ - attributes: ['id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at'], - order: [['name', 'ASC'], ['id', 'ASC']], - }); - return { - data: await adminDtos(reconciledResult.rows), - pagination: requestPaginationDto(page, pageSize, reconciledResult.count), - filterOptions: { - requesters: requesters.map((key) => { - const value = key.toJSON ? key.toJSON() : key; - return { - id: value.id, - name: value.name, - keyPrefix: value.key_prefix, - role: value.role, - isActive: value.is_active === true, - revokedAt: value.revoked_at || null, - }; - }), - }, - }; - } - - async function getAdminRequest(id) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { - throw new RequestError('Request not found', 404); - } - const record = await ExternalRequest.findOne({ - where: { id }, - include: adminIncludes(), - }); - if (!record) throw new RequestError('Request not found', 404); - await reconcile([record]); - return (await adminDtos([record]))[0]; - } - - async function reviewVideoRequest(id, action, input = {}) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { - throw new RequestError('Request not found', 404); - } - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new RequestError('Request body must be an object'); - } - if (!['approve', 'reject'].includes(action)) { - throw new RequestError('Unsupported review action'); - } - const allowedFields = action === 'reject' ? ['reason'] : []; - if (Object.keys(input).some((field) => !allowedFields.includes(field))) { - throw new RequestError('Request body contains unsupported fields'); - } - const reason = action === 'reject' ? sanitizeReason(input.reason) : null; - - const current = await ExternalRequest.findByPk(id); - if (!current || current.request_type !== 'video') throw new RequestError('Request not found', 404); - const review = () => sequelize.transaction(async (transaction) => { - await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); - const record = await ExternalRequest.findOne({ - where: { id, request_type: 'video' }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (!record) throw new RequestError('Request not found', 404); - - if (action === 'reject') { - if (record.status !== 'pending') { - throw new RequestError('Only pending requests can be rejected', 409); - } - const rejectedAt = now(); - await record.update({ - status: 'rejected', - active_dedupe_key: null, - message: reason, - decided_at: rejectedAt, - updated_at: rejectedAt, - }, { transaction }); - return; - } - - if (record.status !== 'pending' && - !(record.status === 'approved' && !record.job_id)) { - throw new RequestError('Only pending requests can be approved', 409); - } - - const failApproval = async (message) => { - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message, - decided_at: record.decided_at || failedAt, - updated_at: failedAt, - }, { transaction }); - }; - - const storedKey = await ApiKey.findByPk(record.api_key_id, { - attributes: [ - 'id', 'name', 'role', 'is_active', 'revoked_at', - 'auto_approve_video_requests', 'auto_approve_channel_requests', - 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', - 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', - 'allowed_media_types', - ], - transaction, - lock: transaction.LOCK.UPDATE, - }); - const key = normalizeStoredKey(storedKey); - if (!key || key.isActive !== true || key.revokedAt || - !hasExternalScope(key, 'video:request')) { - await failApproval('Request is no longer eligible'); - return; - } - try { - await quotas.assertExecutionCapacity( - record.api_key_id, - 'video:request', - record.id, - transaction - ); - } catch (error) { - if (error.name !== 'QuotaError') throw error; - await failApproval('Request exceeds the active job limit'); - return; - } - - let target; - try { - target = await validateTarget(key, record.youtube_id, record.channel_id, transaction); - } catch (error) { - if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; - await failApproval('Request is no longer eligible'); - return; - } - if (target.downloaded) { - const completedAt = now(); - await record.update({ - status: 'completed', - active_dedupe_key: null, - message: 'Video is already downloaded', - decided_at: record.decided_at || completedAt, - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - return; - } - - const duplicate = await ExternalRequest.findOne({ - where: { - id: { [Op.ne]: record.id }, - active_dedupe_key: `${record.api_key_id}:video:${record.youtube_id}`, - status: { [Op.in]: ACTIVE_STATUSES }, - }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (duplicate) { - await failApproval('Another active request already exists'); - return; - } - - const approvedAt = now(); - if (record.status === 'pending') { - await record.update({ - status: 'approved', - decided_at: approvedAt, - updated_at: approvedAt, - }, { transaction }); - } - try { - const existingJob = await Job.findByPk(record.id, { transaction }); - const jobId = existingJob?.id || await executor({ - body: { - urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], - channelId: target.channel.channel_id, - ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, - initiatedBy: { type: 'api_key', name: key.name }, - jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, - externalRequestId: record.id, - }, - }); - if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); - const acceptedAt = now(); - await record.update({ - status: 'processing', - job_id: jobId || record.id, - updated_at: acceptedAt, - }, { transaction }); - } catch (_error) { - await failApproval('Download could not be queued'); - } - }); - try { - await (action === 'approve' ? workLimiter.run(review) : review()); - } catch (error) { - rethrowWorkLimit(error); - throw error; - } - - return getAdminRequest(id); - } - - async function reviewRequest(id, action, input = {}) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { - throw new RequestError('Request not found', 404); - } - const current = await ExternalRequest.findByPk(id); - if (!current) throw new RequestError('Request not found', 404); - if (current.request_type === 'video') return reviewVideoRequest(id, action, input); - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new RequestError('Request body must be an object'); - } - if (!['approve', 'reject'].includes(action)) { - throw new RequestError('Unsupported review action'); - } - const allowedFields = action === 'reject' - ? ['reason'] - : (current.request_type === 'channel' ? ['grantToRequestingKey'] : []); - if (Object.keys(input).some((field) => !allowedFields.includes(field))) { - throw new RequestError('Request body contains unsupported fields'); - } - if (current.request_type === 'channel' && - input.grantToRequestingKey !== undefined && - typeof input.grantToRequestingKey !== 'boolean') { - throw new RequestError('grantToRequestingKey must be a boolean'); - } - const reason = action === 'reject' ? sanitizeReason(input.reason) : null; - let key; - let claimed; - await sequelize.transaction(async (transaction) => { - await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); - const record = await ExternalRequest.findByPk(id, { - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (!record) throw new RequestError('Request not found', 404); - if (record.status !== 'pending') { - throw new RequestError('Only pending requests can be reviewed', 409); - } - const decisionAt = now(); - if (action === 'reject') { - await record.update({ - status: 'rejected', - active_dedupe_key: null, - message: reason, - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - const storedKey = await ApiKey.findByPk(record.api_key_id, { - attributes: [ - 'id', 'name', 'role', 'is_active', 'revoked_at', - 'auto_approve_video_requests', 'auto_approve_channel_requests', - 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', - 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', - 'allowed_media_types', - ], - transaction, - lock: transaction.LOCK.UPDATE, - }); - key = normalizeStoredKey(storedKey); - const requiredScope = record.request_type === 'delete_video' - ? 'video:delete' - : 'channel:request'; - if (!key || key.isActive !== true || key.revokedAt || - !hasExternalScope(key, requiredScope)) { - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - try { - await quotas.assertExecutionCapacity( - record.api_key_id, - requiredScope, - record.id, - transaction - ); - } catch (error) { - if (error.name !== 'QuotaError') throw error; - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request exceeds the active job limit', - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - if (record.request_type === 'delete_video') { - try { - const target = await validateTarget( - key, - record.youtube_id, - record.channel_id, - transaction - ); - if (!target.downloaded) { - await record.update({ - status: 'completed', - active_dedupe_key: null, - message: 'Video is already deleted', - decided_at: decisionAt, - completed_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - } catch (error) { - if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - } - await record.update({ - status: 'approved', - ...(record.request_type === 'channel' - ? { grant_to_requesting_key: input.grantToRequestingKey !== false } - : {}), - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - claimed = record; - }); - if (action === 'approve' && claimed) { - if (claimed.request_type === 'channel') { - await provisionChannelRequest(claimed, key, { - grantToRequestingKey: input.grantToRequestingKey !== false, - }); - } else if (claimed.request_type === 'delete_video') { - await executeDeleteRequest(claimed); - } - } - return getAdminRequest(id); - } - return { createVideoRequest, createChannelRequest, createDeleteVideoRequest, listRequests, getRequest, - listAdminRequests, - getAdminRequest, - reviewVideoRequest, - reviewRequest, + ...reviewService, }; } diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index a2cbac0cb..afd03bbce 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -279,3 +279,71 @@ test('runtime indexes preserve equivalent indexes and survive rollback and parti await migration.up(queryInterface); } }); + +test('manual approval retries reuse the durable job after request persistence fails', async () => { + const created = await service().createVideoRequest(key, { youtubeId, channelId: channel.id }); + models.ExternalRequest.addHook('beforeUpdate', 'simulateApprovalLoss', row => { + if (row.status === 'processing') throw new Error('Synthetic approval connection loss'); + }); + try { + await expect(service().reviewRequest(created.request.id, 'approve')).rejects.toThrow('Synthetic approval'); + } finally { models.ExternalRequest.removeHook('beforeUpdate', 'simulateApprovalLoss'); } + const executor = jest.fn(); + expect((await service({ executor }).reviewRequest(created.request.id, 'approve')).status).toBe('processing'); + expect(executor).not.toHaveBeenCalled(); + expect(await models.Job.count()).toBe(1); +}); + +test('channel approval retries preserve an explicit decision not to grant access', async () => { + await models.ApiKeyChannelGrant.destroy({ where: { api_key_id: key.id } }); + const created = await service().createChannelRequest(key, { channelUrl: 'youtube.com/@synthetic' }); + const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 0 }); + let release; + const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); + try { + await expect(service({ workLimiter: limiter }).reviewRequest(created.request.id, 'approve', + { grantToRequestingKey: false })).rejects.toMatchObject({ status: 503 }); + } finally { release(); await blocker; } + await expect(service().reviewRequest(created.request.id, 'approve', + { grantToRequestingKey: true })).rejects.toMatchObject({ status: 409 }); + const result = await service().reviewRequest(created.request.id, 'approve'); + expect(result).toMatchObject({ status: 'completed', grantToRequestingKey: false }); + expect(await models.ApiKeyChannelGrant.count()).toBe(0); +}); + +test('a completed video idempotency replay returns the original request after downloading', async () => { + await policy({ auto_approve_video_requests: true }); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'completed-video' }; + const first = await service().createVideoRequest(key, input); + await downloadedVideo(); + const replay = await service().createVideoRequest(key, input); + expect(replay).toMatchObject({ outcome: 'duplicate', request: { id: first.request.id, status: 'completed' } }); +}); + +test('an active request cannot shadow an idempotency key used for a different operation', async () => { + const api = service(); + await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + await api.createChannelRequest(key, { channelUrl: 'youtube.com/@synthetic', idempotencyKey: 'used-elsewhere' }); + await expect(api.createVideoRequest(key, { youtubeId, channelId: channel.id, + idempotencyKey: 'used-elsewhere' })).rejects.toMatchObject({ status: 409 }); +}); + +test('catalog request status selects the newest own-key history and preserves duplicate channels', async () => { + const duplicate = await models.Channel.create({ channel_id: youtubeChannelId, + title: 'Unrelated duplicate', enabled: true, default_rating: 'TV-Y' }); + const otherKey = await models.ApiKey.create({ name: 'Other synthetic key', key_hash: 'other-test-only-hash', + key_prefix: 'otherkey', created_at: timestamp, role: 'admin' }); + await models.ExternalRequest.bulkCreate([ + { api_key_id: key.id, channel_id: channel.id, youtube_id: youtubeId, request_type: 'video', + status: 'rejected', created_at: new Date(timestamp.getTime() - 1000), updated_at: timestamp }, + { api_key_id: key.id, channel_id: channel.id, youtube_id: youtubeId, request_type: 'video', + status: 'pending', created_at: timestamp, updated_at: timestamp }, + { api_key_id: otherKey.id, channel_id: duplicate.id, youtube_id: youtubeId, request_type: 'video', + status: 'failed', created_at: new Date(timestamp.getTime() + 1000), updated_at: timestamp }, + ]); + expect((await catalog.listVideos(key)).data).toEqual([ + expect.objectContaining({ channelDatabaseId: channel.id, requestStatus: 'pending' }), + ]); + expect((await catalog.listChannels(key)).data.map(row => row.id)).toEqual([channel.id]); + expect(await models.Channel.count()).toBe(2); +}); From c3de9c366d8805492e3069cf9b479f1016364563 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:29:17 +0000 Subject: [PATCH 12/27] fix: bound channel metadata work without limiting catalog refreshes --- docs/EXTERNAL_API.md | 4 +-- .../__tests__/externalCatalogService.test.js | 4 +-- .../__tests__/channelYtdlpExecutor.test.js | 26 +++++++++++++++++++ .../modules/channel/channelMetadataFetcher.js | 7 +++-- .../modules/channel/channelYtdlpExecutor.js | 17 ++++++++---- 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md index 4b31f434f..8087ec40f 100644 --- a/docs/EXTERNAL_API.md +++ b/docs/EXTERNAL_API.md @@ -409,8 +409,8 @@ deciding whether to offer external API controls. When the shared work queue is full, writes return 503 while retaining the accepted request and its idempotency key. Retry the same body to resume it -without charging another accepted write. An already completed channel or -deletion request returns its original result on an idempotent retry. +without charging another accepted write. Completed video, channel, and +deletion requests return their original result on an idempotent retry. Saving an external key with zero approved channels is intentional and does not backfill or grant access. That key fails closed: catalog reads and requests diff --git a/server/modules/__tests__/externalCatalogService.test.js b/server/modules/__tests__/externalCatalogService.test.js index a2620b6dc..7090f349e 100644 --- a/server/modules/__tests__/externalCatalogService.test.js +++ b/server/modules/__tests__/externalCatalogService.test.js @@ -126,7 +126,7 @@ describe('external cached catalog', () => { channel_database_id: 8, channel_id: 'UCsafe', channel_title: 'Safe Channel', - }]) + }]) .mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', status: 'failed' }]); const result = await catalog.listVideos(key(), { @@ -323,7 +323,7 @@ describe('external cached catalog', () => { channel_database_id: 8, channel_id: 'UCsafe', channel_title: 'Safe Channel', - }]) + }]) .mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', status: 'approved' }]); const metadataService = { getVideoMetadata: jest.fn().mockResolvedValue({ diff --git a/server/modules/channel/__tests__/channelYtdlpExecutor.test.js b/server/modules/channel/__tests__/channelYtdlpExecutor.test.js index cab436c86..ef0517ba5 100644 --- a/server/modules/channel/__tests__/channelYtdlpExecutor.test.js +++ b/server/modules/channel/__tests__/channelYtdlpExecutor.test.js @@ -94,6 +94,32 @@ describe('channelYtdlpExecutor', () => { await expect(promise).resolves.toBeUndefined(); }); + test('bounds metadata calls and kills a child that ignores termination', async () => { + jest.useFakeTimers(); + proc.kill = jest.fn(); + try { + const promise = executor.executeYtDlpCommand(['--dump-json'], null, { timeoutMs: 100 }); + const rejected = expect(promise).rejects.toMatchObject({ code: 'YT_DLP_TIMEOUT' }); + await jest.advanceTimersByTimeAsync(100); + await rejected; + expect(proc.kill).toHaveBeenCalledWith('SIGTERM'); + await jest.advanceTimersByTimeAsync(5000); + expect(proc.kill).toHaveBeenCalledWith('SIGKILL'); + } finally { jest.useRealTimers(); } + }); + + test('does not impose the metadata deadline on large catalog listings', async () => { + jest.useFakeTimers(); + proc.kill = jest.fn(); + try { + const promise = executor.executeYtDlpCommand(['--flat-playlist']); + await jest.advanceTimersByTimeAsync(180000); + expect(proc.kill).not.toHaveBeenCalled(); + proc.emit('exit', 0); + await expect(promise).resolves.toBeUndefined(); + } finally { jest.useRealTimers(); } + }); + test('rejects with COOKIES_REQUIRED when stderr reports a bot check, even on exit code 0', async () => { const promise = executor.executeYtDlpCommand(['--dump-json']); proc.stderr.emit('data', 'ERROR: Sign in to confirm you\'re not a bot'); diff --git a/server/modules/channel/channelMetadataFetcher.js b/server/modules/channel/channelMetadataFetcher.js index abf1d5746..e635b84cd 100644 --- a/server/modules/channel/channelMetadataFetcher.js +++ b/server/modules/channel/channelMetadataFetcher.js @@ -4,6 +4,7 @@ const { sanitizeNameLikeYtDlp } = require('../filesystem'); const youtubeApi = require('../youtubeApi'); const channelYtdlpExecutor = require('./channelYtdlpExecutor'); const { logApiFallback } = require('./apiFallbackLogger'); +const { summarizeCommandArgs } = require('../safeCommandLogging'); class ChannelMetadataFetcher { /** @@ -65,8 +66,10 @@ class ChannelMetadataFetcher { playlistEnd: 1, skipSleepRequests: true, }); - logger.info('fetchChannelMetadata executing yt-dlp with args' + JSON.stringify(args)); - const content = await channelYtdlpExecutor.executeYtDlpCommand(args, outputFilePath); + logger.info(summarizeCommandArgs(args), 'Fetching channel metadata with yt-dlp'); + const content = await channelYtdlpExecutor.executeYtDlpCommand(args, outputFilePath, { + timeoutMs: 120000, + }); logger.info('fetchChannelMetadata received yt-dlp output of length ' + content.length); const metadata = JSON.parse(content); diff --git a/server/modules/channel/channelYtdlpExecutor.js b/server/modules/channel/channelYtdlpExecutor.js index 57855c15a..adde0b473 100644 --- a/server/modules/channel/channelYtdlpExecutor.js +++ b/server/modules/channel/channelYtdlpExecutor.js @@ -17,9 +17,10 @@ class ChannelYtdlpExecutor { * @param {string|null} outputFile - Optional output file path * @param {Object} options - Options object * @param {Function} options.onStdoutData - Called with each raw stdout chunk as it arrives + * @param {number|null} options.timeoutMs - Optional deadline for bounded metadata calls * @returns {Promise} - Output content if outputFile provided */ - async executeYtDlpCommand(args, outputFile = null, { onStdoutData, timeoutMs = 120000 } = {}) { + async executeYtDlpCommand(args, outputFile = null, { onStdoutData, timeoutMs = null } = {}) { const ytDlp = spawnYtDlp(args, { env: { ...process.env, @@ -52,21 +53,27 @@ class ChannelYtdlpExecutor { const exited = new Promise((resolve, reject) => { let settled = false; + let forceKill; const finish = (callback, value) => { if (settled) return; settled = true; clearTimeout(timeout); callback(value); }; - const timeout = setTimeout(() => { + const timeout = timeoutMs > 0 ? setTimeout(() => { const error = new Error(`yt-dlp channel metadata request timed out after ${timeoutMs}ms`); error.code = 'YT_DLP_TIMEOUT'; - if (typeof ytDlp.kill === 'function') ytDlp.kill('SIGTERM'); + if (typeof ytDlp.kill === 'function') { + forceKill = setTimeout(() => ytDlp.kill('SIGKILL'), 5000); + forceKill.unref?.(); + ytDlp.kill('SIGTERM'); + } finish(reject, error); - }, timeoutMs); - timeout.unref?.(); + }, timeoutMs) : null; + timeout?.unref?.(); ytDlp.on('exit', (code) => { + clearTimeout(forceKill); // Check for bot detection if (stderrBuffer.includes('Sign in to confirm you\'re not a bot') || stderrBuffer.includes('Sign in to confirm that you\'re not a bot')) { From 914054885fe9bc306c5e42325fa2b307f440642c Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:29:49 +0000 Subject: [PATCH 13/27] test: align external request model indexes with runtime migration --- server/models/__tests__/externalApiModels.test.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/server/models/__tests__/externalApiModels.test.js b/server/models/__tests__/externalApiModels.test.js index 9f3addfbf..5218fb855 100644 --- a/server/models/__tests__/externalApiModels.test.js +++ b/server/models/__tests__/externalApiModels.test.js @@ -15,9 +15,12 @@ describe('external API persistence models', () => { expect(ExternalRequest.options.indexes.map((index) => index.name)).toEqual(expect.arrayContaining([ 'external_requests_key_created_idx', 'external_requests_key_status_idx', ])); - expect(ExternalRequest.options.indexes.map((index) => index.name)).not.toEqual(expect.arrayContaining([ - 'external_requests_catalog_status_idx', 'external_requests_management_idx', - ])); + const runtimeIndexes = require('../../../migrations/20261007041058-add-external-api-runtime-indexes').INDEXES; + for (const index of runtimeIndexes) { + expect(ExternalRequest.options.indexes).toContainEqual(expect.objectContaining({ + name: index.name, fields: index.fields, + })); + } }); test('register associations for grants, requests, jobs, and quota buckets', () => { From e35e430b47a1ee274451be3dfcdefa8b99285539 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 04:35:20 +0000 Subject: [PATCH 14/27] fix: patch proxy address trust validation for external ingress --- package-lock.json | 11 ++++++++--- server/__tests__/trustProxy.test.js | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 server/__tests__/trustProxy.test.js diff --git a/package-lock.json b/package-lock.json index b2e39da9a..9b18516e2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8399,15 +8399,20 @@ "integrity": "sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==" }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/proxy-from-env": { diff --git a/server/__tests__/trustProxy.test.js b/server/__tests__/trustProxy.test.js new file mode 100644 index 000000000..a318e18dc --- /dev/null +++ b/server/__tests__/trustProxy.test.js @@ -0,0 +1,23 @@ +const express = require('express'); +const request = require('supertest'); + +function appWithTrust(subnet) { + const app = express(); + app.set('trust proxy', subnet); + app.get('/', (req, res) => res.json({ ip: req.ip })); + return app; +} + +test('an IPv4-mapped IPv6 subnet with a short prefix cannot trust an unrelated client', async () => { + // GHSA-jqcg-44mw-7w3h: proxy-addr 2.0.7 treated this as trusting every IPv4 + // client, allowing a forged forwarded address to bypass ingress rate limits. + const response = await request(appWithTrust('::ffff:10.0.0.0/8')) + .get('/').set('X-Forwarded-For', '198.51.100.23').expect(200); + expect(response.body.ip).not.toBe('198.51.100.23'); +}); + +test('an explicitly trusted loopback proxy still forwards the client address', async () => { + const response = await request(appWithTrust('loopback')) + .get('/').set('X-Forwarded-For', '198.51.100.23').expect(200); + expect(response.body.ip).toBe('198.51.100.23'); +}); From ac3ad33aeb0e8c87de20b613daa2bdcde14961e5 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Wed, 7 Oct 2026 05:35:41 +0000 Subject: [PATCH 15/27] fix: return validation errors for unavailable channel grants --- server/modules/apiKeyChannelGrantModule.js | 15 +++++++--- server/routes/apikeys.js | 4 +-- .../testing/externalRuntime.database.test.js | 28 +++++++++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/server/modules/apiKeyChannelGrantModule.js b/server/modules/apiKeyChannelGrantModule.js index a197ee232..db707bc83 100644 --- a/server/modules/apiKeyChannelGrantModule.js +++ b/server/modules/apiKeyChannelGrantModule.js @@ -2,11 +2,18 @@ const { Op } = require('sequelize'); const { sequelize } = require('../db'); const { ApiKey, ApiKeyChannelGrant, Channel } = require('../models'); +class ChannelGrantValidationError extends Error { + constructor(message) { + super(message); + this.name = 'ChannelGrantValidationError'; + } +} + function normalizeChannelIds(channelIds) { - if (!Array.isArray(channelIds)) throw new Error('channelIds must be an array'); + if (!Array.isArray(channelIds)) throw new ChannelGrantValidationError('channelIds must be an array'); const normalized = [...new Set(channelIds)]; if (normalized.some((id) => !Number.isSafeInteger(id) || id < 1)) { - throw new Error('channelIds must contain only positive integer database IDs'); + throw new ChannelGrantValidationError('channelIds must contain only positive integer database IDs'); } return normalized.sort((a, b) => a - b); } @@ -15,7 +22,7 @@ async function requireExternalKey(keyId, transaction) { const key = await ApiKey.findByPk(keyId, { transaction, ...(transaction ? { lock: transaction.LOCK.UPDATE } : {}) }); if (!key) return null; if (!key.is_active || key.revoked_at || key.role === 'legacy_download') { - throw new Error('Only active external API keys can receive channel grants'); + throw new ChannelGrantValidationError('Only active external API keys can receive channel grants'); } return key; } @@ -71,7 +78,7 @@ async function replaceChannelGrants(keyId, channelIds, { transaction: existingTr transaction, }); if (enabledCount !== normalized.length) { - throw new Error('Every channel ID must identify an enabled, non-terminated channel'); + throw new ChannelGrantValidationError('Every channel ID must identify an enabled, non-terminated channel'); } } diff --git a/server/routes/apikeys.js b/server/routes/apikeys.js index 30432e97d..c0c7661c8 100644 --- a/server/routes/apikeys.js +++ b/server/routes/apikeys.js @@ -13,6 +13,7 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { const { sequelize } = require('../db'); const isPolicyValidationError = (error) => + error.name === 'ChannelGrantValidationError' || error.message.includes('Invalid') || error.message.includes('Policy') || error.message.includes('Unsupported') || @@ -267,8 +268,7 @@ module.exports = function createApiKeyRoutes({ verifyToken }) { if (!grants) return res.status(404).json({ error: 'API key not found' }); return res.json({ success: true, ...grants }); } catch (error) { - if (error.message.includes('channelIds') || error.message.includes('enabled channel') || - error.message.includes('Only active external API keys')) { + if (isPolicyValidationError(error)) { return res.status(400).json({ error: error.message }); } req.log.error({ err: error }, 'Failed to replace API key channel grants'); diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index afd03bbce..c33deac72 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -347,3 +347,31 @@ test('catalog request status selects the newest own-key history and preserves du expect((await catalog.listChannels(key)).data.map(row => row.id)).toEqual([channel.id]); expect(await models.Channel.count()).toBe(2); }); + +test.each(['missing', 'disabled', 'terminated'])('management rejects a %s channel grant and rolls back key changes', async state => { + const express = require('express'); + const request = require('supertest'); + const createApiKeyRoutes = require('../routes/apikeys'); + const app = express(); + app.use(express.json()); + app.use(createApiKeyRoutes({ verifyToken: (req, _res, next) => { + req.authType = 'session'; + req.log = { error: () => {} }; + next(); + } })); + if (state === 'disabled') await channel.update({ enabled: false }); + if (state === 'terminated') await channel.update({ terminated_at: timestamp }); + const channelIds = [state === 'missing' ? channel.id + 1000 : channel.id]; + const error = { error: 'Every channel ID must identify an enabled, non-terminated channel' }; + const beforeCount = await models.ApiKey.count(); + await request(app).post('/api/keys') + .send({ name: 'Invalid grant client', policy: { role: 'view' }, channelIds }).expect(400, error); + expect(await models.ApiKey.count()).toBe(beforeCount); + await request(app).put(`/api/keys/${key.id}/external-access`) + .send({ policy: { role: 'view', maxRatingLevel: 1 }, channelIds }).expect(400, error); + await keyRecord.reload(); + expect(keyRecord.role).toBe('admin'); + expect(keyRecord.max_rating_level).toBe(4); + await request(app).put(`/api/keys/${key.id}/channels`).send({ channelIds }).expect(400, error); + expect(await models.ApiKeyChannelGrant.count({ where: { api_key_id: key.id } })).toBe(1); +}); From c0242cce60a41efdfaeda2ba0a53d810789ccfad Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:23:41 +0000 Subject: [PATCH 16/27] fix: keep external API changes out of shared logging and ratings Restore the downloads-directory and cookie temp-dir log lines, the x-request-id correlation header outside the external API namespace, TV-Y7's text rating fallback, and unredacted yt-dlp stderr in server logs. Drop the bare 'key' redaction path so config upgrade logs keep naming the corrected setting. None of these were needed by the API. --- server/__tests__/logger.test.js | 2 +- server/logger.js | 1 - .../__tests__/safeCommandLogging.test.js | 38 ------- .../modules/channel/channelMetadataFetcher.js | 3 +- .../modules/channel/channelYtdlpExecutor.js | 3 +- server/modules/playlistModule.js | 10 +- server/modules/ratingMapper.js | 2 +- server/modules/safeCommandLogging.js | 106 ------------------ .../subscriptionImport/cookiesFetcher.js | 2 +- server/modules/ytDlpRunner.js | 5 +- server/server.js | 11 +- 11 files changed, 17 insertions(+), 166 deletions(-) delete mode 100644 server/modules/__tests__/safeCommandLogging.test.js delete mode 100644 server/modules/safeCommandLogging.js diff --git a/server/__tests__/logger.test.js b/server/__tests__/logger.test.js index d61302bbd..71cac4c9a 100644 --- a/server/__tests__/logger.test.js +++ b/server/__tests__/logger.test.js @@ -107,7 +107,7 @@ describe('logger.js', () => { expect(config.redact.paths).toContain('plexAuthToken'); expect(config.redact.paths).toContain('session_token'); expect(config.redact.paths).toContain('plexApiKey'); - expect(config.redact.paths).toContain('key'); + expect(config.redact.paths).not.toContain('key'); expect(config.redact.paths).toContain('key_hash'); expect(config.redact.paths).toContain('req.body.idempotencyKey'); }); diff --git a/server/logger.js b/server/logger.js index bec4eccdd..9fd58c464 100644 --- a/server/logger.js +++ b/server/logger.js @@ -80,7 +80,6 @@ const pinoConfig = { 'embyApiKey', 'youtubeApiKey', 'apiKey', - 'key', 'key_hash', 'idempotencyKey', 'req.body.apiKey', diff --git a/server/modules/__tests__/safeCommandLogging.test.js b/server/modules/__tests__/safeCommandLogging.test.js deleted file mode 100644 index d3d32b912..000000000 --- a/server/modules/__tests__/safeCommandLogging.test.js +++ /dev/null @@ -1,38 +0,0 @@ -const { - summarizeCommandArgs, - redactSensitiveText, -} = require('../safeCommandLogging'); - -describe('safe command logging', () => { - test('summarizes command arguments without retaining values', () => { - const summary = summarizeCommandArgs([ - '--proxy', 'https://user:secret@proxy.example:8443', - '--cookies', '/private/cookies.txt', - '--add-header', 'Authorization: Bearer sentinel', - 'https://www.youtube.com/watch?v=abcdefghijk', - ]); - expect(summary).toEqual({ - argumentCount: 7, - flags: ['--add-header', '--cookies', '--proxy'], - targetHosts: ['www.youtube.com'], - }); - expect(JSON.stringify(summary)).not.toMatch(/secret|cookies\.txt|sentinel/); - }); - - test('redacts proxy, cookie, header, query, and filesystem sentinels', () => { - const result = redactSensitiveText( - 'failed --proxy https://user:secret@proxy.example ' + - '--cookies /private/cookies.txt --add-header "Cookie: cookie-sentinel" ' + - 'Authorization: Bearer bearer-sentinel\n' + - '--client-certificate-password cert-sentinel ' + - '--client-certificate-key=relative/private-key.pem ' + - 'https://x.test/a?token=query-sentinel&ok=1 ' + - 'Destination: /downloads/private/path-sentinel/video.mp4' - ); - expect(result).not.toMatch( - /user:secret|cookies\.txt|cookie-sentinel|bearer-sentinel|cert-sentinel|private-key|query-sentinel|path-sentinel/ - ); - expect(result).toContain('[REDACTED]'); - expect(result).toContain('[REDACTED_PATH]'); - }); -}); diff --git a/server/modules/channel/channelMetadataFetcher.js b/server/modules/channel/channelMetadataFetcher.js index e635b84cd..9b5718917 100644 --- a/server/modules/channel/channelMetadataFetcher.js +++ b/server/modules/channel/channelMetadataFetcher.js @@ -4,7 +4,6 @@ const { sanitizeNameLikeYtDlp } = require('../filesystem'); const youtubeApi = require('../youtubeApi'); const channelYtdlpExecutor = require('./channelYtdlpExecutor'); const { logApiFallback } = require('./apiFallbackLogger'); -const { summarizeCommandArgs } = require('../safeCommandLogging'); class ChannelMetadataFetcher { /** @@ -66,7 +65,7 @@ class ChannelMetadataFetcher { playlistEnd: 1, skipSleepRequests: true, }); - logger.info(summarizeCommandArgs(args), 'Fetching channel metadata with yt-dlp'); + logger.info('fetchChannelMetadata executing yt-dlp with args' + JSON.stringify(args)); const content = await channelYtdlpExecutor.executeYtDlpCommand(args, outputFilePath, { timeoutMs: 120000, }); diff --git a/server/modules/channel/channelYtdlpExecutor.js b/server/modules/channel/channelYtdlpExecutor.js index adde0b473..0cfd2f1b1 100644 --- a/server/modules/channel/channelYtdlpExecutor.js +++ b/server/modules/channel/channelYtdlpExecutor.js @@ -6,7 +6,6 @@ const { pipeline } = require('stream/promises'); const { v4: uuidv4 } = require('uuid'); const { spawnYtDlp } = require('../ytdlpProcess'); const tempPathManager = require('../download/tempPathManager'); -const { redactSensitiveText } = require('../safeCommandLogging'); class ChannelYtdlpExecutor { /** @@ -100,7 +99,7 @@ class ChannelYtdlpExecutor { const error = new Error(errorMessage); error.code = errorCode; - error.stderr = redactSensitiveText(stderrBuffer); + error.stderr = stderrBuffer; finish(reject, error); } }); diff --git a/server/modules/playlistModule.js b/server/modules/playlistModule.js index 663fefcc0..c13a888fb 100644 --- a/server/modules/playlistModule.js +++ b/server/modules/playlistModule.js @@ -6,7 +6,6 @@ const { Playlist, PlaylistVideo, Channel, Video, Job, JobVideo } = require('../m const youtubeApi = require('./youtubeApi'); const { MAX_PLAYLIST_VIDEOS } = require('./playlistConstants'); const storageGuard = require('./storageGuard'); -const { redactSensitiveText } = require('./safeCommandLogging'); // yt-dlp's flat-playlist listing still returns private/deleted/members-only // videos but strips their metadata: the title comes back null (current yt-dlp) @@ -79,13 +78,13 @@ class PlaylistModule { if (/confirm you.re not a bot|sign in|cookies/i.test(stderr)) { return reject(new Error('COOKIES_REQUIRED')); } - logger.error({ stderr: redactSensitiveText(stderr), code }, 'getPlaylistInfo failed'); + logger.error({ stderr, code }, 'getPlaylistInfo failed'); return reject(new Error('NETWORK_ERROR')); } try { resolve(JSON.parse(stdout)); } catch (err) { - logger.error({ error: redactSensitiveText(err?.message) }, 'getPlaylistInfo parse error'); + logger.error({ err, stdout }, 'getPlaylistInfo parse error'); reject(new Error('PARSE_ERROR')); } }); @@ -493,10 +492,7 @@ class PlaylistModule { child.stderr.on('data', (d) => { stderr += d.toString(); }); child.on('close', (code) => { if (code !== 0) { - logger.error({ - stderr: redactSensitiveText(stderr), - code, - }, '_spawnFlatPlaylist failed'); + logger.error({ stderr, code }, '_spawnFlatPlaylist failed'); return reject(new Error('NETWORK_ERROR')); } try { diff --git a/server/modules/ratingMapper.js b/server/modules/ratingMapper.js index 11ecbe952..f59337d6e 100644 --- a/server/modules/ratingMapper.js +++ b/server/modules/ratingMapper.js @@ -246,7 +246,7 @@ function mapToNumericRating(normalizedRating) { } // Level 2: PG and PG equivalents (family with some caution) - const level2 = ['PG', 'TV-Y7', 'TV-PG']; + const level2 = ['PG', 'TV-PG']; if (level2.includes(normalizedRating)) { return 2; } diff --git a/server/modules/safeCommandLogging.js b/server/modules/safeCommandLogging.js deleted file mode 100644 index b27e8cd0a..000000000 --- a/server/modules/safeCommandLogging.js +++ /dev/null @@ -1,106 +0,0 @@ -const { URL } = require('url'); - -const SENSITIVE_VALUE_FLAGS = new Set([ - '--proxy', - '--cookies', - '--cookies-from-browser', - '--add-header', - '--add-headers', - '--http-header', - '--username', - '--password', - '--video-password', - '--client-certificate', - '--client-certificate-key', - '--client-certificate-password', - '--paths', - '--paths-home', - '--output', - '-o', - '--download-archive', - '--cache-dir', - '--ffmpeg-location', - '--config-locations', - '--exec', -]); -const SENSITIVE_FLAG_PATTERN = [...SENSITIVE_VALUE_FLAGS] - .sort((left, right) => right.length - left.length) - .map((flag) => flag.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')) - .join('|'); -const SENSITIVE_VALUE_PATTERN = new RegExp( - `((${SENSITIVE_FLAG_PATTERN})(?:=|\\s+))(?:"[^"]*"|'[^']*'|\\S+)`, - 'gi' -); - -function safeUrlHost(value) { - try { - const url = new URL(value); - return ['http:', 'https:'].includes(url.protocol) ? url.hostname : null; - } catch (_error) { - return null; - } -} - -function summarizeCommandArgs(args) { - if (!Array.isArray(args)) return { argumentCount: 0, flags: [], targetHosts: [] }; - const flags = []; - const hosts = []; - let redactNext = false; - for (const value of args) { - const token = String(value); - if (redactNext) { - redactNext = false; - continue; - } - if (token.startsWith('-')) { - const flag = token.split('=', 1)[0]; - flags.push(flag); - if (SENSITIVE_VALUE_FLAGS.has(flag) && !token.includes('=')) redactNext = true; - continue; - } - const host = safeUrlHost(token); - if (host) hosts.push(host); - } - return { - argumentCount: args.length, - flags: [...new Set(flags)].sort(), - targetHosts: [...new Set(hosts)].sort(), - }; -} - -function redactSensitiveText(value) { - let text = String(value ?? ''); - text = text.replace( - /((?:--add-header|--add-headers|--http-header)(?:=|\s+))[^\r\n]*/gi, - '$1[REDACTED]' - ); - text = text.replace( - SENSITIVE_VALUE_PATTERN, - '$1[REDACTED]' - ); - text = text.replace( - /\b(Authorization|Proxy-Authorization|Cookie|Set-Cookie)\s*:\s*[^\r\n]*/gi, - '$1: [REDACTED]' - ); - text = text.replace( - /\b(https?:\/\/)([^/\s:@]+):([^@\s/]+)@/gi, - '$1[REDACTED]@' - ); - text = text.replace( - /([?&](?:token|key|api_key|apikey|auth|authorization|signature|sig)=)[^&#\s]*/gi, - '$1[REDACTED]' - ); - text = text.replace(/\bfile:\/\/[^\s"'<>]+/gi, '[REDACTED_PATH]'); - text = text.replace( - /(^|[\s"'=(])(?:\/(?:[^/\s"'<>]+\/)*[^/\s"'<>]+|[A-Za-z]:\\(?:[^\\\s"'<>]+\\)*[^\\\s"'<>]+)/g, - '$1[REDACTED_PATH]' - ); - return text; -} - -module.exports = { - SENSITIVE_VALUE_FLAGS, - summarizeCommandArgs, - redactSensitiveText, - safeUrlHost, -}; diff --git a/server/modules/subscriptionImport/cookiesFetcher.js b/server/modules/subscriptionImport/cookiesFetcher.js index 2e0e1feba..53ebdb775 100644 --- a/server/modules/subscriptionImport/cookiesFetcher.js +++ b/server/modules/subscriptionImport/cookiesFetcher.js @@ -139,7 +139,7 @@ async function fetchWithCookies(buffer) { fs.writeFileSync(cookiesPath, buffer); fs.chmodSync(cookiesPath, COOKIES_FILE_MODE); - logger.info('Running yt-dlp with one-time cookies to fetch subscriptions'); + logger.info({ tempDir }, 'Running yt-dlp with one-time cookies to fetch subscriptions'); const { stdout } = await runYtdlp(cookiesPath); const channels = parseChannelEntries(stdout); diff --git a/server/modules/ytDlpRunner.js b/server/modules/ytDlpRunner.js index 7fbc7a05e..237f49e30 100644 --- a/server/modules/ytDlpRunner.js +++ b/server/modules/ytDlpRunner.js @@ -2,7 +2,6 @@ const { spawnYtDlp } = require('./ytdlpProcess'); const path = require('path'); const fs = require('fs'); const tempPathManager = require('./download/tempPathManager'); -const { redactSensitiveText } = require('./safeCommandLogging'); const MEMBERS_ONLY_PATTERNS = [ /join this channel to get access to members-only content/i, @@ -142,9 +141,7 @@ class YtDlpRunner { timeoutError.code = 'YTDLP_TIMEOUT'; reject(timeoutError); } else { - const errorMessage = redactSensitiveText( - stderr || `yt-dlp process exited with code ${code}` - ); + const errorMessage = stderr || `yt-dlp process exited with code ${code}`; reject(new Error(errorMessage)); } }); diff --git a/server/server.js b/server/server.js index 302ef72ca..9b708f769 100644 --- a/server/server.js +++ b/server/server.js @@ -18,7 +18,6 @@ const STARTUP_TAB_COUNT_REFRESH_DELAY_MS = 2 * 60 * 1000; const { createExternalApiAuth } = require('./middleware/externalApiAuth'); const { sendExternalError } = require('./modules/externalApiResponse'); const app = express(); -app.disable('x-powered-by'); app.set('trust proxy', parseTrustProxySetting(process.env.TRUST_PROXY)); if (process.env.TRUST_PROXY === undefined || process.env.TRUST_PROXY === '') { logger.info('TRUST_PROXY is unset; defaulting Express proxy trust to true for backwards compatibility. Rate limits use the direct peer IP until TRUST_PROXY is explicitly configured.'); @@ -55,7 +54,13 @@ app.use(pinoHttp({ } }, // Generate unique request ID for correlation - genReqId: () => require('crypto').randomUUID(), + genReqId: (req) => { + // External API envelopes promise a server-generated requestId, so the + // public namespace never adopts a caller-supplied correlation header. + const existingId = req.path.startsWith('/external-api') ? null : req.id || req.headers['x-request-id']; + if (existingId) return existingId; + return require('crypto').randomUUID(); + }, // Only log warnings and errors at info level, success at debug customLogLevel: (req, res, err) => { if (res.statusCode >= 400 && res.statusCode < 500) { @@ -373,7 +378,7 @@ const initialize = async () => { subfolderModule, }); - logger.info('YouTube downloads directory configured'); + logger.info({ directoryPath: configModule.directoryPath }, 'YouTube downloads directory configured'); // Degraded mode middleware - checks database health before processing requests const checkDatabaseHealth = function(req, res, next) { From e98ff1d492ed158051157853927bdd3f5a97c66e Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:25:03 +0000 Subject: [PATCH 17/27] fix: allow external request review with auth disabled and reject non-UTF-8 bodies With AUTH_ENABLED=false, verifyToken never sets req.authType, so the review routes' session check returned 403 for every call. Deny API keys instead, matching the key-management routes. External API POST bodies declaring a charset other than UTF-8 now get a 415 rather than a parse error or 500. --- server/routes/__tests__/externalApi.test.js | 21 +++++++++++++++ .../routes/__tests__/externalRequests.test.js | 27 ++++++++++++++++--- server/routes/externalApi.js | 9 +++++++ server/routes/externalRequests.js | 25 +++++++++-------- 4 files changed, 67 insertions(+), 15 deletions(-) diff --git a/server/routes/__tests__/externalApi.test.js b/server/routes/__tests__/externalApi.test.js index 39408b91a..aa656b241 100644 --- a/server/routes/__tests__/externalApi.test.js +++ b/server/routes/__tests__/externalApi.test.js @@ -150,6 +150,27 @@ describe('external API capabilities', () => { .expect(413); }); + test.each(['utf-16', 'iso-8859-1'])('rejects JSON bodies in the %s charset as 415', async (charset) => { + const { app } = makeApp({ key: externalKey() }); + const response = await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .set('Content-Type', `application/json; charset=${charset}`) + .send('{}') + .expect(415); + + expect(response.body.error.code).toBe('unsupported_media_type'); + }); + + test('accepts an explicit UTF-8 charset', async () => { + const requestService = { createVideoRequest: jest.fn().mockResolvedValue({ outcome: 'created', request: {} }) }; + const { app } = makeApp({ key: externalKey(), requestService }); + await request(app).post('/external-api/v1/requests/videos') + .set('x-api-key', 'test-key') + .set('Content-Type', 'application/json; charset=UTF-8') + .send('{}') + .expect(202); + }); + test('keeps unknown v1 paths inside the external JSON contract', async () => { const { app } = makeApp({ key: externalKey() }); const response = await request(app) diff --git a/server/routes/__tests__/externalRequests.test.js b/server/routes/__tests__/externalRequests.test.js index a8b362170..57b1c1f01 100644 --- a/server/routes/__tests__/externalRequests.test.js +++ b/server/routes/__tests__/externalRequests.test.js @@ -7,8 +7,10 @@ const requestId = '9b89e5bc-8c90-4e72-b245-270fed2eacc2'; function makeApp({ authType = 'session', requestService, reviewLimiter } = {}) { const app = express(); app.use(express.json()); + // authType null mimics AUTH_ENABLED=false, where verifyToken passes + // requests through without identifying the caller. const verifyToken = (req, _res, next) => { - req.authType = authType; + if (authType) req.authType = authType; req.sessionId = authType === 'session' ? 'session-1' : undefined; next(); }; @@ -33,7 +35,7 @@ function service() { } describe('external request administrator routes', () => { - test('lists and reads across keys only for session-authenticated callers', async () => { + test('lists and reads across keys for session-authenticated callers', async () => { const requestService = service(); const app = makeApp({ requestService }); @@ -48,10 +50,27 @@ describe('external request administrator routes', () => { page: '2', })); expect(requestService.getAdminRequest).toHaveBeenCalledWith(requestId); + }); - await request(makeApp({ authType: 'api_key', requestService })) + test('rejects API-key callers', async () => { + await request(makeApp({ authType: 'api_key', requestService: service() })) + .get('/api/external-requests') + .expect(403, { error: 'API keys cannot review external requests' }); + }); + + test('allows review when authentication is disabled', async () => { + const requestService = service(); + await request(makeApp({ authType: null, requestService })) .get('/api/external-requests') - .expect(403, { error: 'Session authentication is required' }); + .expect(200); + }); + + test('rejects an x-api-key header even when authentication is disabled', async () => { + await request(makeApp({ authType: null, requestService: service() })) + .post(`/api/external-requests/${requestId}/approve`) + .set('x-api-key', 'some-key') + .send({}) + .expect(403, { error: 'API keys cannot review external requests' }); }); test('runs approve and reject through the dedicated action limiter', async () => { diff --git a/server/routes/externalApi.js b/server/routes/externalApi.js index 2db2ca8b4..6eb179576 100644 --- a/server/routes/externalApi.js +++ b/server/routes/externalApi.js @@ -58,6 +58,15 @@ function createExternalApiRoutes({ requestId: req.id, }); } + // RFC 8259 requires UTF-8 for JSON exchanged between systems. body-parser + // would otherwise decode other utf-* charsets and report a parse error. + const charset = /;\s*charset\s*=\s*"?([^";\s]+)/i.exec(req.get('content-type') || '')?.[1]; + if (charset && charset.toLowerCase() !== 'utf-8') { + return sendExternalError(res, 415, 'Request body must be UTF-8 encoded JSON', { + code: 'unsupported_media_type', + requestId: req.id, + }); + } return externalApiJsonParser(req, res, (error) => { if (!error) return next(); if (error.type === 'entity.too.large') { diff --git a/server/routes/externalRequests.js b/server/routes/externalRequests.js index f69a8e93a..4f7dda8c7 100644 --- a/server/routes/externalRequests.js +++ b/server/routes/externalRequests.js @@ -8,9 +8,12 @@ function createExternalRequestReviewRoutes({ const router = express.Router(); const requests = () => requestService; - const sessionOnly = (req, res, next) => { - if (req.authType !== 'session') { - return res.status(403).json({ error: 'Session authentication is required' }); + // AUTH_ENABLED=false skips verifyToken's session check entirely (access is + // delegated to the deployment's proxy), so deny API keys rather than + // requiring a session; apikeys.js guards key management the same way. + const rejectApiKeys = (req, res, next) => { + if (req.headers['x-api-key'] || req.authType === 'api_key') { + return res.status(403).json({ error: 'API keys cannot review external requests' }); } return next(); }; @@ -50,9 +53,9 @@ function createExternalRequestReviewRoutes({ * schema: { type: string, enum: [video, channel, delete_video] } * responses: * 200: { description: Paginated requests and safe requester filter metadata } - * 403: { description: Session authentication is required } + * 403: { description: API keys cannot review external requests } */ - router.get('/api/external-requests', verifyToken, sessionOnly, async (req, res) => { + router.get('/api/external-requests', verifyToken, rejectApiKeys, async (req, res) => { try { return res.json(await requests().listAdminRequests(req.query)); } catch (error) { @@ -74,10 +77,10 @@ function createExternalRequestReviewRoutes({ * schema: { type: string, format: uuid } * responses: * 200: { description: Safe requester, target, and job metadata } - * 403: { description: Session authentication is required } + * 403: { description: API keys cannot review external requests } * 404: { description: Request not found } */ - router.get('/api/external-requests/:id', verifyToken, sessionOnly, async (req, res) => { + router.get('/api/external-requests/:id', verifyToken, rejectApiKeys, async (req, res) => { try { return res.json(await requests().getAdminRequest(req.params.id)); } catch (error) { @@ -109,7 +112,7 @@ function createExternalRequestReviewRoutes({ * description: Channel requests only. * responses: * 200: { description: Updated request } - * 403: { description: Session authentication is required } + * 403: { description: API keys cannot review external requests } * 404: { description: Request not found } * 409: { description: Request is not pending } * 429: { description: Review action rate limit exceeded } @@ -117,7 +120,7 @@ function createExternalRequestReviewRoutes({ router.post( '/api/external-requests/:id/approve', verifyToken, - sessionOnly, + rejectApiKeys, reviewLimiter, async (req, res) => { try { @@ -154,7 +157,7 @@ function createExternalRequestReviewRoutes({ * responses: * 200: { description: Rejected request } * 400: { description: Invalid reason } - * 403: { description: Session authentication is required } + * 403: { description: API keys cannot review external requests } * 404: { description: Request not found } * 409: { description: Request is not pending } * 429: { description: Review action rate limit exceeded } @@ -162,7 +165,7 @@ function createExternalRequestReviewRoutes({ router.post( '/api/external-requests/:id/reject', verifyToken, - sessionOnly, + rejectApiKeys, reviewLimiter, async (req, res) => { try { From 93801642f2c1221903284fc23a7a792a219078bf Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:50:17 +0000 Subject: [PATCH 18/27] refactor: group external API modules under modules/externalApi Move the external API services, their tests, and the channel grant module into a feature folder like channel/, mediaServers/, and notifications/. Give TV-Y7 its PG-level policy rank inside externalEligibility instead of changing ratingMapper, which also drives NFO output for every video. --- scripts/external-api-index-plans.js | 4 ++-- server/middleware/externalApiAuth.js | 4 ++-- server/modules/__tests__/apiKeyModule.test.js | 4 ++-- server/modules/apiKeyModule.js | 4 ++-- .../__tests__/apiKeyChannelGrantModule.test.js | 6 +++--- .../__tests__/externalApiConfig.test.js | 0 .../__tests__/externalCatalogRequests.test.js | 4 ++-- .../__tests__/externalCatalogService.test.js | 6 +++--- .../__tests__/externalContractFixtures.test.js | 4 ++-- .../__tests__/externalEligibility.test.js | 0 .../__tests__/externalPagination.test.js | 0 .../__tests__/externalPermissions.test.js | 0 .../__tests__/externalQuotaService.test.js | 0 .../__tests__/externalRequestExecution.test.js | 0 .../__tests__/externalRequestPrimitives.test.js | 0 .../externalRequestReconciliation.test.js | 0 .../externalRequestReviewService.test.js | 0 .../__tests__/externalRequestService.test.js | 0 .../__tests__/externalThumbnailProxy.test.js | 0 .../__tests__/fixtures/externalRequests.js | 0 .../apiKeyChannelGrantModule.js | 4 ++-- .../{ => externalApi}/externalApiConfig.js | 0 .../{ => externalApi}/externalApiResponse.js | 0 .../{ => externalApi}/externalCatalogRequests.js | 2 +- .../{ => externalApi}/externalCatalogService.js | 6 +++--- .../{ => externalApi}/externalEligibility.js | 16 ++++++++++++---- .../{ => externalApi}/externalPagination.js | 0 .../{ => externalApi}/externalPermissions.js | 0 .../{ => externalApi}/externalQuotaService.js | 6 +++--- .../externalRequestExecution.js | 4 ++-- .../externalRequestPrimitives.js | 0 .../externalRequestReconciliation.js | 0 .../externalRequestReviewService.js | 2 +- .../{ => externalApi}/externalRequestService.js | 8 ++++---- .../{ => externalApi}/externalThumbnailProxy.js | 0 .../{ => externalApi}/externalWorkLimiter.js | 0 server/routes/__tests__/apikeys.policy.test.js | 4 ++-- server/routes/apikeys.js | 2 +- server/routes/config.js | 2 +- server/routes/externalApi.js | 4 ++-- server/routes/index.js | 14 +++++++------- server/server.js | 2 +- server/testing/externalApiContractApp.js | 2 +- server/testing/externalRuntime.database.test.js | 8 ++++---- 44 files changed, 65 insertions(+), 57 deletions(-) rename server/modules/{ => externalApi}/__tests__/apiKeyChannelGrantModule.test.js (95%) rename server/modules/{ => externalApi}/__tests__/externalApiConfig.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalCatalogRequests.test.js (87%) rename server/modules/{ => externalApi}/__tests__/externalCatalogService.test.js (99%) rename server/modules/{ => externalApi}/__tests__/externalContractFixtures.test.js (98%) rename server/modules/{ => externalApi}/__tests__/externalEligibility.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalPagination.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalPermissions.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalQuotaService.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalRequestExecution.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalRequestPrimitives.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalRequestReconciliation.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalRequestReviewService.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalRequestService.test.js (100%) rename server/modules/{ => externalApi}/__tests__/externalThumbnailProxy.test.js (100%) rename server/modules/{ => externalApi}/__tests__/fixtures/externalRequests.js (100%) rename server/modules/{ => externalApi}/apiKeyChannelGrantModule.js (96%) rename server/modules/{ => externalApi}/externalApiConfig.js (100%) rename server/modules/{ => externalApi}/externalApiResponse.js (100%) rename server/modules/{ => externalApi}/externalCatalogRequests.js (96%) rename server/modules/{ => externalApi}/externalCatalogService.js (99%) rename server/modules/{ => externalApi}/externalEligibility.js (76%) rename server/modules/{ => externalApi}/externalPagination.js (100%) rename server/modules/{ => externalApi}/externalPermissions.js (100%) rename server/modules/{ => externalApi}/externalQuotaService.js (97%) rename server/modules/{ => externalApi}/externalRequestExecution.js (98%) rename server/modules/{ => externalApi}/externalRequestPrimitives.js (100%) rename server/modules/{ => externalApi}/externalRequestReconciliation.js (100%) rename server/modules/{ => externalApi}/externalRequestReviewService.js (99%) rename server/modules/{ => externalApi}/externalRequestService.js (98%) rename server/modules/{ => externalApi}/externalThumbnailProxy.js (100%) rename server/modules/{ => externalApi}/externalWorkLimiter.js (100%) diff --git a/scripts/external-api-index-plans.js b/scripts/external-api-index-plans.js index a7d912a6e..33172ea5c 100644 --- a/scripts/external-api-index-plans.js +++ b/scripts/external-api-index-plans.js @@ -15,10 +15,10 @@ if (!['127.0.0.1', 'localhost', '::1'].includes(host)) { process.env.DB_NAME = databaseName; process.env.DB_HOST = host; const { sequelize } = require('../server/db'); -const catalog = require('../server/modules/externalCatalogService'); +const catalog = require('../server/modules/externalApi/externalCatalogService'); const config = require('../server/modules/configModule'); const logger = require('../server/logger'); -const { createExternalRequestService } = require('../server/modules/externalRequestService'); +const { createExternalRequestService } = require('../server/modules/externalApi/externalRequestService'); const migration = require('../migrations/20261007041058-add-external-api-runtime-indexes'); async function run() { diff --git a/server/middleware/externalApiAuth.js b/server/middleware/externalApiAuth.js index ae195ce9b..c77622fd4 100644 --- a/server/middleware/externalApiAuth.js +++ b/server/middleware/externalApiAuth.js @@ -1,10 +1,10 @@ const EXTERNAL_ROLES = ['view', 'request', 'delete', 'admin']; const ALLOWED_MEDIA_TYPES = ['video', 'short', 'livestream']; -const { sendExternalError } = require('../modules/externalApiResponse'); +const { sendExternalError } = require('../modules/externalApi/externalApiResponse'); const { normalizeExternalPermissions, roleForExternalPermissions, -} = require('../modules/externalPermissions'); +} = require('../modules/externalApi/externalPermissions'); function normalizeAllowedMediaTypes(value) { if (!Array.isArray(value)) return null; diff --git a/server/modules/__tests__/apiKeyModule.test.js b/server/modules/__tests__/apiKeyModule.test.js index 584e914ba..bae1fac22 100644 --- a/server/modules/__tests__/apiKeyModule.test.js +++ b/server/modules/__tests__/apiKeyModule.test.js @@ -1,13 +1,13 @@ jest.mock('../../models/apikey', () => ({ count: jest.fn(), create: jest.fn(), findAll: jest.fn(), findByPk: jest.fn(), })); -jest.mock('../../modules/apiKeyChannelGrantModule', () => ({ +jest.mock('../externalApi/apiKeyChannelGrantModule', () => ({ getEffectiveChannelGrantCounts: jest.fn(), })); jest.mock('../../logger', () => ({ info: jest.fn(), debug: jest.fn() })); const ApiKey = require('../../models/apikey'); -const { getEffectiveChannelGrantCounts } = require('../../modules/apiKeyChannelGrantModule'); +const { getEffectiveChannelGrantCounts } = require('../externalApi/apiKeyChannelGrantModule'); const apiKeyModule = require('../apiKeyModule'); describe('external API key policies', () => { diff --git a/server/modules/apiKeyModule.js b/server/modules/apiKeyModule.js index 83cef5bbe..45d8291f8 100644 --- a/server/modules/apiKeyModule.js +++ b/server/modules/apiKeyModule.js @@ -4,8 +4,8 @@ const logger = require('../logger'); const { normalizeExternalPermissions, roleForExternalPermissions, -} = require('./externalPermissions'); -const { getEffectiveChannelGrantCounts } = require('./apiKeyChannelGrantModule'); +} = require('./externalApi/externalPermissions'); +const { getEffectiveChannelGrantCounts } = require('./externalApi/apiKeyChannelGrantModule'); const MAX_API_KEYS = 20; const ROLES = ['legacy_download', 'view', 'request', 'delete', 'admin']; diff --git a/server/modules/__tests__/apiKeyChannelGrantModule.test.js b/server/modules/externalApi/__tests__/apiKeyChannelGrantModule.test.js similarity index 95% rename from server/modules/__tests__/apiKeyChannelGrantModule.test.js rename to server/modules/externalApi/__tests__/apiKeyChannelGrantModule.test.js index 985090d9d..5aed34519 100644 --- a/server/modules/__tests__/apiKeyChannelGrantModule.test.js +++ b/server/modules/externalApi/__tests__/apiKeyChannelGrantModule.test.js @@ -1,10 +1,10 @@ -jest.mock('../../db', () => ({ +jest.mock('../../../db', () => ({ sequelize: { transaction: jest.fn(async (callback) => callback({ id: 'transaction', LOCK: { UPDATE: 'UPDATE' } })), }, })); const { Op } = require('sequelize'); -jest.mock('../../models', () => ({ +jest.mock('../../../models', () => ({ ApiKey: { findByPk: jest.fn() }, ApiKeyChannelGrant: { findAll: jest.fn(), @@ -14,7 +14,7 @@ jest.mock('../../models', () => ({ Channel: { count: jest.fn() }, })); -const { ApiKey, ApiKeyChannelGrant, Channel } = require('../../models'); +const { ApiKey, ApiKeyChannelGrant, Channel } = require('../../../models'); const grants = require('../apiKeyChannelGrantModule'); describe('API key channel grants', () => { diff --git a/server/modules/__tests__/externalApiConfig.test.js b/server/modules/externalApi/__tests__/externalApiConfig.test.js similarity index 100% rename from server/modules/__tests__/externalApiConfig.test.js rename to server/modules/externalApi/__tests__/externalApiConfig.test.js diff --git a/server/modules/__tests__/externalCatalogRequests.test.js b/server/modules/externalApi/__tests__/externalCatalogRequests.test.js similarity index 87% rename from server/modules/__tests__/externalCatalogRequests.test.js rename to server/modules/externalApi/__tests__/externalCatalogRequests.test.js index a406429f7..476ab46fd 100644 --- a/server/modules/__tests__/externalCatalogRequests.test.js +++ b/server/modules/externalApi/__tests__/externalCatalogRequests.test.js @@ -1,5 +1,5 @@ -jest.mock('../../db', () => ({ sequelize: { query: jest.fn() } })); -const { sequelize } = require('../../db'); +jest.mock('../../../db', () => ({ sequelize: { query: jest.fn() } })); +const { sequelize } = require('../../../db'); const { attachRequestStatuses } = require('../externalCatalogRequests'); beforeEach(() => jest.clearAllMocks()); diff --git a/server/modules/__tests__/externalCatalogService.test.js b/server/modules/externalApi/__tests__/externalCatalogService.test.js similarity index 99% rename from server/modules/__tests__/externalCatalogService.test.js rename to server/modules/externalApi/__tests__/externalCatalogService.test.js index 7090f349e..969ee99c4 100644 --- a/server/modules/__tests__/externalCatalogService.test.js +++ b/server/modules/externalApi/__tests__/externalCatalogService.test.js @@ -1,11 +1,11 @@ -jest.mock('../../db', () => ({ +jest.mock('../../../db', () => ({ sequelize: { query: jest.fn() }, })); -jest.mock('../configModule', () => ({ +jest.mock('../../configModule', () => ({ getImagePath: jest.fn(() => '/safe/images'), })); -const { sequelize } = require('../../db'); +const { sequelize } = require('../../../db'); const catalog = require('../externalCatalogService'); const key = (overrides = {}) => ({ diff --git a/server/modules/__tests__/externalContractFixtures.test.js b/server/modules/externalApi/__tests__/externalContractFixtures.test.js similarity index 98% rename from server/modules/__tests__/externalContractFixtures.test.js rename to server/modules/externalApi/__tests__/externalContractFixtures.test.js index 1b05cfdc7..8b1a9c974 100644 --- a/server/modules/__tests__/externalContractFixtures.test.js +++ b/server/modules/externalApi/__tests__/externalContractFixtures.test.js @@ -3,11 +3,11 @@ const path = require('path'); const crypto = require('crypto'); const { externalErrorBody } = require('../externalApiResponse'); const { dto } = require('../externalRequestService'); -const configModule = require('../configModule'); +const configModule = require('../../configModule'); const fixtureDirectory = path.join( __dirname, - '../../../fixtures/external-api-v1' + '../../../../fixtures/external-api-v1' ); const fixtureBytes = fs.readFileSync(path.join(fixtureDirectory, 'contract.json')); const canonicalFixtureBytes = Buffer.from(fixtureBytes.toString('utf8').replace(/\r\n/g, '\n')); diff --git a/server/modules/__tests__/externalEligibility.test.js b/server/modules/externalApi/__tests__/externalEligibility.test.js similarity index 100% rename from server/modules/__tests__/externalEligibility.test.js rename to server/modules/externalApi/__tests__/externalEligibility.test.js diff --git a/server/modules/__tests__/externalPagination.test.js b/server/modules/externalApi/__tests__/externalPagination.test.js similarity index 100% rename from server/modules/__tests__/externalPagination.test.js rename to server/modules/externalApi/__tests__/externalPagination.test.js diff --git a/server/modules/__tests__/externalPermissions.test.js b/server/modules/externalApi/__tests__/externalPermissions.test.js similarity index 100% rename from server/modules/__tests__/externalPermissions.test.js rename to server/modules/externalApi/__tests__/externalPermissions.test.js diff --git a/server/modules/__tests__/externalQuotaService.test.js b/server/modules/externalApi/__tests__/externalQuotaService.test.js similarity index 100% rename from server/modules/__tests__/externalQuotaService.test.js rename to server/modules/externalApi/__tests__/externalQuotaService.test.js diff --git a/server/modules/__tests__/externalRequestExecution.test.js b/server/modules/externalApi/__tests__/externalRequestExecution.test.js similarity index 100% rename from server/modules/__tests__/externalRequestExecution.test.js rename to server/modules/externalApi/__tests__/externalRequestExecution.test.js diff --git a/server/modules/__tests__/externalRequestPrimitives.test.js b/server/modules/externalApi/__tests__/externalRequestPrimitives.test.js similarity index 100% rename from server/modules/__tests__/externalRequestPrimitives.test.js rename to server/modules/externalApi/__tests__/externalRequestPrimitives.test.js diff --git a/server/modules/__tests__/externalRequestReconciliation.test.js b/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js similarity index 100% rename from server/modules/__tests__/externalRequestReconciliation.test.js rename to server/modules/externalApi/__tests__/externalRequestReconciliation.test.js diff --git a/server/modules/__tests__/externalRequestReviewService.test.js b/server/modules/externalApi/__tests__/externalRequestReviewService.test.js similarity index 100% rename from server/modules/__tests__/externalRequestReviewService.test.js rename to server/modules/externalApi/__tests__/externalRequestReviewService.test.js diff --git a/server/modules/__tests__/externalRequestService.test.js b/server/modules/externalApi/__tests__/externalRequestService.test.js similarity index 100% rename from server/modules/__tests__/externalRequestService.test.js rename to server/modules/externalApi/__tests__/externalRequestService.test.js diff --git a/server/modules/__tests__/externalThumbnailProxy.test.js b/server/modules/externalApi/__tests__/externalThumbnailProxy.test.js similarity index 100% rename from server/modules/__tests__/externalThumbnailProxy.test.js rename to server/modules/externalApi/__tests__/externalThumbnailProxy.test.js diff --git a/server/modules/__tests__/fixtures/externalRequests.js b/server/modules/externalApi/__tests__/fixtures/externalRequests.js similarity index 100% rename from server/modules/__tests__/fixtures/externalRequests.js rename to server/modules/externalApi/__tests__/fixtures/externalRequests.js diff --git a/server/modules/apiKeyChannelGrantModule.js b/server/modules/externalApi/apiKeyChannelGrantModule.js similarity index 96% rename from server/modules/apiKeyChannelGrantModule.js rename to server/modules/externalApi/apiKeyChannelGrantModule.js index db707bc83..45b7ae841 100644 --- a/server/modules/apiKeyChannelGrantModule.js +++ b/server/modules/externalApi/apiKeyChannelGrantModule.js @@ -1,6 +1,6 @@ const { Op } = require('sequelize'); -const { sequelize } = require('../db'); -const { ApiKey, ApiKeyChannelGrant, Channel } = require('../models'); +const { sequelize } = require('../../db'); +const { ApiKey, ApiKeyChannelGrant, Channel } = require('../../models'); class ChannelGrantValidationError extends Error { constructor(message) { diff --git a/server/modules/externalApiConfig.js b/server/modules/externalApi/externalApiConfig.js similarity index 100% rename from server/modules/externalApiConfig.js rename to server/modules/externalApi/externalApiConfig.js diff --git a/server/modules/externalApiResponse.js b/server/modules/externalApi/externalApiResponse.js similarity index 100% rename from server/modules/externalApiResponse.js rename to server/modules/externalApi/externalApiResponse.js diff --git a/server/modules/externalCatalogRequests.js b/server/modules/externalApi/externalCatalogRequests.js similarity index 96% rename from server/modules/externalCatalogRequests.js rename to server/modules/externalApi/externalCatalogRequests.js index 342e06551..169174a4a 100644 --- a/server/modules/externalCatalogRequests.js +++ b/server/modules/externalApi/externalCatalogRequests.js @@ -1,5 +1,5 @@ const { QueryTypes } = require('sequelize'); -const { sequelize } = require('../db'); +const { sequelize } = require('../../db'); async function attachRequestStatuses(key, rows) { if (rows.length === 0) return; diff --git a/server/modules/externalCatalogService.js b/server/modules/externalApi/externalCatalogService.js similarity index 99% rename from server/modules/externalCatalogService.js rename to server/modules/externalApi/externalCatalogService.js index 2d3927668..2269148b0 100644 --- a/server/modules/externalCatalogService.js +++ b/server/modules/externalApi/externalCatalogService.js @@ -3,8 +3,8 @@ const path = require('path'); const crypto = require('crypto'); const { attachRequestStatuses } = require('./externalCatalogRequests'); const { QueryTypes } = require('sequelize'); -const { sequelize } = require('../db'); -const configModule = require('./configModule'); +const { sequelize } = require('../../db'); +const configModule = require('../configModule'); const { normalizePolicy, ratingPolicy } = require('./externalEligibility'); const { publicVideoThumbnail } = require('./externalThumbnailProxy'); const { @@ -609,7 +609,7 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { if (!row) throw new CatalogError('Video not found', 404); await attachRequestStatuses(key, [row]); - const metadataProvider = metadataService || require('./videoMetadataModule'); + const metadataProvider = metadataService || require('../videoMetadataModule'); const metadata = await metadataProvider.getVideoMetadata(youtubeId); const isDownloaded = Boolean(row.downloaded_id) && !row.downloaded_removed; return { diff --git a/server/modules/externalEligibility.js b/server/modules/externalApi/externalEligibility.js similarity index 76% rename from server/modules/externalEligibility.js rename to server/modules/externalApi/externalEligibility.js index b2db66e11..7e6636978 100644 --- a/server/modules/externalEligibility.js +++ b/server/modules/externalApi/externalEligibility.js @@ -1,4 +1,4 @@ -const ratingMapper = require('./ratingMapper'); +const ratingMapper = require('../ratingMapper'); const { CatalogError } = require('./externalPagination'); const MEDIA_TYPES = ['video', 'short', 'livestream']; @@ -6,6 +6,14 @@ const RATING_LABELS = [ 'G', 'TV-Y', 'TV-Y7', 'TV-G', 'PG', 'TV-PG', 'PG-13', 'TV-14', 'R', 'TV-MA', 'NC-17', ]; +// ratingMapper has no numeric level for TV-Y7 (NFO output falls back to the +// text rating), but key policies need one so TV-Y7 videos are neither hidden +// from nor treated as unrated by every key. Place it with PG/TV-PG. +const EXTERNAL_RATING_LEVEL_OVERRIDES = { 'TV-Y7': 2 }; + +function ratingLevel(rating) { + return EXTERNAL_RATING_LEVEL_OVERRIDES[rating] ?? ratingMapper.mapToNumericRating(rating); +} function normalizeExternalPolicy(policy, ErrorType = Error) { const maxRatingLevel = policy?.maxRatingLevel; @@ -36,7 +44,7 @@ function effectiveRating(videoRating, channelDefaultRating) { function isRatingEligible(policy, videoRating, channelDefaultRating) { const rating = effectiveRating(videoRating, channelDefaultRating); - const numericRating = ratingMapper.mapToNumericRating(rating); + const numericRating = ratingLevel(rating); if (numericRating === null) return policy.allowUnrated; return numericRating <= policy.maxRatingLevel; } @@ -47,12 +55,12 @@ function isMediaTypeEligible(policy, mediaType) { function ratingPolicy(policy) { const recognizedRatings = RATING_LABELS.filter( - (rating) => ratingMapper.mapToNumericRating(rating) !== null + (rating) => ratingLevel(rating) !== null ); return { recognizedRatings, allowedRatings: recognizedRatings.filter( - (rating) => ratingMapper.mapToNumericRating(rating) <= policy.maxRatingLevel + (rating) => ratingLevel(rating) <= policy.maxRatingLevel ), }; } diff --git a/server/modules/externalPagination.js b/server/modules/externalApi/externalPagination.js similarity index 100% rename from server/modules/externalPagination.js rename to server/modules/externalApi/externalPagination.js diff --git a/server/modules/externalPermissions.js b/server/modules/externalApi/externalPermissions.js similarity index 100% rename from server/modules/externalPermissions.js rename to server/modules/externalApi/externalPermissions.js diff --git a/server/modules/externalQuotaService.js b/server/modules/externalApi/externalQuotaService.js similarity index 97% rename from server/modules/externalQuotaService.js rename to server/modules/externalApi/externalQuotaService.js index 488bbc1b7..399549127 100644 --- a/server/modules/externalQuotaService.js +++ b/server/modules/externalApi/externalQuotaService.js @@ -1,6 +1,6 @@ const { Op } = require('sequelize'); const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); -const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); +const { normalizeExternalApiKey } = require('../../middleware/externalApiAuth'); const { hasExternalScope } = require('./externalPermissions'); class QuotaError extends Error { @@ -33,8 +33,8 @@ function limitsFor(key) { } function createExternalQuotaService({ - models = require('../models'), - sequelize = require('../db').sequelize, + models = require('../../models'), + sequelize = require('../../db').sequelize, now = () => new Date(), } = {}) { const { ApiKey, ExternalRequest, ExternalApiUsageBucket } = models; diff --git a/server/modules/externalRequestExecution.js b/server/modules/externalApi/externalRequestExecution.js similarity index 98% rename from server/modules/externalRequestExecution.js rename to server/modules/externalApi/externalRequestExecution.js index 3b5c52338..8b127231f 100644 --- a/server/modules/externalRequestExecution.js +++ b/server/modules/externalApi/externalRequestExecution.js @@ -1,5 +1,5 @@ const { Op } = require('sequelize'); -const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); +const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); const { AUXILIARY_RECOVERY_DELAY_MS, requireCurrentAutoApproval, rethrowWorkLimit, } = require('./externalRequestPrimitives'); @@ -115,7 +115,7 @@ function createExternalRequestExecution({ if (requireAutoApproval) { requireCurrentAutoApproval(currentKey, 'channel'); } - const provisioner = channelProvisioner || require('./channel/channelProvisioning'); + const provisioner = channelProvisioner || require('../channel/channelProvisioning'); const result = await provisioner.getChannelInfo( record.channel_url, false, diff --git a/server/modules/externalRequestPrimitives.js b/server/modules/externalApi/externalRequestPrimitives.js similarity index 100% rename from server/modules/externalRequestPrimitives.js rename to server/modules/externalApi/externalRequestPrimitives.js diff --git a/server/modules/externalRequestReconciliation.js b/server/modules/externalApi/externalRequestReconciliation.js similarity index 100% rename from server/modules/externalRequestReconciliation.js rename to server/modules/externalApi/externalRequestReconciliation.js diff --git a/server/modules/externalRequestReviewService.js b/server/modules/externalApi/externalRequestReviewService.js similarity index 99% rename from server/modules/externalRequestReviewService.js rename to server/modules/externalApi/externalRequestReviewService.js index e12a54cac..367977f11 100644 --- a/server/modules/externalRequestReviewService.js +++ b/server/modules/externalApi/externalRequestReviewService.js @@ -1,5 +1,5 @@ const { Op } = require('sequelize'); -const { MANUAL_DOWNLOAD_LABEL } = require('./download/jobTypes'); +const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); const { hasExternalScope } = require('./externalPermissions'); const { RequestError, REQUEST_STATUSES, ACTIVE_STATUSES, normalizeStoredKey, diff --git a/server/modules/externalRequestService.js b/server/modules/externalApi/externalRequestService.js similarity index 98% rename from server/modules/externalRequestService.js rename to server/modules/externalApi/externalRequestService.js index 5d3bb0b33..4edd17434 100644 --- a/server/modules/externalRequestService.js +++ b/server/modules/externalApi/externalRequestService.js @@ -12,12 +12,12 @@ const { createExternalRequestReviewService } = require('./externalRequestReviewS const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); function createExternalRequestService({ - models = require('../models'), - executor = (jobData) => require('./downloadModule').doGroupedManualDownloads(jobData), + models = require('../../models'), + executor = (jobData) => require('../downloadModule').doGroupedManualDownloads(jobData), channelProvisioner = null, - videoDeleter = require('./videoDeletionModule'), + videoDeleter = require('../videoDeletionModule'), now = () => new Date(), - sequelize = require('../db').sequelize, + sequelize = require('../../db').sequelize, quotaService = null, workLimiter = require('./externalWorkLimiter').sharedExternalWorkLimiter, } = {}) { diff --git a/server/modules/externalThumbnailProxy.js b/server/modules/externalApi/externalThumbnailProxy.js similarity index 100% rename from server/modules/externalThumbnailProxy.js rename to server/modules/externalApi/externalThumbnailProxy.js diff --git a/server/modules/externalWorkLimiter.js b/server/modules/externalApi/externalWorkLimiter.js similarity index 100% rename from server/modules/externalWorkLimiter.js rename to server/modules/externalApi/externalWorkLimiter.js diff --git a/server/routes/__tests__/apikeys.policy.test.js b/server/routes/__tests__/apikeys.policy.test.js index ab2e5ea37..f53417f2c 100644 --- a/server/routes/__tests__/apikeys.policy.test.js +++ b/server/routes/__tests__/apikeys.policy.test.js @@ -6,7 +6,7 @@ jest.mock('../../modules/apiKeyModule', () => ({ regenerateApiKey: jest.fn(), serializeApiKey: jest.fn((key) => ({ id: key.id, name: key.name, key_prefix: key.key_prefix })), })); -jest.mock('../../modules/apiKeyChannelGrantModule', () => ({ +jest.mock('../../modules/externalApi/apiKeyChannelGrantModule', () => ({ getChannelGrants: jest.fn(), replaceChannelGrants: jest.fn(), })); @@ -19,7 +19,7 @@ jest.mock('../../db', () => ({ const express = require('express'); const request = require('supertest'); const apiKeyModule = require('../../modules/apiKeyModule'); -const grantModule = require('../../modules/apiKeyChannelGrantModule'); +const grantModule = require('../../modules/externalApi/apiKeyChannelGrantModule'); const createApiKeyRoutes = require('../apikeys'); function createApp(authType = 'session') { diff --git a/server/routes/apikeys.js b/server/routes/apikeys.js index c0c7661c8..f23466fc4 100644 --- a/server/routes/apikeys.js +++ b/server/routes/apikeys.js @@ -9,7 +9,7 @@ const express = require('express'); module.exports = function createApiKeyRoutes({ verifyToken }) { const router = express.Router(); const apiKeyModule = require('../modules/apiKeyModule'); - const apiKeyChannelGrantModule = require('../modules/apiKeyChannelGrantModule'); + const apiKeyChannelGrantModule = require('../modules/externalApi/apiKeyChannelGrantModule'); const { sequelize } = require('../db'); const isPolicyValidationError = (error) => diff --git a/server/routes/config.js b/server/routes/config.js index a217bec18..417f613fb 100644 --- a/server/routes/config.js +++ b/server/routes/config.js @@ -4,7 +4,7 @@ const { SCHEDULES, getScheduleError, getSchedule } = require('../modules/schedul const customArgsParser = require('../modules/download/customArgsParser'); const filenamePreview = require('../modules/filenamePreview'); const { getExternalCookiesPath } = require('../modules/externalCookies'); -const { isExternalApiEnabled } = require('../modules/externalApiConfig'); +const { isExternalApiEnabled } = require('../modules/externalApi/externalApiConfig'); const { isValidLevelSetting } = require('../logging/logLevel'); // Summarize the allowlist check of a saved custom-args string for the client, diff --git a/server/routes/externalApi.js b/server/routes/externalApi.js index 6eb179576..e682e12e0 100644 --- a/server/routes/externalApi.js +++ b/server/routes/externalApi.js @@ -2,8 +2,8 @@ const express = require('express'); const { sendExternalError, setExternalSecurityHeaders, -} = require('../modules/externalApiResponse'); -const { scopesForExternalKey } = require('../modules/externalPermissions'); +} = require('../modules/externalApi/externalApiResponse'); +const { scopesForExternalKey } = require('../modules/externalApi/externalPermissions'); function createExternalApiRoutes({ externalApiAuth, diff --git a/server/routes/index.js b/server/routes/index.js index b6d918121..2d309e16d 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -22,7 +22,7 @@ const createLogRoutes = require('./logs'); const createVideoTrashRoutes = require('./videoTrash'); const createExternalRequestReviewRoutes = require('./externalRequests'); const { createExternalApiRoutes } = require('./externalApi'); -const { sendExternalError } = require('../modules/externalApiResponse'); +const { sendExternalError } = require('../modules/externalApi/externalApiResponse'); const videoMetadataModule = require('../modules/videoMetadataModule'); const videoTrashModule = require('../modules/videoTrashModule'); const videoOembedEnricher = require('../modules/videoOembedEnricher'); @@ -49,12 +49,12 @@ const cookieDetails = require('../modules/cookieDetails'); const cookieTest = require('../modules/cookieTest'); const logger = require('../logger'); const logFilesModule = require('../modules/logFilesModule'); -const externalCatalogService = require('../modules/externalCatalogService'); -const externalThumbnailProxy = require('../modules/externalThumbnailProxy'); -const { sharedExternalWorkLimiter } = require('../modules/externalWorkLimiter'); -const { createExternalRequestService } = require('../modules/externalRequestService'); -const { createExternalQuotaService } = require('../modules/externalQuotaService'); -const { isExternalApiEnabled } = require('../modules/externalApiConfig'); +const externalCatalogService = require('../modules/externalApi/externalCatalogService'); +const externalThumbnailProxy = require('../modules/externalApi/externalThumbnailProxy'); +const { sharedExternalWorkLimiter } = require('../modules/externalApi/externalWorkLimiter'); +const { createExternalRequestService } = require('../modules/externalApi/externalRequestService'); +const { createExternalQuotaService } = require('../modules/externalApi/externalQuotaService'); +const { isExternalApiEnabled } = require('../modules/externalApi/externalApiConfig'); /** * Registers all route modules with the Express app diff --git a/server/server.js b/server/server.js index 9b708f769..ed0ce460a 100644 --- a/server/server.js +++ b/server/server.js @@ -16,7 +16,7 @@ const { isAuthConfigured } = require('./modules/authState'); // Start the channel tab count catch-up after the startup rescan has begun. const STARTUP_TAB_COUNT_REFRESH_DELAY_MS = 2 * 60 * 1000; const { createExternalApiAuth } = require('./middleware/externalApiAuth'); -const { sendExternalError } = require('./modules/externalApiResponse'); +const { sendExternalError } = require('./modules/externalApi/externalApiResponse'); const app = express(); app.set('trust proxy', parseTrustProxySetting(process.env.TRUST_PROXY)); if (process.env.TRUST_PROXY === undefined || process.env.TRUST_PROXY === '') { diff --git a/server/testing/externalApiContractApp.js b/server/testing/externalApiContractApp.js index e76635078..c7ec30664 100644 --- a/server/testing/externalApiContractApp.js +++ b/server/testing/externalApiContractApp.js @@ -6,7 +6,7 @@ const os = require('os'); const path = require('path'); const { createExternalApiAuth } = require('../middleware/externalApiAuth'); const { createExternalApiRoutes } = require('../routes/externalApi'); -const { CatalogError, pagination: validatePagination } = require('../modules/externalPagination'); +const { CatalogError, pagination: validatePagination } = require('../modules/externalApi/externalPagination'); const SYNTHETIC_API_KEY = 'plinx-synthetic-key'; const LANDSCAPE_PNG = Buffer.from( diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index c33deac72..d2d647fc2 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -8,11 +8,11 @@ process.env.DB_NAME = `youtarr_runtime_test_${process.pid}`; const { sequelize } = require('../db'); const models = require('../models'); const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); -const { createExternalRequestService } = require('../modules/externalRequestService'); -const { createExternalQuotaService } = require('../modules/externalQuotaService'); -const { createExternalWorkLimiter } = require('../modules/externalWorkLimiter'); +const { createExternalRequestService } = require('../modules/externalApi/externalRequestService'); +const { createExternalQuotaService } = require('../modules/externalApi/externalQuotaService'); +const { createExternalWorkLimiter } = require('../modules/externalApi/externalWorkLimiter'); const { isSpecificUrlDownloadJob } = require('../modules/download/jobTypes'); -const catalog = require('../modules/externalCatalogService'); +const catalog = require('../modules/externalApi/externalCatalogService'); const configModule = require('../modules/configModule'); const youtubeId = 'abcdefghijk'; From 8e5bcc9fbd02df4a6a99693f14b6530e59433bef Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 04:50:51 +0000 Subject: [PATCH 19/27] fix: match external API paths case-insensitively at the app level Express routes /EXTERNAL-API/v1/... into the external router, but the app-level body parser bypass, error envelopes, degraded-database 503, and request id handling compared req.path case-sensitively. Share one isExternalApiPath check that matches the mount the way Express does. --- .../__tests__/externalApiConfig.test.js | 18 +++++++++++++++++- .../modules/externalApi/externalApiConfig.js | 10 ++++++++++ server/server.js | 11 ++++++----- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/server/modules/externalApi/__tests__/externalApiConfig.test.js b/server/modules/externalApi/__tests__/externalApiConfig.test.js index 02c25caf4..82d0d1044 100644 --- a/server/modules/externalApi/__tests__/externalApiConfig.test.js +++ b/server/modules/externalApi/__tests__/externalApiConfig.test.js @@ -1,6 +1,6 @@ 'use strict'; -const { isExternalApiEnabled } = require('../externalApiConfig'); +const { isExternalApiEnabled, isExternalApiPath } = require('../externalApiConfig'); describe('isExternalApiEnabled', () => { const originalValue = process.env.EXTERNAL_API_ENABLED; @@ -35,3 +35,19 @@ describe('isExternalApiEnabled', () => { expect(isExternalApiEnabled()).toBe(true); }); }); + +describe('isExternalApiPath', () => { + test.each(['/external-api', '/external-api/v1/requests', '/EXTERNAL-API/v1/requests', '/External-Api/v1'])( + 'matches %p like the Express mount does', + (path) => { + expect(isExternalApiPath({ path })).toBe(true); + } + ); + + test.each(['/api/videos', '/external-apis', '/api/external-api/v1', ''])( + 'does not match %p', + (path) => { + expect(isExternalApiPath({ path })).toBe(false); + } + ); +}); diff --git a/server/modules/externalApi/externalApiConfig.js b/server/modules/externalApi/externalApiConfig.js index 1b79b49f5..ebcd3bf9f 100644 --- a/server/modules/externalApi/externalApiConfig.js +++ b/server/modules/externalApi/externalApiConfig.js @@ -4,6 +4,16 @@ function isExternalApiEnabled(value = process.env.EXTERNAL_API_ENABLED) { return String(value ?? '').trim().toLowerCase() === 'true'; } +const EXTERNAL_API_MOUNT = '/external-api'; + +// Express matches the /external-api mount case-insensitively, so app-level +// checks that decide which body parser and error envelope apply must too. +function isExternalApiPath(req) { + const requestPath = String(req?.path || '').toLowerCase(); + return requestPath === EXTERNAL_API_MOUNT || requestPath.startsWith(`${EXTERNAL_API_MOUNT}/`); +} + module.exports = { isExternalApiEnabled, + isExternalApiPath, }; diff --git a/server/server.js b/server/server.js index ed0ce460a..1b7d75cf9 100644 --- a/server/server.js +++ b/server/server.js @@ -17,6 +17,7 @@ const { isAuthConfigured } = require('./modules/authState'); const STARTUP_TAB_COUNT_REFRESH_DELAY_MS = 2 * 60 * 1000; const { createExternalApiAuth } = require('./middleware/externalApiAuth'); const { sendExternalError } = require('./modules/externalApi/externalApiResponse'); +const { isExternalApiPath } = require('./modules/externalApi/externalApiConfig'); const app = express(); app.set('trust proxy', parseTrustProxySetting(process.env.TRUST_PROXY)); if (process.env.TRUST_PROXY === undefined || process.env.TRUST_PROXY === '') { @@ -57,7 +58,7 @@ app.use(pinoHttp({ genReqId: (req) => { // External API envelopes promise a server-generated requestId, so the // public namespace never adopts a caller-supplied correlation header. - const existingId = req.path.startsWith('/external-api') ? null : req.id || req.headers['x-request-id']; + const existingId = isExternalApiPath(req) ? null : req.id || req.headers['x-request-id']; if (existingId) return existingId; return require('crypto').randomUUID(); }, @@ -97,13 +98,13 @@ app.use(pinoHttp({ const defaultJsonParser = express.json(); app.use((req, res, next) => { - if (req.path.startsWith('/external-api')) return next(); + if (isExternalApiPath(req)) return next(); return defaultJsonParser(req, res, next); }); // Express otherwise renders malformed/oversized JSON as an HTML error. Keep // every response in the public external namespace on the versioned envelope. app.use((error, req, res, next) => { - if (!req.path.startsWith('/external-api')) return next(error); + if (!isExternalApiPath(req)) return next(error); if (error?.type === 'entity.too.large') { return sendExternalError(res, 413, 'Request body is too large', { requestId: req.id, @@ -432,7 +433,7 @@ const initialize = async () => { errorMessage = 'A database error has occurred. Please check the logs for details.'; } - if (req.path.startsWith('/external-api')) { + if (isExternalApiPath(req)) { return sendExternalError(res, 503, 'External API is temporarily unavailable', { code: 'service_unavailable', requestId: req.id, @@ -853,7 +854,7 @@ const initialize = async () => { }); app.use((error, req, res, next) => { - if (!req.path.startsWith('/external-api')) return next(error); + if (!isExternalApiPath(req)) return next(error); req.log?.error({ err: error }, 'Unhandled external API request error'); return sendExternalError(res, 500, 'External API request failed', { requestId: req.id, From e2f8fac1fea5a7ae2889025d2480f7cc783f6d33 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:06:08 +0000 Subject: [PATCH 20/27] feat: run external requests in a background executor Creating or approving a request now only records the decision and charges quota. A small in-process executor picks approved rows up outside any lock, re-checks the key's authority, and does the slow work, so no yt-dlp call, download queueing, or file unlink runs under a row lock or inside an HTTP handler: - Channel provisioning resolves the channel without enabling it and takes locks only for the final write, where a terminated channel now fails the request instead of being re-enabled. - Deletions go through deleteVideos, so the channel .m3u, playlists, and storage pause are refreshed like an in-app delete. - A storage pause or a busy video holds the request approved instead of failing it; the executor's sweep retries it. - A jobModule.onJobEnded listener closes video requests when their job ends, so ?status=completed no longer depends on who reads the list. Reads and quota checks no longer reconcile or lock anything. - A new approval_source column records whether a request was auto-approved, so the executor re-checks auto-approval only for those rows. - The download job id is passed as an option to doSpecificDownloads rather than read from the job data, and addJob rejects a preferred job id that is not a UUID. --- ...56-add-external-request-approval-source.js | 22 + .../externalApiDatabase.integration.test.js | 1 + .../externalRequestApprovalSource.test.js | 28 + scripts/external-api-index-plans.js | 4 +- server/models/externalrequest.js | 1 + .../modules/__tests__/downloadModule.test.js | 26 + server/modules/__tests__/jobModule.test.js | 17 +- server/modules/downloadModule.js | 26 +- .../__tests__/externalQuotaService.test.js | 28 +- .../externalRequestExecution.test.js | 321 ------------ .../__tests__/externalRequestExecutor.test.js | 410 +++++++++++++++ .../externalRequestReconciliation.test.js | 72 --- .../externalRequestReviewService.test.js | 151 +++--- .../__tests__/externalRequestService.test.js | 61 ++- .../__tests__/externalRequestTargets.test.js | 58 +++ .../__tests__/fixtures/externalRequests.js | 22 +- .../externalApi/externalQuotaService.js | 21 +- .../externalApi/externalRequestExecution.js | 229 -------- .../externalApi/externalRequestExecutor.js | 342 ++++++++++++ .../externalApi/externalRequestPrimitives.js | 16 +- .../externalRequestReconciliation.js | 119 +++-- .../externalRequestReviewService.js | 391 ++++---------- .../externalApi/externalRequestService.js | 489 +++++------------- .../externalApi/externalRequestTargets.js | 54 ++ server/modules/jobModule.js | 5 +- .../routes/__tests__/externalRequests.test.js | 8 +- server/routes/externalRequests.js | 14 +- server/routes/index.js | 4 +- server/server.js | 10 +- .../testing/externalRuntime.database.test.js | 255 +++++---- 30 files changed, 1601 insertions(+), 1604 deletions(-) create mode 100644 migrations/20261022045156-add-external-request-approval-source.js create mode 100644 migrations/__tests__/externalRequestApprovalSource.test.js delete mode 100644 server/modules/externalApi/__tests__/externalRequestExecution.test.js create mode 100644 server/modules/externalApi/__tests__/externalRequestExecutor.test.js delete mode 100644 server/modules/externalApi/__tests__/externalRequestReconciliation.test.js create mode 100644 server/modules/externalApi/__tests__/externalRequestTargets.test.js delete mode 100644 server/modules/externalApi/externalRequestExecution.js create mode 100644 server/modules/externalApi/externalRequestExecutor.js create mode 100644 server/modules/externalApi/externalRequestTargets.js diff --git a/migrations/20261022045156-add-external-request-approval-source.js b/migrations/20261022045156-add-external-request-approval-source.js new file mode 100644 index 000000000..6629acdb3 --- /dev/null +++ b/migrations/20261022045156-add-external-request-approval-source.js @@ -0,0 +1,22 @@ +'use strict'; + +const { addColumnIfMissing, removeColumnIfExists } = require('./helpers'); + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up (queryInterface, Sequelize) { + // Who approved a request: 'auto' (the key's auto-approval setting at + // create time) or 'admin' (a review decision). The request executor + // re-checks auto-approval only for 'auto' rows, so turning a key's + // auto-approval off stops requests it approved but has not yet run. + await addColumnIfMissing(queryInterface, 'external_requests', 'approval_source', { + type: Sequelize.STRING(10), + allowNull: true, + defaultValue: null + }); + }, + + async down (queryInterface) { + await removeColumnIfExists(queryInterface, 'external_requests', 'approval_source'); + } +}; diff --git a/migrations/__tests__/externalApiDatabase.integration.test.js b/migrations/__tests__/externalApiDatabase.integration.test.js index 907f1b77f..8db34ce64 100644 --- a/migrations/__tests__/externalApiDatabase.integration.test.js +++ b/migrations/__tests__/externalApiDatabase.integration.test.js @@ -19,6 +19,7 @@ const EXTERNAL_MIGRATIONS = [ '20260908101000-create-api-key-channel-grants.js', '20260908102000-create-external-requests.js', '20260908106000-create-external-api-usage-buckets.js', + '20261022045156-add-external-request-approval-source.js', ]; const DATABASE_NAME = `youtarr_external_api_${process.pid}`; const dbOptions = { diff --git a/migrations/__tests__/externalRequestApprovalSource.test.js b/migrations/__tests__/externalRequestApprovalSource.test.js new file mode 100644 index 000000000..7a1f2a4b9 --- /dev/null +++ b/migrations/__tests__/externalRequestApprovalSource.test.js @@ -0,0 +1,28 @@ +const { DataTypes } = require('sequelize'); +const migration = require('../20261022045156-add-external-request-approval-source'); + +function queryInterface(columns = {}) { + const table = { ...columns }; + return { + table, + describeTable: async () => table, + addColumn: jest.fn(async (_table, name, definition) => { table[name] = definition; }), + removeColumn: jest.fn(async (_table, name) => { delete table[name]; }), + }; +} + +test('adds a nullable approval_source column once', async () => { + const query = queryInterface({ id: {} }); + await migration.up(query, DataTypes); + await migration.up(query, DataTypes); + expect(query.addColumn).toHaveBeenCalledTimes(1); + expect(query.table.approval_source).toMatchObject({ allowNull: true, defaultValue: null }); +}); + +test('rollback removes the column and tolerates a missing one', async () => { + const query = queryInterface({ id: {}, approval_source: {} }); + await migration.down(query); + await migration.down(query); + expect(query.removeColumn).toHaveBeenCalledTimes(1); + expect(query.table).not.toHaveProperty('approval_source'); +}); diff --git a/scripts/external-api-index-plans.js b/scripts/external-api-index-plans.js index 33172ea5c..3af5883b8 100644 --- a/scripts/external-api-index-plans.js +++ b/scripts/external-api-index-plans.js @@ -48,7 +48,9 @@ async function run() { await sequelize.query('ANALYZE TABLE channels,channelvideos,external_requests'); const key = { id: 1, role: 'admin', maxRatingLevel: 4, allowUnrated: true, allowedMediaTypes: ['video'] }; - const requests = createExternalRequestService({ executor: () => {}, videoDeleter: {} }); + const requests = createExternalRequestService({ + downloader: () => {}, channelProvisioner: {}, videoDeleter: {}, storageGuard: {}, + }); const reads = async () => [ await catalog.listChannels(key), await catalog.listVideos(key, { pageSize: '50' }), diff --git a/server/models/externalrequest.js b/server/models/externalrequest.js index d1b6070e2..9bbadbd68 100644 --- a/server/models/externalrequest.js +++ b/server/models/externalrequest.js @@ -14,6 +14,7 @@ ExternalRequest.init({ active_dedupe_key: { type: DataTypes.STRING(191), allowNull: true }, idempotency_hash: { type: DataTypes.STRING(64), allowNull: true }, job_id: { type: DataTypes.UUID, allowNull: true }, + approval_source: { type: DataTypes.STRING(10), allowNull: true, validate: { isIn: [['auto', 'admin']] } }, message: { type: DataTypes.STRING(500), allowNull: true }, created_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, updated_at: { type: DataTypes.DATE, allowNull: false, defaultValue: DataTypes.NOW }, diff --git a/server/modules/__tests__/downloadModule.test.js b/server/modules/__tests__/downloadModule.test.js index 4c1fb248c..9e4ca2aa6 100644 --- a/server/modules/__tests__/downloadModule.test.js +++ b/server/modules/__tests__/downloadModule.test.js @@ -1730,6 +1730,32 @@ describe('DownloadModule', () => { ); }); + it('uses a job id passed in options and returns it', async () => { + const jobId = '9b89e5bc-8c90-4e72-b245-270fed2eacc2'; + jobModuleMock.addOrUpdateJob.mockResolvedValue(jobId); + jobModuleMock.getJob.mockReturnValueOnce(undefined).mockReturnValue({ status: 'Pending' }); + + const result = await downloadModule.doSpecificDownloads( + { urls: ['https://youtube.com/watch?v=abc123'] }, false, { jobId } + ); + + expect(jobModuleMock.addOrUpdateJob).toHaveBeenCalledWith(expect.any(Object), false, jobId); + expect(result).toBe(jobId); + }); + + it('ignores a job id supplied in the request body', async () => { + jobModuleMock.getJob.mockReturnValue({ status: 'Pending' }); + + await downloadModule.doSpecificDownloads({ + body: { + urls: ['https://youtube.com/watch?v=abc123'], + externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', + }, + }); + + expect(jobModuleMock.addOrUpdateJob).toHaveBeenCalledWith(expect.any(Object), false); + }); + it('should handle job data directly', async () => { jobModuleMock.getJob.mockReturnValue({ status: 'In Progress' }); const jobData = { diff --git a/server/modules/__tests__/jobModule.test.js b/server/modules/__tests__/jobModule.test.js index 9083c879b..3588fbb15 100644 --- a/server/modules/__tests__/jobModule.test.js +++ b/server/modules/__tests__/jobModule.test.js @@ -2,7 +2,8 @@ // Mock dependencies jest.mock('uuid', () => ({ - v4: jest.fn(() => 'generated-uuid') + v4: jest.fn(() => 'generated-uuid'), + validate: jest.requireActual('uuid').validate })); jest.mock('node-cron'); jest.mock('../messageEmitter.js'); @@ -1253,6 +1254,20 @@ describe('JobModule', () => { await first; }); + test('uses a supplied UUID as the job id', async () => { + const jobId = '9b89e5bc-8c90-4e72-b245-270fed2eacc2'; + await expect(JobModule.addJob({ + jobType: 'Manually Added Urls', data: { urls: ['https://youtu.be/aaaaaaaaaaa'] }, + }, jobId)).resolves.toBe(jobId); + }); + + test('rejects a supplied job id that is not a UUID', async () => { + await expect(JobModule.addJob({ + jobType: 'Manually Added Urls', data: { urls: ['https://youtu.be/aaaaaaaaaaa'] }, + }, '../jobs')).rejects.toThrow('Preferred job id must be a UUID'); + expect(Job.create).not.toHaveBeenCalled(); + }); + test('rolls back queue ownership if creating the job fails', async () => { const activity = require('../download/videoActivity'); Job.create.mockRejectedValueOnce(new Error('offline')); diff --git a/server/modules/downloadModule.js b/server/modules/downloadModule.js index 515536afb..7c3f282cb 100644 --- a/server/modules/downloadModule.js +++ b/server/modules/downloadModule.js @@ -754,7 +754,14 @@ class DownloadModule { } } - async doSpecificDownloads(reqOrJobData, isNextJob = false) { + /** + * @param {Object} reqOrJobData - request (body) or job data + * @param {boolean} [isNextJob] + * @param {{jobId?: string}} [options] - jobId: a stable UUID for the job, from + * trusted internal callers only (the external request executor). Kept out + * of the job data so a request body cannot choose a job id. + */ + async doSpecificDownloads(reqOrJobData, isNextJob = false, { jobId: preferredJobId = null } = {}) { const jobData = reqOrJobData.body ? reqOrJobData.body : reqOrJobData; if (!isNextJob && !jobData.id) { await storageGuard.assertDownloadsAllowed(); @@ -777,14 +784,13 @@ class DownloadModule { const requestedIds = requestedUrls.flatMap(url => { try { return [normalizeUrlToVideoId(url).id]; } catch { return []; } }); - const externalRequestId = this.getJobDataValue(jobData, 'externalRequestId'); - if (!isNextJob && externalRequestId) { - const existingExternalJob = jobModule.getJob(externalRequestId); - if (existingExternalJob) { - if (['Error', 'Killed', 'Terminated'].includes(existingExternalJob.status)) { - throw new Error('Existing external request job is terminal'); + if (!isNextJob && preferredJobId) { + const existingJob = jobModule.getJob(preferredJobId); + if (existingJob) { + if (['Error', 'Killed', 'Terminated'].includes(existingJob.status)) { + throw new Error('Existing job with the requested id is terminal'); } - return externalRequestId; + return preferredJobId; } } const addJobArgs = [ @@ -798,7 +804,7 @@ class DownloadModule { }, isNextJob ]; - if (externalRequestId) addJobArgs.push(externalRequestId); + if (preferredJobId) addJobArgs.push(preferredJobId); const jobId = await jobModule.addOrUpdateJob(...addJobArgs); if (!jobId) return { queued: 0, acceptedIds: [], alreadyActiveIds: [...new Set(requestedIds)] }; @@ -972,7 +978,7 @@ class DownloadModule { await jobModule.startNextJob(); throw err; } - if (externalRequestId) return jobId; + if (preferredJobId) return jobId; return { queued: urls.length, acceptedIds: admission.acceptedIds, alreadyActiveIds: admission.alreadyActiveIds }; } diff --git a/server/modules/externalApi/__tests__/externalQuotaService.test.js b/server/modules/externalApi/__tests__/externalQuotaService.test.js index 6c16410c6..da4590e41 100644 --- a/server/modules/externalApi/__tests__/externalQuotaService.test.js +++ b/server/modules/externalApi/__tests__/externalQuotaService.test.js @@ -1,3 +1,4 @@ +const { Op } = require('sequelize'); const { createExternalQuotaService, QuotaError, @@ -119,23 +120,8 @@ describe('external API durable quotas', () => { ).rejects.toBeInstanceOf(QuotaError); }); - test('reconciles completed downloads before reporting active capacity', async () => { - const completedRequest = { - id: 'request-1', - request_type: 'video', - youtube_id: 'abcdefghijk', - job_id: 'job-1', - status: 'processing', - update: jest.fn(async function update(values) { - Object.assign(this, values); - }), - }; - const { service, models } = fixture({ active: 1 }); - models.ExternalRequest.findAll.mockResolvedValue([completedRequest]); - models.Video.findAll.mockResolvedValue([{ youtubeId: 'abcdefghijk' }]); - models.ExternalRequest.count.mockImplementation(async () => - completedRequest.status === 'processing' ? 1 : 0 - ); + test('counts approved and processing requests as active without modifying them', async () => { + const { service, models } = fixture({ active: 2 }); const result = await service.status({ id: 4, @@ -144,9 +130,9 @@ describe('external API durable quotas', () => { dailyWriteLimit: 200, }); - expect(completedRequest.update).toHaveBeenCalledWith( - expect.objectContaining({ status: 'completed', active_dedupe_key: null }) - ); - expect(result.remaining.activeJobs).toBe(5); + expect(models.ExternalRequest.count).toHaveBeenCalledWith({ + where: { api_key_id: 4, status: { [Op.in]: ['approved', 'processing'] } }, + }); + expect(result.remaining.activeJobs).toBe(3); }); }); diff --git a/server/modules/externalApi/__tests__/externalRequestExecution.test.js b/server/modules/externalApi/__tests__/externalRequestExecution.test.js deleted file mode 100644 index 5d1f9dd49..000000000 --- a/server/modules/externalApi/__tests__/externalRequestExecution.test.js +++ /dev/null @@ -1,321 +0,0 @@ -const { fixture, record, timestamp, youtubeId } = require('./fixtures/externalRequests'); - -describe('external request execution', () => { - test('auto-approval queues only the canonical URL and server-owned channel mapping', async () => { - const { service, models, executor, key, transaction } = fixture({ - key: { autoApproveVideoRequests: true }, - }); - const result = await service.createVideoRequest(key, { - youtubeId, channelId: 8, idempotencyKey: 'external-client-1', - }); - expect(models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ - status: 'pending', - decided_at: timestamp, - idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), - }), { transaction }); - expect(executor).toHaveBeenCalledWith({ - body: { - urls: [`https://www.youtube.com/watch?v=${youtubeId}`], - channelId: 'UC1234567890123456789012', - ownerChannelMap: { [youtubeId]: 'UC1234567890123456789012' }, - initiatedBy: { type: 'api_key', name: 'External Client' }, - jobLabel: 'Manually Added Urls (external request)', - externalRequestId: '9b89e5bc-8c90-4e72-b245-270fed2eacc2', - }, - }); - expect(result.request.status).toBe('processing'); - }); - - test('returns service unavailable when the shared work queue is full', async () => { - const workError = Object.assign(new Error('full'), { - name: 'ExternalWorkLimitError', - }); - const limited = fixture({ - key: { autoApproveVideoRequests: true }, - workLimiter: { run: jest.fn().mockRejectedValue(workError) }, - }); - await expect(limited.service.createVideoRequest(limited.key, { - youtubeId, - channelId: 8, - })).rejects.toMatchObject({ - name: 'RequestError', - status: 503, - code: 'work_queue_full', - }); - expect(limited.created.status).toBe('pending'); - expect(limited.created.active_dedupe_key).toBe(`4:video:${youtubeId}`); - }); - - test('revalidates a channel grant inside the work slot before enqueue', async () => { - const raced = fixture({ - key: { autoApproveVideoRequests: true }, - }); - raced.models.ApiKeyChannelGrant.findOne - .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) - .mockResolvedValueOnce({ api_key_id: 4, channel_id: 8 }) - .mockResolvedValueOnce(null); - - const result = await raced.service.createVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.executor).not.toHaveBeenCalled(); - expect(raced.created.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - })); - expect(result.request.status).toBe('failed'); - }); - - test('does not enqueue when video auto-approval is disabled before execution', async () => { - const raced = fixture({ key: { autoApproveVideoRequests: true } }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - allowVideoRequests: true, - autoApproveVideoRequests: false, - }); - - const result = await raced.service.createVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.executor).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - - test('resumes an interrupted auto-approved dispatch with the stable request job id', async () => { - const interrupted = record({ status: 'pending', decided_at: timestamp, job_id: null }); - const retryExecutor = jest.fn().mockResolvedValue(interrupted.id); - const retry = fixture({ - key: { autoApproveVideoRequests: true }, - executor: retryExecutor, - }); - retry.models.ExternalRequest.findOne.mockResolvedValue(interrupted); - retry.models.ExternalRequest.findByPk.mockResolvedValue(interrupted); - const result = await retry.service.createVideoRequest( - retry.key, - { youtubeId, channelId: 8, idempotencyKey: 'same-request' } - ); - expect(retry.models.ExternalRequest.create).not.toHaveBeenCalled(); - expect(retryExecutor).toHaveBeenCalledWith(expect.objectContaining({ - body: expect.objectContaining({ externalRequestId: interrupted.id }), - })); - expect(interrupted.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'processing', - job_id: interrupted.id, - }), { transaction: retry.transaction }); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { status: 'processing' }, - }); - }); - - test('auto-approved channel requests provision and grant the resulting channel', async () => { - const approved = fixture({ key: { autoApproveChannelRequests: true } }); - const result = await approved.service.createChannelRequest(approved.key, { - channelUrl: 'https://www.youtube.com/channel/UC1234567890123456789012', - }); - - expect(approved.channelProvisioner.getChannelInfo).toHaveBeenCalledWith( - 'https://www.youtube.com/channel/UC1234567890123456789012', - false, - true, - {}, - { skipTabDetection: true } - ); - expect(approved.models.ApiKeyChannelGrant.findOrCreate).toHaveBeenCalledWith({ - where: { api_key_id: 4, channel_id: 8 }, - defaults: { api_key_id: 4, channel_id: 8 }, - transaction: approved.transaction, - }); - expect(result.request).toMatchObject({ - type: 'channel', - status: 'completed', - target: { channelId: 8 }, - }); - }); - - test('does not provision when channel auto-approval is disabled before execution', async () => { - const raced = fixture({ key: { autoApproveChannelRequests: true } }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - allowChannelRequests: true, - autoApproveChannelRequests: false, - }); - - const result = await raced.service.createChannelRequest(raced.key, { - channelUrl: 'https://www.youtube.com/@safe', - }); - - expect(raced.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - - test('resumes an interrupted auto-approved channel request on idempotent retry', async () => { - const interrupted = record({ - request_type: 'channel', - channel_id: null, - youtube_id: null, - channel_url: 'https://www.youtube.com/@safe', - status: 'pending', - decided_at: timestamp, - }); - const recovering = fixture({ - key: { autoApproveChannelRequests: true }, - }); - recovering.models.ExternalRequest.findOne.mockResolvedValue(interrupted); - - const result = await recovering.service.createChannelRequest(recovering.key, { - channelUrl: 'https://www.youtube.com/@safe', - }); - - expect(recovering.channelProvisioner.getChannelInfo).toHaveBeenCalledTimes(1); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { type: 'channel', status: 'completed' }, - }); - }); - - test('auto-approved deletion reaches a completed terminal state', async () => { - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - idempotencyKey: 'delete-1', - }); - - expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( - 91, - expect.objectContaining({ - transaction: deleting.transaction, - video: expect.objectContaining({ id: 91, youtubeId }), - }) - ); - expect(result).toMatchObject({ - outcome: 'created', - request: { type: 'delete_video', status: 'completed' }, - }); - }); - - test('does not delete when delete auto-approval is disabled before execution', async () => { - const raced = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - raced.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - raced.quotaService.assertExecutionCapacity.mockResolvedValue({ - ...raced.key, - role: 'delete', - allowDeleteVideoRequests: true, - autoApproveDeleteRequests: false, - }); - - const result = await raced.service.createDeleteVideoRequest(raced.key, { - youtubeId, - channelId: 8, - }); - - expect(raced.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); - expect(result.request.status).toBe('failed'); - }); - - test('deletion completes idempotently when the target disappears during execution', async () => { - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - videoDeleter: { - deleteVideoById: jest.fn().mockResolvedValue({ - success: false, - error: 'Video not found in database', - }), - }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - }); - - expect(result.request).toMatchObject({ - status: 'completed', - message: 'Video is already deleted', - }); - }); - - test('recovers a stale processing deletion idempotently', async () => { - const stale = record({ - request_type: 'delete_video', - status: 'processing', - channel_url: null, - decided_at: new Date('2026-07-26T11:00:00.000Z'), - updated_at: new Date('2026-07-26T11:00:00.000Z'), - }); - const deleting = fixture({ - key: { role: 'delete', autoApproveDeleteRequests: true }, - }); - deleting.models.Video.findOne.mockResolvedValue({ - id: 91, - youtubeId, - normalized_rating: 'TV-Y', - removed: false, - }); - deleting.models.ExternalRequest.findOne.mockResolvedValue(stale); - - const result = await deleting.service.createDeleteVideoRequest(deleting.key, { - youtubeId, - channelId: 8, - }); - - expect(deleting.videoDeleter.deleteVideoById).toHaveBeenCalledWith( - 91, - expect.objectContaining({ transaction: deleting.transaction }) - ); - expect(result).toMatchObject({ - outcome: 'duplicate', - request: { type: 'delete_video', status: 'completed' }, - }); - }); - - test('marks queue failures terminal so a retry is possible', async () => { - const queueError = new Error('secret executor failure'); - const failed = fixture({ - key: { autoApproveVideoRequests: true }, - executor: jest.fn().mockRejectedValue(queueError), - }); - const result = await failed.service.createVideoRequest( - failed.key, { youtubeId, channelId: 8 } - ); - expect(failed.created.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - message: 'Download could not be queued', - })); - expect(result.request).toMatchObject({ - status: 'failed', - message: 'Download could not be queued', - }); - expect(result.request.message).not.toContain('secret'); - }); - - -}); diff --git a/server/modules/externalApi/__tests__/externalRequestExecutor.test.js b/server/modules/externalApi/__tests__/externalRequestExecutor.test.js new file mode 100644 index 000000000..394d17cee --- /dev/null +++ b/server/modules/externalApi/__tests__/externalRequestExecutor.test.js @@ -0,0 +1,410 @@ +const { + createExternalRequestExecutor, EXTERNAL_REQUEST_JOB_LABEL, +} = require('../externalRequestExecutor'); +const { RequestError } = require('../externalRequestPrimitives'); +const { record, timestamp, youtubeId } = require('./fixtures/externalRequests'); + +const youtubeChannelId = 'UC1234567890123456789012'; +const requestId = '9b89e5bc-8c90-4e72-b245-270fed2eacc2'; + +function apiKey(overrides = {}) { + return { + id: 4, + name: 'External Client', + autoApproveVideoRequests: true, + autoApproveChannelRequests: true, + autoApproveDeleteRequests: true, + ...overrides, + }; +} + +function setup({ rows = [], key = apiKey(), target, claimed = 1 } = {}) { + const events = []; + const transaction = { LOCK: { UPDATE: 'UPDATE' } }; + const channel = { + id: 8, channel_id: youtubeChannelId, enabled: false, terminated_at: null, + update: jest.fn(async function update(values) { Object.assign(this, values); }), + }; + const models = { + ExternalRequest: { + findAll: jest.fn(async ({ where }) => (where.status === 'approved' ? rows : [])), + update: jest.fn(async (values) => { + events.push(['request.update', values.status || values.job_id]); + return [claimed]; + }), + }, + ApiKeyChannelGrant: { findOrCreate: jest.fn().mockResolvedValue([{ id: 1 }, true]) }, + Channel: { findOne: jest.fn().mockResolvedValue(channel) }, + Video: { findAll: jest.fn().mockResolvedValue([]) }, + Job: { findAll: jest.fn().mockResolvedValue([]), findByPk: jest.fn().mockResolvedValue(null) }, + }; + const deps = { + models, + sequelize: { + transaction: jest.fn(async (callback) => { + events.push(['transaction']); + return callback(transaction); + }), + }, + now: () => timestamp, + quotas: { reloadAuthorizedKey: jest.fn().mockResolvedValue(key) }, + validateTarget: jest.fn().mockResolvedValue(target || { + channel: { channel_id: youtubeChannelId }, downloaded: null, + }), + workLimiter: { run: jest.fn((work) => work()) }, + downloader: jest.fn(async (_jobData, { jobId }) => jobId), + channelProvisioner: { + getChannelInfo: jest.fn(async () => { + events.push(['getChannelInfo']); + return { channel_id: youtubeChannelId }; + }), + }, + videoDeleter: { deleteVideos: jest.fn(async (ids) => ({ success: true, deleted: ids, failed: [] })) }, + storageGuard: { isPausedError: (error) => error?.code === 'DOWNLOADS_PAUSED' }, + jobLookup: jest.fn(() => null), + logger: { warn: jest.fn(), error: jest.fn() }, + }; + return { executor: createExternalRequestExecutor(deps), deps, models, channel, transaction, events }; +} + +const approved = (overrides = {}) => record({ + status: 'approved', approval_source: 'admin', decided_at: timestamp, ...overrides, +}); + +describe('external request executor', () => { + describe('video requests', () => { + test('queues the canonical URL with the request id as the job id', async () => { + const row = approved(); + const { executor, deps, models } = setup({ rows: [row] }); + + await executor.runOnce(); + + expect(deps.downloader).toHaveBeenCalledWith({ + urls: [`https://www.youtube.com/watch?v=${youtubeId}`], + channelId: youtubeChannelId, + ownerChannelMap: { [youtubeId]: youtubeChannelId }, + initiatedBy: { type: 'api_key', name: 'External Client' }, + jobLabel: EXTERNAL_REQUEST_JOB_LABEL, + }, { jobId: requestId }); + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ job_id: requestId }), + { where: { id: requestId, status: 'processing' } } + ); + }); + + test('runs without a transaction', async () => { + const { executor, deps } = setup({ rows: [approved()] }); + await executor.runOnce(); + expect(deps.sequelize.transaction).not.toHaveBeenCalled(); + }); + + test('fails an auto-approved request once the key stops auto-approving', async () => { + const row = approved({ approval_source: 'auto' }); + const { executor, deps } = setup({ + rows: [row], key: apiKey({ autoApproveVideoRequests: false }), + }); + + await executor.runOnce(); + + expect(deps.downloader).not.toHaveBeenCalled(); + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', message: 'Request is no longer eligible', active_dedupe_key: null, + })); + }); + + test('runs an administrator-approved request whatever the auto-approval setting', async () => { + const { executor, deps } = setup({ + rows: [approved()], key: apiKey({ autoApproveVideoRequests: false }), + }); + await executor.runOnce(); + expect(deps.downloader).toHaveBeenCalled(); + }); + + test('fails when the key lost access to the target', async () => { + const row = approved(); + const { executor, deps } = setup({ rows: [row] }); + deps.validateTarget.mockRejectedValue(new RequestError('Video not found', 404)); + + await executor.runOnce(); + + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', message: 'Request is no longer eligible', + })); + }); + + test('holds the request while downloads are paused', async () => { + const row = approved(); + const { executor, deps } = setup({ rows: [row] }); + deps.downloader.mockRejectedValue(Object.assign(new Error('paused'), { code: 'DOWNLOADS_PAUSED' })); + + await executor.runOnce(); + + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'approved', message: 'Waiting for downloads to resume', + })); + }); + + test('holds the request while another job downloads the same video', async () => { + const row = approved(); + const { executor, deps } = setup({ rows: [row] }); + deps.downloader.mockResolvedValue({ queued: 0, acceptedIds: [], alreadyActiveIds: [youtubeId] }); + + await executor.runOnce(); + + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'approved', message: 'Waiting for another download of this video to finish', + })); + }); + + test('completes without downloading when the video is already on disk', async () => { + const row = approved(); + const { executor, deps } = setup({ + rows: [row], target: { channel: { channel_id: youtubeChannelId }, downloaded: { id: 91 } }, + }); + + await executor.runOnce(); + + expect(deps.downloader).not.toHaveBeenCalled(); + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', message: 'Video is already downloaded', + })); + }); + + test('does nothing when another sweep claimed the row first', async () => { + const { executor, deps } = setup({ rows: [approved()], claimed: 0 }); + await executor.runOnce(); + expect(deps.downloader).not.toHaveBeenCalled(); + }); + }); + + describe('channel requests', () => { + const channelRow = (overrides = {}) => approved({ + request_type: 'channel', channel_id: null, youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', grant_to_requesting_key: true, ...overrides, + }); + + test('provisions outside the transaction and locks only for the final write', async () => { + const { executor, events } = setup({ rows: [channelRow()] }); + await executor.runOnce(); + const order = events.map(([name]) => name).filter((name) => name !== 'request.update'); + expect(order).toEqual(['getChannelInfo', 'transaction']); + }); + + test('resolves the channel without enabling it, then enables and grants it', async () => { + const row = channelRow(); + const { executor, deps, models, channel } = setup({ rows: [row] }); + + await executor.runOnce(); + + expect(deps.channelProvisioner.getChannelInfo).toHaveBeenCalledWith( + 'https://www.youtube.com/@safe', false, false, {}, { skipTabDetection: true } + ); + expect(channel.enabled).toBe(true); + expect(models.ApiKeyChannelGrant.findOrCreate).toHaveBeenCalledWith(expect.objectContaining({ + where: { api_key_id: 4, channel_id: 8 }, + })); + expect(row.status).toBe('completed'); + }); + + test('fails a terminated channel without re-enabling it', async () => { + const row = channelRow(); + const { executor, channel } = setup({ rows: [row] }); + channel.terminated_at = timestamp; + + await executor.runOnce(); + + expect(channel.update).not.toHaveBeenCalled(); + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', message: 'Channel is unavailable', + })); + }); + + test('skips the grant when the administrator declined it', async () => { + const { executor, models } = setup({ rows: [channelRow({ grant_to_requesting_key: false })] }); + await executor.runOnce(); + expect(models.ApiKeyChannelGrant.findOrCreate).not.toHaveBeenCalled(); + }); + + test('holds the request when the work queue is full', async () => { + const row = channelRow(); + const { executor, deps } = setup({ rows: [row] }); + deps.workLimiter.run.mockRejectedValue(Object.assign(new Error('full'), { name: 'ExternalWorkLimitError' })); + + await executor.runOnce(); + + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'approved' })); + }); + }); + + describe('delete requests', () => { + const deleteRow = () => approved({ request_type: 'delete_video' }); + const downloadedTarget = { channel: { channel_id: youtubeChannelId }, downloaded: { id: 91 } }; + + test('deletes through deleteVideos outside any transaction', async () => { + const row = deleteRow(); + const { executor, deps } = setup({ rows: [row], target: downloadedTarget }); + + await executor.runOnce(); + + expect(deps.videoDeleter.deleteVideos).toHaveBeenCalledWith([91]); + expect(deps.sequelize.transaction).not.toHaveBeenCalled(); + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'completed' })); + }); + + test('completes when the video is already gone', async () => { + const row = deleteRow(); + const { executor, deps } = setup({ rows: [row] }); + + await executor.runOnce(); + + expect(deps.videoDeleter.deleteVideos).not.toHaveBeenCalled(); + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'completed', message: 'Video is already deleted', + })); + }); + + test('fails when the deletion is refused', async () => { + const row = deleteRow(); + const { executor, deps } = setup({ rows: [row], target: downloadedTarget }); + deps.videoDeleter.deleteVideos.mockResolvedValue({ + success: false, deleted: [], failed: [{ videoId: 91, error: 'Library is unavailable' }], + }); + + await executor.runOnce(); + + expect(row.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'failed', message: 'Video could not be deleted', + })); + }); + }); + + describe('recovery and completion', () => { + test('returns an interrupted channel request to the approved queue', async () => { + const stale = record({ request_type: 'channel', status: 'processing', updated_at: new Date(0) }); + const { executor, models } = setup(); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'processing' && where.updated_at ? [stale] : [] + )); + + await executor.runOnce(); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'approved' }), + { where: { id: stale.id, status: 'processing', updated_at: stale.updated_at } } + ); + }); + + test('adopts the job of a video request interrupted before recording it', async () => { + const stale = record({ status: 'processing', job_id: null, updated_at: new Date(0) }); + const { executor, models } = setup(); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'processing' && where.updated_at ? [stale] : [] + )); + models.Job.findByPk.mockResolvedValue({ id: stale.id }); + + await executor.runOnce(); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ job_id: stale.id }), + { where: { id: stale.id, status: 'processing', job_id: null } } + ); + }); + + test('ignores jobs that did not come from an external request', async () => { + const { executor, deps, models } = setup(); + deps.jobLookup.mockReturnValue({ jobType: 'Channel Downloads', status: 'Complete' }); + + await executor.handleJobEnded({ jobId: 'job-1' }); + + expect(models.ExternalRequest.findAll).not.toHaveBeenCalled(); + }); + + test('closes a request as completed when its job ends with the video on disk', async () => { + const processing = record({ status: 'processing', job_id: requestId }); + const { executor, deps, models } = setup(); + deps.jobLookup.mockReturnValue({ jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'Complete' }); + models.ExternalRequest.findAll.mockResolvedValue([processing]); + models.Video.findAll.mockResolvedValue([{ youtubeId }]); + + await executor.handleJobEnded({ jobId: requestId }); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'completed', active_dedupe_key: null }), + { where: { id: requestId, status: 'processing' } } + ); + }); + + test('closes a request as failed when its job ends without the video', async () => { + const processing = record({ status: 'processing', job_id: requestId }); + const { executor, deps, models } = setup(); + deps.jobLookup.mockReturnValue({ jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'Error' }); + models.ExternalRequest.findAll.mockResolvedValue([processing]); + + await executor.handleJobEnded({ jobId: requestId }); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'failed', message: 'Download did not complete' }), + { where: { id: requestId, status: 'processing' } } + ); + }); + + test('the sweep closes a download whose job-ended event was missed', async () => { + const processing = record({ status: 'processing', job_id: 'job-123' }); + const { executor, models } = setup(); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'processing' && where.job_id ? [processing] : [] + )); + models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: 'Killed' }]); + + await executor.runOnce(); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'failed' }), + { where: { id: processing.id, status: 'processing' } } + ); + }); + + test('the sweep fails a download whose job was lost entirely', async () => { + const processing = record({ status: 'processing', job_id: 'job-123' }); + const { executor, models } = setup(); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'processing' && where.job_id ? [processing] : [] + )); + + await executor.runOnce(); + + expect(models.ExternalRequest.update).toHaveBeenCalledWith( + expect.objectContaining({ status: 'failed', message: 'Download did not complete' }), + { where: { id: processing.id, status: 'processing' } } + ); + }); + + test('wake runs approved requests without waiting for the timer', async () => { + const rows = [approved()]; + const { executor, deps, models } = setup(); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'approved' ? rows.splice(0) : [] + )); + + executor.wake(); + for (let tick = 0; tick < 20 && deps.downloader.mock.calls.length === 0; tick += 1) { + await new Promise((resolve) => setImmediate(resolve)); + } + + expect(deps.downloader).toHaveBeenCalledTimes(1); + }); + + test('leaves a request alone while its job is still running', async () => { + const processing = record({ status: 'processing', job_id: requestId }); + const { executor, deps, models } = setup(); + deps.jobLookup.mockReturnValue({ jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'In Progress' }); + models.ExternalRequest.findAll.mockImplementation(async ({ where }) => ( + where.status === 'processing' && where.job_id ? [processing] : [] + )); + + await executor.runOnce(); + + expect(models.ExternalRequest.update).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js b/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js deleted file mode 100644 index 4df2d8ba5..000000000 --- a/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js +++ /dev/null @@ -1,72 +0,0 @@ -const { fixture, record, timestamp, youtubeId } = require('./fixtures/externalRequests'); - -describe('external request reconciliation', () => { - test('reconciles a processing deletion after its video is absent', async () => { - const deleting = fixture({ key: { role: 'delete' } }); - const processing = record({ - request_type: 'delete_video', - status: 'processing', - decided_at: timestamp, - }); - deleting.models.ExternalRequest.findAndCountAll.mockResolvedValue({ - rows: [processing], - count: 1, - }); - deleting.models.Video.findAll.mockResolvedValue([]); - - const result = await deleting.service.listRequests(deleting.key); - - expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'completed', - active_dedupe_key: null, - completed_at: timestamp, - })); - expect(result.data[0]).toMatchObject({ - type: 'delete_video', - status: 'completed', - }); - }); - - test('lists only the calling key and lazily reconciles completed downloads', async () => { - const processing = record({ status: 'processing' }); - const own = fixture(); - own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); - own.models.Video.findAll.mockResolvedValue([{ youtubeId }]); - const result = await own.service.listRequests(own.key, { - page: '1', pageSize: '10', - }); - expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith(expect.objectContaining({ - where: { api_key_id: 4 }, - limit: 10, - offset: 0, - })); - expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'completed', - active_dedupe_key: null, - completed_at: timestamp, - })); - expect(result.data[0].status).toBe('completed'); - }); - - test('reconciles a status-filtered page and requeries its count', async () => { - const stale = record({ status: 'processing', job_id: 'job-123' }); - const own = fixture(); - own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: 'Error' }]); - own.models.ExternalRequest.findAndCountAll - .mockResolvedValueOnce({ rows: [stale], count: 1 }) - .mockResolvedValueOnce({ rows: [], count: 0 }); - - const result = await own.service.listRequests(own.key, { status: 'processing' }); - - expect(stale.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed' })); - expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledWith( - expect.objectContaining({ where: { api_key_id: 4, status: 'processing' } }) - ); - expect(result).toMatchObject({ - data: [], - pagination: { total: 0, totalPages: 0 }, - }); - }); - - -}); diff --git a/server/modules/externalApi/__tests__/externalRequestReviewService.test.js b/server/modules/externalApi/__tests__/externalRequestReviewService.test.js index 0dfc91404..39d9e8448 100644 --- a/server/modules/externalApi/__tests__/externalRequestReviewService.test.js +++ b/server/modules/externalApi/__tests__/externalRequestReviewService.test.js @@ -64,67 +64,96 @@ describe('external request reviewservice', () => { }); }); - test('approves a pending request only after current policy revalidation and queue acceptance', async () => { - const pending = record(); - const hydrated = Object.assign(pending, { + function hydrate(pending, apiKey = {}) { + return Object.assign(pending, { apiKey: { id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: true, revoked_at: null, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: { - id: pending.id, status: 'In Progress', jobType: 'External video request', - timeCreated: timestamp, timeInitiated: timestamp, + is_active: true, revoked_at: null, ...apiKey, }, + channel: { id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel' }, + job: null, }); - const approved = fixture(); - approved.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(null) - .mockResolvedValueOnce(hydrated); - approved.models.ChannelVideo.findAll.mockResolvedValue([{ - youtube_id: youtubeId, - channel_id: 'UC1234567890123456789012', - title: 'Safe video', - media_type: 'video', - }]); + } + + function reviewing(pending, overrides = {}) { + const review = fixture(overrides); + review.models.ExternalRequest.findByPk.mockResolvedValue(pending); + review.models.ExternalRequest.findOne.mockResolvedValue(hydrate(pending)); + return review; + } + + test('approval records the decision and returns without running the request', async () => { + const pending = record(); + const approved = reviewing(pending); - const result = await approved.service.reviewVideoRequest(pending.id, 'approve', {}); + const result = await approved.service.reviewRequest(pending.id, 'approve', {}); expect(approved.models.ApiKey.findByPk).toHaveBeenCalledWith(4, expect.objectContaining({ transaction: approved.transaction, lock: 'UPDATE', attributes: expect.not.arrayContaining(['key_hash']), })); - expect(approved.executor).toHaveBeenCalledWith({ - body: expect.objectContaining({ - urls: [`https://www.youtube.com/watch?v=${youtubeId}`], - channelId: 'UC1234567890123456789012', - externalRequestId: pending.id, - }), + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'approved', + approval_source: 'admin', + decided_at: timestamp, + }), { transaction: approved.transaction }); + expect(approved.downloader).not.toHaveBeenCalled(); + expect(result).toMatchObject({ status: 'approved', approvalSource: 'admin' }); + }); + + test('approval completes a video request whose video was downloaded meanwhile', async () => { + const pending = record(); + const approved = reviewing(pending); + approved.models.Video.findOne.mockResolvedValue({ + id: 91, youtubeId, normalized_rating: 'TV-Y', removed: false, + }); + + const result = await approved.service.reviewRequest(pending.id, 'approve', {}); + + expect(result).toMatchObject({ status: 'completed', message: 'Video is already downloaded' }); + }); + + test('approving an already approved request is a no-op', async () => { + const pending = record({ status: 'approved', decided_at: timestamp }); + const repeated = reviewing(pending); + + const result = await repeated.service.reviewRequest(pending.id, 'approve', {}); + + expect(pending.update).not.toHaveBeenCalled(); + expect(result.status).toBe('approved'); + }); + + test('a repeated channel approval may not change the grant decision', async () => { + const pending = record({ + request_type: 'channel', status: 'approved', channel_id: null, youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', grant_to_requesting_key: true, }); - expect(pending.update.mock.calls.map(([values]) => values.status)).toEqual([ - 'approved', 'processing', - ]); - expect(result.status).toBe('processing'); - expect(result.job.id).toBe(pending.id); + const repeated = reviewing(pending); + + await expect(repeated.service.reviewRequest( + pending.id, 'approve', { grantToRequestingKey: false } + )).rejects.toMatchObject({ status: 409 }); + }); + + test('channel approval records the grant decision', async () => { + const pending = record({ + request_type: 'channel', channel_id: null, youtube_id: null, + channel_url: 'https://www.youtube.com/@safe', + }); + const approved = reviewing(pending); + + await approved.service.reviewRequest(pending.id, 'approve', { grantToRequestingKey: false }); + + expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ + status: 'approved', grant_to_requesting_key: false, + }), { transaction: approved.transaction }); + expect(approved.channelProvisioner.getChannelInfo).not.toHaveBeenCalled(); }); test('fails approval closed when the requester key is revoked', async () => { const pending = record(); - const hydrated = Object.assign(pending, { - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: false, revoked_at: timestamp, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: null, - }); - const revoked = fixture(); + const revoked = reviewing(pending); revoked.models.ApiKey.findByPk.mockResolvedValue({ id: 4, name: 'External Client', @@ -135,14 +164,9 @@ describe('external request reviewservice', () => { allow_unrated: false, allowed_media_types: ['video'], }); - revoked.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(hydrated); - revoked.models.ChannelVideo.findAll.mockResolvedValue([]); - const result = await revoked.service.reviewVideoRequest(pending.id, 'approve', {}); + const result = await revoked.service.reviewRequest(pending.id, 'approve', {}); - expect(revoked.executor).not.toHaveBeenCalled(); expect(pending.update).toHaveBeenCalledWith(expect.objectContaining({ status: 'failed', active_dedupe_key: null, @@ -153,23 +177,9 @@ describe('external request reviewservice', () => { test('rejects pending requests with a bounded sanitized reason and clears dedupe', async () => { const pending = record(); - const hydrated = Object.assign(pending, { - apiKey: { - id: 4, name: 'External Client', key_prefix: 'abcd1234', role: 'request', - is_active: true, revoked_at: null, - }, - channel: { - id: 8, channel_id: 'UC1234567890123456789012', title: 'Safe Channel', - }, - job: null, - }); - const rejected = fixture(); - rejected.models.ExternalRequest.findOne - .mockResolvedValueOnce(pending) - .mockResolvedValueOnce(hydrated); - rejected.models.ChannelVideo.findAll.mockResolvedValue([]); + const rejected = reviewing(pending); - const result = await rejected.service.reviewVideoRequest( + const result = await rejected.service.reviewRequest( pending.id, 'reject', { reason: ' Not\u0000 eligible \n right now. ' } @@ -185,14 +195,13 @@ describe('external request reviewservice', () => { }); test('rejects non-monotonic review transitions and malformed reasons', async () => { - const completed = fixture(); - completed.models.ExternalRequest.findOne.mockResolvedValue(record({ status: 'completed' })); - await expect(completed.service.reviewVideoRequest( + const completed = reviewing(record({ status: 'completed' })); + await expect(completed.service.reviewRequest( '9b89e5bc-8c90-4e72-b245-270fed2eacc2', 'approve', {} )).rejects.toMatchObject({ status: 409 }); - await expect(completed.service.reviewVideoRequest( + await expect(completed.service.reviewRequest( '9b89e5bc-8c90-4e72-b245-270fed2eacc2', 'reject', { reason: 'x'.repeat(301) } diff --git a/server/modules/externalApi/__tests__/externalRequestService.test.js b/server/modules/externalApi/__tests__/externalRequestService.test.js index a24020110..415e108dc 100644 --- a/server/modules/externalApi/__tests__/externalRequestService.test.js +++ b/server/modules/externalApi/__tests__/externalRequestService.test.js @@ -3,7 +3,7 @@ const { fixture, record, youtubeId } = require('./fixtures/externalRequests'); describe('external request service', () => { test('persists a pending request without executing a download', async () => { - const { service, models, executor, key, transaction, quotaService } = fixture(); + const { service, models, downloader, key, transaction, quotaService } = fixture(); const result = await service.createVideoRequest(key, { youtubeId, channelId: 8 }); expect(result).toMatchObject({ outcome: 'created', @@ -19,7 +19,22 @@ describe('external request service', () => { 'video:request', transaction ); - expect(executor).not.toHaveBeenCalled(); + expect(downloader).not.toHaveBeenCalled(); + }); + + test('records an auto-approved request as approved without running it inline', async () => { + const auto = fixture({ key: { autoApproveVideoRequests: true } }); + const result = await auto.service.createVideoRequest(auto.key, { + youtubeId, channelId: 8, idempotencyKey: 'external-client-1', + }); + expect(auto.models.ExternalRequest.create).toHaveBeenCalledWith(expect.objectContaining({ + status: 'approved', + approval_source: 'auto', + decided_at: expect.any(Date), + idempotency_hash: expect.stringMatching(/^[a-f0-9]{64}$/), + }), { transaction: auto.transaction }); + expect(result).toMatchObject({ outcome: 'created', request: { status: 'approved' } }); + expect(auto.downloader).not.toHaveBeenCalled(); }); test('fails closed for missing grants, unknown media, and ratings above policy', async () => { @@ -143,32 +158,22 @@ describe('external request service', () => { channelId: 8, })).rejects.toMatchObject({ status: 404, message: 'Video not found' }); expect(absent.models.ExternalRequest.create).not.toHaveBeenCalled(); - expect(absent.videoDeleter.deleteVideoById).not.toHaveBeenCalled(); + expect(absent.videoDeleter.deleteVideos).not.toHaveBeenCalled(); }); - test.each(['Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings'])( - 'reconciles a %s downloader job to a retryable failed request', - async (jobStatus) => { - const processing = record({ status: 'processing', job_id: 'job-123' }); - const own = fixture(); - own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); - own.models.Job.findAll.mockResolvedValue([{ id: 'job-123', status: jobStatus }]); - const result = await own.service.listRequests(own.key, {}); - expect(own.models.Job.findAll).toHaveBeenCalledWith({ - where: { id: ['job-123'] }, - attributes: ['id', 'status'], - }); - expect(processing.update).toHaveBeenCalledWith(expect.objectContaining({ - status: 'failed', - active_dedupe_key: null, - message: 'Download did not complete', - })); - expect(result.data[0]).toMatchObject({ - status: 'failed', - message: 'Download did not complete', - }); - } - ); + test('reads report stored status without reconciling or writing', async () => { + const processing = record({ status: 'processing', job_id: 'job-123' }); + const own = fixture(); + own.models.ExternalRequest.findAndCountAll.mockResolvedValue({ rows: [processing], count: 1 }); + own.models.ExternalRequest.findOne.mockResolvedValue(processing); + + await own.service.listRequests(own.key, { status: 'completed' }); + await own.service.getRequest(own.key, processing.id); + + expect(processing.update).not.toHaveBeenCalled(); + expect(own.models.ExternalRequest.update).not.toHaveBeenCalled(); + expect(own.models.ExternalRequest.findAndCountAll).toHaveBeenCalledTimes(1); + }); test('validates scope, body fields, paging, and status allowlists', async () => { const { service, key } = fixture(); @@ -189,9 +194,11 @@ describe('external request service', () => { key, { youtubeId, channelId: 8, resolution: '2160' } )).rejects.toThrow('unsupported'); await expect(service.createVideoRequest(key, { youtubeId })).rejects.toThrow('channelId is required'); + await expect(service.createDeleteVideoRequest( + { ...key, role: 'delete', allowDeleteVideoRequests: true }, { youtubeId } + )).rejects.toThrow('channelId is required'); await expect(service.listRequests(key, { status: 'DROP TABLE' })).rejects.toThrow('status'); await expect(service.listRequests(key, { pageSize: '101' })).rejects.toThrow('pageSize'); }); - }); diff --git a/server/modules/externalApi/__tests__/externalRequestTargets.test.js b/server/modules/externalApi/__tests__/externalRequestTargets.test.js new file mode 100644 index 000000000..e3e23def6 --- /dev/null +++ b/server/modules/externalApi/__tests__/externalRequestTargets.test.js @@ -0,0 +1,58 @@ +const { createTargetValidator } = require('../externalRequestTargets'); + +const youtubeId = 'abcdefghijk'; +const key = { id: 4, maxRatingLevel: 2, allowUnrated: false, allowedMediaTypes: ['video'] }; + +function models(overrides = {}) { + return { + ApiKeyChannelGrant: { findOne: jest.fn().mockResolvedValue({ id: 1 }) }, + Channel: { + findByPk: jest.fn().mockResolvedValue({ + id: 8, channel_id: 'UC1234567890123456789012', enabled: true, terminated_at: null, + default_rating: 'TV-Y', + }), + }, + ChannelVideo: { + findOne: jest.fn().mockResolvedValue({ + youtube_id: youtubeId, media_type: 'video', youtube_removed: false, ignored: false, + }), + }, + Video: { findOne: jest.fn().mockResolvedValue(null) }, + ...overrides, + }; +} + +describe('external request target validation', () => { + test('returns the channel and no downloaded video for an eligible target', async () => { + const validateTarget = createTargetValidator({ models: models() }); + await expect(validateTarget(key, youtubeId, 8)).resolves.toMatchObject({ + channel: { id: 8 }, downloaded: null, + }); + }); + + test('returns the downloaded video row with its id', async () => { + const downloaded = { id: 91, youtubeId, normalized_rating: 'TV-Y', removed: false }; + const validateTarget = createTargetValidator({ + models: models({ Video: { findOne: jest.fn().mockResolvedValue(downloaded) } }), + }); + await expect(validateTarget(key, youtubeId, 8)).resolves.toMatchObject({ downloaded }); + }); + + test('hides an ungranted channel behind the same 404', async () => { + const validateTarget = createTargetValidator({ + models: models({ ApiKeyChannelGrant: { findOne: jest.fn().mockResolvedValue(null) } }), + }); + await expect(validateTarget(key, youtubeId, 8)).rejects.toMatchObject({ + status: 404, message: 'Video not found', + }); + }); + + test('locks the grant and catalog rows when given a transaction', async () => { + const stores = models(); + const transaction = { LOCK: { UPDATE: 'UPDATE' } }; + await createTargetValidator({ models: stores })(key, youtubeId, 8, transaction); + expect(stores.ApiKeyChannelGrant.findOne).toHaveBeenCalledWith(expect.objectContaining({ + transaction, lock: 'UPDATE', + })); + }); +}); diff --git a/server/modules/externalApi/__tests__/fixtures/externalRequests.js b/server/modules/externalApi/__tests__/fixtures/externalRequests.js index 54966022d..35f0cdb9c 100644 --- a/server/modules/externalApi/__tests__/fixtures/externalRequests.js +++ b/server/modules/externalApi/__tests__/fixtures/externalRequests.js @@ -33,6 +33,7 @@ function fixture(overrides = {}) { findOne: jest.fn().mockResolvedValue(null), findByPk: jest.fn().mockResolvedValue(created), findAndCountAll: jest.fn().mockResolvedValue({ rows: [], count: 0 }), + findAll: jest.fn().mockResolvedValue([]), }, ApiKeyChannelGrant: { findOne: jest.fn().mockResolvedValue({ id: 1 }), @@ -82,14 +83,18 @@ function fixture(overrides = {}) { const sequelize = { transaction: jest.fn(async (callback) => callback(transaction)), }; - const executor = jest.fn().mockResolvedValue(undefined); + const downloader = jest.fn(async (_jobData, { jobId }) => jobId); const channelProvisioner = overrides.channelProvisioner || { getChannelInfo: jest.fn().mockResolvedValue({ channel_id: 'UC1234567890123456789012' }), }; const videoDeleter = overrides.videoDeleter || { - deleteVideosByYoutubeIds: jest.fn().mockResolvedValue({ success: true, failed: [] }), - deleteVideoById: jest.fn().mockResolvedValue({ success: true }), + deleteVideos: jest.fn(async (ids) => ({ success: true, deleted: ids, failed: [] })), + }; + const storageGuard = overrides.storageGuard || { + isPausedError: jest.fn((error) => error?.code === 'DOWNLOADS_PAUSED'), }; + const jobLookup = overrides.jobLookup || jest.fn(() => null); + const logger = { info: jest.fn(), warn: jest.fn(), error: jest.fn() }; const quotaService = overrides.quotaService || { reserveWrite: jest.fn(async () => ({ id: 4, @@ -137,13 +142,16 @@ function fixture(overrides = {}) { }; const service = createExternalRequestService({ models, - executor: overrides.executor || executor, + downloader: overrides.downloader || downloader, channelProvisioner, videoDeleter, + storageGuard, + jobLookup, + logger, now: overrides.now || (() => timestamp), sequelize: overrides.sequelize || sequelize, quotaService, - workLimiter: overrides.workLimiter, + workLimiter: overrides.workLimiter || { run: (work) => work() }, }); const key = { id: 4, @@ -158,8 +166,8 @@ function fixture(overrides = {}) { ...overrides.key, }; return { - service, models, executor, key, created, sequelize, transaction, - channelProvisioner, videoDeleter, quotaService, + service, models, downloader: overrides.downloader || downloader, key, created, sequelize, + transaction, channelProvisioner, videoDeleter, quotaService, storageGuard, jobLookup, logger, }; } diff --git a/server/modules/externalApi/externalQuotaService.js b/server/modules/externalApi/externalQuotaService.js index 399549127..fb5acfb97 100644 --- a/server/modules/externalApi/externalQuotaService.js +++ b/server/modules/externalApi/externalQuotaService.js @@ -1,5 +1,4 @@ const { Op } = require('sequelize'); -const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); const { normalizeExternalApiKey } = require('../../middleware/externalApiAuth'); const { hasExternalScope } = require('./externalPermissions'); @@ -38,7 +37,6 @@ function createExternalQuotaService({ now = () => new Date(), } = {}) { const { ApiKey, ExternalRequest, ExternalApiUsageBucket } = models; - const reconcile = createExternalRequestReconciliation({ models, now }); async function reloadAuthorizedKey(keyId, requiredScope, transaction = null) { const record = await ApiKey.findByPk(keyId, { @@ -51,28 +49,13 @@ function createExternalQuotaService({ return key; } + // Approved and processing requests hold an active slot until they finish. async function activeJobs(keyId, transaction = null, excludeRequestId = null) { - const processing = await ExternalRequest.findAll({ - where: { - api_key_id: keyId, - status: 'processing', - request_type: { [Op.in]: ['video', 'delete_video'] }, - }, - attributes: [ - 'id', 'request_type', 'youtube_id', 'job_id', 'status', - 'active_dedupe_key', - ], - ...(transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}), - }); - await reconcile(processing, transaction); return ExternalRequest.count({ where: { api_key_id: keyId, ...(excludeRequestId ? { id: { [Op.ne]: excludeRequestId } } : {}), - [Op.or]: [ - { status: { [Op.in]: ['approved', 'processing'] } }, - { status: 'pending', decided_at: { [Op.ne]: null } }, - ], + status: { [Op.in]: ['approved', 'processing'] }, }, ...(transaction ? { transaction } : {}), }); diff --git a/server/modules/externalApi/externalRequestExecution.js b/server/modules/externalApi/externalRequestExecution.js deleted file mode 100644 index 8b127231f..000000000 --- a/server/modules/externalApi/externalRequestExecution.js +++ /dev/null @@ -1,229 +0,0 @@ -const { Op } = require('sequelize'); -const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); -const { - AUXILIARY_RECOVERY_DELAY_MS, requireCurrentAutoApproval, rethrowWorkLimit, -} = require('./externalRequestPrimitives'); - -function createExternalRequestExecution({ - models, sequelize, now, quotas, workLimiter, executor, channelProvisioner, - videoDeleter, validateTarget, -}) { - const { ExternalRequest, ApiKeyChannelGrant, Channel, Job } = models; - async function claimAuxiliaryRequest(record) { - const timestamp = now(); - let whereStatus; - if (['pending', 'approved'].includes(record.status)) { - whereStatus = record.status; - } else if (record.status === 'processing') { - const updatedAt = new Date(record.updated_at).getTime(); - if (!Number.isFinite(updatedAt) || - timestamp.getTime() - updatedAt < AUXILIARY_RECOVERY_DELAY_MS) { - return false; - } - whereStatus = 'processing'; - } else { - return false; - } - const [claimed] = await ExternalRequest.update( - { status: 'processing', updated_at: timestamp }, - { where: { id: record.id, status: whereStatus, updated_at: record.updated_at } } - ); - if (claimed !== 1) return false; - record.status = 'processing'; - record.updated_at = timestamp; - return true; - } - - async function dispatchAutoApproved(record, key) { - let accepted = false; - try { - await workLimiter.run(() => - sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - key.id, - 'video:request', - record.id, - transaction - ); - const locked = await ExternalRequest.findByPk(record.id, { transaction, lock: transaction.LOCK.UPDATE }); - if (!locked || locked.status !== 'pending' || locked.job_id) return; - requireCurrentAutoApproval(currentKey, 'video'); - const target = await validateTarget( - currentKey, - record.youtube_id, - record.channel_id, - transaction - ); - const existingJob = await Job.findByPk(record.id, { transaction }); - const jobId = existingJob?.id || await executor({ - body: { - urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], - channelId: target.channel.channel_id, - ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, - initiatedBy: { type: 'api_key', name: currentKey.name }, - jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, - // The downloader uses this UUID as the job identity. A retry after a - // crash can therefore observe/reuse the accepted job instead of - // starting the same download twice. - externalRequestId: record.id, - }, - }); - if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); - accepted = true; - await record.update({ - status: 'processing', job_id: jobId || record.id, updated_at: now(), - }, { transaction }); - }) - ); - } catch (error) { - // The downloader persists separately. Leave the request retryable if the - // process loses its database connection after enqueueing the stable job. - if (accepted) throw error; - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: 'pending', updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Download could not be queued', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - - async function provisionChannelRequest( - record, - key, - { grantToRequestingKey, requireAutoApproval = false } = {} - ) { - const retryStatus = record.status === 'processing' ? 'approved' : record.status; - if (!(await claimAuxiliaryRequest(record))) return record; - const shouldGrant = grantToRequestingKey ?? - (record.grant_to_requesting_key !== false); - try { - await workLimiter.run(() => sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - key.id, - 'channel:request', - record.id, - transaction - ); - if (requireAutoApproval) { - requireCurrentAutoApproval(currentKey, 'channel'); - } - const provisioner = channelProvisioner || require('../channel/channelProvisioning'); - const result = await provisioner.getChannelInfo( - record.channel_url, - false, - true, - {}, - { skipTabDetection: true } - ); - const channel = await Channel.findOne({ - where: { - [Op.or]: [ - { channel_id: result.channel_id || result.id }, - { url: record.channel_url }, - ], - }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (!channel || channel.enabled !== true || channel.terminated_at) { - throw new Error('Provisioned channel is unavailable'); - } - if (shouldGrant) { - await ApiKeyChannelGrant.findOrCreate({ - where: { api_key_id: currentKey.id, channel_id: channel.id }, - defaults: { api_key_id: currentKey.id, channel_id: channel.id }, - transaction, - }); - } - const completedAt = now(); - await record.update({ - channel_id: channel.id, - status: 'completed', - active_dedupe_key: null, - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - })); - } catch (error) { - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: retryStatus, updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Channel could not be provisioned', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - - async function executeDeleteRequest(record, { requireAutoApproval = false } = {}) { - const retryStatus = record.status === 'processing' ? 'approved' : record.status; - if (!(await claimAuxiliaryRequest(record))) return record; - try { - await workLimiter.run(() => sequelize.transaction(async (transaction) => { - const currentKey = await quotas.assertExecutionCapacity( - record.api_key_id, - 'video:delete', - record.id, - transaction - ); - if (requireAutoApproval) { - requireCurrentAutoApproval(currentKey, 'delete_video'); - } - const target = await validateTarget( - currentKey, - record.youtube_id, - record.channel_id, - transaction - ); - const result = target.downloaded - ? await videoDeleter.deleteVideoById(target.downloaded.id) - : { success: false, error: 'Video is already removed' }; - const alreadyAbsent = result?.success === false && - /not found|already (?:marked as )?removed/i.test(result?.error || ''); - if (result?.success === false && !alreadyAbsent) { - throw new Error('Deletion failed'); - } - const completedAt = now(); - await record.update({ - status: 'completed', - active_dedupe_key: null, - ...(alreadyAbsent ? { message: 'Video is already deleted' } : {}), - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - })); - } catch (error) { - if (error?.name === 'ExternalWorkLimitError') { - await record.update({ status: retryStatus, updated_at: now() }); - rethrowWorkLimit(error); - } - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Video could not be deleted', - updated_at: failedAt, - }); - rethrowWorkLimit(error); - } - return record; - } - - return { dispatchAutoApproved, provisionChannelRequest, executeDeleteRequest }; -} - -module.exports = { createExternalRequestExecution }; diff --git a/server/modules/externalApi/externalRequestExecutor.js b/server/modules/externalApi/externalRequestExecutor.js new file mode 100644 index 000000000..7faf6e8a0 --- /dev/null +++ b/server/modules/externalApi/externalRequestExecutor.js @@ -0,0 +1,342 @@ +const { Op } = require('sequelize'); +const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); +const { requireCurrentAutoApproval } = require('./externalRequestPrimitives'); +const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); + +const EXTERNAL_REQUEST_JOB_LABEL = `${MANUAL_DOWNLOAD_LABEL} (external request)`; +// The sweep is the fallback that retries held requests, recovers work a +// restart interrupted, and closes downloads whose job-ended event was missed. +const SWEEP_INTERVAL_MS = 60 * 1000; +const SWEEP_BATCH_SIZE = 20; +// A processing row this process is not running is only presumed interrupted +// once it is this old, in case another process claimed it moments ago. +const INTERRUPTED_RECOVERY_DELAY_MS = 5 * 60 * 1000; +const REQUEST_TYPES = ['video', 'channel', 'delete_video']; +const REQUIRED_SCOPES = { + video: 'video:request', + channel: 'channel:request', + delete_video: 'video:delete', +}; +const MESSAGES = { + ineligible: 'Request is no longer eligible', + downloadsPaused: 'Waiting for downloads to resume', + videoBusy: 'Waiting for another download of this video to finish', + queueFailed: 'Download could not be queued', + alreadyDownloaded: 'Video is already downloaded', + alreadyDeleted: 'Video is already deleted', + channelUnavailable: 'Channel is unavailable', + channelFailed: 'Channel could not be provisioned', + deleteFailed: 'Video could not be deleted', +}; + +function isAuthorityError(error) { + return [401, 403, 404].includes(error?.status); +} + +/** + * Runs approved external requests in the background. Creating or approving + * a request only records the decision; this executor picks approved rows up + * outside any lock, re-checks the key's authority, and does the slow work + * (queueing a download, provisioning a channel, deleting a video). + */ +function createExternalRequestExecutor({ + models, sequelize, now, quotas, validateTarget, workLimiter, downloader, + channelProvisioner, videoDeleter, storageGuard, jobLookup = () => null, logger, + sweepIntervalMs = SWEEP_INTERVAL_MS, +}) { + const { ExternalRequest, ApiKeyChannelGrant, Channel, Job } = models; + const { resolveVideoRequests } = createExternalRequestReconciliation({ models, now, jobLookup }); + const running = new Map(); + let sweeping = null; + let sweepAgain = false; + let wakeScheduled = false; + let timer = null; + + const stamp = () => now(); + + async function finish(record, values) { + const timestamp = stamp(); + await record.update({ ...values, active_dedupe_key: null, updated_at: timestamp }); + } + + const complete = (record, message = null) => + finish(record, { status: 'completed', message, completed_at: stamp() }); + const fail = (record, message) => finish(record, { status: 'failed', message }); + // Hand the row back to the sweep, keeping its decision and quota slot. + const hold = (record, message) => + record.update({ status: 'approved', message, updated_at: stamp() }); + + async function claim(record) { + const timestamp = stamp(); + const [claimed] = await ExternalRequest.update( + { status: 'processing', updated_at: timestamp }, + { where: { id: record.id, status: 'approved' } } + ); + if (claimed !== 1) return false; + record.status = 'processing'; + record.updated_at = timestamp; + return true; + } + + async function currentKey(record, transaction = null) { + const key = await quotas.reloadAuthorizedKey( + record.api_key_id, REQUIRED_SCOPES[record.request_type], transaction + ); + if (record.approval_source === 'auto') requireCurrentAutoApproval(key, record.request_type); + return key; + } + + async function executeVideo(record) { + const key = await currentKey(record); + const target = await validateTarget(key, record.youtube_id, record.channel_id); + if (target.downloaded) return complete(record, MESSAGES.alreadyDownloaded); + if (!(await claim(record))) return null; + let jobId; + try { + jobId = await downloader({ + urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], + channelId: target.channel.channel_id, + ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, + initiatedBy: { type: 'api_key', name: key.name }, + jobLabel: EXTERNAL_REQUEST_JOB_LABEL, + }, { jobId: record.id }); + } catch (error) { + if (storageGuard.isPausedError(error)) return hold(record, MESSAGES.downloadsPaused); + logger.warn({ err: error, requestId: record.id }, 'External video request could not be queued'); + return fail(record, MESSAGES.queueFailed); + } + if (typeof jobId !== 'string') return hold(record, MESSAGES.videoBusy); + // The job can end before this write; the job-ended hook matches on the + // request id, so never overwrite a row it already closed. If the write + // itself fails, the job is already queued: leave the row processing for + // recovery to adopt the job rather than failing a running download. + try { + await ExternalRequest.update( + { job_id: jobId, message: null, updated_at: stamp() }, + { where: { id: record.id, status: 'processing' } } + ); + } catch (error) { + logger.warn({ err: error, requestId: record.id, jobId }, 'External request job id was not recorded'); + } + return null; + } + + async function executeChannel(record) { + await currentKey(record); + if (!(await claim(record))) return null; + // Resolve or create the channel without enabling it, so a terminated or + // unsubscribed row is never re-enabled for a request that then fails. + const result = await workLimiter.run(() => channelProvisioner.getChannelInfo( + record.channel_url, false, false, {}, { skipTabDetection: true } + )); + let unavailable = false; + await sequelize.transaction(async (transaction) => { + const key = await currentKey(record, transaction); + const channel = await Channel.findOne({ + where: { + [Op.or]: [ + { channel_id: result.channel_id || result.id }, + { url: record.channel_url }, + ], + }, + transaction, + lock: transaction.LOCK.UPDATE, + }); + if (!channel || channel.terminated_at) { + unavailable = true; + return; + } + if (channel.enabled !== true) await channel.update({ enabled: true }, { transaction }); + if (record.grant_to_requesting_key !== false) { + await ApiKeyChannelGrant.findOrCreate({ + where: { api_key_id: key.id, channel_id: channel.id }, + defaults: { api_key_id: key.id, channel_id: channel.id }, + transaction, + }); + } + const completedAt = stamp(); + await record.update({ + channel_id: channel.id, + status: 'completed', + active_dedupe_key: null, + message: null, + completed_at: completedAt, + updated_at: completedAt, + }, { transaction }); + }); + if (unavailable) return fail(record, MESSAGES.channelUnavailable); + return null; + } + + async function executeDelete(record) { + const key = await currentKey(record); + const target = await validateTarget(key, record.youtube_id, record.channel_id); + if (!target.downloaded) return complete(record, MESSAGES.alreadyDeleted); + if (!(await claim(record))) return null; + // deleteVideos also regenerates the channel .m3u, refreshes playlists, + // and re-evaluates a storage pause, like an in-app deletion. + const result = await workLimiter.run(() => videoDeleter.deleteVideos([target.downloaded.id])); + if (result?.deleted?.includes(target.downloaded.id)) return complete(record); + const reason = result?.failed?.[0]?.error || ''; + if (/not found|already (?:marked as )?removed/i.test(reason)) { + return complete(record, MESSAGES.alreadyDeleted); + } + logger.warn({ requestId: record.id, reason }, 'External delete request failed'); + return fail(record, MESSAGES.deleteFailed); + } + + const EXECUTORS = { video: executeVideo, channel: executeChannel, delete_video: executeDelete }; + + async function execute(record) { + try { + await EXECUTORS[record.request_type](record); + } catch (error) { + if (error?.name === 'ExternalWorkLimitError') { + if (record.status === 'processing') await hold(record, null); + return; + } + if (isAuthorityError(error)) { + await fail(record, MESSAGES.ineligible); + return; + } + logger.warn({ err: error, requestId: record.id }, 'External request execution failed'); + await fail(record, record.request_type === 'channel' + ? MESSAGES.channelFailed + : record.request_type === 'delete_video' ? MESSAGES.deleteFailed : MESSAGES.queueFailed); + } + } + + function launch(record) { + if (running.has(record.id)) return; + const work = execute(record) + .catch((error) => logger.error({ err: error, requestId: record.id }, 'External request could not be updated')) + .finally(() => running.delete(record.id)); + running.set(record.id, work); + } + + async function recoverInterrupted() { + const cutoff = new Date(stamp().getTime() - INTERRUPTED_RECOVERY_DELAY_MS); + const stale = await ExternalRequest.findAll({ + where: { + request_type: { [Op.in]: REQUEST_TYPES }, + status: 'processing', + updated_at: { [Op.lt]: cutoff }, + }, + limit: SWEEP_BATCH_SIZE, + }); + for (const record of stale) { + if (running.has(record.id)) continue; + if (record.request_type === 'video') { + if (record.job_id) continue; + // Interrupted between claiming and recording the job: adopt the job + // if it was created (its id is the request id), otherwise retry. + const job = await Job.findByPk(record.id, { attributes: ['id'] }); + if (job) { + await ExternalRequest.update( + { job_id: job.id, updated_at: stamp() }, + { where: { id: record.id, status: 'processing', job_id: null } } + ); + continue; + } + } + // Channel provisioning and deletion are safe to repeat. + await ExternalRequest.update( + { status: 'approved', updated_at: stamp() }, + { where: { id: record.id, status: 'processing', updated_at: record.updated_at } } + ); + } + } + + async function resolveFinishedDownloads() { + const processing = await ExternalRequest.findAll({ + where: { request_type: 'video', status: 'processing', job_id: { [Op.ne]: null } }, + limit: SWEEP_BATCH_SIZE * 5, + }); + await resolveVideoRequests(processing); + } + + async function runSweep() { + await recoverInterrupted(); + await resolveFinishedDownloads(); + const approved = await ExternalRequest.findAll({ + where: { request_type: { [Op.in]: REQUEST_TYPES }, status: 'approved' }, + order: [['created_at', 'ASC'], ['id', 'ASC']], + limit: SWEEP_BATCH_SIZE, + }); + approved.forEach(launch); + } + + /** Start one sweep, or queue another behind the one already running. */ + function sweep() { + if (sweeping) { + sweepAgain = true; + return sweeping; + } + sweeping = runSweep() + .catch((error) => logger.error({ err: error }, 'External request sweep failed')) + .finally(() => { + sweeping = null; + if (sweepAgain) { + sweepAgain = false; + sweep(); + } + }); + return sweeping; + } + + /** Called after a request is approved, so it runs without waiting for the timer. */ + function wake() { + if (wakeScheduled) return; + wakeScheduled = true; + setImmediate(() => { + wakeScheduled = false; + sweep(); + }); + } + + /** jobModule.onJobEnded listener: close the request a finished job served. */ + async function handleJobEnded({ jobId } = {}) { + // Every download job ends through here; only external ones need a query. + if (jobLookup(jobId)?.jobType !== EXTERNAL_REQUEST_JOB_LABEL) return; + try { + const records = await ExternalRequest.findAll({ + where: { + request_type: 'video', + status: 'processing', + [Op.or]: [{ job_id: jobId }, { id: jobId }], + }, + }); + await resolveVideoRequests(records, { endedJobIds: [jobId] }); + } catch (error) { + logger.error({ err: error, jobId }, 'External request could not be closed after its job ended'); + } + } + + function start() { + if (timer) return; + timer = setInterval(sweep, sweepIntervalMs); + timer.unref?.(); + sweep(); + } + + function stop() { + clearInterval(timer); + timer = null; + } + + /** Run one sweep and wait for the work it started (tests and shutdown). */ + async function runOnce() { + await sweep(); + while (sweeping || running.size > 0) { + await Promise.all([sweeping, ...running.values()]); + } + } + + return { start, stop, wake, handleJobEnded, runOnce }; +} + +module.exports = { + createExternalRequestExecutor, + EXTERNAL_REQUEST_JOB_LABEL, + SWEEP_INTERVAL_MS, +}; diff --git a/server/modules/externalApi/externalRequestPrimitives.js b/server/modules/externalApi/externalRequestPrimitives.js index f0ae67150..303fb1dc4 100644 --- a/server/modules/externalApi/externalRequestPrimitives.js +++ b/server/modules/externalApi/externalRequestPrimitives.js @@ -8,7 +8,6 @@ const REQUEST_STATUSES = [ ]; const ACTIVE_STATUSES = ['pending', 'approved', 'processing']; const VIDEO_ID_PATTERN = /^[A-Za-z0-9_-]{11}$/; -const AUXILIARY_RECOVERY_DELAY_MS = 5 * 60 * 1000; const paginationHelpers = require('./externalPagination'); class RequestError extends Error { @@ -20,16 +19,6 @@ class RequestError extends Error { } } -function rethrowWorkLimit(error) { - if (error?.name === 'ExternalWorkLimitError') { - throw new RequestError( - 'External API work capacity is temporarily unavailable', - 503, - 'work_queue_full' - ); - } -} - function requireCurrentAutoApproval(key, requestType) { const enabled = { video: key.autoApproveVideoRequests, @@ -157,6 +146,7 @@ function adminDto(record, catalogVideo = null) { const job = value.job || null; return { ...dto(record), + approvalSource: value.approval_source || null, requester: key ? { id: key.id, name: key.name, @@ -187,8 +177,8 @@ function adminDto(record, catalogVideo = null) { } module.exports = { - REQUEST_STATUSES, ACTIVE_STATUSES, VIDEO_ID_PATTERN, AUXILIARY_RECOVERY_DELAY_MS, - RequestError, rethrowWorkLimit, requireCurrentAutoApproval, parseInteger, + REQUEST_STATUSES, ACTIVE_STATUSES, VIDEO_ID_PATTERN, + RequestError, requireCurrentAutoApproval, parseInteger, requestPagination, requestPaginationDto, normalizeIdempotencyKey, normalizeChannelUrl, dto, normalizeStoredKey, sanitizeReason, isUniqueConstraintError, adminDto, }; diff --git a/server/modules/externalApi/externalRequestReconciliation.js b/server/modules/externalApi/externalRequestReconciliation.js index 5fe6f788c..c2eb67132 100644 --- a/server/modules/externalApi/externalRequestReconciliation.js +++ b/server/modules/externalApi/externalRequestReconciliation.js @@ -1,61 +1,80 @@ -function createExternalRequestReconciliation({ models, now }) { - const { Video, Job } = models; - async function reconcile(records, transaction = null) { - const processing = records.filter((record) => record.status === 'processing'); - const mediaRequests = processing.filter( - (record) => ['video', 'delete_video'].includes(record.request_type) +const { Op } = require('sequelize'); + +const FINAL_JOB_STATUSES = new Set([ + 'Complete', 'Complete with Warnings', 'Error', 'Failed', 'Terminated', 'Killed', +]); + +/** + * Closes processing video requests whose download job has ended: completed + * when the video is now on disk, failed otherwise. Runs from the job-ended + * hook and the executor's sweep, never from reads, so a request's status does + * not depend on who happens to read it. + */ +function createExternalRequestReconciliation({ models, now, jobLookup = () => null }) { + const { ExternalRequest, Video, Job } = models; + + async function endedJobIdsFor(records, knownEnded) { + const ended = new Set(knownEnded); + const unknown = []; + for (const record of records) { + const jobId = record.job_id || record.id; + if (ended.has(jobId)) continue; + const liveJob = jobLookup(jobId); + if (liveJob) { + if (FINAL_JOB_STATUSES.has(liveJob.status)) ended.add(jobId); + } else { + unknown.push(jobId); + } + } + if (unknown.length === 0) return ended; + const stored = await Job.findAll({ + where: { id: { [Op.in]: [...new Set(unknown)] } }, + attributes: ['id', 'status'], + }); + const storedById = new Map(stored.map((job) => [job.id, job.status])); + for (const jobId of unknown) { + // A job neither running nor stored was lost (for example, with a + // restart before it was saved); its request can never complete. + if (!storedById.has(jobId) || FINAL_JOB_STATUSES.has(storedById.get(jobId))) { + ended.add(jobId); + } + } + return ended; + } + + async function resolveVideoRequests(records, { endedJobIds = [] } = {}) { + const processing = records.filter( + (record) => record.request_type === 'video' && record.status === 'processing' ); - const youtubeIds = [...new Set(mediaRequests.map((record) => record.youtube_id))]; - if (youtubeIds.length === 0) return records; + if (processing.length === 0) return []; const videos = await Video.findAll({ - where: { youtubeId: youtubeIds, removed: false }, + where: { youtubeId: [...new Set(processing.map((record) => record.youtube_id))], removed: false }, attributes: ['youtubeId'], - ...(transaction ? { transaction } : {}), }); - const completed = new Set(videos.map((video) => video.youtubeId)); - const completedAt = now(); - const completedRecords = mediaRequests.filter((record) => - (record.request_type === 'video' && completed.has(record.youtube_id)) || - (record.request_type === 'delete_video' && !completed.has(record.youtube_id)) - ); - await Promise.all(completedRecords.map(async (record) => { - await record.update({ - status: 'completed', - active_dedupe_key: null, - completed_at: completedAt, - updated_at: completedAt, - }, ...(transaction ? [{ transaction }] : [])); - })); - const unresolved = mediaRequests.filter( - (record) => record.request_type === 'video' && - !completed.has(record.youtube_id) && record.job_id + const downloaded = new Set(videos.map((video) => video.youtubeId)); + const ended = await endedJobIdsFor( + processing.filter((record) => !downloaded.has(record.youtube_id)), + endedJobIds ); - if (unresolved.length > 0) { - const jobs = await Job.findAll({ - where: { id: [...new Set(unresolved.map((record) => record.job_id))] }, - attributes: ['id', 'status'], - ...(transaction ? { transaction } : {}), - }); - const terminalFailures = new Set( - jobs - .filter((job) => [ - 'Error', 'Killed', 'Terminated', 'Complete', 'Complete with Warnings', - ].includes(job.status)) - .map((job) => job.id) + const resolved = []; + for (const record of processing) { + const finishedAt = now(); + const values = downloaded.has(record.youtube_id) + ? { status: 'completed', completed_at: finishedAt } + : ended.has(record.job_id || record.id) + ? { status: 'failed', message: 'Download did not complete' } + : null; + if (!values) continue; + const [updated] = await ExternalRequest.update( + { ...values, active_dedupe_key: null, updated_at: finishedAt }, + { where: { id: record.id, status: 'processing' } } ); - await Promise.all(unresolved - .filter((record) => terminalFailures.has(record.job_id)) - .map((record) => record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Download did not complete', - updated_at: completedAt, - }, ...(transaction ? [{ transaction }] : [])))); + if (updated === 1) resolved.push(record.id); } - return records; + return resolved; } - return reconcile; + return { resolveVideoRequests }; } -module.exports = { createExternalRequestReconciliation }; +module.exports = { createExternalRequestReconciliation, FINAL_JOB_STATUSES }; diff --git a/server/modules/externalApi/externalRequestReviewService.js b/server/modules/externalApi/externalRequestReviewService.js index 367977f11..936ec1efd 100644 --- a/server/modules/externalApi/externalRequestReviewService.js +++ b/server/modules/externalApi/externalRequestReviewService.js @@ -1,15 +1,31 @@ -const { Op } = require('sequelize'); -const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); const { hasExternalScope } = require('./externalPermissions'); const { - RequestError, REQUEST_STATUSES, ACTIVE_STATUSES, normalizeStoredKey, + RequestError, REQUEST_STATUSES, normalizeStoredKey, parseInteger, requestPagination, requestPaginationDto, adminDto, - sanitizeReason, rethrowWorkLimit, + sanitizeReason, } = require('./externalRequestPrimitives'); +const REQUIRED_SCOPES = { + video: 'video:request', + channel: 'channel:request', + delete_video: 'video:delete', +}; +const KEY_ATTRIBUTES = [ + 'id', 'name', 'role', 'is_active', 'revoked_at', + 'auto_approve_video_requests', 'auto_approve_channel_requests', + 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', + 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', + 'allowed_media_types', +]; +const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; + +/** + * Administrator review. Approving only records the decision; the request + * executor does the work in the background, so an approve call never waits + * on yt-dlp, a download queue, or the filesystem. + */ function createExternalRequestReviewService({ - models, sequelize, now, quotas, workLimiter, executor, reconcile, - validateTarget, provisionChannelRequest, executeDeleteRequest, + models, sequelize, now, quotas, validateTarget, onApproved = () => {}, }) { const { ExternalRequest, ApiKey, Channel, ChannelVideo, Video, Job } = models; const adminIncludes = () => [ @@ -102,24 +118,13 @@ function createExternalRequestReviewService({ limit: pageSize, offset, }); - await reconcile(result.rows); - const reconciledResult = status - ? await ExternalRequest.findAndCountAll({ - where, - include: adminIncludes(), - distinct: true, - order: [['created_at', 'DESC'], ['id', 'DESC']], - limit: pageSize, - offset, - }) - : result; const requesters = await ApiKey.findAll({ attributes: ['id', 'name', 'key_prefix', 'role', 'is_active', 'revoked_at'], order: [['name', 'ASC'], ['id', 'ASC']], }); return { - data: await adminDtos(reconciledResult.rows), - pagination: requestPaginationDto(page, pageSize, reconciledResult.count), + data: await adminDtos(result.rows), + pagination: requestPaginationDto(page, pageSize, result.count), filterOptions: { requesters: requesters.map((key) => { const value = key.toJSON ? key.toJSON() : key; @@ -137,8 +142,7 @@ function createExternalRequestReviewService({ } async function getAdminRequest(id) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + if (typeof id !== 'string' || !UUID_PATTERN.test(id)) { throw new RequestError('Request not found', 404); } const record = await ExternalRequest.findOne({ @@ -146,330 +150,137 @@ function createExternalRequestReviewService({ include: adminIncludes(), }); if (!record) throw new RequestError('Request not found', 404); - await reconcile([record]); return (await adminDtos([record]))[0]; } - async function reviewVideoRequest(id, action, input = {}) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { - throw new RequestError('Request not found', 404); - } + function validateReviewInput(record, action, input) { if (!input || typeof input !== 'object' || Array.isArray(input)) { throw new RequestError('Request body must be an object'); } if (!['approve', 'reject'].includes(action)) { throw new RequestError('Unsupported review action'); } - const allowedFields = action === 'reject' ? ['reason'] : []; + const allowedFields = action === 'reject' + ? ['reason'] + : (record.request_type === 'channel' ? ['grantToRequestingKey'] : []); if (Object.keys(input).some((field) => !allowedFields.includes(field))) { throw new RequestError('Request body contains unsupported fields'); } - const reason = action === 'reject' ? sanitizeReason(input.reason) : null; - - const current = await ExternalRequest.findByPk(id); - if (!current || current.request_type !== 'video') throw new RequestError('Request not found', 404); - const review = () => sequelize.transaction(async (transaction) => { - await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); - const record = await ExternalRequest.findOne({ - where: { id, request_type: 'video' }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (!record) throw new RequestError('Request not found', 404); - - if (action === 'reject') { - if (record.status !== 'pending') { - throw new RequestError('Only pending requests can be rejected', 409); - } - const rejectedAt = now(); - await record.update({ - status: 'rejected', - active_dedupe_key: null, - message: reason, - decided_at: rejectedAt, - updated_at: rejectedAt, - }, { transaction }); - return; - } - - if (record.status !== 'pending' && - !(record.status === 'approved' && !record.job_id)) { - throw new RequestError('Only pending requests can be approved', 409); - } - - const failApproval = async (message) => { - const failedAt = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message, - decided_at: record.decided_at || failedAt, - updated_at: failedAt, - }, { transaction }); - }; - - const storedKey = await ApiKey.findByPk(record.api_key_id, { - attributes: [ - 'id', 'name', 'role', 'is_active', 'revoked_at', - 'auto_approve_video_requests', 'auto_approve_channel_requests', - 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', - 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', - 'allowed_media_types', - ], - transaction, - lock: transaction.LOCK.UPDATE, - }); - const key = normalizeStoredKey(storedKey); - if (!key || key.isActive !== true || key.revokedAt || - !hasExternalScope(key, 'video:request')) { - await failApproval('Request is no longer eligible'); - return; - } - try { - await quotas.assertExecutionCapacity( - record.api_key_id, - 'video:request', - record.id, - transaction - ); - } catch (error) { - if (error.name !== 'QuotaError') throw error; - await failApproval('Request exceeds the active job limit'); - return; - } - - let target; - try { - target = await validateTarget(key, record.youtube_id, record.channel_id, transaction); - } catch (error) { - if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; - await failApproval('Request is no longer eligible'); - return; - } - if (target.downloaded) { - const completedAt = now(); - await record.update({ - status: 'completed', - active_dedupe_key: null, - message: 'Video is already downloaded', - decided_at: record.decided_at || completedAt, - completed_at: completedAt, - updated_at: completedAt, - }, { transaction }); - return; - } - - const duplicate = await ExternalRequest.findOne({ - where: { - id: { [Op.ne]: record.id }, - active_dedupe_key: `${record.api_key_id}:video:${record.youtube_id}`, - status: { [Op.in]: ACTIVE_STATUSES }, - }, - transaction, - lock: transaction.LOCK.UPDATE, - }); - if (duplicate) { - await failApproval('Another active request already exists'); - return; - } + if (record.request_type === 'channel' && + input.grantToRequestingKey !== undefined && + typeof input.grantToRequestingKey !== 'boolean') { + throw new RequestError('grantToRequestingKey must be a boolean'); + } + return action === 'reject' ? sanitizeReason(input.reason) : null; + } - const approvedAt = now(); - if (record.status === 'pending') { - await record.update({ - status: 'approved', - decided_at: approvedAt, - updated_at: approvedAt, - }, { transaction }); - } - let accepted = false; - try { - const existingJob = await Job.findByPk(record.id, { transaction }); - const jobId = existingJob?.id || await executor({ - body: { - urls: [`https://www.youtube.com/watch?v=${record.youtube_id}`], - channelId: target.channel.channel_id, - ownerChannelMap: { [record.youtube_id]: target.channel.channel_id }, - initiatedBy: { type: 'api_key', name: key.name }, - jobLabel: `${MANUAL_DOWNLOAD_LABEL} (external request)`, - externalRequestId: record.id, - }, - }); - if (jobId && typeof jobId !== 'string') throw new Error('Download was not accepted'); - accepted = true; - const acceptedAt = now(); - await record.update({ - status: 'processing', - job_id: jobId || record.id, - updated_at: acceptedAt, - }, { transaction }); - } catch (error) { - if (accepted) throw error; - await failApproval('Download could not be queued'); - } + // Checks that need the current key and catalog, not slow work. Returns the + // terminal values to record instead of approving, or null to approve. + async function approvalOutcome(record, transaction) { + const scope = REQUIRED_SCOPES[record.request_type]; + const storedKey = await ApiKey.findByPk(record.api_key_id, { + attributes: KEY_ATTRIBUTES, + transaction, + lock: transaction.LOCK.UPDATE, }); + const key = normalizeStoredKey(storedKey); + if (!key || key.isActive !== true || key.revokedAt || !hasExternalScope(key, scope)) { + return { status: 'failed', message: 'Request is no longer eligible' }; + } try { - await (action === 'approve' ? workLimiter.run(review) : review()); + await quotas.assertExecutionCapacity(record.api_key_id, scope, record.id, transaction); } catch (error) { - rethrowWorkLimit(error); - throw error; + if (error.name !== 'QuotaError') throw error; + return { status: 'failed', message: 'Request exceeds the active job limit' }; } - - return getAdminRequest(id); + if (record.request_type === 'channel') return null; + let target; + try { + target = await validateTarget(key, record.youtube_id, record.channel_id, transaction); + } catch (error) { + if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; + return { status: 'failed', message: 'Request is no longer eligible' }; + } + if (record.request_type === 'video' && target.downloaded) { + return { status: 'completed', message: 'Video is already downloaded' }; + } + if (record.request_type === 'delete_video' && !target.downloaded) { + return { status: 'completed', message: 'Video is already deleted' }; + } + return null; } async function reviewRequest(id, action, input = {}) { - if (typeof id !== 'string' || - !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { + if (typeof id !== 'string' || !UUID_PATTERN.test(id)) { throw new RequestError('Request not found', 404); } const current = await ExternalRequest.findByPk(id); if (!current) throw new RequestError('Request not found', 404); - if (current.request_type === 'video') return reviewVideoRequest(id, action, input); - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new RequestError('Request body must be an object'); - } - if (!['approve', 'reject'].includes(action)) { - throw new RequestError('Unsupported review action'); - } - const allowedFields = action === 'reject' - ? ['reason'] - : (current.request_type === 'channel' ? ['grantToRequestingKey'] : []); - if (Object.keys(input).some((field) => !allowedFields.includes(field))) { - throw new RequestError('Request body contains unsupported fields'); - } - if (current.request_type === 'channel' && - input.grantToRequestingKey !== undefined && - typeof input.grantToRequestingKey !== 'boolean') { - throw new RequestError('grantToRequestingKey must be a boolean'); - } - const reason = action === 'reject' ? sanitizeReason(input.reason) : null; - let key; - let claimed; + const reason = validateReviewInput(current, action, input); + let approved = false; await sequelize.transaction(async (transaction) => { + // Key before request, the same lock order as request creation. await ApiKey.findByPk(current.api_key_id, { transaction, lock: transaction.LOCK.UPDATE }); const record = await ExternalRequest.findByPk(id, { transaction, lock: transaction.LOCK.UPDATE, }); if (!record) throw new RequestError('Request not found', 404); - const retryingApproval = action === 'approve' && record.status === 'approved'; - if (record.status !== 'pending' && !retryingApproval) { - throw new RequestError('Only pending requests can be reviewed', 409); - } - if (retryingApproval && input.grantToRequestingKey !== undefined && - input.grantToRequestingKey !== record.grant_to_requesting_key) { - throw new RequestError('The original grant decision cannot be changed on retry', 409); - } - const decisionAt = now(); + const decidedAt = now(); + if (action === 'reject') { + if (record.status !== 'pending') { + throw new RequestError('Only pending requests can be rejected', 409); + } await record.update({ status: 'rejected', active_dedupe_key: null, message: reason, - decided_at: decisionAt, - updated_at: decisionAt, + decided_at: decidedAt, + updated_at: decidedAt, }, { transaction }); return; } - const storedKey = await ApiKey.findByPk(record.api_key_id, { - attributes: [ - 'id', 'name', 'role', 'is_active', 'revoked_at', - 'auto_approve_video_requests', 'auto_approve_channel_requests', - 'auto_approve_delete_requests', 'max_rating_level', 'allow_unrated', - 'allow_video_requests', 'allow_channel_requests', 'allow_delete_video_requests', - 'allowed_media_types', - ], - transaction, - lock: transaction.LOCK.UPDATE, - }); - key = normalizeStoredKey(storedKey); - const requiredScope = record.request_type === 'delete_video' - ? 'video:delete' - : 'channel:request'; - if (!key || key.isActive !== true || key.revokedAt || - !hasExternalScope(key, requiredScope)) { - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); + + // Repeating an approval is a no-op, but it may not change the decision. + if (['approved', 'processing'].includes(record.status)) { + if (record.request_type === 'channel' && input.grantToRequestingKey !== undefined && + input.grantToRequestingKey !== (record.grant_to_requesting_key !== false)) { + throw new RequestError('The original grant decision cannot be changed', 409); + } return; } - try { - await quotas.assertExecutionCapacity( - record.api_key_id, - requiredScope, - record.id, - transaction - ); - } catch (error) { - if (error.name !== 'QuotaError') throw error; + if (record.status !== 'pending') { + throw new RequestError('Only pending requests can be approved', 409); + } + + const outcome = await approvalOutcome(record, transaction); + if (outcome) { await record.update({ - status: 'failed', + ...outcome, active_dedupe_key: null, - message: 'Request exceeds the active job limit', - decided_at: decisionAt, - updated_at: decisionAt, + decided_at: decidedAt, + ...(outcome.status === 'completed' ? { completed_at: decidedAt } : {}), + updated_at: decidedAt, }, { transaction }); return; } - if (record.request_type === 'delete_video') { - try { - const target = await validateTarget( - key, - record.youtube_id, - record.channel_id, - transaction - ); - if (!target.downloaded) { - await record.update({ - status: 'completed', - active_dedupe_key: null, - message: 'Video is already deleted', - decided_at: decisionAt, - completed_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - } catch (error) { - if (!(error instanceof RequestError) && error.name !== 'CatalogError') throw error; - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - decided_at: decisionAt, - updated_at: decisionAt, - }, { transaction }); - return; - } - } await record.update({ status: 'approved', - ...(record.request_type === 'channel' && !retryingApproval + approval_source: 'admin', + ...(record.request_type === 'channel' ? { grant_to_requesting_key: input.grantToRequestingKey !== false } : {}), - decided_at: record.decided_at || decisionAt, - updated_at: decisionAt, + decided_at: decidedAt, + updated_at: decidedAt, }, { transaction }); - claimed = record; + approved = true; }); - if (action === 'approve' && claimed) { - if (claimed.request_type === 'channel') { - await provisionChannelRequest(claimed, key, { - grantToRequestingKey: claimed.grant_to_requesting_key !== false, - }); - } else if (claimed.request_type === 'delete_video') { - await executeDeleteRequest(claimed); - } - } + if (approved) onApproved(); return getAdminRequest(id); } - return { listAdminRequests, getAdminRequest, reviewVideoRequest, reviewRequest }; + return { listAdminRequests, getAdminRequest, reviewRequest }; } module.exports = { createExternalRequestReviewService }; diff --git a/server/modules/externalApi/externalRequestService.js b/server/modules/externalApi/externalRequestService.js index 4edd17434..b62690026 100644 --- a/server/modules/externalApi/externalRequestService.js +++ b/server/modules/externalApi/externalRequestService.js @@ -1,5 +1,4 @@ const crypto = require('crypto'); -const { normalizePolicy, isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); const { hasExternalScope } = require('./externalPermissions'); const { @@ -7,121 +6,63 @@ const { parseInteger, requestPagination, requestPaginationDto, normalizeIdempotencyKey, normalizeChannelUrl, dto, adminDto, sanitizeReason, isUniqueConstraintError, } = require('./externalRequestPrimitives'); -const { createExternalRequestExecution } = require('./externalRequestExecution'); +const { createTargetValidator } = require('./externalRequestTargets'); +const { createExternalRequestExecutor } = require('./externalRequestExecutor'); const { createExternalRequestReviewService } = require('./externalRequestReviewService'); -const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); + +const AUTO_APPROVAL_FIELDS = { + 'video:request': 'autoApproveVideoRequests', + 'channel:request': 'autoApproveChannelRequests', + 'video:delete': 'autoApproveDeleteRequests', +}; function createExternalRequestService({ models = require('../../models'), - executor = (jobData) => require('../downloadModule').doGroupedManualDownloads(jobData), - channelProvisioner = null, + downloader = (jobData, options) => + require('../downloadModule').doSpecificDownloads(jobData, false, options), + channelProvisioner = require('../channel/channelProvisioning'), videoDeleter = require('../videoDeletionModule'), + storageGuard = require('../storageGuard'), + jobLookup = (jobId) => require('../jobModule').getJob(jobId), + logger = require('../../logger'), now = () => new Date(), sequelize = require('../../db').sequelize, quotaService = null, workLimiter = require('./externalWorkLimiter').sharedExternalWorkLimiter, + sweepIntervalMs, } = {}) { - const { - ExternalRequest, ApiKeyChannelGrant, Channel, ChannelVideo, Video, - } = models; + const { ExternalRequest } = models; const quotas = quotaService || require('./externalQuotaService').createExternalQuotaService({ models, sequelize, now }); - const reconcile = createExternalRequestReconciliation({ models, now }); - const { dispatchAutoApproved, provisionChannelRequest, executeDeleteRequest } = - createExternalRequestExecution({ - models, sequelize, now, quotas, workLimiter, executor, channelProvisioner, - videoDeleter, validateTarget, - }); + const validateTarget = createTargetValidator({ models }); + const executor = createExternalRequestExecutor({ + models, sequelize, now, quotas, validateTarget, workLimiter, downloader, + channelProvisioner, videoDeleter, storageGuard, jobLookup, logger, sweepIntervalMs, + }); const reviewService = createExternalRequestReviewService({ - models, sequelize, now, quotas, workLimiter, executor, reconcile, - validateTarget, provisionChannelRequest, executeDeleteRequest, + models, sequelize, now, quotas, validateTarget, onApproved: executor.wake, }); - async function createReservedRequest(keyId, scope, buildValues) { - return sequelize.transaction(async (transaction) => { - const currentKey = await quotas.reserveWrite(keyId, scope, transaction); - const record = await ExternalRequest.create( - buildValues(currentKey), - { transaction } - ); - return { currentKey, record }; + // Charge the write quota and persist the request in one transaction. A key + // with auto-approval for this scope records the decision immediately; the + // executor runs it in the background. + async function createReservedRequest(scope, values) { + const result = await sequelize.transaction(async (transaction) => { + const currentKey = await quotas.reserveWrite(values.api_key_id, scope, transaction); + const autoApproved = currentKey[AUTO_APPROVAL_FIELDS[scope]] === true; + const timestamp = now(); + const record = await ExternalRequest.create({ + ...values, + status: autoApproved ? 'approved' : 'pending', + approval_source: autoApproved ? 'auto' : null, + decided_at: autoApproved ? timestamp : null, + created_at: timestamp, + updated_at: timestamp, + }, { transaction }); + return { record, autoApproved }; }); - } - - async function failAuthorityChange(record) { - const timestamp = now(); - await record.update({ - status: 'failed', - active_dedupe_key: null, - message: 'Request is no longer eligible', - decided_at: record.decided_at || timestamp, - updated_at: timestamp, - }); - return record; - } - - async function validateTarget( - key, - youtubeId, - channelId, - transaction = null, - { lockVideo = true } = {} - ) { - const policy = normalizePolicy(key); - const queryOptions = transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}; - const videoQueryOptions = transaction - ? { transaction, ...(lockVideo ? { lock: transaction.LOCK.UPDATE } : {}) } - : {}; - const grant = await ApiKeyChannelGrant.findOne({ - where: { api_key_id: key.id, channel_id: channelId }, - ...queryOptions, - }); - const channel = await Channel.findByPk(channelId, { - attributes: ['id', 'channel_id', 'enabled', 'terminated_at', 'default_rating'], - ...queryOptions, - }); - if (!grant || !channel || channel.enabled !== true || channel.terminated_at || !channel.channel_id) { - throw new RequestError('Video not found', 404); - } - const cached = await ChannelVideo.findOne({ - where: { youtube_id: youtubeId, channel_id: channel.channel_id }, - attributes: ['youtube_id', 'media_type', 'youtube_removed', 'ignored'], - ...queryOptions, - }); - if (!cached || cached.youtube_removed !== false || cached.ignored !== false) { - throw new RequestError('Video not found', 404); - } - if (!isMediaTypeEligible(policy, cached.media_type)) { - throw new RequestError('Video not found', 404); - } - const storedVideo = await Video.findOne({ - where: { youtubeId }, - // Deletion execution reuses this locked instance in the deleter, so it - // needs the complete row (including id/filePath). Read-only validation - // keeps the narrower projection. - ...(!(transaction && lockVideo) - ? { attributes: ['youtubeId', 'normalized_rating', 'removed'] } - : {}), - ...videoQueryOptions, - }); - if (storedVideo && storedVideo.removed !== true && storedVideo.removed !== false) { - throw new RequestError('Video not found', 404); - } - const downloaded = storedVideo?.removed === false ? storedVideo : null; - if (!isRatingEligible(policy, storedVideo?.normalized_rating, channel.default_rating)) { - throw new RequestError('Video not found', 404); - } - return { channel, downloaded }; - } - - async function findDuplicate(keyId, activeDedupeKey, idempotencyHash, youtubeId, channelId) { - const existing = await findExisting(keyId, activeDedupeKey, idempotencyHash); - if (!existing) return null; - if (existing.request_type !== 'video' || existing.youtube_id !== youtubeId || existing.channel_id !== channelId) { - throw new RequestError('Idempotency key was already used for another target', 409); - } - await reconcile([existing]); - return existing; + if (result.autoApproved) executor.wake(); + return result.record; } async function findExisting(keyId, activeDedupeKey, idempotencyHash) { @@ -139,18 +80,27 @@ function createExternalRequestService({ const existing = await findExisting(keyId, activeDedupeKey, idempotencyHash); if (!existing) return null; if (existing.request_type !== requestType || - existing.youtube_id !== youtubeId || + (existing.youtube_id ?? null) !== youtubeId || (requestType !== 'channel' && existing.channel_id !== channelId) || - existing.channel_url !== channelUrl) { + (existing.channel_url ?? null) !== channelUrl) { throw new RequestError('Idempotency key was already used for another target', 409); } return existing; } - async function createVideoRequest(key, input) { - if (!hasExternalScope(key, 'video:request')) { - throw new RequestError('video:request scope is required', 403); + // Persist a new request, or return the row that won a concurrent create. + async function createOrFindDuplicate(scope, values, duplicateLookup) { + try { + return { outcome: 'created', request: dto(await createReservedRequest(scope, values)) }; + } catch (error) { + if (!isUniqueConstraintError(error)) throw error; + const duplicate = await findTypedDuplicate(duplicateLookup); + if (!duplicate) throw error; + return { outcome: 'duplicate', request: dto(duplicate) }; } + } + + function validateMediaInput(input) { if (!input || typeof input !== 'object' || Array.isArray(input)) { throw new RequestError('Request body must be an object'); } @@ -162,85 +112,43 @@ function createExternalRequestService({ throw new RequestError('youtubeId must be an 11-character YouTube video ID'); } if (input.channelId === undefined) throw new RequestError('channelId is required'); - const channelId = parseInteger(input.channelId, null, 1, Number.MAX_SAFE_INTEGER, 'channelId'); - const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); - const activeDedupeKey = `${key.id}:video:${input.youtubeId}`; - const existing = await findDuplicate( - key.id, activeDedupeKey, idempotencyHash, input.youtubeId, channelId - ); - if (existing) { - // Auto-approved requests are created as pending before enqueue. If the - // process stopped in that boundary, an idempotent client retry resumes - // dispatch with the request UUID as the stable downloader job ID. - if (existing.status === 'pending' && existing.decided_at && !existing.job_id) { - try { - const currentKey = await quotas.assertExecutionCapacity( - key.id, 'video:request', existing.id - ); - const currentTarget = await validateTarget( - currentKey, - input.youtubeId, - channelId - ); - await dispatchAutoApproved(existing, currentKey, currentTarget.channel); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(existing); - } - } - return { outcome: 'duplicate', request: dto(existing) }; - } + return { + youtubeId: input.youtubeId, + channelId: parseInteger(input.channelId, null, 1, Number.MAX_SAFE_INTEGER, 'channelId'), + idempotencyHash: normalizeIdempotencyKey(input.idempotencyKey), + }; + } - const { downloaded } = await validateTarget(key, input.youtubeId, channelId); - if (downloaded) return { outcome: 'already_downloaded', request: null }; - const timestamp = now(); - let currentKey; - let record; - try { - ({ currentKey, record } = await createReservedRequest( - key.id, - 'video:request', - (reservedKey) => ({ - api_key_id: key.id, - channel_id: channelId, - youtube_id: input.youtubeId, - request_type: 'video', - status: 'pending', - active_dedupe_key: activeDedupeKey, - idempotency_hash: idempotencyHash, - decided_at: reservedKey.autoApproveVideoRequests ? timestamp : null, - created_at: timestamp, - updated_at: timestamp, - }) - )); - } catch (error) { - if (!isUniqueConstraintError(error)) { - throw error; - } - const duplicate = await findDuplicate( - key.id, activeDedupeKey, idempotencyHash, input.youtubeId, channelId - ); - if (!duplicate) throw error; - return { outcome: 'duplicate', request: dto(duplicate) }; + async function createVideoRequest(apiKey, input) { + if (!hasExternalScope(apiKey, 'video:request')) { + throw new RequestError('video:request scope is required', 403); } + const { youtubeId, channelId, idempotencyHash } = validateMediaInput(input); + const lookup = { + keyId: apiKey.id, + activeDedupeKey: `${apiKey.id}:video:${youtubeId}`, + idempotencyHash, + requestType: 'video', + youtubeId, + channelId, + }; + const existing = await findTypedDuplicate(lookup); + if (existing) return { outcome: 'duplicate', request: dto(existing) }; - if (currentKey.autoApproveVideoRequests) { - try { - const finalKey = await quotas.assertExecutionCapacity( - key.id, 'video:request', record.id - ); - const finalTarget = await validateTarget(finalKey, input.youtubeId, channelId); - await dispatchAutoApproved(record, finalKey, finalTarget.channel); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(record); - } - } - return { outcome: 'created', request: dto(record) }; + const { downloaded } = await validateTarget(apiKey, youtubeId, channelId); + if (downloaded) return { outcome: 'already_downloaded', request: null }; + return createOrFindDuplicate('video:request', { + api_key_id: apiKey.id, + channel_id: channelId, + youtube_id: youtubeId, + request_type: 'video', + active_dedupe_key: lookup.activeDedupeKey, + idempotency_hash: idempotencyHash, + }, lookup); } - async function createChannelRequest(key, input) { - if (!hasExternalScope(key, 'channel:request')) { + async function createChannelRequest(apiKey, input) { + if (!hasExternalScope(apiKey, 'channel:request')) { throw new RequestError('channel:request scope is required', 403); } if (!input || typeof input !== 'object' || Array.isArray(input)) { @@ -252,216 +160,79 @@ function createExternalRequestService({ const channelUrl = normalizeChannelUrl(input.channelUrl); const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); const targetHash = crypto.createHash('sha256').update(channelUrl).digest('hex').slice(0, 32); - const activeDedupeKey = `${key.id}:channel:${targetHash}`; - const existing = await findTypedDuplicate({ - keyId: key.id, - activeDedupeKey, + const lookup = { + keyId: apiKey.id, + activeDedupeKey: `${apiKey.id}:channel:${targetHash}`, idempotencyHash, requestType: 'channel', channelUrl, - }); - if (existing) { - if (existing.decided_at && - ['pending', 'approved', 'processing'].includes(existing.status)) { - try { - const currentKey = await quotas.assertExecutionCapacity( - key.id, 'channel:request', existing.id - ); - await provisionChannelRequest(existing, currentKey, { - requireAutoApproval: existing.status === 'pending', - }); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(existing); - } - } - return { outcome: 'duplicate', request: dto(existing) }; - } - const timestamp = now(); - let currentKey; - let record; - try { - ({ currentKey, record } = await createReservedRequest( - key.id, - 'channel:request', - (reservedKey) => ({ - api_key_id: key.id, - channel_id: null, - youtube_id: null, - channel_url: channelUrl, - request_type: 'channel', - status: 'pending', - grant_to_requesting_key: true, - active_dedupe_key: activeDedupeKey, - idempotency_hash: idempotencyHash, - decided_at: reservedKey.autoApproveChannelRequests ? timestamp : null, - created_at: timestamp, - updated_at: timestamp, - }) - )); - } catch (error) { - if (!isUniqueConstraintError(error)) throw error; - const duplicate = await findTypedDuplicate({ - keyId: key.id, - activeDedupeKey, - idempotencyHash, - requestType: 'channel', - channelUrl, - }); - if (!duplicate) throw error; - return { outcome: 'duplicate', request: dto(duplicate) }; - } - if (currentKey.autoApproveChannelRequests) { - try { - const finalKey = await quotas.assertExecutionCapacity( - key.id, 'channel:request', record.id - ); - await provisionChannelRequest(record, finalKey, { requireAutoApproval: true }); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(record); - } - } - return { outcome: 'created', request: dto(record) }; + }; + const existing = await findTypedDuplicate(lookup); + if (existing) return { outcome: 'duplicate', request: dto(existing) }; + return createOrFindDuplicate('channel:request', { + api_key_id: apiKey.id, + channel_id: null, + youtube_id: null, + channel_url: channelUrl, + request_type: 'channel', + grant_to_requesting_key: true, + active_dedupe_key: lookup.activeDedupeKey, + idempotency_hash: idempotencyHash, + }, lookup); } - async function createDeleteVideoRequest(key, input) { - if (!hasExternalScope(key, 'video:delete')) { + async function createDeleteVideoRequest(apiKey, input) { + if (!hasExternalScope(apiKey, 'video:delete')) { throw new RequestError('video:delete scope is required', 403); } - if (!input || typeof input !== 'object' || Array.isArray(input)) { - throw new RequestError('Request body must be an object'); - } - const supported = ['youtubeId', 'channelId', 'idempotencyKey']; - if (Object.keys(input).some((name) => !supported.includes(name))) { - throw new RequestError('Request body contains unsupported fields'); - } - if (typeof input.youtubeId !== 'string' || !VIDEO_ID_PATTERN.test(input.youtubeId)) { - throw new RequestError('youtubeId must be an 11-character YouTube video ID'); - } - const channelId = parseInteger( - input.channelId, - null, - 1, - Number.MAX_SAFE_INTEGER, - 'channelId' - ); - const idempotencyHash = normalizeIdempotencyKey(input.idempotencyKey); - const activeDedupeKey = `${key.id}:delete_video:${input.youtubeId}`; - const existing = await findTypedDuplicate({ - keyId: key.id, - activeDedupeKey, + const { youtubeId, channelId, idempotencyHash } = validateMediaInput(input); + const lookup = { + keyId: apiKey.id, + activeDedupeKey: `${apiKey.id}:delete_video:${youtubeId}`, idempotencyHash, requestType: 'delete_video', - youtubeId: input.youtubeId, + youtubeId, channelId, - }); - if (existing) { - if (existing.decided_at && - ['pending', 'approved', 'processing'].includes(existing.status)) { - try { - const currentKey = await quotas.assertExecutionCapacity( - key.id, 'video:delete', existing.id - ); - await validateTarget(currentKey, input.youtubeId, channelId); - await executeDeleteRequest(existing, { - requireAutoApproval: existing.status === 'pending', - }); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(existing); - } - } - return { outcome: 'duplicate', request: dto(existing) }; - } - const target = await validateTarget(key, input.youtubeId, channelId); + }; + const existing = await findTypedDuplicate(lookup); + if (existing) return { outcome: 'duplicate', request: dto(existing) }; + const target = await validateTarget(apiKey, youtubeId, channelId); if (!target.downloaded) throw new RequestError('Video not found', 404); - const timestamp = now(); - let currentKey; - let record; - try { - ({ currentKey, record } = await createReservedRequest( - key.id, - 'video:delete', - (reservedKey) => ({ - api_key_id: key.id, - channel_id: channelId, - youtube_id: input.youtubeId, - request_type: 'delete_video', - status: 'pending', - active_dedupe_key: activeDedupeKey, - idempotency_hash: idempotencyHash, - message: null, - decided_at: reservedKey.autoApproveDeleteRequests ? timestamp : null, - completed_at: null, - created_at: timestamp, - updated_at: timestamp, - }) - )); - } catch (error) { - if (!isUniqueConstraintError(error)) throw error; - const duplicate = await findTypedDuplicate({ - keyId: key.id, - activeDedupeKey, - idempotencyHash, - requestType: 'delete_video', - youtubeId: input.youtubeId, - channelId, - }); - if (!duplicate) throw error; - return { outcome: 'duplicate', request: dto(duplicate) }; - } - if (currentKey.autoApproveDeleteRequests) { - try { - const finalKey = await quotas.assertExecutionCapacity( - key.id, 'video:delete', record.id - ); - await validateTarget(finalKey, input.youtubeId, channelId); - await executeDeleteRequest(record, { requireAutoApproval: true }); - } catch (error) { - if (![403, 404].includes(error?.status)) throw error; - await failAuthorityChange(record); - } - } - return { outcome: 'created', request: dto(record) }; + return createOrFindDuplicate('video:delete', { + api_key_id: apiKey.id, + channel_id: channelId, + youtube_id: youtubeId, + request_type: 'delete_video', + active_dedupe_key: lookup.activeDedupeKey, + idempotency_hash: idempotencyHash, + }, lookup); } - async function listRequests(key, query = {}) { + async function listRequests(apiKey, query = {}) { const { page, pageSize, offset } = requestPagination(query); const status = query.status; if (status !== undefined && !REQUEST_STATUSES.includes(status)) { throw new RequestError(`status must be one of: ${REQUEST_STATUSES.join(', ')}`); } - const where = { api_key_id: key.id, ...(status ? { status } : {}) }; const result = await ExternalRequest.findAndCountAll({ - where, + where: { api_key_id: apiKey.id, ...(status ? { status } : {}) }, order: [['created_at', 'DESC'], ['id', 'DESC']], limit: pageSize, offset, }); - await reconcile(result.rows); - const reconciledResult = status - ? await ExternalRequest.findAndCountAll({ - where, - order: [['created_at', 'DESC'], ['id', 'DESC']], - limit: pageSize, - offset, - }) - : result; return { - data: reconciledResult.rows.map(dto), - pagination: requestPaginationDto(page, pageSize, reconciledResult.count), + data: result.rows.map(dto), + pagination: requestPaginationDto(page, pageSize, result.count), }; } - async function getRequest(key, id) { + async function getRequest(apiKey, id) { if (typeof id !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(id)) { throw new RequestError('Request not found', 404); } - const record = await ExternalRequest.findOne({ where: { id, api_key_id: key.id } }); + const record = await ExternalRequest.findOne({ where: { id, api_key_id: apiKey.id } }); if (!record) throw new RequestError('Request not found', 404); - await reconcile([record]); return dto(record); } @@ -472,6 +243,10 @@ function createExternalRequestService({ listRequests, getRequest, ...reviewService, + startExecutor: executor.start, + stopExecutor: executor.stop, + handleJobEnded: executor.handleJobEnded, + runExecutorOnce: executor.runOnce, }; } diff --git a/server/modules/externalApi/externalRequestTargets.js b/server/modules/externalApi/externalRequestTargets.js new file mode 100644 index 000000000..2eb192681 --- /dev/null +++ b/server/modules/externalApi/externalRequestTargets.js @@ -0,0 +1,54 @@ +const { normalizePolicy, isMediaTypeEligible, isRatingEligible } = require('./externalEligibility'); +const { RequestError } = require('./externalRequestPrimitives'); + +/** + * Checks that a key may act on a cached video: the channel is granted, + * enabled, and not terminated, and the video passes the key's media and + * rating policy. Every failure is the same 404 so a key cannot probe for + * videos outside its grants. + */ +function createTargetValidator({ models }) { + const { ApiKeyChannelGrant, Channel, ChannelVideo, Video } = models; + + return async function validateTarget(key, youtubeId, channelId, transaction = null) { + const policy = normalizePolicy(key); + const queryOptions = transaction ? { transaction, lock: transaction.LOCK.UPDATE } : {}; + const grant = await ApiKeyChannelGrant.findOne({ + where: { api_key_id: key.id, channel_id: channelId }, + ...queryOptions, + }); + const channel = await Channel.findByPk(channelId, { + attributes: ['id', 'channel_id', 'enabled', 'terminated_at', 'default_rating'], + ...queryOptions, + }); + if (!grant || !channel || channel.enabled !== true || channel.terminated_at || !channel.channel_id) { + throw new RequestError('Video not found', 404); + } + const cached = await ChannelVideo.findOne({ + where: { youtube_id: youtubeId, channel_id: channel.channel_id }, + attributes: ['youtube_id', 'media_type', 'youtube_removed', 'ignored'], + ...queryOptions, + }); + if (!cached || cached.youtube_removed !== false || cached.ignored !== false) { + throw new RequestError('Video not found', 404); + } + if (!isMediaTypeEligible(policy, cached.media_type)) { + throw new RequestError('Video not found', 404); + } + const storedVideo = await Video.findOne({ + where: { youtubeId }, + attributes: ['id', 'youtubeId', 'normalized_rating', 'removed'], + ...(transaction ? { transaction } : {}), + }); + if (storedVideo && storedVideo.removed !== true && storedVideo.removed !== false) { + throw new RequestError('Video not found', 404); + } + const downloaded = storedVideo?.removed === false ? storedVideo : null; + if (!isRatingEligible(policy, storedVideo?.normalized_rating, channel.default_rating)) { + throw new RequestError('Video not found', 404); + } + return { channel, downloaded }; + }; +} + +module.exports = { createTargetValidator }; diff --git a/server/modules/jobModule.js b/server/modules/jobModule.js index 40bfc4a29..80f266a0e 100644 --- a/server/modules/jobModule.js +++ b/server/modules/jobModule.js @@ -1,4 +1,4 @@ -const { v4: uuidv4 } = require('uuid'); +const { v4: uuidv4, validate: uuidValidate } = require('uuid'); const fs = require('fs'); const fsPromises = fs.promises; const path = require('path'); @@ -1217,6 +1217,9 @@ class JobModule { async addJob(job, preferredJobId = null) { // Trusted internal callers may supply a stable UUID so enqueueing can be // retried safely across an external-request persistence boundary. + if (preferredJobId != null && !uuidValidate(preferredJobId)) { + throw new Error('Preferred job id must be a UUID'); + } const jobId = preferredJobId || uuidv4(); if (isSpecificUrlDownloadJob(job.jobType) && Array.isArray(job.data?.urls)) { const admission = videoActivity.claim(jobId, job.data.urls); diff --git a/server/routes/__tests__/externalRequests.test.js b/server/routes/__tests__/externalRequests.test.js index 57b1c1f01..609f6eb51 100644 --- a/server/routes/__tests__/externalRequests.test.js +++ b/server/routes/__tests__/externalRequests.test.js @@ -30,7 +30,7 @@ function service() { filterOptions: { requesters: [] }, }), getAdminRequest: jest.fn().mockResolvedValue({ id: requestId, status: 'pending' }), - reviewVideoRequest: jest.fn().mockResolvedValue({ id: requestId, status: 'processing' }), + reviewRequest: jest.fn().mockResolvedValue({ id: requestId, status: 'processing' }), }; } @@ -88,17 +88,17 @@ describe('external request administrator routes', () => { .expect(200); expect(reviewLimiter).toHaveBeenCalledTimes(2); - expect(requestService.reviewVideoRequest).toHaveBeenNthCalledWith( + expect(requestService.reviewRequest).toHaveBeenNthCalledWith( 1, requestId, 'approve', {} ); - expect(requestService.reviewVideoRequest).toHaveBeenNthCalledWith( + expect(requestService.reviewRequest).toHaveBeenNthCalledWith( 2, requestId, 'reject', { reason: 'Not approved' } ); }); test('returns bounded service errors without exposing internal failures', async () => { const requestService = service(); - requestService.reviewVideoRequest.mockRejectedValueOnce( + requestService.reviewRequest.mockRejectedValueOnce( Object.assign(new Error('Only pending requests can be approved'), { name: 'RequestError', status: 409, diff --git a/server/routes/externalRequests.js b/server/routes/externalRequests.js index 4f7dda8c7..18979ae71 100644 --- a/server/routes/externalRequests.js +++ b/server/routes/externalRequests.js @@ -93,6 +93,10 @@ function createExternalRequestReviewRoutes({ * /api/external-requests/{id}/approve: * post: * summary: Revalidate and approve a pending external request + * description: > + * Records the approval and returns immediately with status approved; + * the request then runs in the background. Approving an already + * approved or processing request is a no-op. * tags: [External Requests] * security: [{ SessionAuth: [] }] * parameters: @@ -114,7 +118,7 @@ function createExternalRequestReviewRoutes({ * 200: { description: Updated request } * 403: { description: API keys cannot review external requests } * 404: { description: Request not found } - * 409: { description: Request is not pending } + * 409: { description: Request already finished, or a retried approval changes the grant decision } * 429: { description: Review action rate limit exceeded } */ router.post( @@ -124,9 +128,7 @@ function createExternalRequestReviewRoutes({ reviewLimiter, async (req, res) => { try { - const service = requests(); - const review = service.reviewRequest || service.reviewVideoRequest; - return res.json(await review.call(service, req.params.id, 'approve', req.body)); + return res.json(await requests().reviewRequest(req.params.id, 'approve', req.body)); } catch (error) { return sendError(req, res, error); } @@ -169,9 +171,7 @@ function createExternalRequestReviewRoutes({ reviewLimiter, async (req, res) => { try { - const service = requests(); - const review = service.reviewRequest || service.reviewVideoRequest; - return res.json(await review.call(service, req.params.id, 'reject', req.body)); + return res.json(await requests().reviewRequest(req.params.id, 'reject', req.body)); } catch (error) { return sendError(req, res, error); } diff --git a/server/routes/index.js b/server/routes/index.js index 2d309e16d..4d38f57a7 100644 --- a/server/routes/index.js +++ b/server/routes/index.js @@ -62,7 +62,9 @@ const { isExternalApiEnabled } = require('../modules/externalApi/externalApiConf * @param {Object} deps - Dependencies to inject into route modules */ function registerRoutes(app, deps) { - const externalRequestService = createExternalRequestService(); + // server.js owns the request service so it can wire the background + // executor to job completion; standalone callers get their own. + const externalRequestService = deps.externalRequestService || createExternalRequestService(); const externalQuotaService = createExternalQuotaService(); const { verifyToken, diff --git a/server/server.js b/server/server.js index 1b7d75cf9..33c7f70a3 100644 --- a/server/server.js +++ b/server/server.js @@ -17,7 +17,7 @@ const { isAuthConfigured } = require('./modules/authState'); const STARTUP_TAB_COUNT_REFRESH_DELAY_MS = 2 * 60 * 1000; const { createExternalApiAuth } = require('./middleware/externalApiAuth'); const { sendExternalError } = require('./modules/externalApi/externalApiResponse'); -const { isExternalApiPath } = require('./modules/externalApi/externalApiConfig'); +const { isExternalApiEnabled, isExternalApiPath } = require('./modules/externalApi/externalApiConfig'); const app = express(); app.set('trust proxy', parseTrustProxySetting(process.env.TRUST_PROXY)); if (process.env.TRUST_PROXY === undefined || process.env.TRUST_PROXY === '') { @@ -285,6 +285,7 @@ const initialize = async () => { const downloadRunTracker = require('./modules/download/downloadRunTracker'); const { Channel } = require('./models'); const { registerRoutes } = require('./routes'); + const { createExternalRequestService } = require('./modules/externalApi/externalRequestService'); const scheduledTaskRuns = require('./modules/scheduledTaskRuns'); const scheduledTaskManager = require('./modules/scheduledTaskManager'); const storageGuard = require('./modules/storageGuard'); @@ -301,6 +302,12 @@ const initialize = async () => { // Every final status, whichever path set it, so a run never waits on a // job that ended without reporting its results. jobModule.onJobEnded((event) => downloadRunTracker.handleJobEnded(event)); + // External API requests run in the background: approvals wake the + // executor, its sweep retries held work, and finished downloads close + // their requests through the job-ended hook. + const externalRequestService = createExternalRequestService(); + jobModule.onJobEnded((event) => externalRequestService.handleJobEnded(event)); + if (isExternalApiEnabled()) externalRequestService.startExecutor(); // Cache yt-dlp version once during startup to keep the version endpoint fast refreshYtDlpVersionCache(); @@ -821,6 +828,7 @@ const initialize = async () => { // Register all modular routes registerRoutes(app, { + externalRequestService, verifyToken, loginLimiter, setupCreateAuthLimiter, diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index d2d647fc2..167ae5e2f 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -11,6 +11,7 @@ const { normalizeExternalApiKey } = require('../middleware/externalApiAuth'); const { createExternalRequestService } = require('../modules/externalApi/externalRequestService'); const { createExternalQuotaService } = require('../modules/externalApi/externalQuotaService'); const { createExternalWorkLimiter } = require('../modules/externalApi/externalWorkLimiter'); +const { EXTERNAL_REQUEST_JOB_LABEL } = require('../modules/externalApi/externalRequestExecutor'); const { isSpecificUrlDownloadJob } = require('../modules/download/jobTypes'); const catalog = require('../modules/externalApi/externalCatalogService'); const configModule = require('../modules/configModule'); @@ -23,26 +24,33 @@ let channel; let keyRecord; let key; +const services = []; + function service(options = {}) { - return createExternalRequestService({ + const api = createExternalRequestService({ models, sequelize, now: () => timestamp, - executor: async ({ body }) => { - expect(isSpecificUrlDownloadJob(body.jobLabel)).toBe(true); + downloader: async (jobData, { jobId }) => { + expect(isSpecificUrlDownloadJob(jobData.jobLabel)).toBe(true); await models.Job.findOrCreate({ - where: { id: body.externalRequestId }, - defaults: { status: 'Pending', timeCreated: timestamp, timeInitiated: timestamp, jobType: 'Manual Download' }, + where: { id: jobId }, + defaults: { status: 'Pending', timeCreated: timestamp, timeInitiated: timestamp, jobType: jobData.jobLabel }, }); - return body.externalRequestId; + return jobId; }, channelProvisioner: { getChannelInfo: async () => ({ channel_id: youtubeChannelId }) }, videoDeleter: { - deleteVideoById: async (_id, { video, transaction }) => { - await video.update({ removed: true }, { transaction }); - return { success: true }; + deleteVideos: async (ids) => { + await models.Video.update({ removed: true }, { where: { id: ids } }); + return { success: true, deleted: ids, failed: [] }; }, }, + storageGuard: { isPausedError: (error) => error?.code === 'DOWNLOADS_PAUSED' }, + jobLookup: () => null, + logger: { warn: () => {}, error: () => {} }, ...options, }); + services.push(api); + return api; } async function policy(values) { @@ -86,6 +94,12 @@ beforeEach(async () => { key = normalizeExternalApiKey(keyRecord); }); +afterEach(async () => { + // Approvals wake the executor in the background; let that work settle + // before the next test clears the tables. + await Promise.all(services.splice(0).map((api) => api.runExecutorOnce())); +}); + afterAll(async () => { configModule.stopWatchingConfig(); await sequelize.close(); @@ -130,9 +144,10 @@ test('completed channel requests replay the original idempotency key', async () const api = service(); const input = { channelUrl: 'https://www.youtube.com/@synthetic', idempotencyKey: 'channel-replay' }; const first = await api.createChannelRequest(key, input); - expect(first.request.status).toBe('completed'); + expect(first.request.status).toBe('approved'); + await api.runExecutorOnce(); const replay = await api.createChannelRequest(key, input); - expect(replay.request.id).toBe(first.request.id); + expect(replay.request).toMatchObject({ id: first.request.id, status: 'completed' }); }); test('completed deletion requests replay after the video is removed', async () => { @@ -141,27 +156,76 @@ test('completed deletion requests replay after the video is removed', async () = const api = service(); const input = { youtubeId, channelId: channel.id, idempotencyKey: 'delete-replay' }; const first = await api.createDeleteVideoRequest(key, input); - expect(first.request.status).toBe('completed'); + await api.runExecutorOnce(); + expect((await models.Video.findOne({ where: { youtubeId } })).removed).toBe(true); const replay = await api.createDeleteVideoRequest(key, input); - expect(replay.request.id).toBe(first.request.id); + expect(replay.request).toMatchObject({ id: first.request.id, status: 'completed' }); }); -test('a saturated queue can retry the same accepted request without spending another write', async () => { - await policy({ auto_approve_video_requests: true }); +test('channel provisioning holds no lock that blocks key usage updates', async () => { + await policy({ auto_approve_channel_requests: true }); + let release; + let started; + const provisioning = new Promise(resolve => { started = resolve; }); + const provisioner = { getChannelInfo: async () => { + started(); + await new Promise(resolve => { release = resolve; }); + return { channel_id: youtubeChannelId }; + } }; + const api = service({ channelProvisioner: provisioner }); + await api.createChannelRequest(key, { channelUrl: 'https://www.youtube.com/@synthetic' }); + const run = api.runExecutorOnce(); + await provisioning; + try { + // The usage-timestamp UPDATE every external request makes on its key row + // must not wait behind yt-dlp. + await sequelize.transaction(async (transaction) => { + await sequelize.query('SET SESSION innodb_lock_wait_timeout = 2', { transaction }); + await models.ApiKey.update({ last_used_at: timestamp }, { where: { id: key.id }, transaction }); + }); + } finally { + release(); + await run; + } + expect((await models.ExternalRequest.findOne()).status).toBe('completed'); +}); + +test('a full work queue holds an approved request without spending another write', async () => { + await policy({ auto_approve_channel_requests: true }); const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 0 }); let release; const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); const api = service({ workLimiter: limiter }); - const input = { youtubeId, channelId: channel.id, idempotencyKey: 'retry-queue' }; + const created = await api.createChannelRequest(key, { channelUrl: 'https://www.youtube.com/@synthetic' }); try { - await expect(api.createVideoRequest(key, input)).rejects.toMatchObject({ status: 503 }); + await api.runExecutorOnce(); + expect((await api.getRequest(key, created.request.id)).status).toBe('approved'); } finally { release(); await blocker; } - const retried = await api.createVideoRequest(key, input); - expect(retried.request.status).toBe('processing'); - expect(await models.Job.count()).toBe(1); + await api.runExecutorOnce(); + expect((await api.getRequest(key, created.request.id)).status).toBe('completed'); expect((await models.ExternalApiUsageBucket.findAll()).map(bucket => bucket.accepted_writes)).toEqual([1, 1]); }); +test('paused downloads hold an approved video request until they resume', async () => { + await policy({ auto_approve_video_requests: true }); + let paused = true; + const api = service({ + downloader: async (_jobData, { jobId }) => { + if (paused) throw Object.assign(new Error('Downloads are paused'), { code: 'DOWNLOADS_PAUSED' }); + await models.Job.create({ id: jobId, status: 'Pending', jobType: 'External', timeCreated: timestamp, timeInitiated: timestamp }); + return jobId; + }, + }); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + await api.runExecutorOnce(); + expect(await api.getRequest(key, created.request.id)).toMatchObject({ + status: 'approved', message: 'Waiting for downloads to resume', + }); + paused = false; + await api.runExecutorOnce(); + expect((await api.getRequest(key, created.request.id)).status).toBe('processing'); +}); + test('quota windows roll over at UTC midnight without carrying previous writes', async () => { const before = createExternalQuotaService({ models, sequelize, now: () => new Date('2026-10-07T23:59:59Z') }); const after = createExternalQuotaService({ models, sequelize, now: () => new Date('2026-10-08T00:00:00Z') }); @@ -169,84 +233,73 @@ test('quota windows roll over at UTC midnight without carrying previous writes', expect((await after.status(key)).remaining).toMatchObject({ hourlyWrites: 30, dailyWrites: 200 }); }); -test('two stale recovery readers cannot both reclaim a channel request', async () => { +test('two executors cannot both run the same approved request', async () => { await policy({ auto_approve_channel_requests: true }); - const api = service(); - await policy({ auto_approve_channel_requests: false }); - const input = { channelUrl: 'https://www.youtube.com/@synthetic', idempotencyKey: 'stale-recovery' }; - const created = await api.createChannelRequest(key, input); - await models.ExternalRequest.update({ status: 'processing', decided_at: timestamp, - updated_at: new Date(timestamp.getTime() - 600000) }, { where: { id: created.request.id } }); - await policy({ auto_approve_channel_requests: true }); - let readers = 0; - let release; - const bothRead = new Promise(resolve => { release = resolve; }); - models.ExternalRequest.addHook('afterFind', 'coordinateRecovery', async record => { - if (record?.id !== created.request.id || record.updated_at >= timestamp) return; - readers += 1; - if (readers === 2) release(); - await bothRead; - }); const provisioner = { getChannelInfo: jest.fn(async () => ({ channel_id: youtubeChannelId })) }; - try { - await Promise.all([service({ channelProvisioner: provisioner }).createChannelRequest(key, input), - service({ channelProvisioner: provisioner }).createChannelRequest(key, input)]); - expect(provisioner.getChannelInfo).toHaveBeenCalledTimes(1); - } finally { - models.ExternalRequest.removeHook('afterFind', 'coordinateRecovery'); - } + await service().createChannelRequest(key, { channelUrl: 'https://www.youtube.com/@synthetic' }); + await Promise.all([service({ channelProvisioner: provisioner }).runExecutorOnce(), + service({ channelProvisioner: provisioner }).runExecutorOnce()]); + expect(provisioner.getChannelInfo).toHaveBeenCalledTimes(1); }); -test('reconciliation keeps a downloaded video completed when its job is Complete', async () => { +test('a finished download job closes its request through the job-ended hook', async () => { await policy({ auto_approve_video_requests: true }); - const api = service(); + const api = service({ + jobLookup: () => ({ jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'Complete' }), + }); const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + await api.runExecutorOnce(); await downloadedVideo(); - await models.Job.update({ status: 'Complete' }, { where: { id: created.request.id } }); - expect((await api.getRequest(key, created.request.id)).status).toBe('completed'); + await api.handleJobEnded({ jobId: created.request.id }); + expect((await api.listRequests(key, { status: 'completed' })).data.map(row => row.id)) + .toEqual([created.request.id]); }); -test('two administrators cannot enqueue the same approval twice', async () => { +test('two administrators approving at once record one approval and one job', async () => { const api = service(); const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); - const results = await Promise.allSettled([api.reviewRequest(created.request.id, 'approve'), + const results = await Promise.all([api.reviewRequest(created.request.id, 'approve'), api.reviewRequest(created.request.id, 'approve')]); - expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); - expect(results.find(result => result.status === 'rejected').reason.status).toBe(409); + expect(results.map(result => result.status)).toEqual(['approved', 'approved']); + await api.runExecutorOnce(); expect(await models.Job.count()).toBe(1); }); -test('grant revocation while an approval waits for a work slot prevents enqueue', async () => { - const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 1 }); - let release; - const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); - const api = service({ workLimiter: limiter }); +test('grant revocation after approval prevents enqueue', async () => { + const api = service(); const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); - const approval = api.reviewRequest(created.request.id, 'approve'); + await api.reviewRequest(created.request.id, 'approve'); await models.ApiKeyChannelGrant.destroy({ where: { api_key_id: key.id } }); - release(); - await blocker; - expect((await approval).status).toBe('failed'); + await api.runExecutorOnce(); + expect((await api.getAdminRequest(created.request.id)).status).toBe('failed'); expect(await models.Job.count()).toBe(0); }); -test('retry after enqueue but before request persistence reuses the durable job', async () => { +test('a lost job-id write is recovered by adopting the queued job', async () => { await policy({ auto_approve_video_requests: true }); - const api = service(); - const input = { youtubeId, channelId: channel.id, idempotencyKey: 'enqueue-crash' }; - models.ExternalRequest.addHook('beforeUpdate', 'simulateConnectionLoss', row => { - if (row.status === 'processing') throw new Error('Synthetic connection loss after enqueue'); + let downloads = 0; + const api = service({ + downloader: async (_jobData, { jobId }) => { + downloads += 1; + await models.Job.create({ id: jobId, status: 'Pending', jobType: 'External', timeCreated: timestamp, timeInitiated: timestamp }); + return jobId; + }, + }); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + models.ExternalRequest.addHook('beforeBulkUpdate', 'simulateConnectionLoss', options => { + if (options.attributes.job_id) throw new Error('Synthetic connection loss after enqueue'); }); try { - await expect(api.createVideoRequest(key, input)).rejects.toThrow('Synthetic connection loss'); + await api.runExecutorOnce(); } finally { - models.ExternalRequest.removeHook('beforeUpdate', 'simulateConnectionLoss'); + models.ExternalRequest.removeHook('beforeBulkUpdate', 'simulateConnectionLoss'); } - const executor = jest.fn(); - const result = await service({ executor }).createVideoRequest(key, input); - expect(result.request.status).toBe('processing'); - expect(executor).not.toHaveBeenCalled(); - expect(await models.Job.count()).toBe(1); + expect((await models.ExternalRequest.findByPk(created.request.id)).job_id).toBeNull(); + await models.ExternalRequest.update({ updated_at: new Date(timestamp.getTime() - 600000) }, + { where: { id: created.request.id } }); + await api.runExecutorOnce(); + expect((await models.ExternalRequest.findByPk(created.request.id)).job_id).toBe(created.request.id); + expect(downloads).toBe(1); }); test('idempotency keys cannot replay a deletion as a video request', async () => { @@ -280,43 +333,43 @@ test('runtime indexes preserve equivalent indexes and survive rollback and parti } }); -test('manual approval retries reuse the durable job after request persistence fails', async () => { - const created = await service().createVideoRequest(key, { youtubeId, channelId: channel.id }); - models.ExternalRequest.addHook('beforeUpdate', 'simulateApprovalLoss', row => { - if (row.status === 'processing') throw new Error('Synthetic approval connection loss'); - }); - try { - await expect(service().reviewRequest(created.request.id, 'approve')).rejects.toThrow('Synthetic approval'); - } finally { models.ExternalRequest.removeHook('beforeUpdate', 'simulateApprovalLoss'); } - const executor = jest.fn(); - expect((await service({ executor }).reviewRequest(created.request.id, 'approve')).status).toBe('processing'); - expect(executor).not.toHaveBeenCalled(); - expect(await models.Job.count()).toBe(1); -}); - -test('channel approval retries preserve an explicit decision not to grant access', async () => { +test('channel approvals keep an explicit decision not to grant access', async () => { await models.ApiKeyChannelGrant.destroy({ where: { api_key_id: key.id } }); - const created = await service().createChannelRequest(key, { channelUrl: 'youtube.com/@synthetic' }); - const limiter = createExternalWorkLimiter({ concurrency: 1, maxQueue: 0 }); - let release; - const blocker = limiter.run(() => new Promise(resolve => { release = resolve; })); - try { - await expect(service({ workLimiter: limiter }).reviewRequest(created.request.id, 'approve', - { grantToRequestingKey: false })).rejects.toMatchObject({ status: 503 }); - } finally { release(); await blocker; } - await expect(service().reviewRequest(created.request.id, 'approve', + const api = service(); + const created = await api.createChannelRequest(key, { channelUrl: 'youtube.com/@synthetic' }); + await api.reviewRequest(created.request.id, 'approve', { grantToRequestingKey: false }); + await expect(api.reviewRequest(created.request.id, 'approve', { grantToRequestingKey: true })).rejects.toMatchObject({ status: 409 }); - const result = await service().reviewRequest(created.request.id, 'approve'); - expect(result).toMatchObject({ status: 'completed', grantToRequestingKey: false }); + await api.runExecutorOnce(); + const result = await api.getAdminRequest(created.request.id); + expect(result).toMatchObject({ status: 'completed', grantToRequestingKey: false, approvalSource: 'admin' }); expect(await models.ApiKeyChannelGrant.count()).toBe(0); }); +test('a terminated channel is never re-enabled by a channel request', async () => { + await policy({ auto_approve_channel_requests: true }); + await channel.update({ enabled: false, terminated_at: timestamp }); + const api = service(); + const created = await api.createChannelRequest(key, { channelUrl: 'https://www.youtube.com/@synthetic' }); + await api.runExecutorOnce(); + expect((await api.getRequest(key, created.request.id))).toMatchObject({ + status: 'failed', message: 'Channel is unavailable', + }); + await channel.reload(); + expect(channel.enabled).toBe(false); +}); + test('a completed video idempotency replay returns the original request after downloading', async () => { await policy({ auto_approve_video_requests: true }); + const api = service({ + jobLookup: () => ({ jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'Complete' }), + }); const input = { youtubeId, channelId: channel.id, idempotencyKey: 'completed-video' }; - const first = await service().createVideoRequest(key, input); + const first = await api.createVideoRequest(key, input); + await api.runExecutorOnce(); await downloadedVideo(); - const replay = await service().createVideoRequest(key, input); + await api.handleJobEnded({ jobId: first.request.id }); + const replay = await api.createVideoRequest(key, input); expect(replay).toMatchObject({ outcome: 'duplicate', request: { id: first.request.id, status: 'completed' } }); }); From 540260a208bd167c5977bd842c157f61e85a6713 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:21:54 +0000 Subject: [PATCH 21/27] fix: load external request dependencies lazily at startup Creating the request service in server.js eagerly required the channel, deletion and storage modules, which broke the server startup suites. --- .../modules/externalApi/externalRequestService.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/server/modules/externalApi/externalRequestService.js b/server/modules/externalApi/externalRequestService.js index b62690026..f9d9c3dbd 100644 --- a/server/modules/externalApi/externalRequestService.js +++ b/server/modules/externalApi/externalRequestService.js @@ -20,9 +20,17 @@ function createExternalRequestService({ models = require('../../models'), downloader = (jobData, options) => require('../downloadModule').doSpecificDownloads(jobData, false, options), - channelProvisioner = require('../channel/channelProvisioning'), - videoDeleter = require('../videoDeletionModule'), - storageGuard = require('../storageGuard'), + // Resolved on first use so creating the service at startup does not load + // the channel, deletion, and storage modules before they are needed. + channelProvisioner = { + getChannelInfo: (...args) => require('../channel/channelProvisioning').getChannelInfo(...args), + }, + videoDeleter = { + deleteVideos: (...args) => require('../videoDeletionModule').deleteVideos(...args), + }, + storageGuard = { + isPausedError: (error) => require('../storageGuard').isPausedError(error), + }, jobLookup = (jobId) => require('../jobModule').getJob(jobId), logger = require('../../logger'), now = () => new Date(), From 9b87150af1735fe07bd40ab5770a231bfd45f314 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:21:55 +0000 Subject: [PATCH 22/27] fix: keep external catalog reads cached-only and portable - Video detail reads metadata only from the cached .info.json and never runs yt-dlp or writes backfills. - Raw catalog SQL uses snake_case aliases and POSITION() instead of LOCATE(). - Video list ordering and keyset cursors (now v3) tie-break on channel video id. - Single-channel lookups use Channel.findOne with the grant include. --- .../__tests__/videoMetadataModule.test.js | 48 +++++++ .../__tests__/externalCatalogService.test.js | 118 +++++++++++++---- .../externalApi/externalCatalogService.js | 123 ++++++++++-------- server/modules/videoMetadataModule.js | 94 ++++++++----- server/routes/externalApi.js | 2 +- 5 files changed, 267 insertions(+), 118 deletions(-) diff --git a/server/modules/__tests__/videoMetadataModule.test.js b/server/modules/__tests__/videoMetadataModule.test.js index b39ac19d1..0cf2bfe1d 100644 --- a/server/modules/__tests__/videoMetadataModule.test.js +++ b/server/modules/__tests__/videoMetadataModule.test.js @@ -495,6 +495,54 @@ describe('VideoMetadataModule', () => { }); }); + describe('getCachedVideoMetadata', () => { + test('returns curated metadata from a cached .info.json', async () => { + mockFs.readFile.mockResolvedValue(JSON.stringify({ + description: 'Cached description', + view_count: 7, + upload_date: '20240315', + availability: 'public', + })); + mockVideo.findOne.mockResolvedValue(null); + + const result = await videoMetadataModule.getCachedVideoMetadata('abc123'); + + expect(result).toEqual(expect.objectContaining({ + description: 'Cached description', viewCount: 7, uploadDate: '20240315', + })); + expect(mockFs.readFile).toHaveBeenCalledWith('/test/jobs/info/abc123.info.json', 'utf8'); + }); + + test('returns null without running yt-dlp when nothing is cached', async () => { + mockFs.readFile.mockRejectedValue(Object.assign(new Error('missing'), { code: 'ENOENT' })); + + await expect(videoMetadataModule.getCachedVideoMetadata('abc123')).resolves.toBeNull(); + expect(mockYtDlpRunner.fetchMetadata).not.toHaveBeenCalled(); + expect(mockYoutubeApi.client.getVideoMetadata).not.toHaveBeenCalled(); + }); + + test('returns null for an unparseable cache file', async () => { + mockFs.readFile.mockResolvedValue('not json'); + + await expect(videoMetadataModule.getCachedVideoMetadata('abc123')).resolves.toBeNull(); + }); + + test('never writes backfills to the database', async () => { + const update = jest.fn(); + mockFs.readFile.mockResolvedValue(JSON.stringify({ + upload_date: '20240316', availability: 'subscriber_only', + })); + mockVideo.findOne.mockResolvedValue({ youtubeId: 'abc123', originalDate: null, update }); + + await videoMetadataModule.getCachedVideoMetadata('abc123'); + + expect(update).not.toHaveBeenCalled(); + expect(mockChannelVideo.update).not.toHaveBeenCalled(); + expect(mockChannelVideoReanchor.applyExactDateForVideo).not.toHaveBeenCalled(); + expect(mockFs.writeFile).not.toHaveBeenCalled(); + }); + }); + describe('_extractAvailableResolutions', () => { test('extracts supported resolutions from formats with video+audio streams', () => { const formats = [ diff --git a/server/modules/externalApi/__tests__/externalCatalogService.test.js b/server/modules/externalApi/__tests__/externalCatalogService.test.js index 969ee99c4..29c433a2c 100644 --- a/server/modules/externalApi/__tests__/externalCatalogService.test.js +++ b/server/modules/externalApi/__tests__/externalCatalogService.test.js @@ -4,8 +4,13 @@ jest.mock('../../../db', () => ({ jest.mock('../../configModule', () => ({ getImagePath: jest.fn(() => '/safe/images'), })); +jest.mock('../../../models', () => ({ + Channel: { findOne: jest.fn() }, + ApiKeyChannelGrant: {}, +})); const { sequelize } = require('../../../db'); +const { Channel, ApiKeyChannelGrant } = require('../../../models'); const catalog = require('../externalCatalogService'); const key = (overrides = {}) => ({ @@ -20,6 +25,7 @@ describe('external cached catalog', () => { beforeEach(() => { jest.clearAllMocks(); sequelize.query.mockResolvedValue([]); + Channel.findOne.mockResolvedValue(null); }); test('lists only granted enabled channels with SQL paging and policy-filtered counts', async () => { @@ -27,8 +33,8 @@ describe('external cached catalog', () => { .mockResolvedValueOnce([{ total: 1 }]) .mockResolvedValueOnce([{ id: 8, channel_id: 'UCsafe', title: 'Safe Channel', description: 'Description', - sub_folder: 'Kids', lastFetchedByTab: '{"video":"2026-07-20T00:00:00.000Z"}', - videoCount: '12', downloadedCount: '3', + sub_folder: 'Kids', last_fetched_by_tab: '{"video":"2026-07-20T00:00:00.000Z"}', + video_count: '12', downloaded_count: '3', }]); const result = await catalog.listChannels(key(), { page: '2', pageSize: '10', search: 'safe', sortBy: 'videoCount', sortOrder: 'desc', @@ -40,6 +46,7 @@ describe('external cached catalog', () => { thumbnailUrl: '/external-api/v1/assets/channels/8/thumbnail', videoCount: 12, downloadedCount: 3, + lastFetchedAt: '2026-07-20T00:00:00.000Z', })); expect(result.pagination).toEqual({ page: 2, pageSize: 10, total: 1, totalPages: 1, nextCursor: null, @@ -49,6 +56,10 @@ describe('external cached catalog', () => { expect(listSql).toContain('INNER JOIN api_key_channel_grants'); expect(listSql).toContain('g.api_key_id = :keyId'); expect(listSql).toContain('LIMIT :pageSize OFFSET :offset'); + expect(listSql).toContain('ORDER BY video_count DESC, c.id ASC'); + expect(listSql).toContain('POSITION(:search IN LOWER(COALESCE(c.title, c.uploader, \'\'))) > 0'); + expect(listSql).not.toContain('LOCATE('); + expect(listSql).not.toMatch(/ AS [a-z]+[A-Z]/); expect(listSql).toContain( 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))' ); @@ -64,15 +75,15 @@ describe('external cached catalog', () => { }); test('enforces grants, media type, rating, duration, date, and deterministic SQL paging', async () => { + Channel.findOne.mockResolvedValueOnce({ + id: 8, channel_id: 'UCsafe', title: 'Safe Channel', uploader: null, + lastFetchedByTab: '{"short":"2026-07-20T00:00:00.000Z"}', + }); sequelize.query - .mockResolvedValueOnce([{ - id: 8, channel_id: 'UCsafe', title: 'Safe Channel', - lastFetchedByTab: '{"short":"2026-07-20T00:00:00.000Z"}', - }]) .mockResolvedValueOnce([{ total: 1 }]) .mockResolvedValueOnce([{ youtube_id: 'abc', title: 'Allowed', thumbnail: 'https://i.ytimg.com/vi/abc/hqdefault.jpg', - publishedAt: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', + published_at: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', duration: 90, media_type: 'short', description: null, downloaded_id: null, downloaded_removed: null, rating: 'TV-Y', }]) @@ -87,25 +98,28 @@ describe('external cached catalog', () => { ); expect(result.data[0]).toEqual(expect.objectContaining({ - youtubeId: 'abc', rating: 'TV-Y', channelId: 'UCsafe', mediaType: 'short', + youtubeId: 'abc', rating: 'TV-Y', channelId: 'UCsafe', channelTitle: 'Safe Channel', + mediaType: 'short', publishedAt: '2026-07-10T00:00:00.000Z', thumbnailUrl: '/external-api/v1/assets/videos/abc/thumbnail', isDownloaded: false, isRequested: true, requestStatus: 'processing', })); expect(result.isFullyIndexed).toBe(true); - const channelSql = sequelize.query.mock.calls[0][0]; - const listSql = sequelize.query.mock.calls[2][0]; - expect(channelSql).toContain('INNER JOIN api_key_channel_grants'); - expect(channelSql).toContain('c.enabled = true'); - expect(channelSql).toContain('c.terminated_at IS NULL'); + expect(Channel.findOne).toHaveBeenCalledWith(expect.objectContaining({ + where: { id: 8, enabled: true, terminated_at: null }, + include: [expect.objectContaining({ + model: ApiKeyChannelGrant, required: true, where: { api_key_id: 4 }, + })], + })); + const listSql = sequelize.query.mock.calls[1][0]; expect(listSql).toContain('cv.media_type = :mediaType'); expect(listSql).toContain('cv.duration >= :minDuration'); expect(listSql).toContain('cv.duration <= :maxDuration'); expect(listSql).toContain('cv.published_at >= :dateFrom'); expect(listSql).toContain( - 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC' + 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC, cv.id ASC' ); expect(listSql).toContain('LIMIT :fetchLimit OFFSET :offset'); - expect(sequelize.query.mock.calls[3][1].replacements).toMatchObject({ keyId: 4, youtubeIds: ['abc'] }); + expect(sequelize.query.mock.calls[2][1].replacements).toMatchObject({ keyId: 4, youtubeIds: ['abc'] }); }); test('returns the complete deterministic cross-channel catalog beyond the former three-page cap', async () => { @@ -115,7 +129,7 @@ describe('external cached catalog', () => { youtube_id: 'abcdefghijk', title: 'Safe candidate', thumbnail: '/cached/private-thumbnail.jpg', - publishedAt: '2026-07-10T00:00:00.000Z', + published_at: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', duration: 120, media_type: 'video', @@ -153,7 +167,7 @@ describe('external cached catalog', () => { expect(listSql).toContain('INNER JOIN api_key_channel_grants'); expect(listSql).toContain('c.terminated_at IS NULL'); expect(listSql).toContain( - 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC' + 'ORDER BY COALESCE(cv.published_at, \'\') DESC, c.id ASC, cv.youtube_id ASC, cv.id ASC' ); expect(listSql).toContain('LIMIT :fetchLimit OFFSET :offset'); expect(sequelize.query.mock.calls[1][1].replacements).toMatchObject({ @@ -202,7 +216,7 @@ describe('external cached catalog', () => { const first = { youtube_id: 'aaaaaaaaaaa', title: 'Same title', - publishedAt: null, + published_at: null, published_at_source: 'approximate', duration: null, media_type: 'video', @@ -210,6 +224,7 @@ describe('external cached catalog', () => { downloaded_removed: null, rating: 'TV-Y', channel_database_id: 8, + channel_video_id: 70, channel_id: 'UCsafe', channel_title: 'Safe Channel', request_status: null, @@ -219,6 +234,7 @@ describe('external cached catalog', () => { ...first, youtube_id: 'bbbbbbbbbbb', channel_database_id: 9, + channel_video_id: 71, channel_id: 'UCother', channel_title: 'Other Channel', }; @@ -250,6 +266,10 @@ describe('external cached catalog', () => { const seekSql = sequelize.query.mock.calls[1][0]; expect(seekSql).toContain('COALESCE(cv.published_at, \'\') < :cursorSortValue'); expect(seekSql).toContain('c.id > :cursorChannelDatabaseId'); + expect(seekSql).toContain('cv.id > :cursorChannelVideoId'); + expect(sequelize.query.mock.calls[1][1].replacements).toMatchObject({ + cursorChannelDatabaseId: 8, cursorYoutubeId: 'aaaaaaaaaaa', cursorChannelVideoId: 70, + }); expect(seekSql).not.toContain('OFFSET :offset'); await expect(catalog.listVideos(key(), { @@ -264,6 +284,16 @@ describe('external cached catalog', () => { await expect(catalog.listVideos(key(), { cursor: Buffer.from(JSON.stringify({ v: 1, page: 2 })).toString('base64url'), })).rejects.toThrow('cursor is invalid'); + const legacyCursor = JSON.parse( + Buffer.from(firstPage.pagination.nextCursor, 'base64url').toString('utf8') + ); + delete legacyCursor.channelVideoId; + legacyCursor.v = 2; + await expect(catalog.listVideos(key(), { + pageSize: '1', + status: 'all', + cursor: Buffer.from(JSON.stringify(legacyCursor)).toString('base64url'), + })).rejects.toThrow('cursor is invalid'); await expect(catalog.listVideos(key(), { cursor: 'not-a-cursor', })).rejects.toThrow('cursor is invalid'); @@ -281,6 +311,7 @@ describe('external cached catalog', () => { await expect(catalog.listChannelVideos(key(), '8', { tabType: 'shorts' })) .rejects.toMatchObject({ status: 403 }); expect(sequelize.query).not.toHaveBeenCalled(); + expect(Channel.findOne).not.toHaveBeenCalled(); }); test('keeps TV-Y7 in the rated policy branch when unrated content is allowed', async () => { @@ -296,7 +327,6 @@ describe('external cached catalog', () => { }); test('returns the same 404 for missing and ungranted channels', async () => { - sequelize.query.mockResolvedValueOnce([]); await expect(catalog.listChannelVideos(key(), '99', {})) .rejects.toMatchObject({ status: 404, message: 'Channel not found' }); }); @@ -305,7 +335,7 @@ describe('external cached catalog', () => { sequelize.query.mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', title: 'Detailed video', - publishedAt: '2026-07-10T00:00:00.000Z', + published_at: '2026-07-10T00:00:00.000Z', published_at_source: 'exact', duration: 120, media_type: 'video', @@ -314,8 +344,8 @@ describe('external cached catalog', () => { downloaded_id: 42, downloaded_removed: false, last_downloaded_at: '2026-07-20T12:00:00.000Z', - fileSize: '12345', - audioFileSize: '678', + file_size: '12345', + audio_file_size: '678', protected: true, rating_source: 'manual', video_resolution: '1920x1080', @@ -326,7 +356,7 @@ describe('external cached catalog', () => { }]) .mockResolvedValueOnce([{ youtube_id: 'abcdefghijk', status: 'approved' }]); const metadataService = { - getVideoMetadata: jest.fn().mockResolvedValue({ + getCachedVideoMetadata: jest.fn().mockResolvedValue({ description: 'full description', viewCount: 1000, likeCount: 50, @@ -340,6 +370,7 @@ describe('external cached catalog', () => { expect(result).toMatchObject({ youtubeId: 'abcdefghijk', thumbnailUrl: '/external-api/v1/assets/videos/abcdefghijk/thumbnail', + publishedAt: '2026-07-10T00:00:00.000Z', isDownloaded: true, isRequested: true, requestStatus: 'approved', @@ -356,17 +387,17 @@ describe('external cached catalog', () => { }); expect(result).not.toHaveProperty('filePath'); expect(result).not.toHaveProperty('audioFilePath'); - expect(metadataService.getVideoMetadata).toHaveBeenCalledWith('abcdefghijk'); + expect(metadataService.getCachedVideoMetadata).toHaveBeenCalledWith('abcdefghijk'); expect(sequelize.query.mock.calls[0][0]).toContain('INNER JOIN api_key_channel_grants'); }); test('returns 404 without fetching metadata when video is not eligible', async () => { sequelize.query.mockResolvedValueOnce([]); - const metadataService = { getVideoMetadata: jest.fn() }; + const metadataService = { getCachedVideoMetadata: jest.fn() }; await expect(catalog.getVideoDetail(key(), 'abcdefghijk', metadataService)) .rejects.toMatchObject({ status: 404, message: 'Video not found' }); - expect(metadataService.getVideoMetadata).not.toHaveBeenCalled(); + expect(metadataService.getCachedVideoMetadata).not.toHaveBeenCalled(); }); test('falls back to the stored description when extended metadata has none', async () => { @@ -380,19 +411,50 @@ describe('external cached catalog', () => { channel_title: 'Safe Channel', }]); const metadataService = { - getVideoMetadata: jest.fn().mockResolvedValue({ description: null }), + getCachedVideoMetadata: jest.fn().mockResolvedValue({ description: null }), }; const result = await catalog.getVideoDetail(key(), 'abcdefghijk', metadataService); expect(result.metadata.description).toBe('database fallback'); }); + test('keeps the stored description when no metadata is cached', async () => { + sequelize.query.mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', + title: 'Detailed video', + media_type: 'video', + description: 'database fallback', + }]); + const metadataService = { getCachedVideoMetadata: jest.fn().mockResolvedValue(null) }; + + const result = await catalog.getVideoDetail(key(), 'abcdefghijk', metadataService); + expect(result.metadata).toEqual({ description: 'database fallback' }); + }); + + test('returns null metadata when nothing is cached or stored', async () => { + sequelize.query.mockResolvedValueOnce([{ + youtube_id: 'abcdefghijk', title: 'Uncached video', media_type: 'video', description: null, + }]); + const metadataService = { getCachedVideoMetadata: jest.fn().mockResolvedValue(null) }; + + const result = await catalog.getVideoDetail(key(), 'abcdefghijk', metadataService); + expect(result.metadata).toBeNull(); + }); + test('rejects unsafe cached thumbnail identifiers without exposing a path', async () => { - sequelize.query.mockResolvedValueOnce([{ channel_id: '../../secret' }]); + Channel.findOne.mockResolvedValueOnce({ channel_id: '../../secret' }); await expect(catalog.getChannelThumbnail(key(), '8')) .rejects.toMatchObject({ status: 404, message: 'Thumbnail not found' }); }); + test('returns 404 for an ungranted channel thumbnail', async () => { + await expect(catalog.getChannelThumbnail(key(), '8')) + .rejects.toMatchObject({ status: 404, message: 'Thumbnail not found' }); + expect(Channel.findOne).toHaveBeenCalledWith(expect.objectContaining({ + where: { id: 8, enabled: true, terminated_at: null }, + })); + }); + test('only exposes HTTPS thumbnail URLs from known YouTube image hosts', () => { expect(catalog.publicVideoThumbnail('https://i.ytimg.com/vi/abc/hqdefault.jpg')).toContain('ytimg.com'); expect(catalog.publicVideoThumbnail('http://i.ytimg.com/vi/abc/hqdefault.jpg')).toBeNull(); diff --git a/server/modules/externalApi/externalCatalogService.js b/server/modules/externalApi/externalCatalogService.js index 2269148b0..8838ca6f5 100644 --- a/server/modules/externalApi/externalCatalogService.js +++ b/server/modules/externalApi/externalCatalogService.js @@ -5,6 +5,7 @@ const { attachRequestStatuses } = require('./externalCatalogRequests'); const { QueryTypes } = require('sequelize'); const { sequelize } = require('../../db'); const configModule = require('../configModule'); +const { Channel, ApiKeyChannelGrant } = require('../../models'); const { normalizePolicy, ratingPolicy } = require('./externalEligibility'); const { publicVideoThumbnail } = require('./externalThumbnailProxy'); const { @@ -21,6 +22,7 @@ const ACTIVE_REQUEST_STATUSES = ['pending', 'approved', 'processing']; const MAX_PAGE = 100; const MAX_DESCRIPTION_LENGTH = 2000; const MAX_TITLE_LENGTH = 500; +const CATALOG_CURSOR_VERSION = 3; function boundedString(value, maximum) { if (value === null || value === undefined) return null; @@ -76,7 +78,7 @@ function decodeCatalogCursor(value, expected) { } try { const parsed = JSON.parse(Buffer.from(value, 'base64url').toString('utf8')); - if (parsed?.v !== 2 || + if (parsed?.v !== CATALOG_CURSOR_VERSION || parsed.endpoint !== expected.endpoint || parsed.sortBy !== expected.sortBy || parsed.sortOrder !== expected.sortOrder || @@ -85,6 +87,7 @@ function decodeCatalogCursor(value, expected) { !Number.isSafeInteger(parsed.channelDatabaseId) || parsed.channelDatabaseId < 1 || typeof parsed.youtubeId !== 'string' || parsed.youtubeId.length === 0 || parsed.youtubeId.length > 32 || + !Number.isSafeInteger(parsed.channelVideoId) || parsed.channelVideoId < 1 || (expected.sortBy === 'duration' ? !Number.isFinite(parsed.sortValue) || parsed.sortValue < -1 : typeof parsed.sortValue !== 'string' || parsed.sortValue.length > MAX_TITLE_LENGTH)) { @@ -98,10 +101,10 @@ function decodeCatalogCursor(value, expected) { function encodeCatalogCursor({ endpoint, sortBy, sortOrder, fingerprint, page, - sortValue, channelDatabaseId, youtubeId, + sortValue, channelDatabaseId, youtubeId, channelVideoId, }) { return Buffer.from(JSON.stringify({ - v: 2, + v: CATALOG_CURSOR_VERSION, endpoint, sortBy, sortOrder, @@ -110,6 +113,7 @@ function encodeCatalogCursor({ sortValue, channelDatabaseId, youtubeId, + channelVideoId, }), 'utf8').toString('base64url'); } @@ -136,6 +140,8 @@ function catalogSeekSql(sortExpression, sortOrder, cursor) { AND ( c.id > :cursorChannelDatabaseId OR (c.id = :cursorChannelDatabaseId AND cv.youtube_id > :cursorYoutubeId) + OR (c.id = :cursorChannelDatabaseId AND cv.youtube_id = :cursorYoutubeId + AND cv.id > :cursorChannelVideoId) ) ) )`; @@ -156,14 +162,29 @@ function catalogPaginationDto({ sortValue: lastRow.cursor_sort_value, channelDatabaseId: Number(lastRow.channel_database_id), youtubeId: lastRow.youtube_id, + channelVideoId: Number(lastRow.channel_video_id), }) : null, }; } -function lastFetched(channel, mediaType = null) { +function findGrantedChannel(keyId, channelDatabaseId, attributes) { + return Channel.findOne({ + where: { id: channelDatabaseId, enabled: true, terminated_at: null }, + attributes, + include: [{ + model: ApiKeyChannelGrant, + as: 'apiKeyGrants', + required: true, + where: { api_key_id: keyId }, + attributes: [], + }], + }); +} + +function lastFetched(lastFetchedByTab, mediaType = null) { try { - const values = JSON.parse(channel.lastFetchedByTab || '{}'); + const values = JSON.parse(lastFetchedByTab || '{}'); if (mediaType) return values[mediaType] || null; const dates = Object.values(values).filter(Boolean).map((value) => new Date(value)); if (dates.length === 0 || dates.some((value) => Number.isNaN(value.getTime()))) return null; @@ -183,8 +204,8 @@ async function listChannels(key, query = {}) { const search = normalizeSearch(query.search); const sortColumns = { title: 'title', - videoCount: 'videoCount', - downloadedCount: 'downloadedCount', + videoCount: 'video_count', + downloadedCount: 'downloaded_count', id: 'id', }; const sortBy = query.sortBy || 'title'; @@ -201,8 +222,8 @@ async function listChannels(key, query = {}) { AND cv.media_type IN (:allowedMediaTypes) AND ${ratings.sql}`; const whereSearch = search ? `AND ( - LOCATE(:search, LOWER(COALESCE(c.title, c.uploader, ''))) > 0 - OR LOCATE(:search, LOWER(COALESCE(c.description, ''))) > 0 + POSITION(:search IN LOWER(COALESCE(c.title, c.uploader, ''))) > 0 + OR POSITION(:search IN LOWER(COALESCE(c.description, ''))) > 0 )` : ''; const subfolder = query.subfolder === undefined || query.subfolder === '' @@ -227,13 +248,13 @@ async function listChannels(key, query = {}) { ); const rows = await sequelize.query( `SELECT c.id, c.channel_id, COALESCE(c.title, c.uploader, '') AS title, - c.description, c.sub_folder, c.last_fetched_by_tab AS lastFetchedByTab, + c.description, c.sub_folder, c.last_fetched_by_tab, (SELECT COUNT(*) FROM channelvideos cv LEFT JOIN videos v ON v.youtube_id = cv.youtube_id - WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS videoCount, + WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS video_count, (SELECT COUNT(*) FROM channelvideos cv INNER JOIN videos v ON v.youtube_id = cv.youtube_id AND v.removed = false - WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS downloadedCount + WHERE cv.channel_id = c.channel_id AND ${videoPolicy}) AS downloaded_count FROM channels c INNER JOIN api_key_channel_grants g ON g.channel_id = c.id AND g.api_key_id = :keyId @@ -251,9 +272,9 @@ async function listChannels(key, query = {}) { descriptionSummary: row.description ? row.description.trim().slice(0, 240) : null, thumbnailUrl: `/external-api/v1/assets/channels/${row.id}/thumbnail`, subfolder: boundedString(row.sub_folder, 255), - videoCount: Number(row.videoCount || 0), - downloadedCount: Number(row.downloadedCount || 0), - lastFetchedAt: lastFetched(row), + videoCount: Number(row.video_count || 0), + downloadedCount: Number(row.downloaded_count || 0), + lastFetchedAt: lastFetched(row.last_fetched_by_tab), })), pagination: paginationDto(page, pageSize, total), dataSource: 'cache', @@ -308,17 +329,11 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { }; const { page, pageSize, offset, cursor } = catalogPagination(query, cursorSpec); - const channelRows = await sequelize.query( - `SELECT c.id, c.channel_id, COALESCE(c.title, c.uploader, '') AS title, c.last_fetched_by_tab AS lastFetchedByTab - FROM channels c - INNER JOIN api_key_channel_grants g - ON g.channel_id = c.id AND g.api_key_id = :keyId - WHERE c.id = :channelDatabaseId AND c.enabled = true AND c.terminated_at IS NULL - LIMIT 1`, - { replacements: { keyId: key.id, channelDatabaseId: id }, type: QueryTypes.SELECT } + const channel = await findGrantedChannel( + key.id, id, ['id', 'channel_id', 'title', 'uploader', 'lastFetchedByTab'] ); - if (channelRows.length === 0) throw new CatalogError('Channel not found', 404); - const channel = channelRows[0]; + if (!channel) throw new CatalogError('Channel not found', 404); + const channelTitle = channel.title || channel.uploader || ''; const effectiveRating = 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; const ratings = ratingSql(policy, effectiveRating); @@ -331,7 +346,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { ]; if (minDuration !== null) filters.push('cv.duration >= :minDuration'); if (maxDuration !== null) filters.push('cv.duration <= :maxDuration'); - if (search) filters.push('LOCATE(:search, LOWER(COALESCE(cv.title, \'\'))) > 0'); + if (search) filters.push('POSITION(:search IN LOWER(COALESCE(cv.title, \'\'))) > 0'); if (dateFrom) filters.push('cv.published_at >= :dateFrom'); if (dateTo) filters.push('cv.published_at <= :dateTo'); if (status === 'downloaded') filters.push('v.id IS NOT NULL AND v.removed = false'); @@ -354,6 +369,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { cursorSortValue: cursor?.sortValue, cursorChannelDatabaseId: cursor?.channelDatabaseId, cursorYoutubeId: cursor?.youtubeId, + cursorChannelVideoId: cursor?.channelVideoId, ...ratings.replacements, }; const from = `FROM channelvideos cv @@ -366,14 +382,14 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { { replacements, type: QueryTypes.SELECT } ); const fetchedRows = await sequelize.query( - `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at AS publishedAt, cv.published_at_source, + `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at, cv.published_at_source, cv.duration, cv.media_type, v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, ${effectiveRating} AS rating, - c.id AS channel_database_id, + c.id AS channel_database_id, cv.id AS channel_video_id, ${sortExpressions[sortBy]} AS cursor_sort_value ${from} ${seekSql} - ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC + ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC, cv.id ASC LIMIT :fetchLimit${cursor ? '' : ' OFFSET :offset'}`, { replacements, type: QueryTypes.SELECT } ); @@ -381,13 +397,13 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { const rows = fetchedRows.slice(0, pageSize); await attachRequestStatuses(key, rows); const total = Number(countRows[0]?.total || 0); - const lastIndexedAt = lastFetched(channel, mediaType); + const lastIndexedAt = lastFetched(channel.lastFetchedByTab, mediaType); return { data: rows.map((row) => ({ youtubeId: row.youtube_id, title: boundedString(row.title, MAX_TITLE_LENGTH), thumbnailUrl: videoThumbnailUrl(row.youtube_id), - publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + publishedAt: row.published_at_source === 'estimated' ? null : row.published_at, duration: row.duration, description: boundedString(row.description, MAX_DESCRIPTION_LENGTH), isDownloaded: Boolean(row.downloaded_id) && !row.downloaded_removed, @@ -395,7 +411,7 @@ async function listChannelVideos(key, channelDatabaseId, query = {}) { requestStatus: row.request_status || null, rating: row.rating || null, channelId: channel.channel_id, - channelTitle: channel.title, + channelTitle, mediaType: row.media_type, })), pagination: catalogPaginationDto({ @@ -473,7 +489,7 @@ async function listVideos(key, query = {}) { ratings.sql, ]; if (mediaType) filters.push('cv.media_type = :mediaType'); - if (search) filters.push('LOCATE(:search, LOWER(COALESCE(cv.title, \'\'))) > 0'); + if (search) filters.push('POSITION(:search IN LOWER(COALESCE(cv.title, \'\'))) > 0'); if (minDuration !== null) filters.push('cv.duration >= :minDuration'); if (maxDuration !== null) filters.push('cv.duration <= :maxDuration'); if (dateFrom) filters.push('cv.published_at >= :dateFrom'); @@ -506,6 +522,7 @@ async function listVideos(key, query = {}) { cursorSortValue: cursor?.sortValue, cursorChannelDatabaseId: cursor?.channelDatabaseId, cursorYoutubeId: cursor?.youtubeId, + cursorChannelVideoId: cursor?.channelVideoId, ...ratings.replacements, }; const from = `FROM channelvideos cv @@ -520,14 +537,14 @@ async function listVideos(key, query = {}) { { replacements, type: QueryTypes.SELECT } ); const fetchedRows = await sequelize.query( - `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at AS publishedAt, cv.published_at_source, + `SELECT cv.youtube_id, cv.title, cv.thumbnail, cv.published_at, cv.published_at_source, cv.duration, cv.media_type, v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, ${effectiveRating} AS rating, c.id AS channel_database_id, c.channel_id, COALESCE(c.title, c.uploader, '') AS channel_title, - ${sortExpressions[sortBy]} AS cursor_sort_value + cv.id AS channel_video_id, ${sortExpressions[sortBy]} AS cursor_sort_value ${from} ${seekSql} - ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC + ORDER BY ${sortExpressions[sortBy]} ${sortOrder.toUpperCase()}, c.id ASC, cv.youtube_id ASC, cv.id ASC LIMIT :fetchLimit${cursor ? '' : ' OFFSET :offset'}`, { replacements, type: QueryTypes.SELECT } ); @@ -540,7 +557,7 @@ async function listVideos(key, query = {}) { youtubeId: row.youtube_id, title: boundedString(row.title, MAX_TITLE_LENGTH), thumbnailUrl: videoThumbnailUrl(row.youtube_id), - publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + publishedAt: row.published_at_source === 'estimated' ? null : row.published_at, duration: row.duration, description: boundedString(row.description, MAX_DESCRIPTION_LENGTH), isDownloaded: Boolean(row.downloaded_id) && !row.downloaded_removed, @@ -576,10 +593,10 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { 'COALESCE(NULLIF(v.normalized_rating, \'\'), NULLIF(c.default_rating, \'\'))'; const ratings = ratingSql(policy, effectiveRating); const rows = await sequelize.query( - `SELECT cv.youtube_id, cv.title, cv.published_at AS publishedAt, cv.published_at_source, + `SELECT cv.youtube_id, cv.title, cv.published_at, cv.published_at_source, cv.duration, cv.media_type, cv.availability, v.description, v.id AS downloaded_id, v.removed AS downloaded_removed, - v.last_downloaded_at, v.file_size AS fileSize, v.audio_file_size AS audioFileSize, v.protected, + v.last_downloaded_at, v.file_size, v.audio_file_size, v.protected, v.rating_source, v.video_resolution, ${effectiveRating} AS rating, c.id AS channel_database_id, c.channel_id, COALESCE(c.title, c.uploader, '') AS channel_title @@ -610,13 +627,14 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { await attachRequestStatuses(key, [row]); const metadataProvider = metadataService || require('../videoMetadataModule'); - const metadata = await metadataProvider.getVideoMetadata(youtubeId); + // Cached only: the catalog must never spawn yt-dlp for an external read. + const metadata = await metadataProvider.getCachedVideoMetadata(youtubeId); const isDownloaded = Boolean(row.downloaded_id) && !row.downloaded_removed; return { youtubeId: row.youtube_id, title: boundedString(row.title, MAX_TITLE_LENGTH), thumbnailUrl: videoThumbnailUrl(row.youtube_id), - publishedAt: row.published_at_source === 'estimated' ? null : row.publishedAt, + publishedAt: row.published_at_source === 'estimated' ? null : row.published_at, duration: row.duration, isDownloaded, isRequested: ACTIVE_REQUEST_STATUSES.includes(row.request_status), @@ -631,30 +649,21 @@ async function getVideoDetail(key, youtubeId, metadataService = null) { downloadedAt: isDownloaded && row.last_downloaded_at ? new Date(row.last_downloaded_at).toISOString() : null, - fileSize: isDownloaded && row.fileSize != null ? Number(row.fileSize) : null, - audioFileSize: isDownloaded && row.audioFileSize != null ? Number(row.audioFileSize) : null, + fileSize: isDownloaded && row.file_size != null ? Number(row.file_size) : null, + audioFileSize: isDownloaded && row.audio_file_size != null ? Number(row.audio_file_size) : null, isProtected: isDownloaded ? Boolean(row.protected) : false, videoResolution: isDownloaded ? row.video_resolution || null : null, - metadata: { - ...(metadata || {}), - description: metadata?.description ?? row.description ?? null, - }, + metadata: metadata || row.description != null + ? { ...(metadata || {}), description: metadata?.description ?? row.description } + : null, }; } async function getChannelThumbnail(key, channelDatabaseId) { normalizePolicy(key); const id = parseInteger(channelDatabaseId, null, 1, Number.MAX_SAFE_INTEGER, 'channel id'); - const rows = await sequelize.query( - `SELECT c.channel_id - FROM channels c - INNER JOIN api_key_channel_grants g - ON g.channel_id = c.id AND g.api_key_id = :keyId - WHERE c.id = :channelDatabaseId AND c.enabled = true AND c.terminated_at IS NULL - LIMIT 1`, - { replacements: { keyId: key.id, channelDatabaseId: id }, type: QueryTypes.SELECT } - ); - const channelId = rows[0]?.channel_id; + const channel = await findGrantedChannel(key.id, id, ['channel_id']); + const channelId = channel?.channel_id; if (!channelId || !/^[A-Za-z0-9_-]+$/.test(channelId)) throw new CatalogError('Thumbnail not found', 404); try { const imageDirectory = await fs.realpath(path.resolve(configModule.getImagePath())); diff --git a/server/modules/videoMetadataModule.js b/server/modules/videoMetadataModule.js index 095399eb9..78353d58b 100644 --- a/server/modules/videoMetadataModule.js +++ b/server/modules/videoMetadataModule.js @@ -212,44 +212,74 @@ class VideoMetadataModule { } } - // Use the numeric aspect_ratio from yt-dlp (e.g. 1.78 for 16:9) - const aspectRatio = rawData.aspect_ratio ?? null; - - // Collect related files on disk (thumbnail, subtitles, nfo, etc.) - const relatedFiles = await this._getVideoRelatedFiles(youtubeId); - - // Extract available resolutions from the formats array - const availableResolutions = this._extractAvailableResolutions(rawData.formats); - - return { - description: rawData.description ?? null, - viewCount: rawData.view_count ?? null, - likeCount: rawData.like_count ?? null, - commentCount: rawData.comment_count ?? null, - tags: rawData.tags ?? null, - categories: rawData.categories ?? null, - uploadDate: rawData.upload_date ?? null, - resolution: rawData.resolution ?? null, - width: rawData.width ?? null, - height: rawData.height ?? null, - fps: rawData.fps ?? null, - aspectRatio, - language: rawData.language ?? null, - isLive: rawData.is_live ?? null, - wasLive: rawData.was_live ?? null, - availability: rawData.availability ?? null, - channelFollowerCount: rawData.channel_follower_count ?? null, - ageLimit: rawData.age_limit ?? null, - webpageUrl: rawData.webpage_url ?? null, - relatedFiles, - availableResolutions, - }; + return this._curateMetadata(youtubeId, rawData); } catch (err) { logger.error({ err, youtubeId }, 'Unexpected error in getVideoMetadata'); return NULL_METADATA; } } + /** + * Curated metadata from an already-cached .info.json only. Unlike + * getVideoMetadata it never runs yt-dlp and never writes to the database, + * so callers outside the app (the external API) cannot trigger YouTube + * fetches. Returns null when no .info.json is cached. + * + * @param {string} youtubeId - YouTube video ID + * @returns {Promise} Curated metadata object, or null + */ + async getCachedVideoMetadata(youtubeId) { + let rawData; + try { + const infoPath = path.join(configModule.getJobsPath(), 'info', `${youtubeId}.info.json`); + rawData = JSON.parse(await fs.readFile(infoPath, 'utf8')); + } catch { + return null; + } + if (!rawData || typeof rawData !== 'object') return null; + try { + return await this._curateMetadata(youtubeId, rawData); + } catch (err) { + logger.error({ err, youtubeId }, 'Unexpected error in getCachedVideoMetadata'); + return null; + } + } + + async _curateMetadata(youtubeId, rawData) { + // Use the numeric aspect_ratio from yt-dlp (e.g. 1.78 for 16:9) + const aspectRatio = rawData.aspect_ratio ?? null; + + // Collect related files on disk (thumbnail, subtitles, nfo, etc.) + const relatedFiles = await this._getVideoRelatedFiles(youtubeId); + + // Extract available resolutions from the formats array + const availableResolutions = this._extractAvailableResolutions(rawData.formats); + + return { + description: rawData.description ?? null, + viewCount: rawData.view_count ?? null, + likeCount: rawData.like_count ?? null, + commentCount: rawData.comment_count ?? null, + tags: rawData.tags ?? null, + categories: rawData.categories ?? null, + uploadDate: rawData.upload_date ?? null, + resolution: rawData.resolution ?? null, + width: rawData.width ?? null, + height: rawData.height ?? null, + fps: rawData.fps ?? null, + aspectRatio, + language: rawData.language ?? null, + isLive: rawData.is_live ?? null, + wasLive: rawData.was_live ?? null, + availability: rawData.availability ?? null, + channelFollowerCount: rawData.channel_follower_count ?? null, + ageLimit: rawData.age_limit ?? null, + webpageUrl: rawData.webpage_url ?? null, + relatedFiles, + availableResolutions, + }; + } + /** * Fallback when yt-dlp fails: pull the fields the public API can provide * (description, view/like/comment counts, tags, categories, uploadDate, diff --git a/server/routes/externalApi.js b/server/routes/externalApi.js index e682e12e0..cb99a6e71 100644 --- a/server/routes/externalApi.js +++ b/server/routes/externalApi.js @@ -264,7 +264,7 @@ function createExternalApiRoutes({ * /external-api/v1/videos/{youtubeId}: * get: * summary: Read full curated metadata for one eligible cached video - * description: Returns the catalog identity and status plus the metadata used by Youtarr's video detail modal. + * description: Returns the catalog identity and status plus the metadata used by Youtarr's video detail modal. Metadata comes only from Youtarr's local cache and is null when nothing is cached; this endpoint never contacts YouTube. * tags: [External API] * security: [{ ExternalApiKeyAuth: [] }] * parameters: From 5f48d9ce65aceb72fc4adc2848d2c7f627940c49 Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:21:55 +0000 Subject: [PATCH 23/27] docs: describe the external request lifecycle and executor Document approval, background execution and held requests, cached-only video detail, auto-approved channel grants, delete and idempotency responses, the work queue env vars and the approval_source column. Bump the contract fixture to version 5 for the approved create response. --- CLAUDE.md | 6 +- docs/DATABASE.md | 2 +- docs/ENVIRONMENT_VARIABLES.md | 16 ++++ docs/EXTERNAL_API.md | 90 +++++++++++++------ fixtures/external-api-v1/SHA256SUMS | 2 +- fixtures/external-api-v1/contract.json | 4 +- .../externalContractFixtures.test.js | 2 +- 7 files changed, 88 insertions(+), 34 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index a72123e67..5de23ee09 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,9 +29,9 @@ For multi-part requests (e.g., "review this PR AND explain WebSocket handling"), ## Architecture ### Backend (server/) -- `routes/externalApi.js` and `middleware/externalApiAuth.js`: opt-in `/external-api/v1` routes with mandatory external-key authentication, bounded JSON bodies, private responses, and per-key limits. `modules/externalApiConfig.js`, `externalApiResponse.js`, `externalPermissions.js`, and `externalEligibility.js` centralize feature availability, errors, scopes, and media/rating policy. -- `modules/externalCatalogService.js`: cached, grant-filtered catalog and asset lookup; `externalCatalogRequests.js` batches current-page request status, `externalPagination.js` shares bounded page cursors, and `externalThumbnailProxy.js` enforces upstream image URL, size, and timeout limits. -- `modules/externalRequestService.js`: request creation and owner history; `externalRequestExecution.js` owns dispatch/recovery, `externalRequestReviewService.js` owns administrator decisions, `externalRequestPrimitives.js` shares validation/DTOs, and `externalRequestReconciliation.js` reconciles durable jobs for both request reads and `externalQuotaService.js`. Key-before-request locks serialize decisions; `externalWorkLimiter.js` bounds expensive work. `routes/externalRequests.js` exposes session-authenticated management endpoints; their UI is a separate follow-up. +- `routes/externalApi.js` and `middleware/externalApiAuth.js`: opt-in `/external-api/v1` routes with mandatory external-key authentication, bounded UTF-8 JSON bodies, private responses, and per-key limits. All External API business logic lives in `modules/externalApi/`; `externalApiConfig.js` (feature availability and `isExternalApiPath`), `externalApiResponse.js`, `externalPermissions.js`, and `externalEligibility.js` (media/rating policy; maps TV-Y7 to level 2 locally since `ratingMapper` has no numeric TV-Y7) centralize the shared rules, and `apiKeyChannelGrantModule.js` manages per-key channel grants. +- `modules/externalApi/externalCatalogService.js`: cached, grant-filtered catalog and asset lookup. Reads never spawn yt-dlp or write: video detail uses `videoMetadataModule.getCachedVideoMetadata` (cached `.info.json` only). Raw SQL stays portable (snake_case aliases, `POSITION(... IN ...)`), keyset cursors (v3) tie-break on `cv.id`, and single-channel lookups go through `Channel.findOne` with the `apiKeyGrants` include. `externalCatalogRequests.js` batches current-page request status, `externalPagination.js` shares bounded page cursors, and `externalThumbnailProxy.js` enforces upstream image URL, size, and timeout limits. +- `modules/externalApi/externalRequestService.js`: request creation (auto-approved requests are stored `approved` with `approval_source='auto'`) and owner history; reads never change request state. `externalRequestExecutor.js` is the background executor (started in `server.js` only when the External API is enabled; sweeps every minute, wakes on create/approve, and on `jobModule.onJobEnded`): it rechecks authority, claims `approved` -> `processing`, queues video downloads through `downloadModule.doSpecificDownloads` with the request id as the job id, provisions/enables/grants channels, and deletes videos through `videoDeletionModule`, holding requests with a message while downloads are paused or the video is busy, and recovering interrupted rows. `externalRequestTargets.js` validates grant/channel/video/policy targets, `externalRequestReconciliation.js` closes processing video requests whose job ended, `externalRequestReviewService.js` owns idempotent administrator approve/reject (approve only marks `approved` and wakes the executor), `externalRequestPrimitives.js` shares validation/DTOs, and `externalQuotaService.js` counts `approved` + `processing` as active jobs. Key-before-request locks serialize decisions; `externalWorkLimiter.js` bounds expensive work. `routes/externalRequests.js` exposes admin management endpoints that reject every API key; their UI is a separate follow-up. - `modules/externalCookies.js`: validates external cookie snapshots through the installed yt-dlp parser in `utils/validate-cookies.py`, memoizes unchanged results, and reports unusable files for omission; `modules/uploadedCookies.js` gives the uploaded `config/cookies.user.txt` a disposable private copy per run (yt-dlp rewrites its cookie file in place on exit, so concurrent runs sharing it could read a half-written jar or undo a Settings upload; nothing is written back, and Settings uploads are written to a temp file and renamed into place; if the copy fails, the run goes without cookies (error-level log; info when the upload was just deleted) and yt-dlp is never handed the uploaded file itself, whose exit-time jar save could overwrite a newer upload or recreate deleted cookies); `modules/ytdlpProcess.js` owns both kinds of copy's lifetime around yt-dlp processes. No persistent cookie fallback. - `modules/rescanRowUpdate.js` + `modules/videoRowGuard.js`: the filesystem rescan (`videosModule.backfillVideoMetadata`) walks the disk once and reads rows in chunks, so downloads and deletions in between make the walk stale. The walk only nominates rows (`isRescanCandidate`); `resolveRescanUpdate` recomputes each nominated row's write from a fresh per-format stat (present only for a regular file / missing on `ENOENT` or `ENOTDIR` or a non-file / unknown on any other error; both formats missing marks the row missing and keeps its stored paths; a missing format is cleared only when the other is present; any present format means `removed: false` even if the other is unknown; an unknown format is never cleared, and with nothing present an unknown writes nothing), and the write only lands if the row is unchanged since read (`videoRowGuard.unchangedSinceRead`: removed, both paths and sizes, `video_resolution`, `last_downloaded_at`; zero affected rows means a conflict only when the write includes a column that really differs, because Sequelize's MySQL dialect connects with -FOUND_ROWS and mysql2 then counts changed rows, not matched rows). Nominated rows left as recorded (files still at a stored path, e.g. outside the downloads folder, or files that could not be checked; re-checked every run) are counted as `kept` (`videosKept` in the run details, not the message). The walk's count of folders and files it could not read is `unreadableOnDisk` in the run details and log (not the message: those rows are re-checked by path). Rows another writer changed are counted as `skippedChanged` (`videosSkipped` in the run record) and failed writes as `failed` (`videosFailed`); both appear in the run message, and a finished scan with failed writes records as `error`/`partial` (a timed-out scan stays `timed-out` so Maintenance can continue it); the completion `rescanStatus` broadcast is built through the same `rescanRunSummary` record so the live Maintenance view matches the run history. Library repair (`jobModule.backfillFromCompleteList`) uses the same guard: it reads the row before searching the disk, writes file details and `removed: false` only for a file it actually found, and looks up a lost row's files by YouTube id with one `videosModule.scanForVideoFiles` walk per run (only when a lost row exists; video and audio, any folder layout), recreating it as missing when the walk finds nothing. The walk skips and counts (`unreadable`) folders and files it cannot read rather than stopping, logging unreadable files once per folder; repair still recreates unmatched rows as missing after a partial walk, because deferring would block recovery wherever a folder is never readable (lost+found, NAS recycle bins), and the next rescan restores a row whose file it finds. Such a run records as `error`/`partial`, naming the count in its message (`recreatedUnverified` and `unreadableOnDisk` in the details); a partial walk that affected no row stays `completed`. - `modules/cookieDetails.js`: summarizes the YouTube login cookies (names, counts, earliest expiry; never values) in the active cookie file as `details` on `GET /api/cookies/status` and the upload/delete responses, reading external files through `externalCookies.readExternalCookies`. `modules/cookieTest.js`: `POST /api/cookies/test`, one yt-dlp request to the subscriptions feed with `buildCommonArgs` (same proxy/IP family/cookies/cache as downloads) through `ytDlpRunner`, against a private temp copy of uploaded cookies (yt-dlp writes its jar back on exit, which would otherwise undo a replace/delete made mid-test; external files get spawnYtDlp's snapshot) and never logs stderr (yt-dlp echoes malformed cookie lines with their values); any exit 0 with JSON is signed in (even zero subscriptions), and the feed's "Incomplete yt initial data" warnings mean signed out (EXPIRED_COOKIES), a meaning specific to this feed, so it is not in the shared subscription-import classifier. One test at a time (409), rate-limited by `cookieTestRateLimiter`. diff --git a/docs/DATABASE.md b/docs/DATABASE.md index edca3037d..6c4827184 100644 --- a/docs/DATABASE.md +++ b/docs/DATABASE.md @@ -31,7 +31,7 @@ Youtarr uses MariaDB/MySQL for storing: | `sessions` | `Session` | User authentication sessions | | `apikeys` | `ApiKey` | API key credentials for external integrations (bookmarklets, shortcuts, automation). External API policy fields store the key role, request permissions and auto-approval settings, allowed media types, rating limits, revocation time, and write quotas. Existing keys are assigned the `legacy_download` role. | | `api_key_channel_grants` | `ApiKeyChannelGrant` | Channels granted to an API key. Unique index on `(api_key_id, channel_id)`. | -| `external_requests` | `ExternalRequest` | External API request lifecycle records, including the requesting key, optional channel and job, status, idempotency hash, and completion timestamps. | +| `external_requests` | `ExternalRequest` | External API request lifecycle records, including the requesting key, optional channel and job, status, idempotency hash, and completion timestamps. `approval_source` records how an approved request was approved (`auto` for key auto-approval, `admin` for an administrator decision); the background executor rechecks auto-approval only for `auto` rows. | | `external_api_usage_buckets` | `ExternalApiUsageBucket` | Per-key hourly and daily accepted-write counters. Unique index on `(api_key_id, window_type, window_start)`. | | `playlists` | `Playlist` | Subscribed YouTube playlists with per-playlist sync targets and seeded settings. `auto_download_baseline_at` (DATETIME, nullable): saved starting time for automatic following. `auto_download_baseline_id` (INTEGER, nullable): last known playlist entry id for new setups, avoiding same-second timestamp ambiguity. Existing timestamp-only cutoffs are preserved on upgrade. `auto_download_setup_error` (STRING, nullable): persisted first-setup failure (`PLAYLIST_TOO_LARGE` or `PLAYLIST_REFRESH_INCOMPLETE`), cleared when a baseline is successfully established. `sort_order` (STRING NOT NULL, default `'default'`): saved output order for the `.m3u` file and media server sync; `'reversed'` flips the YouTube playlist order. `title_filter_regex` (TEXT, nullable): matched against video titles on refresh as a case-insensitive JavaScript regex; the settings endpoint rejects a pattern that does not compile. It also seeds auto-created source channels, whose title filter runs through yt-dlp's Python matching (case sensitive). | | `playlistvideos` | `PlaylistVideo` | One row per (playlist, video) with the YouTube playlist position. `first_seen_at`: immutable local discovery time, backfilled from row creation on upgrade. `downloaded_at`: recorded download/job time, independently updated. `auto_download_requested`: explicitly selected existing videos pending successful download, saved only with auto-download enabled and an established baseline. `auto_download_last_attempt_at` (DATETIME(3), nullable): scheduling time for an explicit saved batch or a selected scheduled retry, written before queue submission. Used to rotate older saved requests, independently of discovery and download dates; legacy requests with no known attempt remain null and join the bounded retry pool. First initialization preserves pending requests; successful downloads clear their own flags, and an explicit starting-point reset clears the remaining requests. Legacy `added_at` is retained for pre-upgrade timestamp cutoffs; it is no longer overwritten on download. | diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index 640c8910b..f69ed3c31 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -115,6 +115,22 @@ origin port private and publish only the exact v1 path through a TLS reverse proxy. Exposing the complete port also exposes unrelated application routes; see [Youtarr External API](EXTERNAL_API.md). +### EXTERNAL_API_WORK_CONCURRENCY +**Required**: No +**Default**: `2` +**Options**: `1` or `2` (larger values are capped at `2`) +**Description**: How many expensive External API tasks run at once: video-detail +lookups, channel provisioning, and video deletion. Download jobs are queued +separately through Youtarr's normal download queue. + +### EXTERNAL_API_WORK_QUEUE +**Required**: No +**Default**: `20` +**Options**: `0`-`100` (larger values are capped at `100`) +**Description**: How many External API tasks may wait for a free slot. When the +queue is full, video-detail lookups return 503 and approved requests stay +`approved` until the next background sweep. + ### AUTH_PRESET_USERNAME **Required**: No **Default**: None diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md index 8087ec40f..716ded293 100644 --- a/docs/EXTERNAL_API.md +++ b/docs/EXTERNAL_API.md @@ -45,6 +45,7 @@ Every external key has: permission is enabled; - durable workload ceilings, defaulting to 5 active jobs, 30 accepted writes per UTC hour, and 200 per UTC day (administrators may select lower values); + active jobs are the key's `approved` and `processing` requests; - an explicit set of granted Youtarr channel database IDs. The HTTP contract uses a compact `maxRatingLevel` value with these rating ceilings: @@ -96,7 +97,7 @@ request bodies are not accepted. CORS is disabled. The sanitized cross-client decoding fixture is `fixtures/external-api-v1/contract.json`; its reviewable checksum is recorded -beside it in `SHA256SUMS`. Fixture version 3 covers capabilities, channels, +beside it in `SHA256SUMS`. Fixture version 5 covers capabilities, channels, cursor-paginated mixed-media catalog pages, rich and sparse video details, every request type and status, video-request outcomes, and forward-compatible unknown enum samples. The fixture contains no credentials or private @@ -210,10 +211,13 @@ Returns one policy-filtered video with the catalog identity and request status plus the full curated metadata used by Youtarr's video detail modal. This includes the complete description, engagement counts, tags, categories, availability, upload data, technical video fields, related-file summaries, and -available resolutions. The lookup is intentionally one video at a time and may -populate Youtarr's metadata cache when no cached `.info.json` exists. These -lookups share Youtarr's bounded external-work queue and return 503 when that -queue is full. +available resolutions. Metadata is read only from Youtarr's local cache (the +`.info.json` written when the video was downloaded or opened in Youtarr); this +endpoint never contacts YouTube and never writes to Youtarr's database. When +nothing is cached, `metadata` contains only the stored description, or is +`null` when there is none. The lookup is intentionally one video at a time, +shares Youtarr's bounded external-work queue, and returns 503 when that queue +is full. Downloaded-video details include timestamps, sizes, protection state, and resolution. Youtarr filesystem paths are never exposed because they are neither @@ -267,11 +271,16 @@ POST /external-api/v1/requests/channels ``` Only canonical YouTube handle, `/channel/`, `/c/`, and `/user/` URLs are -accepted. Approval provisions/enables the channel. By default it also grants -the resulting channel to the requesting key; the administrator may turn that -off during approval. That decision is stored with the request so an -idempotent recovery cannot broaden the grant after interrupted execution. -Uncached channel metadata resolution terminates after two minutes. +accepted. Approval provisions the channel, or re-enables it when it was +previously unsubscribed, and keeps its saved settings. By default it also +grants the resulting channel to the requesting key; the administrator may turn +that off during approval. That decision is stored with the request so an +idempotent recovery cannot broaden the grant after interrupted execution. A +key with channel auto-approval skips the administrator entirely: its requests +are subscribed, re-enabled, and granted to the key with no review, so enable +it only for fully trusted clients. A channel YouTube reports as terminated +fails with `Channel is unavailable`. Uncached channel metadata resolution +terminates after two minutes. Downloaded-video deletion: @@ -288,12 +297,17 @@ POST /external-api/v1/requests/delete-videos ``` Deletion applies only to the downloaded video asset. It never removes or -disables a channel subscription. Missing and already-removed targets complete -idempotently and retain a terminal request record for audit. - -Created requests return HTTP 202 and `outcome: "created"`. Duplicates and -already-terminal targets return HTTP 200 with `duplicate`, -`already_downloaded`, or `already_deleted`. +disables a channel subscription. A video that is not currently downloaded +returns 404. If the video is removed by other means before the request runs, +the request completes with `Video is already deleted` and keeps its record for +audit. + +Created requests return HTTP 202 and `outcome: "created"`, with status +`approved` when the key auto-approves that request type and `pending` +otherwise. Duplicates return HTTP 200 with `duplicate`, and a video request +for a video that is already downloaded returns HTTP 200 with +`already_downloaded`. Reusing an `idempotencyKey` for a different target +returns 409. ### Owner-scoped request reads @@ -305,25 +319,49 @@ an opaque `cursor` or the compatible `page` parameter, is limited to 100 rows, rejects pages beyond 100, and accepts an exact `status` filter. Statuses are `pending`, `approved`, `processing`, `completed`, `rejected`, -`failed`, or `cancelled`. Youtarr reconciles downloader state before returning -results. A terminal downloader job without a produced video becomes `failed`; -it does not remain stuck in `processing`. Video deletion is reconciled to -completed when the asset is already absent. Interrupted channel/deletion -execution can be reclaimed after a five-minute stale window and rerun through -its idempotent operation. +`failed`, or `cancelled`. Reads never change a request; they report the state +Youtarr's background executor has recorded. + +The lifecycle is: + +1. `pending`: waiting for an administrator (keys without auto-approval). +2. `approved`: approved by the key's auto-approval or by an administrator, and + waiting for the background executor. The executor runs about once a minute + and immediately after each approval. +3. `processing`: the executor has claimed the request. A video request is + queued as a normal Youtarr download job; channel and deletion requests run + in the bounded external-work queue. +4. `completed` or `failed`: a video request completes when its job ends with + the video on disk, and fails with `Download did not complete` otherwise. + +An approved request can stay `approved` with an explanatory `message` while it +waits: `Waiting for downloads to resume` while Youtarr's storage guard has +paused downloads, and `Waiting for another download of this video to finish` +when the video is already in another download. Before running, the executor +reloads the key and target; a request whose key, permission, grant, or policy +no longer allows it fails with `Request is no longer eligible`. Requests +interrupted by a restart return to `approved` after a five-minute stale window +and rerun through their idempotent operation. ## Administrator review -Administrator clients use session-only endpoints under `/api/external-requests`: +Administrator clients use endpoints under `/api/external-requests`. They use +Youtarr's normal admin authentication and reject every API key, including +external keys, with 403: - list with `status`, `requestType`, and `apiKeyId` filters; - request detail; - approve; - reject with a 1–300 character reason. -Approval locks the key before the request, reloads the current key policy, and -revalidates authorization and target state immediately before execution. -Revocation or a policy/grant change therefore takes effect immediately. +Approval locks the key before the request, reloads the current key policy, +checks the key's active-job capacity, and revalidates the target. It then marks +the request `approved` and hands it to the background executor, so the +approve call returns without waiting for a download, channel fetch, or +deletion. Approving an already-approved request again is a no-op; approving it +with a different channel-grant decision returns 409. The executor revalidates +authorization and target state again immediately before running, so +revocation or a policy/grant change still takes effect before any work starts. ## Rate limits diff --git a/fixtures/external-api-v1/SHA256SUMS b/fixtures/external-api-v1/SHA256SUMS index 212b99f94..d72a4be94 100644 --- a/fixtures/external-api-v1/SHA256SUMS +++ b/fixtures/external-api-v1/SHA256SUMS @@ -1 +1 @@ -5a175ad47f088c91723df3907dc80f3a5c13b4d5cd51962793fab266fdd88ac2 contract.json +bc5fb694e468d5b9cff96c57418b009dae41a58da22220c100eb292538cd89a6 contract.json diff --git a/fixtures/external-api-v1/contract.json b/fixtures/external-api-v1/contract.json index 6d2588451..e6d5f8ea6 100644 --- a/fixtures/external-api-v1/contract.json +++ b/fixtures/external-api-v1/contract.json @@ -1,5 +1,5 @@ { - "fixtureVersion": 4, + "fixtureVersion": 5, "capabilities": { "apiVersion": "1", "serverVersion": "1.77.0", @@ -526,7 +526,7 @@ "request": { "id": "00000000-0000-4000-8000-000000000010", "type": "video", - "status": "processing", + "status": "approved", "target": { "youtubeId": "abcdefghijk", "channelId": 8, diff --git a/server/modules/externalApi/__tests__/externalContractFixtures.test.js b/server/modules/externalApi/__tests__/externalContractFixtures.test.js index 8b1a9c974..c3fee8704 100644 --- a/server/modules/externalApi/__tests__/externalContractFixtures.test.js +++ b/server/modules/externalApi/__tests__/externalContractFixtures.test.js @@ -19,7 +19,7 @@ describe('external API shared contract fixture', () => { }); test('covers the complete Plinx-consumed v1 surface', () => { - expect(fixture.fixtureVersion).toBe(4); + expect(fixture.fixtureVersion).toBe(5); expect(fixture.capabilities).toEqual(expect.objectContaining({ apiVersion: '1', policy: expect.objectContaining({ From 60e5a654f9c30dff3d9e73021a404e1f3640873c Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:09:21 +0000 Subject: [PATCH 24/27] fix: point the external fixture check at the moved test --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6ce58a3ce..b1e28fe95 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,7 @@ "docs:check": "npm --prefix website run check", "docs:build": "npm --prefix website run build", "docs:serve": "npm --prefix website run serve", - "test:external-fixture": "jest --config jest.config.js server/modules/__tests__/externalContractFixtures.test.js --runInBand", + "test:external-fixture": "jest --config jest.config.js server/modules/externalApi/__tests__/externalContractFixtures.test.js --runInBand", "test:external-contract-server": "node scripts/external-api-contract-server.js" }, "lint-staged": { From 8af011119fb187d798999ec55e9c1ea57dce206d Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:09:21 +0000 Subject: [PATCH 25/27] test: run the request executor only when database tests ask A background wake after approval could sweep before the test changed state (for example revoking a grant), which made the database job fail at random on either engine. The wake scheduler is now injectable and the database tests turn it off. --- server/modules/externalApi/externalRequestExecutor.js | 5 ++++- server/modules/externalApi/externalRequestService.js | 2 ++ server/testing/externalRuntime.database.test.js | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/server/modules/externalApi/externalRequestExecutor.js b/server/modules/externalApi/externalRequestExecutor.js index 7faf6e8a0..cda4c7fcb 100644 --- a/server/modules/externalApi/externalRequestExecutor.js +++ b/server/modules/externalApi/externalRequestExecutor.js @@ -43,6 +43,9 @@ function createExternalRequestExecutor({ models, sequelize, now, quotas, validateTarget, workLimiter, downloader, channelProvisioner, videoDeleter, storageGuard, jobLookup = () => null, logger, sweepIntervalMs = SWEEP_INTERVAL_MS, + // How a wake defers its sweep; the database tests pass a no-op so each + // sweep runs only when the test asks for it. + scheduleWake = setImmediate, }) { const { ExternalRequest, ApiKeyChannelGrant, Channel, Job } = models; const { resolveVideoRequests } = createExternalRequestReconciliation({ models, now, jobLookup }); @@ -288,7 +291,7 @@ function createExternalRequestExecutor({ function wake() { if (wakeScheduled) return; wakeScheduled = true; - setImmediate(() => { + scheduleWake(() => { wakeScheduled = false; sweep(); }); diff --git a/server/modules/externalApi/externalRequestService.js b/server/modules/externalApi/externalRequestService.js index f9d9c3dbd..86af39e5a 100644 --- a/server/modules/externalApi/externalRequestService.js +++ b/server/modules/externalApi/externalRequestService.js @@ -38,6 +38,7 @@ function createExternalRequestService({ quotaService = null, workLimiter = require('./externalWorkLimiter').sharedExternalWorkLimiter, sweepIntervalMs, + scheduleWake, } = {}) { const { ExternalRequest } = models; const quotas = quotaService || @@ -46,6 +47,7 @@ function createExternalRequestService({ const executor = createExternalRequestExecutor({ models, sequelize, now, quotas, validateTarget, workLimiter, downloader, channelProvisioner, videoDeleter, storageGuard, jobLookup, logger, sweepIntervalMs, + scheduleWake, }); const reviewService = createExternalRequestReviewService({ models, sequelize, now, quotas, validateTarget, onApproved: executor.wake, diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index 167ae5e2f..64f91b72a 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -47,6 +47,9 @@ function service(options = {}) { storageGuard: { isPausedError: (error) => error?.code === 'DOWNLOADS_PAUSED' }, jobLookup: () => null, logger: { warn: () => {}, error: () => {} }, + // Background wakes would race the test's own steps; each test runs the + // executor explicitly with runExecutorOnce. + scheduleWake: () => {}, ...options, }); services.push(api); @@ -95,8 +98,7 @@ beforeEach(async () => { }); afterEach(async () => { - // Approvals wake the executor in the background; let that work settle - // before the next test clears the tables. + // Let any work a test started settle before the next test clears the tables. await Promise.all(services.splice(0).map((api) => api.runExecutorOnce())); }); From d698fb3066b23e659e0e02a9212eece93d782ead Mon Sep 17 00:00:00 2001 From: Philip Davis <46248315+bballdavis@users.noreply.github.com> Date: Thu, 22 Oct 2026 06:09:21 +0000 Subject: [PATCH 26/27] fix(deps): update Storybook to 10.6.2 for GHSA-f2ph-cmg5-wxc8 --- client/package-lock.json | 1309 ++++++++++++++++++++++++++++++-------- 1 file changed, 1057 insertions(+), 252 deletions(-) diff --git a/client/package-lock.json b/client/package-lock.json index aea3b1eee..97f59ffb7 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -806,21 +806,21 @@ } }, "node_modules/@emnapi/core": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.8.1.tgz", - "integrity": "sha512-AvT9QFpxK0Zd8J0jopedNm+w/2fIzvtPKPjqyw9jwvBaReTTqPBk9Hixaz7KbjimP+QNz605/XnjFcDAL2pqBg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.9.2.tgz", + "integrity": "sha512-UC+ZhH3XtczQYfOlu3lNEkdW/p4dsJ1r/bP7H8+rhao3TTTMO1ATq/4DdIi23XuGoFY+Cz0JmCbdVl0hz9jZcA==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/wasi-threads": "1.1.0", + "@emnapi/wasi-threads": "1.2.1", "tslib": "^2.4.0" } }, "node_modules/@emnapi/runtime": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.8.1.tgz", - "integrity": "sha512-mehfKSMWjjNol8659Z8KxEMrdSJDDot5SXMq00dM8BN4o+CLNXQ0xH2V7EchNHV4RmbZLmmPdEaXZc5H2FXmDg==", + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.9.2.tgz", + "integrity": "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw==", "dev": true, "license": "MIT", "optional": true, @@ -829,9 +829,9 @@ } }, "node_modules/@emnapi/wasi-threads": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.1.0.tgz", - "integrity": "sha512-WI0DdZ8xFSbgMjR1sFsKABJ/C5OnRrjT06JXbZKexJGrDuPTzZdDYfFlsgcCXCyf+suG5QU2e/y1Wo2V/OapLQ==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", "dev": true, "license": "MIT", "optional": true, @@ -2000,9 +2000,9 @@ } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript": { - "version": "0.6.4", - "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.6.4.tgz", - "integrity": "sha512-6PyZBYKnnVNqOSB0YFly+62R7dmov8segT27A+RVTBVd4iAE6kbW9QBJGlyR2yG4D4ohzhZSTIu7BK1UTtmFFA==", + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/@joshwooding/vite-plugin-react-docgen-typescript/-/vite-plugin-react-docgen-typescript-0.7.0.tgz", + "integrity": "sha512-qvsTEwEFefhdirGOPnu9Wp6ChfIwy2dBCRuETU3uE+4cC+PFoxMSiiEhxk4lOluA34eARHA0OxqsEUYDqRMgeQ==", "dev": true, "license": "MIT", "dependencies": { @@ -2011,7 +2011,7 @@ }, "peerDependencies": { "typescript": ">= 4.3.x", - "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0" + "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" }, "peerDependenciesMeta": { "typescript": { @@ -2043,27 +2043,27 @@ } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript/node_modules/glob": { - "version": "13.0.5", - "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.5.tgz", - "integrity": "sha512-BzXxZg24Ibra1pbQ/zE7Kys4Ua1ks7Bn6pKLkVPZ9FZe4JQS6/Q7ef3LG1H+k7lUf5l4T3PLSyYyYJVYUvfgTw==", + "version": "13.0.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-13.0.6.tgz", + "integrity": "sha512-Wjlyrolmm8uDpm/ogGyXZXb1Z+Ca2B8NbJwqBVg0axK9GbBeoS7yGV6vjXnYdGm6X53iehEuxxbyiKp8QmN4Vw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "minimatch": "^10.2.1", - "minipass": "^7.1.2", - "path-scurry": "^2.0.0" + "minimatch": "^10.2.2", + "minipass": "^7.1.3", + "path-scurry": "^2.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript/node_modules/lru-cache": { - "version": "11.2.6", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.2.6.tgz", - "integrity": "sha512-ESL2CrkS/2wTPfuend7Zhkzo2u0daGJ/A2VucJOgQ/C48S/zB8MMeMHSGKYpXhIjbPxfuezITkaBH1wqv00DDQ==", + "version": "11.5.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.3.tgz", + "integrity": "sha512-U4N8FgzmWxc8k1VH8Kr6lQg18U7Fjvby6wXHVRX/ZZ7IwWbRMgrRbP0Wrb5q5NVinryp4SQampHKdvtecItxUg==", "dev": true, "license": "BlueOak-1.0.0", "engines": { @@ -2071,13 +2071,13 @@ } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript/node_modules/minimatch": { - "version": "10.2.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.5.tgz", - "integrity": "sha512-MULkVLfKGYDFYejP07QOurDLLQpcjk7Fw+7jXS2R2czRQzR56yHRveU5NDJEOviH+hETZKSkIk5c+T23GjFUMg==", + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.5" + "brace-expansion": "^5.0.8" }, "engines": { "node": "18 || 20 || >=22" @@ -2087,9 +2087,9 @@ } }, "node_modules/@joshwooding/vite-plugin-react-docgen-typescript/node_modules/path-scurry": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.1.tgz", - "integrity": "sha512-oWyT4gICAu+kaA7QWk/jvCHWarMKNs6pXOGWKDTr7cw4IGcUbW+PeTfbaQiLGheFRpjo6O9J0PmyMfQPjH71oA==", + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.2.tgz", + "integrity": "sha512-3O/iVVsJAPsOnpwWIeD+d6z/7PmqApyQePUtCndjatj/9I5LylHvt5qluFaBT3I5h3r1ejfR056c+FCv+NnNXg==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { @@ -2097,7 +2097,7 @@ "minipass": "^7.1.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -2114,137 +2114,886 @@ "@jridgewell/trace-mapping": "^0.3.24" } }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", + "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "dev": true, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mswjs/interceptors": { + "version": "0.41.3", + "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.41.3.tgz", + "integrity": "sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@open-draft/deferred-promise": "^2.2.0", + "@open-draft/logger": "^0.3.0", + "@open-draft/until": "^2.0.0", + "is-node-process": "^1.2.0", + "outvariant": "^1.4.3", + "strict-event-emitter": "^0.5.1" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", + "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.10.0" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@open-draft/deferred-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", + "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@open-draft/logger": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", + "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-node-process": "^1.2.0", + "outvariant": "^1.4.0" + } + }, + "node_modules/@open-draft/until": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", + "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@oxc-parser/binding-android-arm-eabi": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm-eabi/-/binding-android-arm-eabi-0.127.0.tgz", + "integrity": "sha512-0LC7ye4hvqbIKxAzThzvswgHLFu2AURKzYLeSVvLdu2TBOYWQDmHnTqPLeA597BcUCxiLqLsS4CJ5uoI5WYWCQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-android-arm64": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-android-arm64/-/binding-android-arm64-0.127.0.tgz", + "integrity": "sha512-b5jtVTH6AU5CJXHNdj7Jj9IEiR9yVjjnwHzPJhGyHGPdcsZSzBCkS9GBbV33niRMvKthDwQRFRJfI4a+k4PvYg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-arm64": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-arm64/-/binding-darwin-arm64-0.127.0.tgz", + "integrity": "sha512-obCE8B7ISKkJidjlhv9xRGJPOSDG2Yu6PRga9Ruaz35uintHxbp1Ki/Yc71wx4rj3Edrm0a1kzG1TAwit0wFpg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-darwin-x64": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-darwin-x64/-/binding-darwin-x64-0.127.0.tgz", + "integrity": "sha512-JL6Xb5IwPQT8rUzlpsX7E+AgfcdNklXNPFp8pjCQQ5MQOQo5rtEB2ui+3Hgg9Sn7Y9Egj6YOLLiHhLpdAe12Aw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-freebsd-x64": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-freebsd-x64/-/binding-freebsd-x64-0.127.0.tgz", + "integrity": "sha512-SDQ/3MQFw58fqQz3Z1PhSKFF3JoCF4gmlNjziDm8X02tTahCw0qJbd7FGPDKw1i4VTBZene9JPyC3mHtSvi+wA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-gnueabihf": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-0.127.0.tgz", + "integrity": "sha512-Av+D1MIqzV0YMGPT9we2SIZaMKD7Cxs4CvXSx/yxaWHewZjYEjScpOf5igc8IILASViw4WTnjlwUdI1KzVtDHQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm-musleabihf": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-0.127.0.tgz", + "integrity": "sha512-Cs2fdJ8cPpFdeebj6p4dag8A4+56hPvZ0AhQQzlaLswGz1tz7bXt1nETLeorrM9+AMcWFFkqxcXwDGfTVidY8g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-gnu": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-0.127.0.tgz", + "integrity": "sha512-qdOfTcT6SY8gsJrrV92uyEUyjqMGPpIB5JZUG6QN5dukYd+7/j0kX6MwK1DgQj39jtUYixxPiaRUiEN1+0CXgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-arm64-musl": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-arm64-musl/-/binding-linux-arm64-musl-0.127.0.tgz", + "integrity": "sha512-EoTCZneNFU/P2qrpEM+RHmQwt+CvDkyGESG6qhr7KaegXLZwePfbrkCDfAk8/rhxbDUVGsZILX+2tqPzFtoFWA==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-ppc64-gnu": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-0.127.0.tgz", + "integrity": "sha512-zALjmZYgxFLHjXeudcDF0xFGNydTAtkAeXAr2EuC17ywCyFxcmQra4w0BMde0Yi/re4Bi4iwEoEXtYN7l6eBLQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-gnu": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-0.127.0.tgz", + "integrity": "sha512-fPP8M6zQLS7Jz7o9d5ArUSuAuSK3e+WCYVrCpdzeCOejidtZExJ9tjhDrAd3HEPqARBCPmdpqxESPFqy44vkBQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-riscv64-musl": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-0.127.0.tgz", + "integrity": "sha512-7IcC4Ao02oGpfnjt+X/oF4U2mllo2qoSkw5xxiXNKL9MCTsTiAC6616beOuehdxGcnz1bRoPC1RQ2f1GQDdN+g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-s390x-gnu": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-0.127.0.tgz", + "integrity": "sha512-pbXIhiNFHoqWeqDNLiJ9JkpHz1IM9k4DXa66x+1GTWMG7iLxtkXgE53iiuKSXwmk3zIYmaPVfBvgcAhS583K4Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-gnu": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-gnu/-/binding-linux-x64-gnu-0.127.0.tgz", + "integrity": "sha512-MYCguB9RvBvlSd6gbuNI7QwiLoCCAlGnlRJFPrzLI6U1/9wkC/WK6LtBAUln55H1Ctqw45PWmqrobKoMhsYQzQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-linux-x64-musl": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-linux-x64-musl/-/binding-linux-x64-musl-0.127.0.tgz", + "integrity": "sha512-5eY0B/bxf1xIUxb4NOTvOI3KWtBQfPWYyKAzgcrCt0mDibSZygVpO1Pz8bkeiSZ5Jj9+M09dkggG3H8I5d0Uyg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-openharmony-arm64": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-openharmony-arm64/-/binding-openharmony-arm64-0.127.0.tgz", + "integrity": "sha512-Gld0ajrFTUXNtdw20fVBuTQx66FA75nIVg+//pPfR3sXkuABB4mTBhl3r9JNzrJpgW//qiwxf0nWXUWGJSL3UQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-wasm32-wasi/-/binding-wasm32-wasi-0.127.0.tgz", + "integrity": "sha512-T6KVD7rhLzFlwGRXMnxUFfkCZD8FHnb968wVXW1mXzgRFc5RNXOBY2mPPDZ77x5Ln76ltLMgtPg0cOkU1NSrEQ==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.9.2", + "@emnapi/runtime": "1.9.2", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.5.tgz", + "integrity": "sha512-HshSvXzmBxNzqejd3k/KemgQ3hdREUYFRRrgxOZ+gPljDIsO2SDZrVZMd4Z/HsxudQbE62OVyVFE8z9BCurfjA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.3" + }, + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" + } + }, + "node_modules/@oxc-parser/binding-win32-arm64-msvc": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-0.127.0.tgz", + "integrity": "sha512-Ujvw4X+LD1CCGULcsQcvb4YNVoBGqt+JHgNNzGGaCImELiZLk477ifUH53gIbE7EKd933NdTi25JWEr9K2HwXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-ia32-msvc": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-ia32-msvc/-/binding-win32-ia32-msvc-0.127.0.tgz", + "integrity": "sha512-0cwxKO7KHQQQfo4Uf4B2SQrhgm+cJaP9OvFFhx52Tkg4bezsacu83GB2/In5bC415Ueeym+kXdnge/57rbSfTw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-parser/binding-win32-x64-msvc": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-parser/binding-win32-x64-msvc/-/binding-win32-x64-msvc-0.127.0.tgz", + "integrity": "sha512-rOrnSQSCbhI2kowr9XxE7m9a8oQXnBHjnS6j95LxxAnEZ0+Fz20WlRXG4ondQb+ejjt2KOsa65sE6++L6kUd+w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", + "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@oxc-resolver/binding-android-arm-eabi": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm-eabi/-/binding-android-arm-eabi-11.21.2.tgz", + "integrity": "sha512-xQoCRv+gKax9KTdwdaQNnAFOai8neay7g3jExDIORzhbrejwGSJaZNTdOJHR5ziLg2joMxOCMOFMo4zuxza2uQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@oxc-resolver/binding-android-arm64": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-android-arm64/-/binding-android-arm64-11.21.2.tgz", + "integrity": "sha512-HF5oiE2L05yInPYCFD/4uxSrEZW4SuIfn99Y6L1xnJnzl066JR+MJs2rIdstw8A2MPlAKH+13dpFPNycjqzvGg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@oxc-resolver/binding-darwin-arm64": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-arm64/-/binding-darwin-arm64-11.21.2.tgz", + "integrity": "sha512-UX4u49CVCAD8QZNELaW8eMGgMAGwFWYEPbvNsh+3r/gs4NX3KfpiACMVwRQT0EuH3uat9hM5Zl+Ppm9pJD8tgg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oxc-resolver/binding-darwin-x64": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-darwin-x64/-/binding-darwin-x64-11.21.2.tgz", + "integrity": "sha512-J9xPx7YBkrRmJ+xl561ztnMWEc1aOyjEIxBiGX1dVb3u7bGSnfObfcZk+Pd+uM0HZAPNsQ1xvD8j52A/uOSqNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@oxc-resolver/binding-freebsd-x64": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-freebsd-x64/-/binding-freebsd-x64-11.21.2.tgz", + "integrity": "sha512-fRlt7OvSaQkWj6+EDTVxawVxOlqJB2QSnBfkeCyK4RTvsGctbw3BiH2Tb7DzMs7bikc4BRBpvWP5zF9K8b54Zg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@oxc-resolver/binding-linux-arm-gnueabihf": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-11.21.2.tgz", + "integrity": "sha512-gK+vPUcPQITkGwBKpZGrcDHSlU6eDGl7AQacxS2CEKAZIBHWkOVFeJwLZ4tYnA1acJqRM5lt7yYwPCVqGHIJ7A==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-arm-musleabihf": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm-musleabihf/-/binding-linux-arm-musleabihf-11.21.2.tgz", + "integrity": "sha512-L/Rgas7SrOKy/z7IH+HxSFRqVO4PuLDKLEGKvnhoCBBq3UJ0YzGBou3qMzaAGgkJwsIvX2pBF+7ojzfUhLiZxQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-arm64-gnu": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-11.21.2.tgz", + "integrity": "sha512-CUEYvlX1Fk7E9kUMzuswru1J9HLxMwnpDeQGjQuI4ZH+iNCoa2X9T+pvyzrbsgl7WnIeTFTlNlHsRfVdf5g9/g==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-arm64-musl": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-arm64-musl/-/binding-linux-arm64-musl-11.21.2.tgz", + "integrity": "sha512-ViN1ZibQyxwC67GpoP33oo+S9UyUnkog13vzQb9+v9bCNvrVzJuk0MRdacjtv/9xfRMVF/eqHFyl8YOeykI10Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-ppc64-gnu": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-11.21.2.tgz", + "integrity": "sha512-CU1sCqWnhGqYD5I1HedHk5pujrn7ssDkNB/AEQd/pd3D/EojVSgJUlpbafwIbyhia3PgIfkvdFpRPWSALzVumA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-riscv64-gnu": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-gnu/-/binding-linux-riscv64-gnu-11.21.2.tgz", + "integrity": "sha512-jWVyZtIHca4Gb96x7dag+y69vlei7ffjrsveLkmf2ZhqEAz6ZSBnY1GWvgZUaZlwZP62A3xD092BW97Q5VGc+g==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-riscv64-musl": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-riscv64-musl/-/binding-linux-riscv64-musl-11.21.2.tgz", + "integrity": "sha512-LF29obFqNgBUgDX7rmUK7M4D0JQG5LxhYzn3xXmECcHU9aQAdWG7NiY052qybtesEdwHQXKNTWYQ7mTsybNvWg==", + "cpu": [ + "riscv64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@oxc-resolver/binding-linux-s390x-gnu": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-11.21.2.tgz", + "integrity": "sha512-+NYcm+cCHBbtdQQ3A4phQTSuVRYnNHz7wrl9XRAPEovcdoqi0mb1K5ZOl+jN54ZD+q1zz3V0vltbFJmzecJKmw==", + "cpu": [ + "s390x" + ], "dev": true, + "libc": [ + "glibc" + ], "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } + "optional": true, + "os": [ + "linux" + ] }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "node_modules/@oxc-resolver/binding-linux-x64-gnu": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-gnu/-/binding-linux-x64-gnu-11.21.2.tgz", + "integrity": "sha512-UQqZDdG2r2HhAOsZEgufkIWHPQ886IUyuJQkoZByvzhW8j51R4UNzGBJFkTiTnLhQnggwJRdJgFWK4uY6ZVIMw==", + "cpu": [ + "x64" + ], "dev": true, - "engines": { - "node": ">=6.0.0" - } + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "node_modules/@oxc-resolver/binding-linux-x64-musl": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-linux-x64-musl/-/binding-linux-x64-musl-11.21.2.tgz", + "integrity": "sha512-3Q9PMRjalWkT6NZ4jfujuqTCFwoWErg3y3BnOgb544B8IMw4PktiwWOigMfOHNRLMghZeJ7hpfpZf4CP7rV7Og==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "node_modules/@oxc-resolver/binding-openharmony-arm64": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-openharmony-arm64/-/binding-openharmony-arm64-11.21.2.tgz", + "integrity": "sha512-Eljeq3ndtyKhM+Es8LITi4Zl2htzuRZcrPMF3kMCsrILztvU6AjZ3FuEhHHKobPUB9rMpzLpJP5bDqXI7r+iog==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } + "optional": true, + "os": [ + "openharmony" + ] }, - "node_modules/@mswjs/interceptors": { - "version": "0.41.3", - "resolved": "https://registry.npmjs.org/@mswjs/interceptors/-/interceptors-0.41.3.tgz", - "integrity": "sha512-cXu86tF4VQVfwz8W1SPbhoRyHJkti6mjH/XJIxp40jhO4j2k1m4KYrEykxqWPkFF3vrK4rgQppBh//AwyGSXPA==", + "node_modules/@oxc-resolver/binding-wasm32-wasi": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-wasm32-wasi/-/binding-wasm32-wasi-11.21.2.tgz", + "integrity": "sha512-HGDbNsIywqc4LxU38+CTJNnB/6BF7rheWOJ259b4eE0aEnelYblC8x+1tEd63bp31fYne63RQz9Jb4yVnC7Yig==", + "cpu": [ + "wasm32" + ], "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@open-draft/deferred-promise": "^2.2.0", - "@open-draft/logger": "^0.3.0", - "@open-draft/until": "^2.0.0", - "is-node-process": "^1.2.0", - "outvariant": "^1.4.3", - "strict-event-emitter": "^0.5.1" + "@emnapi/core": "1.11.0", + "@emnapi/runtime": "1.11.0", + "@napi-rs/wasm-runtime": "^1.1.5" }, "engines": { - "node": ">=18" + "node": ">=14.0.0" } }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "0.2.12", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", - "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "node_modules/@oxc-resolver/binding-wasm32-wasi/node_modules/@emnapi/core": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.0.tgz", + "integrity": "sha512-l9Oo58x0HOP5znGzVhYW9U3e5wVuA4LAZU2AGezTmkhO1CgQRFDhDg4nneHsu/t3WniXg9QrG2nIXL/ZS8ln8Q==", "dev": true, "license": "MIT", "optional": true, "dependencies": { - "@emnapi/core": "^1.4.3", - "@emnapi/runtime": "^1.4.3", - "@tybys/wasm-util": "^0.10.0" + "@emnapi/wasi-threads": "1.2.2", + "tslib": "^2.4.0" } }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "node_modules/@oxc-resolver/binding-wasm32-wasi/node_modules/@emnapi/runtime": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.0.tgz", + "integrity": "sha512-55coeOFKHv1ywEcUXJtWU5f+Jr/W5tZDvZig8DLKSwUN1JpROQ4rk/SNOQiFWmaR/VKF4zuFyW1B8JduOSv6Pg==", "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" + "tslib": "^2.4.0" } }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "node_modules/@oxc-resolver/binding-wasm32-wasi/node_modules/@emnapi/wasi-threads": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", + "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", "dev": true, "license": "MIT", - "engines": { - "node": ">= 8" + "optional": true, + "dependencies": { + "tslib": "^2.4.0" } }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "node_modules/@oxc-resolver/binding-wasm32-wasi/node_modules/@napi-rs/wasm-runtime": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.5.tgz", + "integrity": "sha512-HshSvXzmBxNzqejd3k/KemgQ3hdREUYFRRrgxOZ+gPljDIsO2SDZrVZMd4Z/HsxudQbE62OVyVFE8z9BCurfjA==", "dev": true, "license": "MIT", + "optional": true, "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" + "@tybys/wasm-util": "^0.10.3" }, "engines": { - "node": ">= 8" + "node": "^20.19.0 || ^22.13.0 || >=23.5.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", + "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" } }, - "node_modules/@open-draft/deferred-promise": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@open-draft/deferred-promise/-/deferred-promise-2.2.0.tgz", - "integrity": "sha512-CecwLWx3rhxVQF6V4bAgPS5t+So2sTbPgAzafKkVizyi7tlwpcFpdFqq+wqF2OwNBmqFuu6tOyouTuxgpMfzmA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@open-draft/logger": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@open-draft/logger/-/logger-0.3.0.tgz", - "integrity": "sha512-X2g45fzhxH238HKO4xbSr7+wBS8Fvw6ixhTDuvLd5mqh6bJJCFAPwU9mPDxbcrRtfxv4u5IHCEH77BmxvXmmxQ==", + "node_modules/@oxc-resolver/binding-win32-arm64-msvc": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-11.21.2.tgz", + "integrity": "sha512-iWx25CBEgH49iE9q5coEGI/jb1jl5kkCY9z6U5Og67xCkQ/WFMDc2J5U78+AE91SUxM2NqSLhJC8/PLfWnImww==", + "cpu": [ + "arm64" + ], "dev": true, "license": "MIT", - "dependencies": { - "is-node-process": "^1.2.0", - "outvariant": "^1.4.0" - } + "optional": true, + "os": [ + "win32" + ] }, - "node_modules/@open-draft/until": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@open-draft/until/-/until-2.1.0.tgz", - "integrity": "sha512-U69T3ItWHvLwGg5eJ0n3I62nWuE6ilHlmz7zM0npLBRvPRd7e6NYmg54vvRtP5mZG7kZqZCFVdsTWo7BPtBujg==", + "node_modules/@oxc-resolver/binding-win32-x64-msvc": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/@oxc-resolver/binding-win32-x64-msvc/-/binding-win32-x64-msvc-11.21.2.tgz", + "integrity": "sha512-VPoCAhKvCQTlG7vxqaBXcmuvbh77BfnGXj8g0pbvVXpm1F/R8rDVwqIWcEbMzrI1JvJlm8v7T9uMVQb6UctMRg==", + "cpu": [ + "x64" + ], "dev": true, - "license": "MIT" + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", @@ -4047,9 +4796,9 @@ } }, "node_modules/@storybook/addon-a11y": { - "version": "10.2.17", - "resolved": "https://registry.npmjs.org/@storybook/addon-a11y/-/addon-a11y-10.2.17.tgz", - "integrity": "sha512-J0ogEc4/XFC+Ytz+X1we6TOKreEk/shgUs/mtxdsLa0xJ6bp2n2OQPSjNtQHH/nK4SRBSfHWPm8ztfcXTzeG9w==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/addon-a11y/-/addon-a11y-10.6.2.tgz", + "integrity": "sha512-zleS+btwrTRcUf3opbZntKsEt6PTqAtxsek08tx5jh5fRvd6wWmtnwW3uaBcypfIKnheTB3KGljJwFX+Mc+Umg==", "dev": true, "license": "MIT", "dependencies": { @@ -4061,13 +4810,13 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^10.2.17" + "storybook": "^10.6.2" } }, "node_modules/@storybook/addon-links": { - "version": "10.2.17", - "resolved": "https://registry.npmjs.org/@storybook/addon-links/-/addon-links-10.2.17.tgz", - "integrity": "sha512-KY2usxhPpt9AAzD22uBEfdPj1NZyCNyaYXgKkr8r/UeCNt7E7OdVBLNA1QMYZZ5dtIWj9EtY8c55OPuBM7aUkQ==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/addon-links/-/addon-links-10.6.2.tgz", + "integrity": "sha512-pWwWp+KAcqdX9Cg5i3yGaKcD3+7OEDhHkaiYv8GHv1JIOl7Ao/CaoY07or8BdM785akOqxk5Z+UrY7zNKZJ6Gw==", "dev": true, "license": "MIT", "dependencies": { @@ -4078,23 +4827,26 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.2.17" + "storybook": "^10.6.2" }, "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, "react": { "optional": true } } }, "node_modules/@storybook/builder-vite": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-10.2.10.tgz", - "integrity": "sha512-Wd6CYL7LvRRNiXMz977x9u/qMm7nmMw/7Dow2BybQo+Xbfy1KhVjIoZ/gOiG515zpojSozctNrJUbM0+jH1jwg==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/builder-vite/-/builder-vite-10.6.2.tgz", + "integrity": "sha512-SzeQEGjvybdhP2coF4aIX6JX/7rZTrQCpRU3vAEO/R98Vr3xM8jdg7a3nvt91TeF/P8iWqljLIpaDjTdk2r/sQ==", "dev": true, "license": "MIT", "dependencies": { - "@storybook/csf-plugin": "10.2.10", "ts-dedent": "^2.0.0" }, "funding": { @@ -4102,43 +4854,8 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "storybook": "^10.2.10", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0" - } - }, - "node_modules/@storybook/csf-plugin": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/@storybook/csf-plugin/-/csf-plugin-10.2.10.tgz", - "integrity": "sha512-aFvgaNDAnKMjuyhPK5ialT22pPqMN0XfPBNPeeNVPYztngkdKBa8WFqF/umDd47HxAjebq+vn6uId1xHyOHH3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "unplugin": "^2.3.5" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/storybook" - }, - "peerDependencies": { - "esbuild": "*", - "rollup": "*", - "storybook": "^10.2.10", - "vite": "*", - "webpack": "*" - }, - "peerDependenciesMeta": { - "esbuild": { - "optional": true - }, - "rollup": { - "optional": true - }, - "vite": { - "optional": true - }, - "webpack": { - "optional": true - } + "storybook": "^10.6.2", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" } }, "node_modules/@storybook/global": { @@ -4149,47 +4866,55 @@ "license": "MIT" }, "node_modules/@storybook/icons": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@storybook/icons/-/icons-2.0.1.tgz", - "integrity": "sha512-/smVjw88yK3CKsiuR71vNgWQ9+NuY2L+e8X7IMrFjexjm6ZR8ULrV2DRkTA61aV6ryefslzHEGDInGpnNeIocg==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@storybook/icons/-/icons-2.2.0.tgz", + "integrity": "sha512-FpUB/xizIrV1ILhCIYFo+TtQNN9YEoLjfcnAW0JtOXL+BcM2aisSysI0B54JMWDnljriXySBHgN6ps4s9lb75A==", "dev": true, "license": "MIT", "peerDependencies": { - "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "node_modules/@storybook/react": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/@storybook/react/-/react-10.2.10.tgz", - "integrity": "sha512-PcsChzPI8lhllB9exV7nFb96093i6sTwIl0jpPjaTFPQCRoueR9E/YeP3qSKQL9xt4cmii0cW7F0RUx25rW93Q==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/react/-/react-10.6.2.tgz", + "integrity": "sha512-kNWUU6Q8TrLmEqAzYmNYhC9A3i3kbjFxqIcLL3eu+o4gDu0afp1Of6Sy6jIF2qPGH05A3r6FWMzD03y4Mm5xRw==", "dev": true, "license": "MIT", "dependencies": { "@storybook/global": "^5.0.0", - "@storybook/react-dom-shim": "10.2.10", - "react-docgen": "^8.0.2" + "@storybook/react-dom-shim": "10.6.2", + "react-docgen": "^8.0.2", + "react-docgen-typescript": "^2.2.2" }, "funding": { "type": "opencollective", "url": "https://opencollective.com/storybook" }, "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.2.10", + "storybook": "^10.6.2", "typescript": ">= 4.9.x" }, "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + }, "typescript": { "optional": true } } }, "node_modules/@storybook/react-dom-shim": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/@storybook/react-dom-shim/-/react-dom-shim-10.2.10.tgz", - "integrity": "sha512-TmBrhyLHn8B8rvDHKk5uW5BqzO1M1T+fqFNWg88NIAJOoyX4Uc90FIJjDuN1OJmWKGwB5vLmPwaKBYsTe1yS+w==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/react-dom-shim/-/react-dom-shim-10.6.2.tgz", + "integrity": "sha512-k0262Vqxjk3wHeg3g/QO/ENDrkC7kkbuwsFSXOdYr7Sl888tsF1DAS7uQ/gr+s/BRz1FLYRX7wBY3smEUV4MJg==", "dev": true, "license": "MIT", "funding": { @@ -4197,25 +4922,35 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "@types/react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.2.10" + "storybook": "^10.6.2" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } } }, "node_modules/@storybook/react-vite": { - "version": "10.2.10", - "resolved": "https://registry.npmjs.org/@storybook/react-vite/-/react-vite-10.2.10.tgz", - "integrity": "sha512-C652GhZHXURi+gFqqLKmZPskEq1FQto4VCf/eQea2exmdVS0nOB+FFWQZNCivX6mpkDHza8UxRZNFpDB0mWcJQ==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/@storybook/react-vite/-/react-vite-10.6.2.tgz", + "integrity": "sha512-RfYSW9eefxRXk7IXKBTBkBYAspHKaxNVBR7mlXQAt+Z2bJZzj+S21mv6kBcpFIfAIT9wSIq8xMg0RbeR7CneVQ==", "dev": true, "license": "MIT", "dependencies": { - "@joshwooding/vite-plugin-react-docgen-typescript": "^0.6.4", + "@joshwooding/vite-plugin-react-docgen-typescript": "^0.7.0", "@rollup/pluginutils": "^5.0.2", - "@storybook/builder-vite": "10.2.10", - "@storybook/react": "10.2.10", + "@storybook/builder-vite": "10.6.2", + "@storybook/react": "10.6.2", "empathic": "^2.0.0", - "magic-string": "^0.30.0", - "react-docgen": "^8.0.0", + "magic-string": "^1.1.0", + "react-docgen": "^8.0.2", "resolve": "^1.22.8", "tsconfig-paths": "^4.2.0" }, @@ -4226,8 +4961,14 @@ "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", - "storybook": "^10.2.10", - "vite": "^5.0.0 || ^6.0.0 || ^7.0.0" + "storybook": "^10.6.2", + "typescript": ">= 4.9.x", + "vite": "^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } } }, "node_modules/@swc/core": { @@ -4479,7 +5220,6 @@ "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-10.4.1.tgz", "integrity": "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==", "dev": true, - "peer": true, "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", @@ -4495,10 +5235,11 @@ } }, "node_modules/@testing-library/jest-dom": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.8.0.tgz", - "integrity": "sha512-WgXcWzVM6idy5JaftTVC8Vs83NKRmGJz4Hqs4oyOuO2J4r/y79vvKZsb+CaGyCSEbUPI6OsewfPd0G1A0/TUZQ==", + "version": "6.9.1", + "resolved": "https://registry.npmjs.org/@testing-library/jest-dom/-/jest-dom-6.9.1.tgz", + "integrity": "sha512-zIcONa+hVtVSSep9UT3jZ5rizo2BsxgyDYU7WFD5eICBE7no3881HGeb/QkGfsJs6JTkY1aQhT7rIPC7e+0nnA==", "dev": true, + "license": "MIT", "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", @@ -4547,10 +5288,11 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.1", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.1.tgz", - "integrity": "sha512-vq7fv0rnt+QTXgPxr5Hjc210p6YKq2kmdziLgnsZGgLJ9e6VAShx1pACLuRjd/AS/sr7phAR58OIIpf0LlmQNw==", + "version": "14.6.8", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.8.tgz", + "integrity": "sha512-vgSSqnANMnkcYR/vUNDbwS7aM3OErRLNUG8vCVVlj4oc6cKffUrccX7UAS5AmJQQNZTSibi9z5GvvRZ0aYgUCQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=12", "npm": ">=6" @@ -4588,9 +5330,9 @@ "license": "MIT" }, "node_modules/@tybys/wasm-util": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.1.tgz", - "integrity": "sha512-9tTaPJLSiejZKx+Bmog4uSubteqTvFrVrURwkmHixBo0G4seD0zUxp98E1DzUBJxLQ3NPwXrGKDiVjwx/DpPsg==", + "version": "0.10.4", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.4.tgz", + "integrity": "sha512-W3c4gRigFS0T/Ma4qIYF3GDAc5AQdHb1yL5znJT1Zv1YaD9Kitx656wBjvr19qbiosmZT8lWDM5BEMynUqX65A==", "dev": true, "license": "MIT", "optional": true, @@ -4602,8 +5344,7 @@ "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", "integrity": "sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==", - "dev": true, - "peer": true + "dev": true }, "node_modules/@types/babel__core": { "version": "7.20.5", @@ -5218,6 +5959,13 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@webcontainer/env": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@webcontainer/env/-/env-1.1.1.tgz", + "integrity": "sha512-6aN99yL695Hi9SuIk1oC88l9o0gmxL1nGWWQ/kNy81HigJ0FoaoTXpytCj6ItzgyCEwA9kF1wixsTuv5cjsgng==", + "dev": true, + "license": "MIT" + }, "node_modules/acorn": { "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", @@ -6418,8 +7166,7 @@ "version": "0.5.16", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", "integrity": "sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==", - "dev": true, - "peer": true + "dev": true }, "node_modules/dom-helpers": { "version": "5.2.1", @@ -8826,19 +9573,18 @@ "resolved": "https://registry.npmjs.org/lz-string/-/lz-string-1.5.0.tgz", "integrity": "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==", "dev": true, - "peer": true, "bin": { "lz-string": "bin/bin.js" } }, "node_modules/magic-string": { - "version": "0.30.21", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", - "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.3.tgz", + "integrity": "sha512-z12OxmaPGE0F4xlpGdjuAUckipLpQlTAuAmyGhNoD7ODpYUzvDfvtxUbjM/jwznyP178oju/KDdyi220wNJ0RQ==", "dev": true, "license": "MIT", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" + "@jridgewell/sourcemap-codec": "^1.6.0" } }, "node_modules/make-dir": { @@ -9572,11 +10318,11 @@ } }, "node_modules/minipass": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", - "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.3.tgz", + "integrity": "sha512-tEBHqDnIoM/1rXME1zgka9g6Q2lcoCkxHLuc7ODJ5BxbP5d4c2Z5cGgtXAku59200Cx7diuHTOYfSBD8n6mm8A==", "dev": true, - "license": "ISC", + "license": "BlueOak-1.0.0", "engines": { "node": ">=16 || 14 >=14.17" } @@ -9874,6 +10620,75 @@ "dev": true, "license": "MIT" }, + "node_modules/oxc-parser": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/oxc-parser/-/oxc-parser-0.127.0.tgz", + "integrity": "sha512-bkgD4qHlN7WxLdX8bLXdaU54TtQtAIg/ZBAfm0aje/mo3MRDo3P0hZSgr4U7O3xfX+fQmR5AP04JS/TGcZLcFA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "^0.127.0" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-parser/binding-android-arm-eabi": "0.127.0", + "@oxc-parser/binding-android-arm64": "0.127.0", + "@oxc-parser/binding-darwin-arm64": "0.127.0", + "@oxc-parser/binding-darwin-x64": "0.127.0", + "@oxc-parser/binding-freebsd-x64": "0.127.0", + "@oxc-parser/binding-linux-arm-gnueabihf": "0.127.0", + "@oxc-parser/binding-linux-arm-musleabihf": "0.127.0", + "@oxc-parser/binding-linux-arm64-gnu": "0.127.0", + "@oxc-parser/binding-linux-arm64-musl": "0.127.0", + "@oxc-parser/binding-linux-ppc64-gnu": "0.127.0", + "@oxc-parser/binding-linux-riscv64-gnu": "0.127.0", + "@oxc-parser/binding-linux-riscv64-musl": "0.127.0", + "@oxc-parser/binding-linux-s390x-gnu": "0.127.0", + "@oxc-parser/binding-linux-x64-gnu": "0.127.0", + "@oxc-parser/binding-linux-x64-musl": "0.127.0", + "@oxc-parser/binding-openharmony-arm64": "0.127.0", + "@oxc-parser/binding-wasm32-wasi": "0.127.0", + "@oxc-parser/binding-win32-arm64-msvc": "0.127.0", + "@oxc-parser/binding-win32-ia32-msvc": "0.127.0", + "@oxc-parser/binding-win32-x64-msvc": "0.127.0" + } + }, + "node_modules/oxc-resolver": { + "version": "11.21.2", + "resolved": "https://registry.npmjs.org/oxc-resolver/-/oxc-resolver-11.21.2.tgz", + "integrity": "sha512-w5tLwYN3Zo24w5EeWJjJWZOwhYqTtC8PS2B1tIt7BZUuqTIcU07sQValbDw+rq7+AuAGzOHklgK+ifsy4lpXfw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + }, + "optionalDependencies": { + "@oxc-resolver/binding-android-arm-eabi": "11.21.2", + "@oxc-resolver/binding-android-arm64": "11.21.2", + "@oxc-resolver/binding-darwin-arm64": "11.21.2", + "@oxc-resolver/binding-darwin-x64": "11.21.2", + "@oxc-resolver/binding-freebsd-x64": "11.21.2", + "@oxc-resolver/binding-linux-arm-gnueabihf": "11.21.2", + "@oxc-resolver/binding-linux-arm-musleabihf": "11.21.2", + "@oxc-resolver/binding-linux-arm64-gnu": "11.21.2", + "@oxc-resolver/binding-linux-arm64-musl": "11.21.2", + "@oxc-resolver/binding-linux-ppc64-gnu": "11.21.2", + "@oxc-resolver/binding-linux-riscv64-gnu": "11.21.2", + "@oxc-resolver/binding-linux-riscv64-musl": "11.21.2", + "@oxc-resolver/binding-linux-s390x-gnu": "11.21.2", + "@oxc-resolver/binding-linux-x64-gnu": "11.21.2", + "@oxc-resolver/binding-linux-x64-musl": "11.21.2", + "@oxc-resolver/binding-openharmony-arm64": "11.21.2", + "@oxc-resolver/binding-wasm32-wasi": "11.21.2", + "@oxc-resolver/binding-win32-arm64-msvc": "11.21.2", + "@oxc-resolver/binding-win32-x64-msvc": "11.21.2" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -11050,24 +11865,29 @@ } }, "node_modules/storybook": { - "version": "10.2.17", - "resolved": "https://registry.npmjs.org/storybook/-/storybook-10.2.17.tgz", - "integrity": "sha512-yueTpl5YJqLzQqs3CanxNdAAfFU23iP0j+JVJURE4ghfEtRmWfWoZWLGkVcyjmgum7UmjwAlqRuOjQDNvH89kw==", + "version": "10.6.2", + "resolved": "https://registry.npmjs.org/storybook/-/storybook-10.6.2.tgz", + "integrity": "sha512-qHDINMiapxFUxYb+xC3j8u7Y+85X3u+/PlytE/wmnyg+bU0agCW3jUmWP4vIh2MioO2jTK/2B3Lcw6IpRFzQBA==", "dev": true, "license": "MIT", "dependencies": { "@storybook/global": "^5.0.0", - "@storybook/icons": "^2.0.1", - "@testing-library/jest-dom": "^6.6.3", - "@testing-library/user-event": "^14.6.1", + "@storybook/icons": "^2.0.2", + "@testing-library/dom": "^10.4.1", + "@testing-library/jest-dom": "6.9.1", + "@testing-library/user-event": "^14.6.3", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", - "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0", + "@webcontainer/env": "^1.1.1", + "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0 || ^0.28.0", + "jsonc-parser": "^3.3.1", "open": "^10.2.0", + "oxc-parser": "^0.127.0", + "oxc-resolver": "11.21.2", "recast": "^0.23.5", "semver": "^7.7.3", "use-sync-external-store": "^1.5.0", - "ws": "^8.18.0" + "ws": "^8.21.1" }, "bin": { "storybook": "dist/bin/dispatcher.js" @@ -11077,11 +11897,19 @@ "url": "https://opencollective.com/storybook" }, "peerDependencies": { - "prettier": "^2 || ^3" + "@types/react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "prettier": "^2 || ^3", + "vite-plus": "^0.1.15 || ^0.2.0" }, "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, "prettier": { "optional": true + }, + "vite-plus": { + "optional": true } } }, @@ -11653,9 +12481,9 @@ } }, "node_modules/ts-dedent": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.2.0.tgz", - "integrity": "sha512-q5W7tVM71e2xjHZTlgfTDoPF/SmqKG5hddq9SzR49CH2hayqRKJtQ4mtRlSxKaJlR/+9rEM+mnBHf7I2/BQcpQ==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/ts-dedent/-/ts-dedent-2.3.0.tgz", + "integrity": "sha512-JfJeIHke7y2egdGGgRAvpCwYFUsHlM2gPcrVOxFkznt/4uzQ7HFmvE63iFHVLBJNDuyDOQgijDK/tXH/f6Msjg==", "dev": true, "license": "MIT", "engines": { @@ -11878,22 +12706,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/unplugin": { - "version": "2.3.11", - "resolved": "https://registry.npmjs.org/unplugin/-/unplugin-2.3.11.tgz", - "integrity": "sha512-5uKD0nqiYVzlmCRs01Fhs2BdkEgBS3SAVP6ndrBsuK42iC2+JHyxM05Rm9G8+5mkmRtzMZGY8Ct5+mliZxU/Ww==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/remapping": "^2.3.5", - "acorn": "^8.15.0", - "picomatch": "^4.0.3", - "webpack-virtual-modules": "^0.6.2" - }, - "engines": { - "node": ">=18.12.0" - } - }, "node_modules/unrs-resolver": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/unrs-resolver/-/unrs-resolver-1.11.1.tgz", @@ -12228,13 +13040,6 @@ "node": ">=12" } }, - "node_modules/webpack-virtual-modules": { - "version": "0.6.2", - "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", - "integrity": "sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==", - "dev": true, - "license": "MIT" - }, "node_modules/whatwg-encoding": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", From a423d5c164739dff15c70d973e75d4ce32dddb96 Mon Sep 17 00:00:00 2001 From: Philip Davis Date: Thu, 29 Oct 2026 15:05:38 -0500 Subject: [PATCH 27/27] Fix external request retry recovery and held-work fairness --- .../Configuration/sections/ApiKeysSection.tsx | 3 +- .../__tests__/ApiKeysSection.test.tsx | 5 +- docs/EXTERNAL_API.md | 18 ++- .../modules/__tests__/downloadModule.test.js | 34 +++++- .../__tests__/videoDeletionModule.test.js | 4 +- .../__tests__/downloadJobFinalizer.test.js | 29 +++-- .../modules/download/downloadJobFinalizer.js | 7 +- server/modules/downloadModule.js | 45 +++---- .../__tests__/externalRequestExecutor.test.js | 45 ++++++- .../externalRequestReconciliation.test.js | 115 ++++++++++++++++++ .../externalApi/externalRequestExecutor.js | 16 ++- .../externalRequestReconciliation.js | 114 ++++++++++------- server/modules/videoDeletionModule.js | 9 +- .../testing/externalRuntime.database.test.js | 87 +++++++++++++ 14 files changed, 428 insertions(+), 103 deletions(-) create mode 100644 server/modules/externalApi/__tests__/externalRequestReconciliation.test.js diff --git a/client/src/components/Configuration/sections/ApiKeysSection.tsx b/client/src/components/Configuration/sections/ApiKeysSection.tsx index ae84c1ab6..56bc84c66 100644 --- a/client/src/components/Configuration/sections/ApiKeysSection.tsx +++ b/client/src/components/Configuration/sections/ApiKeysSection.tsx @@ -36,6 +36,7 @@ interface ApiKey { created_at: string; last_used_at: string | null; is_active: boolean; + revoked_at?: string | null; usage_count: number; } @@ -82,7 +83,7 @@ const ApiKeysSection: React.FC = ({ token, apiKeyRateLimit, if (response.ok) { const data = await response.json(); - setApiKeys(data.keys || []); + setApiKeys((data.keys || []).filter((key: ApiKey) => !key.revoked_at)); } else { const errData = await response.json(); setError(errData.error || 'Failed to fetch API keys'); diff --git a/client/src/components/Configuration/sections/__tests__/ApiKeysSection.test.tsx b/client/src/components/Configuration/sections/__tests__/ApiKeysSection.test.tsx index 7693d4383..5af7e3245 100644 --- a/client/src/components/Configuration/sections/__tests__/ApiKeysSection.test.tsx +++ b/client/src/components/Configuration/sections/__tests__/ApiKeysSection.test.tsx @@ -510,7 +510,7 @@ describe('ApiKeysSection Component', () => { }) .mockResolvedValueOnce({ ok: true, - json: jest.fn().mockResolvedValue({ keys: [mockApiKeys[1]] }), + json: jest.fn().mockResolvedValue({ keys: [{ ...mockApiKeys[0], is_active: false, revoked_at: '2026-10-29T12:00:00Z' }, mockApiKeys[1]] }), }); const props = createSectionProps(); @@ -533,6 +533,9 @@ describe('ApiKeysSection Component', () => { method: 'DELETE', })); }); + await waitFor(() => expect(screen.queryByText('My Bookmarklet')).not.toBeInTheDocument()); + expect(screen.getByText('iPhone Shortcut')).toBeInTheDocument(); + expect(screen.getAllByRole('button', { name: /Delete/i })).toHaveLength(1); }); }); diff --git a/docs/EXTERNAL_API.md b/docs/EXTERNAL_API.md index 716ded293..a945bfb3a 100644 --- a/docs/EXTERNAL_API.md +++ b/docs/EXTERNAL_API.md @@ -91,8 +91,8 @@ differences. The standardized statuses are 400 (malformed or invalid request), 401 (authentication), 403 (missing scope), 404 (unknown or hidden resource), 405 -(method), 413 (body size), 415 (content type), 429 (rate/quota), 500 -(unexpected failure), and 503 (bounded downstream capacity). Compressed +(method), 409 (idempotency key reused for a different target), 413 (body size), +415 (content type), 429 (rate/quota), 500 (unexpected failure), and 503 (bounded downstream capacity). Compressed request bodies are not accepted. CORS is disabled. The sanitized cross-client decoding fixture is @@ -332,17 +332,27 @@ The lifecycle is: queued as a normal Youtarr download job; channel and deletion requests run in the bounded external-work queue. 4. `completed` or `failed`: a video request completes when its job ends with - the video on disk, and fails with `Download did not complete` otherwise. + the video on disk. Transient 403 and cookie-related retries keep the same + request `processing`, dedupe key, and quota charge; the executor follows + persisted retry job identities, including after a restart. Only the final + attempt failing without a downloaded video produces `Download did not complete`. An approved request can stay `approved` with an explanatory `message` while it waits: `Waiting for downloads to resume` while Youtarr's storage guard has paused downloads, and `Waiting for another download of this video to finish` -when the video is already in another download. Before running, the executor +when the video is already in another download. A deletion refused by the library +root/subfolder check stays approved with `Waiting for the downloads folder to +become available`, retaining its dedupe key and quota charge. Held work rotates +behind other approved work so downloads waiting for storage cannot block +deletions that free space. Before every attempt, the executor reloads the key and target; a request whose key, permission, grant, or policy no longer allows it fails with `Request is no longer eligible`. Requests interrupted by a restart return to `approved` after a five-minute stale window and rerun through their idempotent operation. +The current API Keys & Bookmarklet screen hides revoked keys after deletion; +management responses retain those records for audit history. + ## Administrator review Administrator clients use endpoints under `/api/external-requests`. They use diff --git a/server/modules/__tests__/downloadModule.test.js b/server/modules/__tests__/downloadModule.test.js index 9e4ca2aa6..03d719578 100644 --- a/server/modules/__tests__/downloadModule.test.js +++ b/server/modules/__tests__/downloadModule.test.js @@ -1359,17 +1359,18 @@ describe('DownloadModule', () => { ChannelVideoMock.findAll.mockResolvedValue([]); doSpecificDownloadsSpy = jest .spyOn(downloadModule, 'doSpecificDownloads') - .mockResolvedValue(undefined); + .mockResolvedValue({ jobId: 'retry-job-1', acceptedIds: [retryVideo.youtubeId] }); }); it('enqueues a URL-list job with the auto-retry label, attempt counter, and run id', async () => { - await downloadModule.enqueueAutoRetryJob({ + const links = await downloadModule.enqueueAutoRetryJob({ retryVideos: [retryVideo], autoRetryAttempt: 1, runId: 'run-1', sourceJobData: { overrideSettings: { resolution: '720' } }, }); + expect(links).toEqual({ abc123def45: 'retry-job-1' }); expect(doSpecificDownloadsSpy).toHaveBeenCalledWith({ body: expect.objectContaining({ urls: ['https://www.youtube.com/watch?v=abc123def45'], @@ -1378,7 +1379,7 @@ describe('DownloadModule', () => { autoRetryAttempt: 1, runId: 'run-1', }), - }); + }, false, { includeJobId: true }); }); it('marks cookie-specific retries as anonymous', async () => { @@ -1399,7 +1400,7 @@ describe('DownloadModule', () => { autoRetryAttempt: 1, runId: 'run-cookie', }), - }); + }, false, { includeJobId: true }); }); it('splits mixed authenticated and anonymous retries into separate jobs', async () => { @@ -1441,6 +1442,22 @@ describe('DownloadModule', () => { ])); }); + it('retains the successful handoff when the other retry group fails', async () => { + doSpecificDownloadsSpy.mockRejectedValueOnce(new Error('queue unavailable')) + .mockResolvedValueOnce({ jobId: 'anonymous-job', acceptedIds: ['zzz999xxx11'] }); + const links = await downloadModule.enqueueAutoRetryJob({ + retryVideos: [retryVideo, { youtubeId: 'zzz999xxx11', url: 'https://youtu.be/zzz999xxx11', anonymousRetry: true }], + autoRetryAttempt: 1, sourceJobData: {}, + }); + expect(links).toEqual({ zzz999xxx11: 'anonymous-job' }); + }); + + it('does not claim a handoff when video admission was refused', async () => { + doSpecificDownloadsSpy.mockResolvedValue({ queued: 0, acceptedIds: [], alreadyActiveIds: [retryVideo.youtubeId] }); + await expect(downloadModule.enqueueAutoRetryJob({ retryVideos: [retryVideo], autoRetryAttempt: 1 })) + .resolves.toEqual({}); + }); + it('resolves owning channels for unmapped videos and passes channelId when unique', async () => { ChannelVideoMock.findAll.mockResolvedValue([ { youtube_id: 'abc123def45', channel_id: 'UC-owner' }, @@ -1743,6 +1760,15 @@ describe('DownloadModule', () => { expect(result).toBe(jobId); }); + it('returns the admitted identity only to internal retry callers', async () => { + jobModuleMock.getJob.mockReturnValue({ status: 'Pending' }); + const request = { body: { urls: ['https://youtu.be/abcdefghijk'], includeJobId: true } }; + const legacy = await downloadModule.doSpecificDownloads(request); + expect(legacy).not.toHaveProperty('jobId'); + const internal = await downloadModule.doSpecificDownloads(request, false, { includeJobId: true }); + expect(internal).toMatchObject({ jobId: mockJobId, acceptedIds: ['abcdefghijk'] }); + }); + it('ignores a job id supplied in the request body', async () => { jobModuleMock.getJob.mockReturnValue({ status: 'Pending' }); diff --git a/server/modules/__tests__/videoDeletionModule.test.js b/server/modules/__tests__/videoDeletionModule.test.js index d96dbeaf4..6a0962563 100644 --- a/server/modules/__tests__/videoDeletionModule.test.js +++ b/server/modules/__tests__/videoDeletionModule.test.js @@ -806,7 +806,7 @@ describe('VideoDeletionModule', () => { const result = await VideoDeletionModule.deleteVideos([1, 2]); const error = 'The downloads folder is empty. Nothing was deleted; check that the drive is mounted.'; - expect(result).toEqual({ success: false, deleted: [], failed: [{ videoId: 1, error }, { videoId: 2, error }] }); + expect(result).toEqual({ success: false, deleted: [], failed: [{ videoId: 1, error, code: 'LIBRARY_UNAVAILABLE' }, { videoId: 2, error, code: 'LIBRARY_UNAVAILABLE' }] }); expect(mockVideo1.update).not.toHaveBeenCalled(); expect(mockFilesystem.removeDirectoryResilient).not.toHaveBeenCalled(); }); @@ -820,7 +820,7 @@ describe('VideoDeletionModule', () => { const result = await VideoDeletionModule.deleteVideos([1, 2]); expect(result.deleted).toEqual([2]); - expect(result.failed).toEqual([{ videoId: 1, error: 'The subfolder __Music is empty. Nothing was deleted; check that the drive is mounted.' }]); + expect(result.failed).toEqual([{ videoId: 1, error: 'The subfolder __Music is empty. Nothing was deleted; check that the drive is mounted.', code: 'LIBRARY_UNAVAILABLE' }]); expect(inMusic.update).not.toHaveBeenCalled(); expect(inRoot.update).toHaveBeenCalledWith(USER_DELETED_COLUMNS); }); diff --git a/server/modules/download/__tests__/downloadJobFinalizer.test.js b/server/modules/download/__tests__/downloadJobFinalizer.test.js index 63094b35c..e25d807d1 100644 --- a/server/modules/download/__tests__/downloadJobFinalizer.test.js +++ b/server/modules/download/__tests__/downloadJobFinalizer.test.js @@ -437,7 +437,7 @@ describe('downloadJobFinalizer', () => { it('reports no issue when every failed video went to the retry', async () => { endWith([failure('vid403aaaa1')]); - await finalizeDownloadJob(makeContext({ runId: 'run-1', code: 1, enqueueAutoRetry: jest.fn().mockResolvedValue() })); + await finalizeDownloadJob(makeContext({ runId: 'run-1', code: 1, enqueueAutoRetry: jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }) })); const [, , summary] = downloadRunTracker.recordJobResult.mock.calls[0]; expect(summary).toMatchObject({ totalFailed: 0 }); @@ -447,7 +447,7 @@ describe('downloadJobFinalizer', () => { it('still reports the error when a failed video was not retried', async () => { endWith([failure('vid403aaaa1'), failure('vidprivate1', 'Private video. Sign in if you have been granted access')]); - await finalizeDownloadJob(makeContext({ runId: 'run-1', code: 1, enqueueAutoRetry: jest.fn().mockResolvedValue() })); + await finalizeDownloadJob(makeContext({ runId: 'run-1', code: 1, enqueueAutoRetry: jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }) })); expect(downloadRunTracker.recordJobResult).toHaveBeenCalledWith('run-1', mockJobId, expect.objectContaining({ totalFailed: 1, @@ -560,7 +560,7 @@ describe('downloadJobFinalizer', () => { const failure = make403Failure(); primeFailure(failure); jobModule.getJob.mockReturnValue({ data: { overrideSettings: { resolution: '720' } } }); - const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const enqueueAutoRetry = jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }); await finalizeDownloadJob(makeContext({ code: 1, enqueueAutoRetry, runId: 'run-9' })); @@ -584,13 +584,15 @@ describe('downloadJobFinalizer', () => { const failure = make403Failure(); primeFailure(failure); jobModule.getJob.mockReturnValue({ data: {} }); - const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const enqueueAutoRetry = jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }); await finalizeDownloadJob(makeContext({ enqueueAutoRetry })); expect(failure.autoRetryQueued).toBe(true); const [, fields] = jobModule.updateJob.mock.calls[0]; - expect(fields.data.failedVideos[0].autoRetryQueued).toBe(true); + expect(fields.data.failedVideos[0]).toMatchObject({ + autoRetryQueued: true, autoRetryJobId: 'retry-job-1', + }); const finalCall = MessageEmitter.emitMessage.mock.calls.find( (call) => call[4] && call[4].finalSummary @@ -607,7 +609,7 @@ describe('downloadJobFinalizer', () => { primeFailure(failure); jobModule.getJob.mockReturnValue({ data: {} }); downloadRunTracker.isActive.mockReturnValue(true); - const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const enqueueAutoRetry = jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }); await finalizeDownloadJob(makeContext({ enqueueAutoRetry, runId: 'run-1' })); @@ -623,7 +625,7 @@ describe('downloadJobFinalizer', () => { }); primeFailure(failure); configModule.getCookiesPath.mockReturnValue('/app/config/cookies.user.txt'); - const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const enqueueAutoRetry = jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }); await finalizeDownloadJob(makeContext({ code: 1, @@ -657,6 +659,17 @@ describe('downloadJobFinalizer', () => { expect(failure.autoRetryQueued).toBeUndefined(); }); + it('does not mark failed videos as handed off when no retry was admitted', async () => { + const failure = make403Failure(); + primeFailure(failure); + await finalizeDownloadJob(makeContext({ enqueueAutoRetry: jest.fn().mockResolvedValue({}) })); + expect(failure.autoRetryQueued).toBeUndefined(); + expect(failure.autoRetryJobId).toBeUndefined(); + expect(jobModule.updateJob.mock.calls[0][1].data.failedVideos).toContainEqual( + expect.objectContaining({ youtubeId: failure.youtubeId }) + ); + }); + it('does not enqueue when bot detection fired', async () => { primeFailure(make403Failure()); const enqueueAutoRetry = jest.fn(); @@ -803,7 +816,7 @@ describe('downloadJobFinalizer', () => { const failure = make403Failure(); primeFailure(failure); jobModule.getJob.mockReturnValue({ data: {} }); - const enqueueAutoRetry = jest.fn().mockResolvedValue(); + const enqueueAutoRetry = jest.fn().mockResolvedValue({ vid403aaaa1: 'retry-job-1' }); await finalizeDownloadJob(makeContext({ enqueueAutoRetry })); diff --git a/server/modules/download/downloadJobFinalizer.js b/server/modules/download/downloadJobFinalizer.js index 19ee4df8b..b2309d974 100644 --- a/server/modules/download/downloadJobFinalizer.js +++ b/server/modules/download/downloadJobFinalizer.js @@ -247,19 +247,20 @@ async function finalizeDownloadJob({ }); if (retryPlan) { try { - await enqueueAutoRetry({ + const retryJobs = await enqueueAutoRetry({ retryVideos: retryPlan.retryVideos, autoRetryAttempt: retryPlan.nextAttempt, runId, sourceJobData, }); - const retryIds = new Set(retryPlan.retryVideos.map((video) => video.youtubeId)); + const retryIds = new Set(Object.keys(retryJobs || {})); for (const failedVideo of failedVideosList) { if (retryIds.has(failedVideo.youtubeId)) { failedVideo.autoRetryQueued = true; + failedVideo.autoRetryJobId = retryJobs[failedVideo.youtubeId]; } } - autoRetryQueuedCount = retryPlan.retryVideos.length; + autoRetryQueuedCount = retryIds.size; logger.info( { jobId, count: autoRetryQueuedCount, attempt: retryPlan.nextAttempt }, 'Queued auto-retry job for retryable download failures' diff --git a/server/modules/downloadModule.js b/server/modules/downloadModule.js index 7c3f282cb..5fb998efc 100644 --- a/server/modules/downloadModule.js +++ b/server/modules/downloadModule.js @@ -146,7 +146,7 @@ class DownloadModule { * @param {Object} params.sourceJobData - the failing job's data payload */ async enqueueAutoRetryJob({ retryVideos, autoRetryAttempt, runId, sourceJobData = {} }) { - if (!Array.isArray(retryVideos) || retryVideos.length === 0) return; + if (!Array.isArray(retryVideos) || retryVideos.length === 0) return {}; // Keep normal transient-403 retries authenticated, while videos that // specifically failed with cookie-induced "Video unavailable" are retried @@ -159,21 +159,21 @@ class DownloadModule { ); if (anonymousVideos.length > 0 && authenticatedVideos.length > 0) { - await this.enqueueAutoRetryJob({ - retryVideos: authenticatedVideos, - autoRetryAttempt, - runId, - sourceJobData, - }); - - await this.enqueueAutoRetryJob({ - retryVideos: anonymousVideos, - autoRetryAttempt, - runId, - sourceJobData, - }); - - return; + const links = {}; + let lastError; + for (const videos of [authenticatedVideos, anonymousVideos]) { + try { + Object.assign(links, await this.enqueueAutoRetryJob({ + retryVideos: videos, autoRetryAttempt, runId, sourceJobData, + })); + } catch (err) { + lastError = err; + logger.error({ err }, 'Failed to enqueue one auto-retry group'); + } + } + // Preserve the successful group's handoff even if the other failed. + if (Object.keys(links).length === 0 && lastError) throw lastError; + return links; } const ownerChannelMap = { ...(this.getJobDataValue(sourceJobData, 'ownerChannelMap') || {}) }; @@ -226,7 +226,9 @@ class DownloadModule { { videoCount: retryVideos.length, autoRetryAttempt, channelId, runId }, 'Enqueueing auto-retry job for retryable download failures' ); - await this.doSpecificDownloads({ body }); + const admission = await this.doSpecificDownloads({ body }, false, { includeJobId: true }); + if (!admission?.jobId) return {}; + return Object.fromEntries(admission.acceptedIds.map((id) => [id, admission.jobId])); } async doChannelDownloads(jobData = {}, isNextJob = false) { @@ -757,11 +759,12 @@ class DownloadModule { /** * @param {Object} reqOrJobData - request (body) or job data * @param {boolean} [isNextJob] - * @param {{jobId?: string}} [options] - jobId: a stable UUID for the job, from + * @param {{jobId?: string, includeJobId?: boolean}} [options] - jobId: a stable UUID for the job, from * trusted internal callers only (the external request executor). Kept out - * of the job data so a request body cannot choose a job id. + * of the job data so a request body cannot choose a job id. includeJobId + * returns the admitted job identity to the internal retry finalizer. */ - async doSpecificDownloads(reqOrJobData, isNextJob = false, { jobId: preferredJobId = null } = {}) { + async doSpecificDownloads(reqOrJobData, isNextJob = false, { jobId: preferredJobId = null, includeJobId = false } = {}) { const jobData = reqOrJobData.body ? reqOrJobData.body : reqOrJobData; if (!isNextJob && !jobData.id) { await storageGuard.assertDownloadsAllowed(); @@ -979,7 +982,7 @@ class DownloadModule { throw err; } if (preferredJobId) return jobId; - return { queued: urls.length, acceptedIds: admission.acceptedIds, alreadyActiveIds: admission.alreadyActiveIds }; + return { queued: urls.length, ...admission, ...(includeJobId ? { jobId } : {}) }; } /** diff --git a/server/modules/externalApi/__tests__/externalRequestExecutor.test.js b/server/modules/externalApi/__tests__/externalRequestExecutor.test.js index 394d17cee..38b981316 100644 --- a/server/modules/externalApi/__tests__/externalRequestExecutor.test.js +++ b/server/modules/externalApi/__tests__/externalRequestExecutor.test.js @@ -88,7 +88,7 @@ describe('external request executor', () => { }, { jobId: requestId }); expect(models.ExternalRequest.update).toHaveBeenCalledWith( expect.objectContaining({ job_id: requestId }), - { where: { id: requestId, status: 'processing' } } + { where: { id: requestId, status: 'processing', job_id: null } } ); }); @@ -263,6 +263,21 @@ describe('external request executor', () => { })); }); + test('holds an unavailable library and completes after it returns', async () => { + const row = deleteRow(); + row.active_dedupe_key = 'delete-dedupe'; + const { executor, deps } = setup({ rows: [row], target: downloadedTarget }); + deps.videoDeleter.deleteVideos.mockResolvedValueOnce({ + success: false, deleted: [], failed: [{ videoId: 91, + code: 'LIBRARY_UNAVAILABLE', error: 'The downloads folder does not exist.' }], + }); + await executor.runOnce(); + expect(row).toMatchObject({ status: 'approved', active_dedupe_key: 'delete-dedupe', + message: 'Waiting for the downloads folder to become available' }); + await executor.runOnce(); + expect(row).toMatchObject({ status: 'completed', active_dedupe_key: null }); + }); + test('fails when the deletion is refused', async () => { const row = deleteRow(); const { executor, deps } = setup({ rows: [row], target: downloadedTarget }); @@ -330,7 +345,7 @@ describe('external request executor', () => { expect(models.ExternalRequest.update).toHaveBeenCalledWith( expect.objectContaining({ status: 'completed', active_dedupe_key: null }), - { where: { id: requestId, status: 'processing' } } + { where: { id: requestId, status: 'processing', job_id: requestId } } ); }); @@ -344,10 +359,30 @@ describe('external request executor', () => { expect(models.ExternalRequest.update).toHaveBeenCalledWith( expect.objectContaining({ status: 'failed', message: 'Download did not complete' }), - { where: { id: requestId, status: 'processing' } } + { where: { id: requestId, status: 'processing', job_id: requestId } } ); }); + test('the original event hands off to the retry and the retry event completes it', async () => { + const processing = record({ status: 'processing', job_id: requestId, active_dedupe_key: 'dedupe' }); + const { executor, deps, models } = setup(); + const jobs = { + [requestId]: { jobType: EXTERNAL_REQUEST_JOB_LABEL, status: 'Error', data: { failedVideos: [ + { youtubeId, autoRetryQueued: true, autoRetryJobId: 'retry-1' }, + ] } }, + 'retry-1': { jobType: 'Auto-retry: 1 video (no cookies)', status: 'Pending' }, + }; + deps.jobLookup.mockImplementation(id => jobs[id]); + models.ExternalRequest.findAll.mockImplementation(async () => [{ ...processing }]); + models.ExternalRequest.update.mockImplementation(async values => { Object.assign(processing, values); return [1]; }); + await executor.handleJobEnded({ jobId: requestId }); + expect(processing).toMatchObject({ job_id: 'retry-1', status: 'processing', active_dedupe_key: 'dedupe' }); + jobs['retry-1'].status = 'Complete'; + models.Video.findAll.mockResolvedValue([{ youtubeId }]); + await executor.handleJobEnded({ jobId: 'retry-1' }); + expect(processing).toMatchObject({ status: 'completed', active_dedupe_key: null }); + }); + test('the sweep closes a download whose job-ended event was missed', async () => { const processing = record({ status: 'processing', job_id: 'job-123' }); const { executor, models } = setup(); @@ -360,7 +395,7 @@ describe('external request executor', () => { expect(models.ExternalRequest.update).toHaveBeenCalledWith( expect.objectContaining({ status: 'failed' }), - { where: { id: processing.id, status: 'processing' } } + { where: { id: processing.id, status: 'processing', job_id: 'job-123' } } ); }); @@ -375,7 +410,7 @@ describe('external request executor', () => { expect(models.ExternalRequest.update).toHaveBeenCalledWith( expect.objectContaining({ status: 'failed', message: 'Download did not complete' }), - { where: { id: processing.id, status: 'processing' } } + { where: { id: processing.id, status: 'processing', job_id: 'job-123' } } ); }); diff --git a/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js b/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js new file mode 100644 index 000000000..2946fd4f8 --- /dev/null +++ b/server/modules/externalApi/__tests__/externalRequestReconciliation.test.js @@ -0,0 +1,115 @@ +const { Op } = require('sequelize'); +const { createExternalRequestReconciliation } = require('../externalRequestReconciliation'); + +const youtubeId = 'abcdefghijk'; +function setup({ stored = false } = {}) { + const jobs = new Map(); + const request = { id: 'original', job_id: 'original', youtube_id: youtubeId, + request_type: 'video', status: 'processing', active_dedupe_key: 'dedupe' }; + const models = { + ExternalRequest: { update: jest.fn(async (values, { where }) => { + if (where.job_id !== request.job_id || request.status !== where.status) return [0]; + Object.assign(request, values); + return [1]; + }) }, + Video: { findAll: jest.fn().mockResolvedValue([]) }, + Job: { findAll: jest.fn(async ({ where }) => where.id[Op.in].flatMap(id => { + const job = jobs.get(id); + return job ? [{ id, status: job.status, aux_data: JSON.stringify(job.data) }] : []; + })) }, + }; + const api = createExternalRequestReconciliation({ models, now: () => new Date(), + jobLookup: id => stored ? null : jobs.get(id) }); + const handoff = (next, id = youtubeId) => ({ status: 'Error', data: { failedVideos: [ + { youtubeId: id, autoRetryQueued: true, autoRetryJobId: next }, + ] } }); + const resolve = () => api.resolveVideoRequests([{ ...request }]); + return { jobs, models, request, handoff, resolve, api }; +} + +describe.each([false, true])('retry reconciliation (stored jobs: %s)', stored => { + test('keeps pending retries processing with dedupe and closes the successful retry', async () => { + const { jobs, models, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', { status: 'Pending' }); + await resolve(); + expect(request).toMatchObject({ job_id: 'retry-1', status: 'processing', active_dedupe_key: 'dedupe' }); + jobs.set('retry-1', { status: 'Complete' }); + models.Video.findAll.mockResolvedValue([{ youtubeId }]); + await resolve(); + expect(request).toMatchObject({ status: 'completed', active_dedupe_key: null }); + }); + + test.each(['Error', 'Killed', 'Terminated'])('fails only after the last retry ends in %s', async status => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', handoff('retry-2')); + jobs.set('retry-2', { status: 'In Progress' }); + await resolve(); + expect(request).toMatchObject({ job_id: 'retry-2', status: 'processing', active_dedupe_key: 'dedupe' }); + jobs.set('retry-2', { status }); + await resolve(); + expect(request).toMatchObject({ status: 'failed', active_dedupe_key: null, message: 'Download did not complete' }); + }); + + test('completes when a retry persists its video between the initial read and terminal lookup', async () => { + const { jobs, models, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', { status: 'Complete' }); + models.Video.findAll.mockResolvedValueOnce([]).mockResolvedValue([{ youtubeId }]); + await resolve(); + expect(models.Video.findAll).toHaveBeenCalledTimes(2); + expect(request).toMatchObject({ status: 'completed', active_dedupe_key: null }); + }); + + test('does not follow a handoff for a different video in a mixed job', async () => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('unrelated', 'lmnopqrstuv')); + jobs.set('unrelated', { status: 'Pending' }); + await resolve(); + expect(request.status).toBe('failed'); + }); + + test('fails when the queued retry was lost entirely', async () => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('lost-retry')); + await resolve(); + expect(request).toMatchObject({ status: 'failed', job_id: 'lost-retry', active_dedupe_key: null }); + }); + + test('the maximum retry chain can still be pending', async () => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', handoff('retry-2')); + jobs.set('retry-2', handoff('retry-3')); + jobs.set('retry-3', { status: 'Pending' }); + await resolve(); + expect(request).toMatchObject({ status: 'processing', job_id: 'retry-3' }); + }); + + test('does not hang on cyclic handoffs', async () => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', handoff('original')); + await resolve(); + expect(request.status).toBe('failed'); + }); + + test('handles a retry that ended before the original event was processed', async () => { + const { jobs, request, handoff, resolve } = setup({ stored }); + jobs.set('original', handoff('retry-1')); + jobs.set('retry-1', { status: 'Error' }); + await resolve(); + expect(request).toMatchObject({ status: 'failed', job_id: 'retry-1' }); + }); +}); + +test('a stale event cannot close or overwrite a newer retry handoff', async () => { + const { jobs, request, models, api } = setup(); + const stale = { ...request }; + request.job_id = 'retry-1'; + jobs.set('original', { status: 'Error' }); + await api.resolveVideoRequests([stale], { endedJobIds: ['original'] }); + expect(models.ExternalRequest.update).toHaveBeenCalled(); + expect(request).toMatchObject({ job_id: 'retry-1', status: 'processing', active_dedupe_key: 'dedupe' }); +}); diff --git a/server/modules/externalApi/externalRequestExecutor.js b/server/modules/externalApi/externalRequestExecutor.js index cda4c7fcb..297a4bda4 100644 --- a/server/modules/externalApi/externalRequestExecutor.js +++ b/server/modules/externalApi/externalRequestExecutor.js @@ -1,5 +1,5 @@ const { Op } = require('sequelize'); -const { MANUAL_DOWNLOAD_LABEL } = require('../download/jobTypes'); +const { MANUAL_DOWNLOAD_LABEL, AUTO_RETRY_LABEL_PREFIX } = require('../download/jobTypes'); const { requireCurrentAutoApproval } = require('./externalRequestPrimitives'); const { createExternalRequestReconciliation } = require('./externalRequestReconciliation'); @@ -27,6 +27,7 @@ const MESSAGES = { channelUnavailable: 'Channel is unavailable', channelFailed: 'Channel could not be provisioned', deleteFailed: 'Video could not be deleted', + libraryUnavailable: 'Waiting for the downloads folder to become available', }; function isAuthorityError(error) { @@ -116,7 +117,7 @@ function createExternalRequestExecutor({ try { await ExternalRequest.update( { job_id: jobId, message: null, updated_at: stamp() }, - { where: { id: record.id, status: 'processing' } } + { where: { id: record.id, status: 'processing', job_id: null } } ); } catch (error) { logger.warn({ err: error, requestId: record.id, jobId }, 'External request job id was not recorded'); @@ -180,7 +181,9 @@ function createExternalRequestExecutor({ // and re-evaluates a storage pause, like an in-app deletion. const result = await workLimiter.run(() => videoDeleter.deleteVideos([target.downloaded.id])); if (result?.deleted?.includes(target.downloaded.id)) return complete(record); - const reason = result?.failed?.[0]?.error || ''; + const failure = result?.failed?.find((item) => item.videoId === target.downloaded.id); + if (failure?.code === 'LIBRARY_UNAVAILABLE') return hold(record, MESSAGES.libraryUnavailable); + const reason = failure?.error || ''; if (/not found|already (?:marked as )?removed/i.test(reason)) { return complete(record, MESSAGES.alreadyDeleted); } @@ -263,7 +266,9 @@ function createExternalRequestExecutor({ await resolveFinishedDownloads(); const approved = await ExternalRequest.findAll({ where: { request_type: { [Op.in]: REQUEST_TYPES }, status: 'approved' }, - order: [['created_at', 'ASC'], ['id', 'ASC']], + // A held request goes to the back so paused downloads cannot starve + // newer work, especially deletions that can free up storage. + order: [['updated_at', 'ASC'], ['id', 'ASC']], limit: SWEEP_BATCH_SIZE, }); approved.forEach(launch); @@ -300,7 +305,8 @@ function createExternalRequestExecutor({ /** jobModule.onJobEnded listener: close the request a finished job served. */ async function handleJobEnded({ jobId } = {}) { // Every download job ends through here; only external ones need a query. - if (jobLookup(jobId)?.jobType !== EXTERNAL_REQUEST_JOB_LABEL) return; + const jobType = jobLookup(jobId)?.jobType; + if (jobType !== EXTERNAL_REQUEST_JOB_LABEL && !jobType?.startsWith(AUTO_RETRY_LABEL_PREFIX)) return; try { const records = await ExternalRequest.findAll({ where: { diff --git a/server/modules/externalApi/externalRequestReconciliation.js b/server/modules/externalApi/externalRequestReconciliation.js index c2eb67132..14e6c2514 100644 --- a/server/modules/externalApi/externalRequestReconciliation.js +++ b/server/modules/externalApi/externalRequestReconciliation.js @@ -1,47 +1,19 @@ const { Op } = require('sequelize'); +const { parseAuxData } = require('../jobAuxData'); +const { MAX_AUTO_RETRY_COUNT } = require('../download/transient403RetryPlanner'); const FINAL_JOB_STATUSES = new Set([ 'Complete', 'Complete with Warnings', 'Error', 'Failed', 'Terminated', 'Killed', ]); /** - * Closes processing video requests whose download job has ended: completed - * when the video is now on disk, failed otherwise. Runs from the job-ended - * hook and the executor's sweep, never from reads, so a request's status does - * not depend on who happens to read it. + * Follow each video's durable retry handoffs before deciding whether its + * download ended. Both job-ended events and the recovery sweep use this path; + * reads never drive request state. No transactions span the download work. */ function createExternalRequestReconciliation({ models, now, jobLookup = () => null }) { const { ExternalRequest, Video, Job } = models; - async function endedJobIdsFor(records, knownEnded) { - const ended = new Set(knownEnded); - const unknown = []; - for (const record of records) { - const jobId = record.job_id || record.id; - if (ended.has(jobId)) continue; - const liveJob = jobLookup(jobId); - if (liveJob) { - if (FINAL_JOB_STATUSES.has(liveJob.status)) ended.add(jobId); - } else { - unknown.push(jobId); - } - } - if (unknown.length === 0) return ended; - const stored = await Job.findAll({ - where: { id: { [Op.in]: [...new Set(unknown)] } }, - attributes: ['id', 'status'], - }); - const storedById = new Map(stored.map((job) => [job.id, job.status])); - for (const jobId of unknown) { - // A job neither running nor stored was lost (for example, with a - // restart before it was saved); its request can never complete. - if (!storedById.has(jobId) || FINAL_JOB_STATUSES.has(storedById.get(jobId))) { - ended.add(jobId); - } - } - return ended; - } - async function resolveVideoRequests(records, { endedJobIds = [] } = {}) { const processing = records.filter( (record) => record.request_type === 'video' && record.status === 'processing' @@ -52,24 +24,74 @@ function createExternalRequestReconciliation({ models, now, jobLookup = () => nu attributes: ['youtubeId'], }); const downloaded = new Set(videos.map((video) => video.youtubeId)); - const ended = await endedJobIdsFor( - processing.filter((record) => !downloaded.has(record.youtube_id)), - endedJobIds - ); - const resolved = []; + const knownEnded = new Set(endedJobIds); + const jobs = new Map(); + + async function loadJobs(ids) { + const missing = []; + for (const id of new Set(ids)) { + if (jobs.has(id)) continue; + const live = jobLookup(id); + if (live) jobs.set(id, live); + else { jobs.set(id, null); missing.push(id); } + } + if (missing.length === 0) return; + const stored = await Job.findAll({ + where: { id: { [Op.in]: missing } }, + attributes: ['id', 'status', 'aux_data'], + }); + for (const job of stored) jobs.set(job.id, { status: job.status, data: parseAuxData(job.aux_data) }); + } + + await loadJobs(processing.filter((record) => !downloaded.has(record.youtube_id)) + .map((record) => record.job_id || record.id)); + const decisions = []; for (const record of processing) { const finishedAt = now(); - const values = downloaded.has(record.youtube_id) + let jobId = record.job_id || record.id; + let ended = false; + if (!downloaded.has(record.youtube_id)) { + const visited = new Set(); + // The planner permits at most three retries. Bound malformed/cyclic + // history too, so corrupted job data cannot stall the executor. + for (let hop = 0; hop <= MAX_AUTO_RETRY_COUNT; hop += 1) { + if (visited.has(jobId)) { ended = true; break; } + visited.add(jobId); + await loadJobs([jobId]); + const job = jobs.get(jobId); + ended = !job || knownEnded.has(jobId) || FINAL_JOB_STATUSES.has(job.status); + if (!ended) break; + const failure = job?.data?.failedVideos?.find((video) => video.youtubeId === record.youtube_id); + if (!failure?.autoRetryQueued || typeof failure.autoRetryJobId !== 'string') break; + jobId = failure.autoRetryJobId; + } + } + decisions.push({ record, jobId, ended, finishedAt }); + } + // A retry can persist its video and turn terminal after the first video + // query. Observe the final job first, then recheck presence before failing; + // otherwise a stale absence can win the CAS against the completion event. + const failedCandidates = decisions.filter(({ record, ended }) => ended && !downloaded.has(record.youtube_id)); + if (failedCandidates.length > 0) { + const finishedVideos = await Video.findAll({ + where: { youtubeId: [...new Set(failedCandidates.map(({ record }) => record.youtube_id))], removed: false }, + attributes: ['youtubeId'], + }); + finishedVideos.forEach((video) => downloaded.add(video.youtubeId)); + } + const resolved = []; + for (const { record, jobId, ended, finishedAt } of decisions) { + const terminal = downloaded.has(record.youtube_id) ? { status: 'completed', completed_at: finishedAt } - : ended.has(record.job_id || record.id) - ? { status: 'failed', message: 'Download did not complete' } - : null; - if (!values) continue; + : ended ? { status: 'failed', message: 'Download did not complete' } : null; + if (!terminal && jobId === (record.job_id || record.id)) continue; const [updated] = await ExternalRequest.update( - { ...values, active_dedupe_key: null, updated_at: finishedAt }, - { where: { id: record.id, status: 'processing' } } + { ...(terminal || {}), job_id: jobId, + ...(terminal ? { active_dedupe_key: null } : {}), updated_at: finishedAt }, + // An old event/sweep cannot overwrite a newer retry handoff. + { where: { id: record.id, status: 'processing', job_id: record.job_id || null } } ); - if (updated === 1) resolved.push(record.id); + if (updated === 1 && terminal) resolved.push(record.id); } return resolved; } diff --git a/server/modules/videoDeletionModule.js b/server/modules/videoDeletionModule.js index f8ad0c680..59db164a9 100644 --- a/server/modules/videoDeletionModule.js +++ b/server/modules/videoDeletionModule.js @@ -151,7 +151,8 @@ class VideoDeletionModule { return { success: false, videoId, - error: `${libraryCheck.message} Nothing was deleted; check that the drive is mounted.` + error: `${libraryCheck.message} Nothing was deleted; check that the drive is mounted.`, + code: 'LIBRARY_UNAVAILABLE' }; } @@ -288,7 +289,8 @@ class VideoDeletionModule { } else { failed.push({ videoId, - error: result.error || 'Unknown error' + error: result.error || 'Unknown error', + ...(result.code ? { code: result.code } : {}) }); } } @@ -340,7 +342,8 @@ class VideoDeletionModule { } else { failed.push({ youtubeId, - error: result.error || 'Unknown error' + error: result.error || 'Unknown error', + ...(result.code ? { code: result.code } : {}) }); } } catch (error) { diff --git a/server/testing/externalRuntime.database.test.js b/server/testing/externalRuntime.database.test.js index 64f91b72a..9f79dedc2 100644 --- a/server/testing/externalRuntime.database.test.js +++ b/server/testing/externalRuntime.database.test.js @@ -430,3 +430,90 @@ test.each(['missing', 'disabled', 'terminated'])('management rejects a %s channe await request(app).put(`/api/keys/${key.id}/channels`).send({ channelIds }).expect(400, error); expect(await models.ApiKeyChannelGrant.count({ where: { api_key_id: key.id } })).toBe(1); }); + +test('held video batches rotate so a newer deletion can free storage', async () => { + await policy({ auto_approve_delete_requests: true }); + let clock = timestamp; + const downloader = jest.fn(async () => { + throw Object.assign(new Error('paused'), { code: 'DOWNLOADS_PAUSED' }); + }); + const api = service({ now: () => clock, downloader }); + // Five keys stay within the real per-key cap of five active requests. + const downloadKeys = []; + for (let index = 0; index < 5; index += 1) { + const record = await models.ApiKey.create({ name: `Download key ${index}`, key_hash: `synthetic-${index}`, + key_prefix: `syn0000${index}`, created_at: timestamp, role: 'request', + allow_video_requests: true, auto_approve_video_requests: true, allow_unrated: true }); + await models.ApiKeyChannelGrant.create({ api_key_id: record.id, channel_id: channel.id }); + downloadKeys.push(normalizeExternalApiKey(record)); + } + for (let index = 0; index < 21; index += 1) { + const id = `v${String(index).padStart(10, '0')}`; + await models.ChannelVideo.create({ youtube_id: id, channel_id: youtubeChannelId, title: id }); + await api.createVideoRequest(downloadKeys[Math.floor(index / 5)], { youtubeId: id, channelId: channel.id }); + } + clock = new Date(timestamp.getTime() + 1000); + const video = await downloadedVideo(); + const deletion = await api.createDeleteVideoRequest(key, { youtubeId, channelId: channel.id }); + clock = new Date(timestamp.getTime() + 2000); + await api.runExecutorOnce(); + expect(downloader).toHaveBeenCalledTimes(20); + expect((await api.getRequest(key, deletion.request.id)).status).toBe('approved'); + clock = new Date(timestamp.getTime() + 3000); + await api.runExecutorOnce(); + expect((await api.getRequest(key, deletion.request.id)).status).toBe('completed'); + expect((await video.reload()).removed).toBe(true); + expect(await models.ExternalRequest.count({ where: { request_type: 'video', status: 'approved' } })).toBe(21); + expect((await models.ExternalApiUsageBucket.findAll()).reduce((sum, bucket) => sum + bucket.accepted_writes, 0)).toBe(44); +}); + +test('an unavailable library retains deletion identity, approval and quota until recovery', async () => { + await policy({ auto_approve_delete_requests: true }); + const video = await downloadedVideo(); + let mounted = false; + const api = service({ videoDeleter: { deleteVideos: async ids => { + if (!mounted) return { deleted: [], failed: [{ videoId: ids[0], code: 'LIBRARY_UNAVAILABLE', + error: 'The downloads folder does not exist. Nothing was deleted; check that the drive is mounted.' }] }; + await models.Video.update({ removed: true }, { where: { id: ids } }); + return { deleted: ids, failed: [] }; + } } }); + const input = { youtubeId, channelId: channel.id, idempotencyKey: 'nas-recovery' }; + const created = await api.createDeleteVideoRequest(key, input); + await api.runExecutorOnce(); + const held = await models.ExternalRequest.findByPk(created.request.id); + expect(held).toMatchObject({ status: 'approved', approval_source: 'auto', + message: 'Waiting for the downloads folder to become available' }); + expect(held.active_dedupe_key).toBeTruthy(); + expect((await video.reload()).removed).toBe(false); + const duplicate = await api.createDeleteVideoRequest(key, input); + expect(duplicate.request.id).toBe(created.request.id); + mounted = true; + await api.runExecutorOnce(); + expect(await held.reload()).toMatchObject({ status: 'completed', active_dedupe_key: null }); + expect((await models.ExternalApiUsageBucket.findAll()).map(bucket => bucket.accepted_writes)).toEqual([1, 1]); +}); + +test.each(['Complete', 'Error'])('persisted retry chain recovers after restart and ends in %s', async status => { + await policy({ auto_approve_video_requests: true }); + const api = service(); + const created = await api.createVideoRequest(key, { youtubeId, channelId: channel.id }); + await api.runExecutorOnce(); + const retryId = require('crypto').randomUUID(); + await models.Job.update({ status: 'Error', aux_data: JSON.stringify({ failedVideos: [ + { youtubeId, autoRetryQueued: true, autoRetryJobId: retryId }, + ] }) }, { where: { id: created.request.id } }); + await models.Job.create({ id: retryId, status: 'Pending', jobType: 'Auto-retry: 1 video (no cookies)', + timeCreated: timestamp, timeInitiated: timestamp }); + // Fresh service, no live jobs: only persisted history can supply the handoff. + const restarted = service(); + await restarted.runExecutorOnce(); + expect(await models.ExternalRequest.findByPk(created.request.id)).toMatchObject({ + status: 'processing', job_id: retryId, + }); + await models.Job.update({ status }, { where: { id: retryId } }); + if (status === 'Complete') await downloadedVideo(); + await restarted.runExecutorOnce(); + expect((await restarted.getRequest(key, created.request.id)).status) + .toBe(status === 'Complete' ? 'completed' : 'failed'); + expect((await models.ExternalApiUsageBucket.findAll()).map(bucket => bucket.accepted_writes)).toEqual([1, 1]); +});