diff --git a/api/src/app.module.ts b/api/src/app.module.ts index ff1ea16511..25df9b3e6b 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -15,6 +15,7 @@ import { QueryController } from "./endpoints/query.controller"; import { FtsSearchService } from "./endpoints/ftsSearch.service"; import { FtsSearchController } from "./endpoints/ftsSearch.controller"; import { StorageStatusController } from "./endpoints/storageStatus.controller"; +import { EncoderConfigController } from "./endpoints/encoderConfig.controller"; import { AuthIdentityService } from "./auth/authIdentity.service"; import { QueryRateLimiterService } from "./ratelimit/queryRateLimiter.service"; @@ -57,6 +58,7 @@ if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") { QueryController, FtsSearchController, StorageStatusController, + EncoderConfigController, ], providers: [ DbService, diff --git a/api/src/changeRequests/documentProcessing/deleteMediaCollection.spec.ts b/api/src/changeRequests/documentProcessing/deleteMediaCollection.spec.ts new file mode 100644 index 0000000000..e2a97643b4 --- /dev/null +++ b/api/src/changeRequests/documentProcessing/deleteMediaCollection.spec.ts @@ -0,0 +1,99 @@ +import { resolveCollectionPrefix } from "./deleteMediaCollection"; + +/** A real collection URL: MinIO, where the bucket name is part of the public path. */ +const PUBLIC = "http://localhost:9000/media"; +const SESSION = "c5829f07-4ba8-42ed-a449-80d83e6c0b53"; +const HLS = `${PUBLIC}/${SESSION}/master.m3u8`; + +const prefixOf = (r: ReturnType) => + "prefix" in r ? r.prefix : undefined; +const refusalOf = (r: ReturnType) => + "refusal" in r ? r.refusal : undefined; + +describe("resolveCollectionPrefix", () => { + describe("resolves a collection this API wrote", () => { + it("strips the bucket's public base and the master filename", () => { + expect(prefixOf(resolveCollectionPrefix(HLS, PUBLIC))).toBe(SESSION); + }); + + it("tolerates a trailing slash on the configured public URL", () => { + expect(prefixOf(resolveCollectionPrefix(HLS, `${PUBLIC}/`))).toBe(SESSION); + expect(prefixOf(resolveCollectionPrefix(HLS, `${PUBLIC}///`))).toBe(SESSION); + }); + + it("keeps a nested path prefix intact", () => { + // pathPrefix on the session puts the collection in a subfolder. + const url = `${PUBLIC}/tenant-a/videos/${SESSION}/master.m3u8`; + expect(prefixOf(resolveCollectionPrefix(url, PUBLIC))).toBe( + `tenant-a/videos/${SESSION}`, + ); + }); + + it("ignores a query string or fragment", () => { + expect(prefixOf(resolveCollectionPrefix(`${HLS}?v=2`, PUBLIC))).toBe(SESSION); + expect(prefixOf(resolveCollectionPrefix(`${HLS}#top`, PUBLIC))).toBe(SESSION); + }); + + it("handles a bucket published at a bare host", () => { + const base = "https://cdn.example.com"; + expect( + prefixOf(resolveCollectionPrefix(`${base}/${SESSION}/master.m3u8`, base)), + ).toBe(SESSION); + }); + }); + + describe("refuses anything it cannot prove it wrote", () => { + it("refuses a URL in a different bucket", () => { + const other = "https://someone-elses-cdn.example.com/media"; + expect(refusalOf(resolveCollectionPrefix(`${other}/${SESSION}/master.m3u8`, PUBLIC))) + .toMatch(/not in this bucket/); + }); + + it("refuses a bucket whose name merely prefixes another", () => { + // The separator is part of the match, or `…/media` would claim + // `…/media-archive//master.m3u8`. + const url = `${PUBLIC}-archive/${SESSION}/master.m3u8`; + expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/not in this bucket/); + }); + + it("refuses a URL that is not a master playlist", () => { + expect(refusalOf(resolveCollectionPrefix(`${PUBLIC}/${SESSION}/`, PUBLIC))) + .toMatch(/master playlist/); + expect( + refusalOf( + resolveCollectionPrefix(`${PUBLIC}/${SESSION}/stream/playlist.m3u8`, PUBLIC), + ), + ).toMatch(/master playlist/); + }); + + it("refuses the bucket root", () => { + expect(refusalOf(resolveCollectionPrefix(`${PUBLIC}/master.m3u8`, PUBLIC))) + .toMatch(/bucket root/); + }); + + it("refuses a path that tries to climb out", () => { + const url = `${PUBLIC}/../other-tenant/${SESSION}/master.m3u8`; + expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/suspicious path/); + }); + + it("refuses a folder that is not a session id", () => { + // The case the tick box makes possible: hlsUrl is editable, so someone + // can paste a URL naming a folder shared with other content. + const url = `${PUBLIC}/shared-videos/master.m3u8`; + expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/not a session id/); + }); + + it("refuses when either side is missing", () => { + expect(refusalOf(resolveCollectionPrefix(undefined, PUBLIC))).toMatch(/no media URL/); + expect(refusalOf(resolveCollectionPrefix(HLS, undefined))).toMatch(/no public URL/); + expect(refusalOf(resolveCollectionPrefix("", PUBLIC))).toMatch(/no media URL/); + }); + }); + + it("never returns a prefix with a leading or trailing slash", () => { + // The caller appends '/' to scope the listing; a stray slash would widen it. + const prefix = prefixOf(resolveCollectionPrefix(HLS, PUBLIC))!; + expect(prefix.startsWith("/")).toBe(false); + expect(prefix.endsWith("/")).toBe(false); + }); +}); diff --git a/api/src/changeRequests/documentProcessing/deleteMediaCollection.ts b/api/src/changeRequests/documentProcessing/deleteMediaCollection.ts new file mode 100644 index 0000000000..b73f076e0b --- /dev/null +++ b/api/src/changeRequests/documentProcessing/deleteMediaCollection.ts @@ -0,0 +1,169 @@ +import { MediaDto } from "../../dto/MediaDto"; +import { DbService } from "../../db/db.service"; +import { S3Service } from "../../s3/s3.service"; + +/** + * Where a collection lives in its bucket, or why we will not touch it. + * + * A refusal is not an error: it is the safe answer for a URL we cannot prove we + * wrote, and the caller reports it as a warning rather than failing the request. + */ +export type PrefixResolution = { prefix: string } | { refusal: string }; + +/** + * The encoder names every collection prefix after its session id. + * + * Checked because `hlsUrl` is an editable field: someone can paste a URL naming a + * shared folder, and "delete everything under it" would then be a data-loss bug + * wearing a tick box. A collection this API did not produce is one it must not + * remove — and every collection the encoder has ever written satisfies this. + */ +const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** What the encoder publishes at the root of a collection. */ +const MASTER = "/master.m3u8"; + +/** + * Turn a published `hlsUrl` into the object prefix holding that collection. + * + * The guard falls out of the arithmetic rather than being bolted on: the only way + * to get an object key from a public URL is to strip the bucket's own public base + * from it, so a URL that does not start with that base cannot be resolved at all. + * That is precisely the "never delete a prefix we did not create" rule, and unlike + * a marker object it also protects every collection already in a bucket. + */ +export function resolveCollectionPrefix( + hlsUrl: string | undefined, + publicUrl: string | undefined, +): PrefixResolution { + if (!hlsUrl) return { refusal: "the document has no media URL" }; + if (!publicUrl) return { refusal: "the bucket has no public URL configured" }; + + // Query strings and fragments are addressing, not location. + const url = hlsUrl.split(/[?#]/)[0]; + const base = publicUrl.replace(/\/+$/, ""); + + // The separator has to be part of the match, or a bucket published at + // `https://cdn/media` would claim URLs belonging to `https://cdn/media-archive`. + if (!url.startsWith(`${base}/`)) { + return { + refusal: `the media URL is not in this bucket (expected it to start with ${base}/)`, + }; + } + + const key = url.slice(base.length + 1); + + // Named before the suffix check below, which would otherwise report a master + // at the bucket root as "not a master playlist" — true but unhelpful for the + // one input where being clear matters most. + if (key === MASTER.slice(1)) return { refusal: "the media URL names the bucket root" }; + + if (!key.endsWith(MASTER)) { + return { + refusal: `the media URL does not name a master playlist (expected it to end with ${MASTER})`, + }; + } + + const prefix = key.slice(0, -MASTER.length); + if (!prefix) return { refusal: "the media URL names the bucket root" }; + + // A traversal cannot reach outside the bucket, but it can certainly reach a + // sibling prefix, and there is no legitimate reason for one to be here. + const segments = prefix.split("/"); + if (segments.some((s) => s === "" || s === "." || s === "..")) { + return { refusal: `the media URL has a suspicious path (${prefix})` }; + } + + const last = segments[segments.length - 1]; + if (!SESSION_ID.test(last)) { + return { + refusal: + `the media URL was not written by the encoder — its last folder ` + + `(${last}) is not a session id, so this API did not create it`, + }; + } + + return { prefix }; +} + +/** + * Delete the collection a document points at, if we can prove we wrote it. + * + * Best-effort by design, matching how images are handled: the caller is deleting a + * document, and refusing to do that because a bucket was unreachable would be + * worse than leaving objects behind. Everything that goes wrong comes back as a + * warning the CMS shows, and every key removed is logged first — the first real + * deletion in any bucket should be auditable after the fact. + */ +export async function deleteMediaCollection( + media: MediaDto | undefined, + bucketId: string | undefined, + db: DbService, +): Promise { + const warnings: string[] = []; + + if (!media?.hlsUrl) return warnings; + if (!bucketId) { + warnings.push( + "Media files were not deleted: the document has no storage bucket. " + + "Please remove them on the storage provider.", + ); + return warnings; + } + + let bucket: { publicUrl?: string; name?: string }; + try { + const result = await db.getDoc(bucketId); + if (!result.docs?.length) { + warnings.push( + `Media files were not deleted: bucket ${bucketId} no longer exists. ` + + "Please remove them on the storage provider.", + ); + return warnings; + } + bucket = result.docs[0]; + } catch (error) { + warnings.push(`Media files were not deleted: ${error.message}`); + return warnings; + } + + const resolved = resolveCollectionPrefix(media.hlsUrl, bucket.publicUrl); + if ("refusal" in resolved) { + warnings.push( + `Media files were not deleted because ${resolved.refusal}. ` + + "Please remove them on the storage provider if they are no longer needed.", + ); + return warnings; + } + + try { + const s3 = await S3Service.create(bucketId, db); + const keys = await s3.listObjectsUnder(`${resolved.prefix}/`); + + if (keys.length === 0) { + // Already gone, or never uploaded. Not worth a warning: the caller + // asked for the files to be absent and they are. + return warnings; + } + + // Named before removal, so a deletion that turns out to be wrong can be + // reconstructed from the log rather than guessed at. + console.log( + `Deleting ${keys.length} media object(s) under ${resolved.prefix}/ ` + + `in bucket ${bucket.name ?? bucketId}: ${keys.join(", ")}`, + ); + + // Batched: removeObjects takes a list, and a collection can run to + // hundreds of objects. 1000 is the S3 API's own limit per call. + for (let i = 0; i < keys.length; i += 1000) { + await s3.removeObjects(keys.slice(i, i + 1000)); + } + } catch (error) { + warnings.push( + `Some media files could not be deleted from storage: ${error.message}. ` + + "Please check the storage provider.", + ); + } + + return warnings; +} diff --git a/api/src/changeRequests/documentProcessing/migrateMediaCollection.spec.ts b/api/src/changeRequests/documentProcessing/migrateMediaCollection.spec.ts new file mode 100644 index 0000000000..06557b314f --- /dev/null +++ b/api/src/changeRequests/documentProcessing/migrateMediaCollection.spec.ts @@ -0,0 +1,244 @@ +import { migrateMediaCollection } from "./migrateMediaCollection"; +import { S3Service } from "../../s3/s3.service"; +import { DbService } from "../../db/db.service"; +import { MediaDto } from "../../dto/MediaDto"; + +jest.mock("../../s3/s3.service", () => ({ S3Service: { create: jest.fn() } })); + +const SESSION = "c5829f07-4ba8-42ed-a449-80d83e6c0b53"; +const OLD_BASE = "http://old.example.com/media"; +const NEW_BASE = "http://new.example.com/media"; +const OLD_URL = `${OLD_BASE}/${SESSION}/master.m3u8`; + +const KEYS = [ + `${SESSION}/master.m3u8`, + `${SESSION}/stream_1080p/playlist.m3u8`, + `${SESSION}/media/v0_0.m4s`, +]; + +/** Buckets keyed by id, as `db.getDoc` would return them. */ +const stubDb = (buckets: Record) => + ({ + getDoc: jest.fn(async (id: string) => + buckets[id] ? { docs: [buckets[id]] } : { docs: [] }, + ), + }) as unknown as DbService; + +const defaultDb = () => + stubDb({ + "bucket-old": { publicUrl: OLD_BASE, name: "old-bucket" }, + "bucket-new": { publicUrl: NEW_BASE, name: "new-bucket" }, + }); + +/** + * A pair of fake buckets. Sizes are recorded per key so a truncated copy can be + * simulated by returning a different size from the destination. + */ +const stubS3 = ( + opts: { + keys?: string[]; + sizes?: Record; + destinationSizes?: Record; + putRejects?: string; + removeRejects?: boolean; + } = {}, +) => { + const keys = opts.keys ?? KEYS; + const sizes = opts.sizes ?? Object.fromEntries(keys.map((k) => [k, 100])); + + const source = { + listObjectsUnder: jest.fn().mockResolvedValue(keys), + statObject: jest.fn(async (k: string) => ({ + size: sizes[k], + metaData: { "content-type": "video/iso.segment" }, + })), + getObject: jest.fn(async (k: string) => `stream:${k}`), + removeObjects: opts.removeRejects + ? jest.fn().mockRejectedValue(new Error("bucket is read-only")) + : jest.fn().mockResolvedValue(undefined), + }; + + const destination = { + putStream: jest.fn(async (k: string) => { + if (opts.putRejects === k) throw new Error("connection reset"); + }), + statObject: jest.fn(async (k: string) => ({ + size: (opts.destinationSizes ?? sizes)[k], + })), + }; + + (S3Service.create as jest.Mock).mockImplementation(async (bucketId: string) => + bucketId === "bucket-old" ? source : destination, + ); + + return { source, destination }; +}; + +const media = (): MediaDto => ({ hlsUrl: OLD_URL }) as MediaDto; + +const migrate = (m: MediaDto, db: DbService) => + migrateMediaCollection(m, OLD_URL, "bucket-old", "bucket-new", db); + +describe("migrateMediaCollection", () => { + beforeEach(() => jest.clearAllMocks()); + + it("copies every object, then repoints the document at the new bucket", async () => { + const { source, destination } = stubS3(); + const m = media(); + + const result = await migrate(m, defaultDb()); + + expect(result.failed).toBe(false); + expect(destination.putStream).toHaveBeenCalledTimes(KEYS.length); + expect(m.hlsUrl).toBe(`${NEW_BASE}/${SESSION}/master.m3u8`); + expect(source.removeObjects).toHaveBeenCalledWith(KEYS); + }); + + it("preserves each object's key, so the playlists' relative paths still resolve", async () => { + // Media playlists reference segments as `../media/_.m4s`. Renaming + // anything on the way across would break playback silently. + const { destination } = stubS3(); + + await migrate(media(), defaultDb()); + + expect(destination.putStream.mock.calls.map((c) => c[0])).toEqual(KEYS); + }); + + it("streams rather than buffering, and passes the source's size and type", async () => { + const { destination } = stubS3({ sizes: Object.fromEntries(KEYS.map((k) => [k, 512])) }); + + await migrate(media(), defaultDb()); + + expect(destination.putStream).toHaveBeenCalledWith( + KEYS[0], + `stream:${KEYS[0]}`, + 512, + "video/iso.segment", + ); + }); + + it("does not delete the source or move the URL when a copy fails", async () => { + // The guarantee that matters: a failed migration leaves a whole, reachable + // collection where the document already says it is. + const { source } = stubS3({ putRejects: KEYS[1] }); + const m = media(); + + const result = await migrate(m, defaultDb()); + + expect(result.failed).toBe(true); + expect(source.removeObjects).not.toHaveBeenCalled(); + expect(m.hlsUrl).toBe(OLD_URL); + }); + + it("treats a truncated copy as a failure", async () => { + const { source } = stubS3({ + sizes: { [KEYS[0]]: 100, [KEYS[1]]: 100, [KEYS[2]]: 100 }, + destinationSizes: { [KEYS[0]]: 100, [KEYS[1]]: 40, [KEYS[2]]: 100 }, + }); + const m = media(); + + const result = await migrate(m, defaultDb()); + + expect(result.failed).toBe(true); + expect(result.warnings.join(" ")).toContain(KEYS[1]); + expect(source.removeObjects).not.toHaveBeenCalled(); + expect(m.hlsUrl).toBe(OLD_URL); + }); + + it("succeeds when the copy worked but the originals could not be removed", async () => { + // Leftovers cost storage; they do not break playback, and the document + // already points at the new bucket. + stubS3({ removeRejects: true }); + const m = media(); + + const result = await migrate(m, defaultDb()); + + expect(result.failed).toBe(false); + expect(m.hlsUrl).toBe(`${NEW_BASE}/${SESSION}/master.m3u8`); + expect(result.warnings.join(" ")).toContain("could not be removed"); + }); + + it("refuses a URL it cannot prove the encoder wrote", async () => { + stubS3(); + const m = { hlsUrl: `${OLD_BASE}/shared-folder/master.m3u8` } as MediaDto; + + const result = await migrateMediaCollection( + m, + `${OLD_BASE}/shared-folder/master.m3u8`, + "bucket-old", + "bucket-new", + defaultDb(), + ); + + expect(result.failed).toBe(true); + expect(result.warnings.join(" ")).toContain("not a session id"); + }); + + it("refuses when the destination has no public URL to publish under", async () => { + stubS3(); + const db = stubDb({ + "bucket-old": { publicUrl: OLD_BASE, name: "old" }, + "bucket-new": { name: "new" }, + }); + + const result = await migrate(media(), db); + + expect(result.failed).toBe(true); + expect(result.warnings.join(" ")).toContain("no public URL"); + expect(S3Service.create).not.toHaveBeenCalled(); + }); + + it("fails when a bucket document has gone", async () => { + stubS3(); + const db = stubDb({ "bucket-old": { publicUrl: OLD_BASE } }); + + const result = await migrate(media(), db); + + expect(result.failed).toBe(true); + expect(result.warnings.join(" ")).toContain("no longer exists"); + }); + + it("reports an empty source rather than repointing at nothing", async () => { + stubS3({ keys: [] }); + const m = media(); + + const result = await migrate(m, defaultDb()); + + expect(result.failed).toBe(true); + expect(m.hlsUrl).toBe(OLD_URL); + }); + + it("leaves a hand-edited URL alone instead of moving files under it", async () => { + // Changing the URL and the bucket together is repointing the document, not + // asking for a migration. + stubS3(); + const m = { hlsUrl: "http://elsewhere/x/master.m3u8" } as MediaDto; + + const result = await migrateMediaCollection( + m, + OLD_URL, + "bucket-old", + "bucket-new", + defaultDb(), + ); + + expect(result.failed).toBe(false); + expect(m.hlsUrl).toBe("http://elsewhere/x/master.m3u8"); + expect(S3Service.create).not.toHaveBeenCalled(); + }); + + it("does nothing for a document that never had media", async () => { + stubS3(); + + const result = await migrateMediaCollection( + {} as unknown as MediaDto, + undefined, + "bucket-old", + "bucket-new", + defaultDb(), + ); + + expect(result.failed).toBe(false); + expect(S3Service.create).not.toHaveBeenCalled(); + }); +}); diff --git a/api/src/changeRequests/documentProcessing/migrateMediaCollection.ts b/api/src/changeRequests/documentProcessing/migrateMediaCollection.ts new file mode 100644 index 0000000000..36e564aceb --- /dev/null +++ b/api/src/changeRequests/documentProcessing/migrateMediaCollection.ts @@ -0,0 +1,171 @@ +import { MediaDto } from "../../dto/MediaDto"; +import { DbService } from "../../db/db.service"; +import { S3Service } from "../../s3/s3.service"; +import { resolveCollectionPrefix } from "./deleteMediaCollection"; + +/** What the encoder publishes at the root of a collection. */ +const MASTER = "master.m3u8"; + +/** The S3 API's own ceiling on keys per delete call. */ +const DELETE_BATCH = 1000; + +/** + * Where a bucket publishes its objects, and what to call it in a warning. + */ +type Bucket = { publicUrl?: string; name?: string }; + +async function loadBucket( + bucketId: string, + db: DbService, +): Promise<{ bucket: Bucket } | { error: string }> { + try { + const result = await db.getDoc(bucketId); + if (!result.docs?.length) return { error: `bucket ${bucketId} no longer exists` }; + return { bucket: result.docs[0] }; + } catch (error) { + return { error: error.message }; + } +} + +/** + * Move a media collection from one bucket to another, then point the document at + * its new home. + * + * Ordering is the whole design. Copy everything, prove every object arrived, only + * then rewrite `hlsUrl`, and only then delete the source. A collection is not a set + * of independent files — a master playlist without its segments is a broken video — + * so this deliberately does not follow the per-file "upload then delete" of + * `migrateImagesBetweenBuckets`, where a partial result costs one thumbnail. + * + * On any failure the caller reverts `mediaBucketId`, which is what keeps the + * document honest: `mediaBucketId` and `hlsUrl` must always name the same bucket, + * or a later delete cannot resolve the collection and the files leak. + */ +export async function migrateMediaCollection( + media: MediaDto, + previousHlsUrl: string | undefined, + oldBucketId: string, + newBucketId: string, + db: DbService, +): Promise<{ failed: boolean; warnings: string[] }> { + const warnings: string[] = []; + + if (!previousHlsUrl) return { failed: false, warnings }; + + // A URL edited in the same save as a bucket change is the user repointing the + // document by hand, not asking for a move. Moving files then overwriting their + // edit would undo a deliberate action. + if (media.hlsUrl && media.hlsUrl !== previousHlsUrl) { + warnings.push( + "The media URL and the storage bucket were changed together, so no files were " + + "moved. Change the bucket on its own if you want the existing files migrated.", + ); + return { failed: false, warnings }; + } + + const oldResult = await loadBucket(oldBucketId, db); + if ("error" in oldResult) { + warnings.push(`Media files were not moved: ${oldResult.error}.`); + return { failed: true, warnings }; + } + const newResult = await loadBucket(newBucketId, db); + if ("error" in newResult) { + warnings.push(`Media files were not moved: ${newResult.error}.`); + return { failed: true, warnings }; + } + + const oldBucket = oldResult.bucket; + const newBucket = newResult.bucket; + + if (!newBucket.publicUrl) { + warnings.push( + "Media files were not moved: the destination bucket has no public URL configured, " + + "so the new media URL cannot be built.", + ); + return { failed: true, warnings }; + } + + // The same proof used before deleting: a prefix we cannot derive from the + // bucket's own public base is a collection we did not write. + const resolved = resolveCollectionPrefix(previousHlsUrl, oldBucket.publicUrl); + if ("refusal" in resolved) { + warnings.push(`Media files were not moved because ${resolved.refusal}.`); + return { failed: true, warnings }; + } + const prefix = resolved.prefix; + + try { + const source = await S3Service.create(oldBucketId, db); + const destination = await S3Service.create(newBucketId, db); + + const keys = await source.listObjectsUnder(`${prefix}/`); + if (keys.length === 0) { + warnings.push( + `Media files were not moved: nothing was found under ${prefix}/ in ` + + `${oldBucket.name ?? oldBucketId}.`, + ); + return { failed: true, warnings }; + } + + // Copy first, whole collection, streaming each object. Sequential on + // purpose: these are large objects and the point is to keep memory flat. + for (const key of keys) { + const stat = await source.statObject(key); + const stream = await source.getObject(key); + const contentType = + (stat.metaData && stat.metaData["content-type"]) || "application/octet-stream"; + + await destination.putStream(key, stream, stat.size, contentType); + + // Verified per object rather than at the end: the size is the one thing + // a truncated copy gets wrong, and checking it here names the object + // that failed instead of reporting the collection as generally bad. + const copied = await destination.statObject(key); + if (copied.size !== stat.size) { + throw new Error( + `${key} copied as ${copied.size} bytes but the source is ${stat.size}`, + ); + } + } + + // Only now is the new location real, so only now may the document name it. + media.hlsUrl = `${newBucket.publicUrl.replace(/\/+$/, "")}/${prefix}/${MASTER}`; + + // Last, and its failure is not the migration's failure: the files are in + // the new bucket and the document points at them. Leftovers in the old + // bucket cost storage, not playback. + try { + console.log( + `Moved ${keys.length} media object(s) under ${prefix}/ from ` + + `${oldBucket.name ?? oldBucketId} to ${newBucket.name ?? newBucketId}; ` + + "removing the originals", + ); + for (let i = 0; i < keys.length; i += DELETE_BATCH) { + await source.removeObjects(keys.slice(i, i + DELETE_BATCH)); + } + } catch (error) { + warnings.push( + `Media files were copied to ${newBucket.name ?? newBucketId} but the originals ` + + `could not be removed from ${oldBucket.name ?? oldBucketId}: ${error.message}. ` + + "Please remove them on the storage provider.", + ); + } + + warnings.push( + `Successfully moved ${keys.length} media file(s) from ` + + `${oldBucket.name ?? oldBucketId} to ${newBucket.name ?? newBucketId}.`, + ); + return { failed: false, warnings }; + } catch (error) { + // Nothing was deleted and the URL was not rewritten, so the collection is + // still whole and still where the document says it is. Anything already + // copied is left in place: it is unreferenced, harmless, and overwritten by + // a retry — whereas deleting it on the way out of a failure risks removing + // objects we did not put there. + warnings.push( + `Media migration failed: ${error.message}. The files were left in ` + + `${oldBucket.name ?? oldBucketId}.`, + ); + return { failed: true, warnings }; + } +} diff --git a/api/src/changeRequests/documentProcessing/processMediaDto.spec.ts b/api/src/changeRequests/documentProcessing/processMediaDto.spec.ts index ddf28f8195..a7456ab5c6 100644 --- a/api/src/changeRequests/documentProcessing/processMediaDto.spec.ts +++ b/api/src/changeRequests/documentProcessing/processMediaDto.spec.ts @@ -1,393 +1,80 @@ import { processMedia } from "./processMediaDto"; -import { S3Service } from "../../s3/s3.service"; import { createTestingModule } from "../../test/testingModule"; -import * as fs from "fs"; -import * as path from "path"; -import { v4 as uuidv4 } from "uuid"; import { MediaDto } from "../../dto/MediaDto"; -import { MediaPreset, MediaType, DocType, StorageType } from "../../enums"; import { DbService } from "../../db/db.service"; -import { storeCryptoData } from "../../util/encryption"; -import { s3TestConfig, createTestCredentials } from "../../test/s3TestConfig"; +import { retrieveCryptoData } from "../../util/encryption"; + +const HLS_URL = "https://cdn.example.com/media/post-1/master.m3u8"; +const HLS_KEY = "0123456789abcdef0123456789abcdef"; describe("processMediaDto", () => { let db: DbService; - let s3Service: S3Service; - let testBucketId: string; - let testBucket: string; - const resMedia: MediaDto[] = []; - - const testCredentials = createTestCredentials(); beforeAll(async () => { const module = await createTestingModule("process-media-dto"); db = module.dbService; - - testBucket = `test-media-${uuidv4()}`; - testBucketId = `storage-test-${uuidv4()}`; - testCredentials.bucketName = testBucket; - - // Create encrypted credentials for the test bucket - const encryptedCredId = await storeCryptoData(db, testCredentials); - - // Create a bucket document - const bucketDoc = { - _id: testBucketId, - type: DocType.Storage, - name: "Test Media Bucket", - mimeTypes: ["audio/*"], - publicUrl: `${s3TestConfig.publicUrl}/${testBucket}`, - storageType: StorageType.Media, - credential_id: encryptedCredId, - memberOf: ["group-super-admins"], - updatedTimeUtc: Date.now(), - }; - - await db.upsertDoc(bucketDoc); - - // Create S3Service instance and create the bucket - s3Service = await S3Service.create(testBucketId, db); - await s3Service.makeBucket(); }); - afterAll(async () => { - // Cleanup uploaded media files - const removeFiles = Array.from( - new Set( - resMedia.flatMap((r) => - r.fileCollections.map((f) => f.fileUrl.split("/").pop()!).filter(Boolean), - ), - ), - ); - if (removeFiles.length > 0) { - try { - await s3Service.removeObjects(removeFiles); - } catch { - // Ignore errors during cleanup - } - } - try { - await s3Service.removeBucket(); - } catch { - // Ignore errors if bucket is not empty or doesn't exist - } - - // Clean up storage document - if (testBucketId) { - const storageDoc = (await db.getDoc(testBucketId)).docs[0]; - if (storageDoc) { - storageDoc.deleteReq = 1; - await db.upsertDoc(storageDoc); - } - } + it("stores a submitted HLS key as a crypto object and keeps only the reference", async () => { + const media: MediaDto = { hlsUrl: HLS_URL, hlsKey: HLS_KEY }; - S3Service.clearCache(); - }); + const warnings = await processMedia(media, db); - it("should be defined", () => { - expect(processMedia).toBeDefined(); + expect(warnings).toEqual([]); + expect(media.hlsKey_id).toBeDefined(); + // The key itself must not survive onto the document. + expect(media.hlsKey).toBeUndefined(); + expect(media.hlsUrl).toBe(HLS_URL); }); - it("can process and upload a media file", async () => { - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - const warnings = await processMedia(media, undefined, db, testBucketId); - expect(warnings.warnings.length).toBe(0); + it("stores a key that can be read back and decrypted", async () => { + const media: MediaDto = { hlsUrl: HLS_URL, hlsKey: HLS_KEY }; - // Check if files are uploaded (allow informational warnings like S3_PUBLIC_ACCESS_URL not configured) - const files = media.fileCollections.map((f) => f.fileUrl.split("/").pop()!); - expect(files.length).toBeGreaterThan(0); + await processMedia(media, db); - for (const file of files) { - const exists = await s3Service.objectExists(file); - expect(exists).toBe(true); - } - resMedia.push(media); + await expect(retrieveCryptoData(db, media.hlsKey_id!)).resolves.toBe(HLS_KEY); }); - it("can delete a removed media from S3", async () => { - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - await processMedia(media, undefined, db, testBucketId); - const originalFiles = media.fileCollections.map((f) => f.fileUrl.split("/").pop()!); - - // Simulate removing the media - const prevMedia = JSON.parse(JSON.stringify(media)) as MediaDto; - media.fileCollections = []; + it("gives each submission its own crypto object", async () => { + const first: MediaDto = { hlsUrl: HLS_URL, hlsKey: HLS_KEY }; + const second: MediaDto = { hlsUrl: HLS_URL, hlsKey: HLS_KEY }; - // Process with previous media - await processMedia(media, prevMedia, db, testBucketId); + await processMedia(first, db); + await processMedia(second, db); - // Check if removed files are gone - for (const file of originalFiles) { - const exists = await s3Service.objectExists(file); - expect(exists).toBe(false); - } + expect(first.hlsKey_id).not.toBe(second.hlsKey_id); }); - it("discards user-added file collection objects", async () => { - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - await processMedia(media, undefined, db, testBucketId); - - const media2 = JSON.parse(JSON.stringify(media)) as MediaDto; - media2.fileCollections.push({ - languageId: "invalid", - fileUrl: "http://example.com/invalid.mp3", - bitrate: 128, - mediaType: MediaType.Audio, - }); + it("leaves an unencrypted collection alone", async () => { + const media: MediaDto = { hlsUrl: HLS_URL }; - await processMedia(media2, media, db, testBucketId); + const warnings = await processMedia(media, db); - // Check if the client-added file collection is removed - expect(media2.fileCollections.length).toBe(1); - - resMedia.push(media); + expect(warnings).toEqual([]); + expect(media.hlsKey_id).toBeUndefined(); }); - it("should allow uploading media for different languages independently", async () => { - // First, upload media for English - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - await processMedia(media, undefined, db, testBucketId); - expect(media.fileCollections.length).toBe(1); - expect(media.fileCollections[0].languageId).toBe("lang-eng"); - - const englishFileUrl = media.fileCollections[0].fileUrl; - - // Now upload media for Spanish, keeping the English media - const media2 = JSON.parse(JSON.stringify(media)) as MediaDto; - media2.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-spa", - }, - ]; - - await processMedia(media2, media, db, testBucketId); - - // Should have both English and Spanish media - expect(media2.fileCollections.length).toBe(2); - expect(media2.fileCollections.find((f) => f.languageId === "lang-eng")).toBeDefined(); - expect(media2.fileCollections.find((f) => f.languageId === "lang-spa")).toBeDefined(); - expect(media2.fileCollections.find((f) => f.languageId === "lang-eng")?.fileUrl).toBe( - englishFileUrl, - ); + it("keeps an existing key reference when no new key is submitted", async () => { + const media: MediaDto = { hlsUrl: HLS_URL, hlsKey_id: "crypto-existing" }; - // Verify both files exist in S3 - for (const fileCollection of media2.fileCollections) { - const filename = fileCollection.fileUrl.split("/").pop()!; - const exists = await s3Service.objectExists(filename); - expect(exists).toBe(true); - } + await processMedia(media, db); - resMedia.push(media2); + expect(media.hlsKey_id).toBe("crypto-existing"); }); - it("should replace media when uploading for same language", async () => { - // First, upload media for English - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - await processMedia(media, undefined, db, testBucketId); - expect(media.fileCollections.length).toBe(1); + it("drops the key rather than persisting it in plain text when storing fails", async () => { + const media: MediaDto = { hlsUrl: HLS_URL, hlsKey: HLS_KEY }; + const failingDb = { + upsertDoc: () => Promise.reject(new Error("database unavailable")), + } as unknown as DbService; - const firstFileUrl = media.fileCollections[0].fileUrl; - - // Upload a new media for English (should replace the old one) - const media2 = JSON.parse(JSON.stringify(media)) as MediaDto; - media2.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - - await processMedia(media2, media, db, testBucketId); - - // Should still have only one media file (the new one) - expect(media2.fileCollections.length).toBe(1); - expect(media2.fileCollections[0].languageId).toBe("lang-eng"); - expect(media2.fileCollections[0].fileUrl).not.toBe(firstFileUrl); - - resMedia.push(media2); - }); - - it("should warn when parentBucketId is not provided for upload", async () => { - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - - // Call without parentBucketId (undefined) - const result = await processMedia(media, undefined, db, undefined); - - // Should have a warning about missing bucket - expect(result.warnings.length).toBeGreaterThan(0); - }); - - it("should warn when bucket document is not found", async () => { - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - ]; - - // Call with a non-existent bucket ID - const result = await processMedia(media, undefined, db, "nonexistent-bucket-id"); - - // Should have warnings about bucket not found - expect(result.warnings.length).toBeGreaterThan(0); - }); - - it("should warn when db is not provided for file deletion", async () => { - const media = new MediaDto(); - media.fileCollections = []; - - const prevMedia = new MediaDto(); - prevMedia.fileCollections = [ - { - languageId: "lang-eng", - fileUrl: `http://localhost:9000/test/some-file-key`, - bitrate: 128, - mediaType: MediaType.Audio, - }, - ]; - - // Call with no db and no parentBucketId - files to delete but no way to delete them - const result = await processMedia(media, prevMedia, undefined as any, undefined); - - expect(result.warnings.some((w) => w.includes("cannot be automatically deleted"))).toBe( - true, + await expect(processMedia(media, failingDb)).rejects.toThrow( + /Failed to encrypt the HLS key/, ); - }); - - it("should delete media file from S3 when removed from fileCollections", async () => { - // First, upload media for English and Spanish - const media = new MediaDto(); - media.fileCollections = []; - media.uploadData = [ - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-eng", - }, - { - fileData: fs.readFileSync( - path.resolve(__dirname + "/../../test/" + "silence.wav"), - ) as unknown as ArrayBuffer, - preset: MediaPreset.Default, - mediaType: MediaType.Audio, - languageId: "lang-spa", - }, - ]; - await processMedia(media, undefined, db, testBucketId); - expect(media.fileCollections.length).toBe(2); - - const englishFile = media.fileCollections.find((f) => f.languageId === "lang-eng"); - const spanishFile = media.fileCollections.find((f) => f.languageId === "lang-spa"); - expect(englishFile).toBeDefined(); - expect(spanishFile).toBeDefined(); - - const englishKey = englishFile!.fileUrl.split("/").pop()!; - const spanishKey = spanishFile!.fileUrl.split("/").pop()!; - - // Verify both files exist in S3 - expect(await s3Service.objectExists(englishKey)).toBe(true); - expect(await s3Service.objectExists(spanishKey)).toBe(true); - - // Remove English media from fileCollections (simulate user deletion) - const media2 = JSON.parse(JSON.stringify(media)) as MediaDto; - media2.fileCollections = media2.fileCollections.filter((f) => f.languageId !== "lang-eng"); - - await processMedia(media2, media, db, testBucketId); - - // Should only have Spanish media now - expect(media2.fileCollections.length).toBe(1); - expect(media2.fileCollections[0].languageId).toBe("lang-spa"); - - // Verify English file is deleted from S3 - expect(await s3Service.objectExists(englishKey)).toBe(false); - - // Verify Spanish file still exists in S3 - expect(await s3Service.objectExists(spanishKey)).toBe(true); - resMedia.push(media2); + // The whole point of the finally: a key that could not be encrypted must + // not reach the document by way of the caller's error handling. + expect(media.hlsKey).toBeUndefined(); + expect(media.hlsKey_id).toBeUndefined(); }); }); diff --git a/api/src/changeRequests/documentProcessing/processMediaDto.ts b/api/src/changeRequests/documentProcessing/processMediaDto.ts index b6572455d9..eba63f26e8 100644 --- a/api/src/changeRequests/documentProcessing/processMediaDto.ts +++ b/api/src/changeRequests/documentProcessing/processMediaDto.ts @@ -1,451 +1,37 @@ import { MediaDto } from "../../dto/MediaDto"; -import { MediaUploadDataDto } from "../../dto/MediaUploadDataDto"; -import { MediaFileDto } from "../../dto/MediaFileDto"; -import { v4 as uuidv4 } from "uuid"; -import { S3Service } from "../../s3/s3.service"; import { DbService } from "../../db/db.service"; -import { StorageDto } from "../../dto/StorageDto"; -import { DocType } from "../../enums"; -import { getAudioFormatInfo } from "../../s3-audio/audioFormatDetection"; +import { storeCryptoData } from "../../util/encryption"; /** - * Migrates all media files from one bucket to another - * Supports migration between different S3 systems (e.g., MinIO to AWS S3, or different MinIO instances) - * Each bucket uses its own credentials and endpoint, enabling cross-system transfers - * Only deletes from old bucket if migration is successful + * Processes the media object on a content parent document. * - * @param media - The media DTO containing file collections to migrate - * @param oldBucketId - The ID of the source bucket - * @param newBucketId - The ID of the destination bucket - * @param db - Database service to retrieve bucket configurations - * @returns Object with migration failure status and warnings + * Media is an HLS collection produced by the Luminary Media Convert desktop app. + * That app writes to the storage bucket itself, so there is nothing to upload here — + * the document carries a URL to a collection this API never handles the bytes of on + * the way in. + * + * What this function handles is the decryption key. It arrives once, on the change + * request that first saves the collection, and is stored as a crypto object so it + * never rests in plain text on the content document. + * + * Moving and removing the collection are the caller's, in `processPostTagDto`: + * `migrateMediaCollection` on a bucket change and `deleteMediaCollection` when the + * document is deleted and the user asked for the files to go with it. */ -async function migrateMediaBetweenBuckets( - media: MediaDto, - oldBucketId: string, - newBucketId: string, - db: DbService, -): Promise<{ failed: boolean; warnings: string[] }> { +export async function processMedia(media: MediaDto, db: DbService): Promise { const warnings: string[] = []; - try { - // Create S3Service instances for each bucket - const oldS3Service = await S3Service.create(oldBucketId, db); - const newS3Service = await S3Service.create(newBucketId, db); - - // Get all media files to migrate - const allFiles = media.fileCollections; - - if (allFiles.length === 0) { - warnings.push("No media files to migrate."); - return { failed: false, warnings }; - } - - const oldBucketName = oldS3Service.getBucketName(); - const newBucketName = newS3Service.getBucketName(); - - let successfulMigrations = 0; - let failedMigrations = 0; - - // Migrate each file - for (const fileCollection of allFiles) { - try { - // Extract filename from URL - const urlParts = fileCollection.fileUrl.split("/"); - const filename = urlParts[urlParts.length - 1]; - - // Download from old bucket - const fileStream = await oldS3Service.getObject(filename); - const chunks: Uint8Array[] = []; - - // Collect all chunks - await new Promise((resolve, reject) => { - fileStream.on("data", (chunk: Uint8Array) => chunks.push(chunk)); - fileStream.on("end", () => resolve()); - fileStream.on("error", (err) => reject(err)); - }); - - const fileBuffer = Buffer.concat(chunks); - - // Get metadata from old bucket - const stat = await oldS3Service.getClient().statObject(oldBucketName, filename); - const metadata = stat.metaData || { "Content-Type": "audio/mpeg" }; - - // Upload to new bucket - await newS3Service.uploadFile( - filename, - fileBuffer, - metadata["Content-Type"] || "audio/mpeg", - ); - - // Delete from old bucket only after successful upload - await oldS3Service.getClient().removeObject(oldBucketName, filename); - - successfulMigrations++; - } catch (error) { - failedMigrations++; - warnings.push( - `Failed to migrate media file from bucket ${oldBucketName} to ${newBucketName}: ${error.message}`, - ); - } - } - - if (successfulMigrations > 0) { - warnings.push( - `Successfully migrated ${successfulMigrations} media file(s) from bucket ${oldBucketName} to ${newBucketName}`, - ); - } - - if (failedMigrations > 0) { - warnings.push( - `Failed to migrate ${failedMigrations} media file(s). These files remain in the old bucket.`, - ); - } - - // Migration is considered failed if ANY files failed to migrate - return { failed: failedMigrations > 0, warnings }; - } catch (error) { - warnings.push(`Media migration failed: ${error.message}`); - return { failed: true, warnings }; - } -} - -/** - * Processes an embedded media upload by uploading to S3 - * Requires bucket-specific credentials configured at the post/tag level - * Bucket ID is passed from the parent post/tag document for consistency - * Returns object with migration failure status and warnings - */ -export async function processMedia( - media: MediaDto, - prevMedia: MediaDto | undefined, - db: DbService, - parentBucketId?: string, - prevParentBucketId?: string, -): Promise<{ migrationFailed: boolean; warnings: string[] }> { - const warnings: string[] = []; - let migrationFailed = false; + if (!media.hlsKey) return warnings; try { - // Detect bucket change and migrate media if needed - if ( - prevMedia && - prevParentBucketId && - parentBucketId && - prevParentBucketId !== parentBucketId && - media.fileCollections.length > 0 - ) { - const migrationResult = await migrateMediaBetweenBuckets( - media, - prevParentBucketId, - parentBucketId, - db, - ); - warnings.push(...migrationResult.warnings); - migrationFailed = migrationResult.failed; - } - - if (prevMedia) { - // Track files to delete from S3 - const filesToDelete: string[] = []; - - // Strategy: The client sends ALL fileCollections it wants to keep - // We need to: - // 1. Delete files that are not in the client's list - // 2. Discard invalid files the client may have added - // 3. Replace files when uploading for the same language - - const languagesBeingUploaded = - media.uploadData?.map((u) => u.languageId).filter(Boolean) || []; - - // Get fileUrls from previous media (valid files) - const prevFileUrls = new Set(prevMedia.fileCollections.map((c) => c.fileUrl)); - - // Get fileUrls that the client is keeping (only keep if they were in prevMedia) - // BUT exclude files for languages that are being uploaded (they'll be replaced) - const keptFileUrls = new Set( - media.fileCollections - .filter((c) => { - // Only keep if it was in prevMedia - if (!prevFileUrls.has(c.fileUrl)) return false; - - // Don't keep if its language is being replaced by an upload - if (languagesBeingUploaded.includes(c.languageId)) return false; - - return true; - }) - .map((c) => c.fileUrl), - ); - - // Check each previous file collection - prevMedia.fileCollections.forEach((collection) => { - // If the file is not in the kept list, mark it for deletion - if (!keptFileUrls.has(collection.fileUrl)) { - // Extract key from URL - const urlParts = collection.fileUrl.split("/"); - const key = urlParts[urlParts.length - 1]; - if (key && key.length > 0) { - filesToDelete.push(key); - } - } - }); - - // Delete files from S3 using the parent bucket ID - if (filesToDelete.length > 0 && db && parentBucketId) { - try { - const result = await db.getDoc(parentBucketId); - if (!result.docs || result.docs.length === 0) { - warnings.push( - `Bucket ${parentBucketId} not found. Cannot delete ${ - filesToDelete.length - } files. Manual cleanup required for: ${filesToDelete.join(", ")}`, - ); - } else { - const bucketS3Service = await S3Service.create(parentBucketId, db); - - // Delete files from the bucket - for (const key of filesToDelete) { - try { - await bucketS3Service - .getClient() - .removeObject(bucketS3Service.getBucketName(), key); - } catch (error) { - warnings.push( - `Failed to delete ${key} from bucket ${bucketS3Service.getBucketName()}: ${ - error.message - }`, - ); - } - } - } - } catch (error) { - warnings.push( - `Failed to connect to bucket ${parentBucketId}: ${error.message}. Cannot delete ${filesToDelete.length} files.`, - ); - } - } else if (filesToDelete.length > 0 && (!db || !parentBucketId)) { - warnings.push( - `Warning: ${ - filesToDelete.length - } old media files cannot be automatically deleted without ${ - !db ? "database access" : "parent bucket ID" - }. ` + `Please manually clean up files on the storage provider`, - ); - } - - // Start with only valid files that the client is keeping - media.fileCollections = media.fileCollections.filter((c) => - keptFileUrls.has(c.fileUrl), - ); - } - - // Upload new files - if (media.uploadData) { - if (!db) { - warnings.push("Unable to upload media - system configuration error."); - return { migrationFailed, warnings }; - } - - if (!parentBucketId) { - warnings.push("Parent bucket ID is required for media uploads."); - return { migrationFailed, warnings }; - } - - const promises: Promise<{ success: boolean; warnings: string[] }>[] = []; - media.uploadData?.forEach((uploadData) => { - promises.push(processMediaUpload(uploadData, media, db, parentBucketId)); - }); - - const results = await Promise.all(promises); - - // Collect all warnings from uploads - results.forEach((result) => { - warnings.push(...result.warnings); - }); - - // If any uploads failed completely, we should clean up any successful uploads - const successfulUploads = results.filter((r) => r.success).length; - const failedUploads = results.filter((r) => !r.success).length; - - if (failedUploads > 0) { - warnings.push(`${failedUploads} of ${results.length} media uploads failed`); - - if (successfulUploads === 0) { - warnings.push("All media uploads failed - no media were processed"); - } - } - - delete media.uploadData; // Remove upload data after processing - } + media.hlsKey_id = await storeCryptoData(db, media.hlsKey); } catch (error) { - warnings.push(`Media processing failed: ${error.message}`); + throw new Error(`Failed to encrypt the HLS key: ${error.message}`); + } finally { + // Dropped whether or not it was stored, and before the caller can catch: + // a key that failed to encrypt must not reach the document either. + delete media.hlsKey; } - return { migrationFailed, warnings }; -} - -async function processMediaUpload( - uploadData: MediaUploadDataDto, - media: MediaDto, - db: DbService, - bucketId: string, -): Promise<{ success: boolean; warnings: string[] }> { - const warnings: string[] = []; - - try { - let preset = uploadData?.preset || "default"; - if (preset != "default" && preset != "audio" && preset != "speech") { - preset = "default"; - } - - // Bucket ID is required - if (!bucketId) { - warnings.push( - "No bucket specified for media upload. Each post/tag must specify a target bucket with proper credentials.", - ); - return { success: false, warnings }; - } - - // Look up the bucket and create bucket-specific S3 client - let storage: StorageDto; - - try { - const bucketDocs = await db.getDocsByType(DocType.Storage); - const foundBucket = bucketDocs.docs.find( - (doc: any) => doc._id === bucketId, - ) as StorageDto; - - if (!foundBucket || !foundBucket.name) { - warnings.push( - `Bucket with ID ${bucketId} not found. Please configure a storage bucket with proper credentials before uploading media.`, - ); - return { success: false, warnings }; - } - - storage = foundBucket; - - // Validate file type against bucket's allowed mimeTypes (if specified) - // Use audio format detection to determine mimetype - if (storage.mimeTypes && storage.mimeTypes.length > 0) { - // Parse metadata to determine format - let detectedMimetype = "audio/mpeg"; // default - - try { - const { parseBuffer } = await import("music-metadata"); - const metadata = await parseBuffer(new Uint8Array(uploadData.fileData)); - const formatInfo = getAudioFormatInfo(metadata); - detectedMimetype = formatInfo.mime; - } catch { - // Fall back to default - } - - const isAllowed = storage.mimeTypes.some((allowedType) => { - // Support wildcards like "audio/*" - if (allowedType.endsWith("/*")) { - const prefix = allowedType.slice(0, -2); - return detectedMimetype.startsWith(prefix + "/"); - } - // Exact match - return detectedMimetype === allowedType; - }); - - if (!isAllowed) { - warnings.push( - `File type "${detectedMimetype}" is not allowed for bucket "${ - storage.name - }". Allowed types: ${storage.mimeTypes.join(", ")}`, - ); - return { success: false, warnings }; - } - } - - // Create bucket-specific S3 service with bucket's credentials - const s3Service = await S3Service.create(bucketId, db); - - // Process and upload the media file - const uploadResult = await uploadMediaFile(uploadData, s3Service, media, storage); - warnings.push(...uploadResult.warnings); - - if (!uploadResult.success) { - return { success: false, warnings }; - } - - return { success: true, warnings }; - } catch (error) { - warnings.push( - `Failed to connect to bucket ${bucketId}: ${error.message}. Please ensure the bucket has valid credentials configured.`, - ); - return { success: false, warnings }; - } - } catch (error) { - warnings.push(`Media upload failed: ${error.message}`); - return { success: false, warnings }; - } -} - -async function uploadMediaFile( - uploadData: MediaUploadDataDto, - s3Service: S3Service, - media: MediaDto, - storage: StorageDto, -): Promise<{ success: boolean; warnings: string[] }> { - const warnings: string[] = []; - - try { - // Parse metadata to infer bitrate and format info - let formatInfo = { ext: "", mime: "application/octet-stream", isValidAudio: false }; - let bitrate = 0; - const u8 = new Uint8Array(uploadData.fileData); - - try { - const { parseBuffer } = await import("music-metadata"); - const metadata = await parseBuffer(u8); - - // Use robust format detection - formatInfo = getAudioFormatInfo(metadata); - bitrate = Math.round(metadata.format.bitrate || 0); - } catch { - // Fall back; format/bitrate unknown in this environment - } - - // Fallback to generic audio if we couldn't determine format - if (!formatInfo.ext) { - formatInfo.ext = "mp3"; // Use mp3 as safe default extension - formatInfo.mime = "audio/mpeg"; // safe default - } - - // Include file extension in the key for proper MIME type handling - const key = `${uuidv4()}-default.${formatInfo.ext}`; - - // Upload original buffer as-is - const buf = Buffer.from(u8); - - await s3Service.uploadFile(key, buf, formatInfo.mime); - - // Validate upload accessibility - const validateRes = await s3Service.objectExists(key); - if (!validateRes) { - warnings.push("Media file uploaded but not accessible"); - } - - // Construct the public URL using the bucket's publicUrl from StorageDto - // Remove trailing slash from publicUrl if present - const baseUrl = storage.publicUrl.replace(/\/$/, ""); - const fileUrl = `${baseUrl}/${key}`; - - const file = new MediaFileDto(); - file.languageId = uploadData.languageId; - file.fileUrl = fileUrl; - file.bitrate = bitrate; - file.mediaType = uploadData.mediaType; - - media.fileCollections.push(file); - - return { success: true, warnings }; - } catch (error) { - return { - success: false, - warnings: [`Failed to upload media file: ${error.message}\n`], - }; - } + return warnings; } diff --git a/api/src/changeRequests/documentProcessing/processPostTagDto.deleteMedia.spec.ts b/api/src/changeRequests/documentProcessing/processPostTagDto.deleteMedia.spec.ts new file mode 100644 index 0000000000..dd39e9c1f0 --- /dev/null +++ b/api/src/changeRequests/documentProcessing/processPostTagDto.deleteMedia.spec.ts @@ -0,0 +1,125 @@ +import processPostTagDto from "./processPostTagDto"; +import { deleteMediaCollection } from "./deleteMediaCollection"; +import { deleteImage, processImage } from "./processImageDto"; +import { processMedia } from "./processMediaDto"; +import { DbService } from "../../db/db.service"; +import { PostDto } from "../../dto/PostDto"; + +jest.mock("./deleteMediaCollection", () => ({ deleteMediaCollection: jest.fn() })); +jest.mock("./processImageDto", () => ({ processImage: jest.fn(), deleteImage: jest.fn() })); +jest.mock("./processMediaDto", () => ({ processMedia: jest.fn() })); + +/** + * A database stub, so the one branch under test does not need CouchDB. + * + * The delete path returns as soon as it has cascaded to the children, so + * `getContentByParentId` and `upsertDoc` are all it reaches. + */ +const stubDb = () => + ({ + getContentByParentId: jest.fn().mockResolvedValue({ docs: [] }), + upsertDoc: jest.fn().mockResolvedValue({ id: "x" }), + getDocs: jest.fn().mockResolvedValue({ docs: [] }), + getDoc: jest.fn().mockResolvedValue({ docs: [] }), + }) as unknown as DbService; + +const HLS = "http://localhost:9000/media/c5829f07-4ba8-42ed-a449-80d83e6c0b53/master.m3u8"; + +/** The document as saved, which is the authority on where the files are. */ +const saved = () => + ({ + _id: "post-1", + type: "post", + memberOf: ["group-public-content"], + tags: [], + publishDateVisible: true, + postType: "blog", + mediaBucketId: "bucket-media", + media: { hlsUrl: HLS, fileCollections: [] }, + }) as unknown as PostDto; + +/** The delete request, carrying the user's answer from the confirmation. */ +const deleteRequest = (deleteFiles?: boolean) => { + const doc = saved(); + doc.deleteReq = 1; + if (deleteFiles !== undefined) doc.media!.deleteFiles = deleteFiles; + return doc; +}; + +describe("processPostTagDto — deleting media files from storage", () => { + beforeEach(() => { + jest.clearAllMocks(); + (deleteMediaCollection as jest.Mock).mockResolvedValue([]); + (deleteImage as jest.Mock).mockResolvedValue([]); + (processImage as jest.Mock).mockResolvedValue({ warnings: [] }); + (processMedia as jest.Mock).mockResolvedValue([]); + }); + + it("leaves storage alone when the user did not opt in", async () => { + // The guarantee that matters most: deleting a document must never remove + // anyone's video unless they asked for it in the confirmation. + await processPostTagDto(deleteRequest(), saved(), stubDb()); + + expect(deleteMediaCollection).not.toHaveBeenCalled(); + }); + + it("leaves storage alone when the box was explicitly unticked", async () => { + await processPostTagDto(deleteRequest(false), saved(), stubDb()); + + expect(deleteMediaCollection).not.toHaveBeenCalled(); + }); + + it("deletes the collection when asked", async () => { + const db = stubDb(); + await processPostTagDto(deleteRequest(true), saved(), db); + + expect(deleteMediaCollection).toHaveBeenCalledWith( + expect.objectContaining({ hlsUrl: HLS }), + "bucket-media", + db, + ); + }); + + it("takes the location from the saved document, not the incoming one", async () => { + // A hlsUrl edited in the same breath as the delete must not redirect the + // deletion at something else. Intent comes from the request; the target + // comes from what was actually saved. + const incoming = deleteRequest(true); + incoming.media!.hlsUrl = "http://localhost:9000/media/somewhere-else/master.m3u8"; + + await processPostTagDto(incoming, saved(), stubDb()); + + expect(deleteMediaCollection).toHaveBeenCalledWith( + expect.objectContaining({ hlsUrl: HLS }), + "bucket-media", + expect.anything(), + ); + }); + + it("does not call it at all for a document with no media", async () => { + const doc = deleteRequest(); + delete doc.media; + + await processPostTagDto(doc, saved(), stubDb()); + + expect(deleteMediaCollection).not.toHaveBeenCalled(); + }); + + it("reports what storage could not remove, without failing the delete", async () => { + (deleteMediaCollection as jest.Mock).mockResolvedValueOnce([ + "Media files were not deleted: bucket is unreachable", + ]); + + const warnings = await processPostTagDto(deleteRequest(true), saved(), stubDb()); + + expect(warnings.some((w) => w.includes("unreachable"))).toBe(true); + }); + + it("still cascades the delete to the child content documents", async () => { + // The media work must not displace what the delete path is actually for. + const db = stubDb(); + await processPostTagDto(deleteRequest(true), saved(), db); + + expect(db.getContentByParentId).toHaveBeenCalledWith("post-1"); + }); +}); diff --git a/api/src/changeRequests/documentProcessing/processPostTagDto.migrateMedia.spec.ts b/api/src/changeRequests/documentProcessing/processPostTagDto.migrateMedia.spec.ts new file mode 100644 index 0000000000..d3d568703c --- /dev/null +++ b/api/src/changeRequests/documentProcessing/processPostTagDto.migrateMedia.spec.ts @@ -0,0 +1,114 @@ +import processPostTagDto from "./processPostTagDto"; +import { migrateMediaCollection } from "./migrateMediaCollection"; +import { deleteMediaCollection } from "./deleteMediaCollection"; +import { processImage } from "./processImageDto"; +import { processMedia } from "./processMediaDto"; +import { DbService } from "../../db/db.service"; +import { PostDto } from "../../dto/PostDto"; + +jest.mock("./migrateMediaCollection", () => ({ migrateMediaCollection: jest.fn() })); +jest.mock("./deleteMediaCollection", () => ({ deleteMediaCollection: jest.fn() })); +jest.mock("./processImageDto", () => ({ processImage: jest.fn(), deleteImage: jest.fn() })); +jest.mock("./processMediaDto", () => ({ processMedia: jest.fn() })); + +const stubDb = () => + ({ + getContentByParentId: jest.fn().mockResolvedValue({ docs: [] }), + upsertDoc: jest.fn().mockResolvedValue({ id: "x" }), + getDocs: jest.fn().mockResolvedValue({ docs: [] }), + getDoc: jest.fn().mockResolvedValue({ docs: [] }), + }) as unknown as DbService; + +const HLS = "http://old.example.com/media/c5829f07-4ba8-42ed-a449-80d83e6c0b53/master.m3u8"; + +const post = (bucketId: string, hlsUrl = HLS) => + ({ + _id: "post-1", + type: "post", + memberOf: ["group-public-content"], + tags: [], + publishDateVisible: true, + postType: "blog", + mediaBucketId: bucketId, + media: { hlsUrl }, + }) as unknown as PostDto; + +describe("processPostTagDto — migrating media between buckets", () => { + beforeEach(() => { + jest.clearAllMocks(); + (migrateMediaCollection as jest.Mock).mockResolvedValue({ failed: false, warnings: [] }); + (processImage as jest.Mock).mockResolvedValue({ warnings: [] }); + (processMedia as jest.Mock).mockResolvedValue([]); + (deleteMediaCollection as jest.Mock).mockResolvedValue([]); + }); + + it("migrates when the bucket changes, from the saved URL", async () => { + const db = stubDb(); + const incoming = post("bucket-new"); + + await processPostTagDto(incoming, post("bucket-old"), db); + + expect(migrateMediaCollection).toHaveBeenCalledWith( + incoming.media, + HLS, + "bucket-old", + "bucket-new", + db, + ); + }); + + it("does not migrate when the bucket is unchanged", async () => { + await processPostTagDto(post("bucket-old"), post("bucket-old"), stubDb()); + + expect(migrateMediaCollection).not.toHaveBeenCalled(); + }); + + it("does not migrate on first save, when there is nothing to move from", async () => { + await processPostTagDto(post("bucket-new"), undefined, stubDb()); + + expect(migrateMediaCollection).not.toHaveBeenCalled(); + }); + + it("reverts the bucket when the migration fails, so the files stay reachable", async () => { + // The invariant: mediaBucketId and hlsUrl must name the same bucket. If the + // move failed, the files are still in the old one, so the document has to be. + (migrateMediaCollection as jest.Mock).mockResolvedValue({ + failed: true, + warnings: ["Media migration failed: connection reset."], + }); + const incoming = post("bucket-new"); + + const warnings = await processPostTagDto(incoming, post("bucket-old"), stubDb()); + + expect(incoming.mediaBucketId).toBe("bucket-old"); + expect(warnings.some((w) => w.includes("Reverted to previous bucket"))).toBe(true); + expect(warnings.some((w) => w.includes("connection reset"))).toBe(true); + }); + + it("keeps the new bucket when the migration succeeded", async () => { + const incoming = post("bucket-new"); + + await processPostTagDto(incoming, post("bucket-old"), stubDb()); + + expect(incoming.mediaBucketId).toBe("bucket-new"); + }); + + it("still stores the key after migrating", async () => { + // The migration must not displace what the media path is otherwise for. + const incoming = post("bucket-new"); + + await processPostTagDto(incoming, post("bucket-old"), stubDb()); + + expect(processMedia).toHaveBeenCalledWith(incoming.media, expect.anything()); + }); + + it("does not migrate on a delete request", async () => { + // A delete removes files; it does not move them somewhere first. + const incoming = post("bucket-new"); + incoming.deleteReq = 1; + + await processPostTagDto(incoming, post("bucket-old"), stubDb()); + + expect(migrateMediaCollection).not.toHaveBeenCalled(); + }); +}); diff --git a/api/src/changeRequests/documentProcessing/processPostTagDto.spec.ts b/api/src/changeRequests/documentProcessing/processPostTagDto.spec.ts index 8c43e0cb12..d111ad15fc 100644 --- a/api/src/changeRequests/documentProcessing/processPostTagDto.spec.ts +++ b/api/src/changeRequests/documentProcessing/processPostTagDto.spec.ts @@ -5,7 +5,7 @@ import { PermissionSystem } from "../../permissions/permissions.service"; import { processChangeRequest } from "../processChangeRequest"; import { changeRequest_content, changeRequest_post } from "../../test/changeRequestDocuments"; import { ChangeReqDto } from "../../dto/ChangeReqDto"; -import { DocType, MediaType } from "../../enums"; +import { DocType } from "../../enums"; import { deleteImage, processImage } from "./processImageDto"; import { processMedia } from "./processMediaDto"; @@ -33,7 +33,7 @@ describe("processPostTagDto", () => { jest.clearAllMocks(); (deleteImage as jest.Mock).mockResolvedValue([]); (processImage as jest.Mock).mockResolvedValue({ warnings: [] }); - (processMedia as jest.Mock).mockResolvedValue({ warnings: [] }); + (processMedia as jest.Mock).mockResolvedValue([]); }); it("should cascade Post/Tag delete request to content documents", async () => { @@ -316,14 +316,7 @@ describe("processPostTagDto", () => { changeRequest.doc._id = "post-blog6"; (changeRequest.doc as PostDto).mediaBucketId = "test-bucket-id"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { - languageId: "lang-eng", - fileUrl: "http://test.com/test-audio.mp3", - bitrate: 128, - mediaType: MediaType.Audio, - }, - ], + hlsUrl: "http://test.com/media/post-blog6/master.m3u8", }; // This should not throw an error even though prevDoc is undefined @@ -343,14 +336,7 @@ describe("processPostTagDto", () => { changeRequest.doc.deleteReq = 1; (changeRequest.doc as PostDto).mediaBucketId = "test-bucket-id"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { - languageId: "lang-eng", - fileUrl: "test-audio.mp3", - bitrate: 128, - mediaType: MediaType.Audio, - }, - ], + hlsUrl: "http://test.com/media/post-blog7/master.m3u8", }; // This should not throw an error even though prevDoc is undefined @@ -389,21 +375,17 @@ describe("processPostTagDto", () => { expect(result.warnings).toContain("Image cleanup warning"); }); - it("warns when media processing returns warnings during deletion", async () => { - (processMedia as jest.Mock).mockResolvedValueOnce({ - warnings: ["Media cleanup warning"], - }); - + it("does not process media when a document is deleted", async () => { const changeRequest = changeRequest_post(); changeRequest.doc._id = "post-delete-med-warn"; (changeRequest.doc as PostDto).mediaBucketId = "test-bucket"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { languageId: "en", fileUrl: "test.mp3", bitrate: 128, mediaType: MediaType.Audio }, - ], + hlsUrl: "http://test.com/media/post-delete-med-warn/master.m3u8", }; changeRequest.doc.deleteReq = 1; + (processMedia as jest.Mock).mockClear(); + const result = await processChangeRequest( "test-user", changeRequest, @@ -411,7 +393,10 @@ describe("processPostTagDto", () => { db, ); - expect(result.warnings).toContain("Media cleanup warning"); + // The collection is written by the encoder, not by this API, and nothing + // here knows which objects belong to it — so deletion leaves it alone. + expect(processMedia).not.toHaveBeenCalled(); + expect(result.result.ok).toBe(true); }); it("calls processImage without bucketId when imageBucketId is not specified", async () => { @@ -506,9 +491,7 @@ describe("processPostTagDto", () => { const changeRequest = changeRequest_post(); changeRequest.doc._id = "post-no-med-bucket"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { languageId: "en", fileUrl: "test.mp3", bitrate: 128, mediaType: MediaType.Audio }, - ], + hlsUrl: "http://test.com/media/post-no-med-bucket/master.m3u8", }; delete (changeRequest.doc as PostDto).mediaBucketId; @@ -517,51 +500,16 @@ describe("processPostTagDto", () => { ).rejects.toThrow("Bucket is not specified for media processing"); }); - it("reverts mediaBucketId when media migration fails", async () => { - // First create the post with old bucket - const changeRequest = changeRequest_post(); - changeRequest.doc._id = "post-med-migrate-fail"; - (changeRequest.doc as PostDto).mediaBucketId = "old-media-bucket"; - (changeRequest.doc as PostDto).media = { - fileCollections: [ - { languageId: "en", fileUrl: "test.mp3", bitrate: 128, mediaType: MediaType.Audio }, - ], - }; - (processMedia as jest.Mock).mockResolvedValueOnce({ warnings: [] }); - await processChangeRequest("test-user", changeRequest, ["group-super-admins"], db); - - // Now update with new bucket that fails migration - (processMedia as jest.Mock).mockResolvedValueOnce({ - warnings: [], - migrationFailed: true, - }); - (changeRequest.doc as PostDto).mediaBucketId = "new-media-bucket"; - const result = await processChangeRequest( - "test-user", - changeRequest, - ["group-super-admins"], - db, - ); - - expect(result.warnings.some((w) => w.includes("Media migration failed"))).toBe(true); - }); - - it("reverts mediaBucketId when processMedia throws an error", async () => { - // First create the post with old bucket + it("warns rather than failing the save when media processing throws", async () => { const changeRequest = changeRequest_post(); changeRequest.doc._id = "post-med-throw"; - (changeRequest.doc as PostDto).mediaBucketId = "old-media-bucket"; + (changeRequest.doc as PostDto).mediaBucketId = "media-bucket"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { languageId: "en", fileUrl: "test.mp3", bitrate: 128, mediaType: MediaType.Audio }, - ], + hlsUrl: "http://test.com/media/post-med-throw/master.m3u8", }; - (processMedia as jest.Mock).mockResolvedValueOnce({ warnings: [] }); - await processChangeRequest("test-user", changeRequest, ["group-super-admins"], db); - // Now update with new bucket where processMedia throws - (processMedia as jest.Mock).mockRejectedValueOnce(new Error("Media processing failed")); - (changeRequest.doc as PostDto).mediaBucketId = "new-media-bucket"; + (processMedia as jest.Mock).mockRejectedValueOnce(new Error("key store unavailable")); + const result = await processChangeRequest( "test-user", changeRequest, @@ -569,9 +517,10 @@ describe("processPostTagDto", () => { db, ); - expect(result.warnings.some((w) => w.includes("Bucket media processing failed"))).toBe( - true, - ); + // A key that could not be stored costs the encryption, not the document — + // processMedia has already dropped the plaintext key by this point. + expect(result.warnings.some((w) => w.includes("Media processing failed"))).toBe(true); + expect(result.result.ok).toBe(true); }); it("copies tag properties to content documents for Tag type", async () => { @@ -609,33 +558,19 @@ describe("processPostTagDto", () => { } }); - it("can remove media from S3 when a post/tag document is marked for deletion", async () => { + it("passes the media object and the db to processMedia on save", async () => { const changeRequest = changeRequest_post(); changeRequest.doc._id = "post-blog8"; (changeRequest.doc as PostDto).mediaBucketId = "test-bucket-id"; (changeRequest.doc as PostDto).media = { - fileCollections: [ - { - languageId: "lang-eng", - fileUrl: "test-audio.mp3", - bitrate: 128, - mediaType: MediaType.Audio, - }, - ], + hlsUrl: "http://test.com/media/post-blog8/master.m3u8", + hlsKey: "0123456789abcdef0123456789abcdef", }; - await processChangeRequest("test-user", changeRequest, ["group-super-admins"], db); + (processMedia as jest.Mock).mockClear(); - // Mark the post document for deletion - const deleteRequest = JSON.parse(JSON.stringify(changeRequest)) as ChangeReqDto; - deleteRequest.doc.deleteReq = 1; - await processChangeRequest("test-user", deleteRequest, ["group-super-admins"], db); + await processChangeRequest("test-user", changeRequest, ["group-super-admins"], db); - expect(processMedia).toHaveBeenCalledWith( - { fileCollections: [] }, // Empty fileCollections to remove the media from S3 - (changeRequest.doc as PostDto).media, - db, - (changeRequest.doc as PostDto).mediaBucketId, - ); + expect(processMedia).toHaveBeenCalledWith((changeRequest.doc as PostDto).media, db); }); }); diff --git a/api/src/changeRequests/documentProcessing/processPostTagDto.ts b/api/src/changeRequests/documentProcessing/processPostTagDto.ts index dfbd06fd45..97d767a6b1 100644 --- a/api/src/changeRequests/documentProcessing/processPostTagDto.ts +++ b/api/src/changeRequests/documentProcessing/processPostTagDto.ts @@ -5,6 +5,8 @@ import { DbService } from "../../db/db.service"; import { DocType, Uuid } from "../../enums"; import { deleteImage, processImage } from "./processImageDto"; import { processMedia } from "./processMediaDto"; +import { deleteMediaCollection } from "./deleteMediaCollection"; +import { migrateMediaCollection } from "./migrateMediaCollection"; /** * Process Post / Tag DTO @@ -35,17 +37,19 @@ export default async function processPostTagDto( warnings.push(...imageWarnings); } - // Remove medias from S3 - if (doc.media) { - const mediaResult = await processMedia( - { fileCollections: [] }, - prevDoc?.media, - db, - prevDoc?.mediaBucketId, // Delete from the bucket where files currently exist + // Media files go only when the user asked for them in the delete + // confirmation. Opt-in because it is irreversible and because the + // collection may be referenced somewhere this API cannot see; the previous + // document is the authority on where the files are, and the incoming one + // carries the intent. + if (doc.media?.deleteFiles) { + warnings.push( + ...(await deleteMediaCollection( + prevDoc?.media, + prevDoc?.mediaBucketId, + db, + )), ); - if (mediaResult && mediaResult.warnings && mediaResult.warnings.length > 0) { - warnings.push(...mediaResult.warnings); - } } return warnings; // no need to process further @@ -104,43 +108,40 @@ export default async function processPostTagDto( delete (doc as any).image; // Remove the legacy image field } - // Process media uploads + // Process media if (doc.media) { - let mediaWarnings: string[] = []; - - // Check if bucket is specified for this upload + // The bucket is where the encoder was told to write, and is what a later + // edit of the collection has to be pointed back at. if (!doc.mediaBucketId) { throw new Error("Bucket is not specified for media processing."); } - // Use the new bucket processing with db service for bucket lookup - try { - const result = await processMedia( + // A bucket change has to take the files with it. `mediaBucketId` and + // `hlsUrl` must name the same bucket: if they diverge, the collection can no + // longer be resolved from the URL, and deleting the document then leaves the + // files behind for good. + if (prevDoc?.mediaBucketId && prevDoc.mediaBucketId !== doc.mediaBucketId) { + const migration = await migrateMediaCollection( doc.media, - prevDoc?.media, - db, + prevDoc.media?.hlsUrl, + prevDoc.mediaBucketId, doc.mediaBucketId, - prevDoc?.mediaBucketId, // Pass previous bucket ID for migration + db, ); - mediaWarnings = result.warnings; + warnings.push(...migration.warnings); - // If migration failed, revert to the old bucket ID to keep files accessible - if (result.migrationFailed && prevDoc?.mediaBucketId) { + if (migration.failed) { doc.mediaBucketId = prevDoc.mediaBucketId; warnings.push( "Media migration failed. Reverted to previous bucket configuration to ensure files remain accessible.", ); } - } catch (error) { - // If processing throws an error, also revert bucket ID - if (prevDoc?.mediaBucketId && doc.mediaBucketId !== prevDoc.mediaBucketId) { - doc.mediaBucketId = prevDoc.mediaBucketId; - } - mediaWarnings.push(`Bucket media processing failed: ${error.message}`); } - if (mediaWarnings && mediaWarnings.length > 0) { - warnings.push(...mediaWarnings); + try { + warnings.push(...(await processMedia(doc.media, db))); + } catch (error) { + warnings.push(`Media processing failed: ${error.message}`); } } diff --git a/api/src/changeRequests/validateChangeRequest.spec.ts b/api/src/changeRequests/validateChangeRequest.spec.ts index 3fe2b4e6aa..88e1092433 100644 --- a/api/src/changeRequests/validateChangeRequest.spec.ts +++ b/api/src/changeRequests/validateChangeRequest.spec.ts @@ -282,9 +282,7 @@ describe("validateChangeRequest", () => { expect(result.validatedData.acl).toHaveLength(2); }); - it("validates a post with valid audio upload data for multiple languages", async () => { - const audioFile = fs.readFileSync(path.resolve(__dirname + "/../test/" + "silence.wav")); - + it("validates a post with an HLS media collection", async () => { const changeRequest = { id: 42, doc: { @@ -295,27 +293,8 @@ describe("validateChangeRequest", () => { tags: [], publishDateVisible: true, media: { - fileCollections: [], - uploadData: [ - { - fileData: audioFile, - preset: "default", - mediaType: "audio", - languageId: "lang-eng", - }, - { - fileData: audioFile, - preset: "default", - mediaType: "audio", - languageId: "lang-spa", - }, - { - fileData: audioFile, - preset: "default", - mediaType: "audio", - languageId: "lang-fra", - }, - ], + hlsUrl: "https://cdn.example.com/media/post-test/master.m3u8", + hlsKey: "0123456789abcdef0123456789abcdef", }, }, }; @@ -326,7 +305,7 @@ describe("validateChangeRequest", () => { expect(result.error).toBe(undefined); }); - it("fails validation for post with invalid audio upload data", async () => { + it("fails validation for media with no playlist URL", async () => { const changeRequest = { id: 42, doc: { @@ -336,24 +315,15 @@ describe("validateChangeRequest", () => { postType: "blog", tags: [], publishDateVisible: true, - media: { - fileCollections: [], - uploadData: [ - { - fileData: Buffer.from("not an audio file"), - preset: "default", - mediaType: "audio", - languageId: "lang-eng", - }, - ], - }, + // A key with nothing to decrypt is not a media object. + media: { hlsKey: "0123456789abcdef0123456789abcdef" }, }, }; const result = await validateChangeRequest(changeRequest, ["group-super-admins"], db); expect(result.validated).toBe(false); - expect(result.error).toContain("isAudio"); + expect(result.error).toContain("hlsUrl"); }); it("rejects a redirect whose slug has published content", async () => { diff --git a/api/src/dto/MediaDto.ts b/api/src/dto/MediaDto.ts index b2e8e05938..562638cfa5 100644 --- a/api/src/dto/MediaDto.ts +++ b/api/src/dto/MediaDto.ts @@ -1,28 +1,45 @@ import "reflect-metadata"; // https://stackoverflow.com/questions/72009995/typeerror-reflect-getmetadata-is-not-a-function -import { IsArray, IsOptional, IsString, ValidateNested } from "class-validator"; -import { Expose, Type } from "class-transformer"; -import { MediaFileDto } from "./MediaFileDto"; -import { MediaUploadDataDto } from "./MediaUploadDataDto"; +import { IsBoolean, IsOptional, IsString } from "class-validator"; +import { Expose } from "class-transformer"; +import { Uuid } from "src/enums"; /** * Database structured Media object */ export class MediaDto { - @IsOptional() @IsString() @Expose() - hlsUrl?: string; + hlsUrl: string; - @IsArray() - @ValidateNested({ each: true }) - @Type(() => MediaFileDto) // This throws an exception on validation failure, so we need to catch the error on validation. The message is less user-friendly but at least the validator fails and will protect our data. + /** + * ID to the CryptoObject where the (optional) encryption key is stored + */ + @IsOptional() + @IsString() @Expose() - fileCollections: MediaFileDto[] = []; + hlsKey_id?: Uuid; + /** + * Optional field for submitting an HLS encryption key for a newly added HLS URL. + * When set, this key is stored as a crypto object, and the crypto object ID is + * exposed as the hlsKey_id. + */ @IsOptional() - @IsArray() - @ValidateNested({ each: true }) - @Type(() => MediaUploadDataDto) // This throws an exception on validation failure, so we need to catch the error on validation. The message is less user-friendly but at least the validator fails and will protect our data. - @Expose() - uploadData?: MediaUploadDataDto[]; + @IsString() + @Expose({ toClassOnly: true }) + hlsKey?: string; + + /** + * Write-only: the user asked, in the delete confirmation, for the files in + * storage to go with the document. + * + * Carried on the document rather than as a separate call because a delete *is* + * a change request — the whole document arrives with `deleteReq` set, so the + * intent travels with the thing it applies to and cannot be separated from it + * in flight. Never persisted, like `hlsKey` above. + */ + @IsOptional() + @IsBoolean() + @Expose({ toClassOnly: true }) + deleteFiles?: boolean; } diff --git a/api/src/endpoints/encoderConfig.controller.ts b/api/src/endpoints/encoderConfig.controller.ts new file mode 100644 index 0000000000..edf706b172 --- /dev/null +++ b/api/src/endpoints/encoderConfig.controller.ts @@ -0,0 +1,128 @@ +import { Controller, Get, Query, UseGuards, Req, HttpException, HttpStatus } from "@nestjs/common"; +import { AuthGuard } from "../auth/auth.guard"; +import { DbService } from "../db/db.service"; +import { validateApiVersion } from "../validation/apiVersion"; +import { PermissionSystem } from "../permissions/permissions.service"; +import { AclPermission, DocType } from "../enums"; +import { S3CredentialDto } from "../dto/S3CredentialDto"; +import { retrieveCryptoData } from "../util/encryption"; +import { FastifyRequest } from "fastify"; + +/** + * Everything the local media encoder needs to write a collection to a bucket and + * to publish a URL for it. Shaped for the encoder's `POST /api/cms/sessions` body + * rather than for our own storage model, so the CMS forwards it without reshaping. + */ +export type EncoderConfigResponseDto = { + s3: { + endPoint: string; + port: number; + useSSL: boolean; + bucket: string; + accessKey: string; + secretKey: string; + }; + publicBaseUrl: string; +}; + +/** + * Hands out the S3 credentials for a media bucket. + * + * The encoder runs on the editor's own machine and uploads straight to the bucket, + * so it needs real credentials — there is no path where the server does the upload + * on its behalf. Credentials are stored encrypted and are not replicated to + * clients, which is why they are fetched here rather than read off the Storage + * document the CMS already holds. + * + * Gated on `Assign` rather than `View`: assigning a bucket is the right to publish + * into it, which is exactly what these credentials confer. `View` is what the + * status endpoint needs to render a connectivity dot, and is held far more widely. + */ +@Controller("storage") +export class EncoderConfigController { + constructor(private readonly dbService: DbService) {} + + @Get("encoderconfig") + @UseGuards(AuthGuard) + async getEncoderConfig( + @Query("bucketId") bucketId: string, + @Query("apiVersion") apiVersion: string, + @Req() request: FastifyRequest, + ): Promise { + await validateApiVersion(apiVersion); + + const userDetails = request.user; + + if (!bucketId) { + throw new HttpException("bucketId query parameter is required", HttpStatus.BAD_REQUEST); + } + + const bucketResult = await this.dbService.getDoc(bucketId); + if (!bucketResult.docs || bucketResult.docs.length === 0) { + throw new HttpException( + `Bucket configuration not found: ${bucketId}`, + HttpStatus.NOT_FOUND, + ); + } + + const bucket = bucketResult.docs[0]; + + const hasPermission = PermissionSystem.verifyAccess( + bucket.memberOf, + DocType.Storage, + AclPermission.Assign, + userDetails.groups, + ); + + if (!hasPermission) { + throw new HttpException( + "Insufficient permissions to encode to this bucket", + HttpStatus.FORBIDDEN, + ); + } + + if (!bucket.credential_id) { + throw new HttpException( + `No credentials configured for bucket: ${bucket.name}`, + HttpStatus.CONFLICT, + ); + } + + if (!bucket.publicUrl) { + throw new HttpException( + `No public URL configured for bucket: ${bucket.name}. The encoder needs one to ` + + "publish a playable address for the collection it writes.", + HttpStatus.CONFLICT, + ); + } + + const credentials = await retrieveCryptoData( + this.dbService, + bucket.credential_id, + ); + + if (!credentials?.accessKey || !credentials?.secretKey || !credentials?.bucketName) { + throw new HttpException( + `Stored credentials for bucket ${bucket.name} are incomplete`, + HttpStatus.CONFLICT, + ); + } + + // The encoder takes host, port and TLS as separate fields; we store one URL. + // Split the same way S3Service does, so both reach the same endpoint. + const url = new URL(credentials.endpoint); + const useSSL = url.protocol === "https:"; + + return { + s3: { + endPoint: url.hostname, + port: parseInt(url.port) || (useSSL ? 443 : 80), + useSSL, + bucket: credentials.bucketName, + accessKey: credentials.accessKey, + secretKey: credentials.secretKey, + }, + publicBaseUrl: bucket.publicUrl, + }; + } +} diff --git a/api/src/s3/s3.service.ts b/api/src/s3/s3.service.ts index 4010673bcd..a0869ba76c 100644 --- a/api/src/s3/s3.service.ts +++ b/api/src/s3/s3.service.ts @@ -1,4 +1,5 @@ import * as Minio from "minio"; +import { Readable } from "stream"; import { S3CredentialDto } from "../dto/S3CredentialDto"; import { DbService } from "../db/db.service"; import { retrieveCryptoData, decryptObject } from "../util/encryption"; @@ -283,6 +284,30 @@ export class S3Service { return this.client.putObject(this.bucketName, key, file, file.length, metadata); } + /** + * Uploads a stream of a known length, without holding the object in memory. + * + * `uploadFile` above takes a Buffer, which is right for an image and wrong for + * media: a byte-range chunk chain is capped at 500 MB by default, so buffering + * one to copy it would trade a bounded stream for an unbounded heap. The size is + * required because S3 needs either a length or a multipart upload, and the + * source's own `statObject` already knows it. + */ + public async putStream(key: string, stream: Readable, size: number, mimetype: string) { + this.touch(); + return this.client.putObject(this.bucketName, key, stream, size, { + "Content-Type": mimetype, + }); + } + + /** + * Size and metadata of a single object, without fetching its body. + */ + public async statObject(key: string): Promise { + this.touch(); + return this.client.statObject(this.bucketName, key); + } + /** * Removes objects from a bucket */ @@ -331,6 +356,27 @@ export class S3Service { return this.client.listObjects(this.bucketName); } + /** + * Every object key under a prefix, recursively. + * + * `listObjects` above returns the whole bucket as a stream, which cannot answer + * "what belongs to this collection" — and a caller that means to delete needs + * an exact list, not a stream it might abandon half-read. Pass the prefix with + * its trailing slash: `foo` would also match `foo-archive/…`. + */ + public async listObjectsUnder(prefix: string): Promise { + this.touch(); + return new Promise((resolve, reject) => { + const keys: string[] = []; + const stream = this.client.listObjectsV2(this.bucketName, prefix, true); + stream.on("data", (item) => { + if (item.name) keys.push(item.name); + }); + stream.on("end", () => resolve(keys)); + stream.on("error", reject); + }); + } + /** * Check if an S3 service is reachable */ diff --git a/app/src/components/content/ContentTile.vue b/app/src/components/content/ContentTile.vue index 96da7f69ed..cc1f43ea66 100644 --- a/app/src/components/content/ContentTile.vue +++ b/app/src/components/content/ContentTile.vue @@ -9,6 +9,7 @@ import { computed } from "vue"; import { useI18n } from "vue-i18n"; import { cmsLanguages, cmsDefaultLanguage } from "@/globalConfig"; import { sessionNow } from "@/util/sessionNow"; +import { hasVideoSource, videoSourceFor } from "@/util/videoSource"; const { t } = useI18n(); @@ -53,9 +54,9 @@ const publishDateText = computed(() => { ).toLocaleString(DateTime.DATETIME_MED); }); -const hasVideo = computed(() => Boolean(props.content.video)); +const hasVideo = computed(() => hasVideoSource(props.content)); const hasAudio = computed( - () => !props.content.video && Boolean(props.content.parentMedia?.fileCollections?.length), + () => !hasVideo.value && Boolean(props.content.parentMedia?.fileCollections?.length), ); const mediaIconClass = computed(() => @@ -81,8 +82,9 @@ const isComingSoon = computed(() => { const mediaProgress = computed(() => { if (!props.showProgress) return 0; - const mediaIds = props.content.video - ? [props.content.video] + const videoSource = videoSourceFor(props.content); + const mediaIds = videoSource + ? [videoSource] : (props.content.parentMedia?.fileCollections ?? []).map((f) => f.fileUrl); for (const mediaId of mediaIds) { diff --git a/app/src/components/content/VideoPlayer.spec.ts b/app/src/components/content/VideoPlayer.spec.ts index a06363d7a6..9498c70091 100644 --- a/app/src/components/content/VideoPlayer.spec.ts +++ b/app/src/components/content/VideoPlayer.spec.ts @@ -150,8 +150,8 @@ describe("VideoPlayer", () => { it("renders the poster image for regular video", async () => { const content = { ...mockEnglishContentDto, - // VideoPlayer reads `content.video`; mock data only defines parentMedia.hlsUrl - video: mockEnglishContentDto.parentMedia!.hlsUrl!, + // The encoded collection is the source; no typed URL is involved. + video: undefined, }; const wrapper = mount(VideoPlayer, { @@ -162,7 +162,9 @@ describe("VideoPlayer", () => { }); await waitForExpect(() => { - expect(srcMock).toHaveBeenCalledWith(expect.objectContaining({ src: content.video })); + expect(srcMock).toHaveBeenCalledWith( + expect.objectContaining({ src: content.parentMedia!.hlsUrl }), + ); }); await waitForExpect(() => { @@ -186,6 +188,8 @@ describe("VideoPlayer", () => { it("handles YouTube videos correctly", async () => { const youtubeContent = { ...mockEnglishContentDto, + // No encoded collection: this post's video is the link somebody typed. + parentMedia: undefined, video: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", }; @@ -236,6 +240,7 @@ describe("VideoPlayer", () => { it("sets HLS source for regular video", async () => { const contentWithVideo = { ...mockEnglishContentDto, + parentMedia: undefined, video: "https://example.com/stream.m3u8", }; @@ -349,6 +354,7 @@ describe("VideoPlayer", () => { it("disposes player on unmount", async () => { const contentWithVideo = { ...mockEnglishContentDto, + parentMedia: undefined, video: "https://example.com/stream.m3u8", }; @@ -372,6 +378,8 @@ describe("VideoPlayer", () => { it("hides audio toggle for YouTube videos", async () => { const youtubeContent = { ...mockEnglishContentDto, + // No encoded collection: this post's video is the link somebody typed. + parentMedia: undefined, video: "https://www.youtube.com/watch?v=dQw4w9WgXcQ", }; diff --git a/app/src/components/content/VideoPlayer.vue b/app/src/components/content/VideoPlayer.vue index 8dba8d1443..12486fd141 100644 --- a/app/src/components/content/VideoPlayer.vue +++ b/app/src/components/content/VideoPlayer.vue @@ -5,12 +5,13 @@ import "videojs-mobile-ui"; import type Player from "video.js/dist/types/player"; import { type ContentDto } from "luminary-shared"; import px from "./px.png"; -import { matchTrackLanguage } from "./audioTrackLanguage"; +import { pickAudioTrack } from "./audioTrackLanguage"; import LImage from "../images/LImage.vue"; import { appLanguagesPreferredAsRef, queryParams } from "@/globalConfig"; import { getMediaProgress, removeMediaProgress, setMediaProgress } from "@/contentProgress"; import { extractAndBuildAudioMaster } from "./extractAndBuildAudioMaster"; import { isYouTubeUrl, convertToVideoJSYouTubeUrl } from "@/util/youtube"; +import { videoSourceFor } from "@/util/videoSource"; type Props = { content: ContentDto; @@ -35,8 +36,8 @@ const isRestoringTrack = ref(false); const isYouTube = ref(false); // Check if the current video is a YouTube video -if (props.content.video) { - isYouTube.value = isYouTubeUrl(props.content.video); +if (videoSourceFor(props.content)) { + isYouTube.value = isYouTubeUrl(videoSourceFor(props.content)!); if (isYouTube.value) { // hides audio mode toggle for YouTube videos as it's not supported showAudioModeToggle.value = false; @@ -88,21 +89,18 @@ function setAudioTrackLanguage(languageCode: string | null) { return; } - let trackFound = false; - for (let i = 0; i < audioTracks.length; i++) { - const track = audioTracks[i]; + // Decide before changing anything: disabling as we go leaves every track + // disabled when none matches, which silently starves the stream of audio. + const tracks: any[] = []; + for (let i = 0; i < audioTracks.length; i++) tracks.push(audioTracks[i]); - if (matchTrackLanguage(track.language, languageCode)) { - track.enabled = true; - trackFound = true; - } else { - track.enabled = false; - } - } - - if (!trackFound) { + const match = pickAudioTrack(tracks, languageCode); + if (!match) { console.warn(`No matching audio track found for language: ${languageCode}`); + return; } + + for (const track of tracks) track.enabled = track === match; } function syncKeepAudioStateAlive() { @@ -184,16 +182,59 @@ onMounted(async () => { // If statement is to protect from SSR issues if (typeof window !== "undefined" && window.document) { - player = videojs(playerElement.value!, options); + // videojs() throws on an element it does not recognise, and the throw + // escapes the mounted hook as an unhandled rejection rather than anything + // catchable by the caller. The ref is empty whenever the