Api cms delete and migrate media storage

api/src/app.module.ts

@@ -15,6 +15,7 @@ import { QueryController } from "./endpoints/query.controller";
 import { FtsSearchService } from "./endpoints/ftsSearch.service";
 import { FtsSearchController } from "./endpoints/ftsSearch.controller";
 import { StorageStatusController } from "./endpoints/storageStatus.controller";
+import { EncoderConfigController } from "./endpoints/encoderConfig.controller";
 import { AuthIdentityService } from "./auth/authIdentity.service";
 import { QueryRateLimiterService } from "./ratelimit/queryRateLimiter.service";
 
@@ -57,6 +58,7 @@ if (!process.env.NODE_ENV || process.env.NODE_ENV === "development") {
         QueryController,
         FtsSearchController,
         StorageStatusController,
+        EncoderConfigController,
     ],
     providers: [
         DbService,

api/src/changeRequests/documentProcessing/deleteMediaCollection.spec.ts

@@ -0,0 +1,99 @@
+import { resolveCollectionPrefix } from "./deleteMediaCollection";
+
+/** A real collection URL: MinIO, where the bucket name is part of the public path. */
+const PUBLIC = "http://localhost:9000/media";
+const SESSION = "c5829f07-4ba8-42ed-a449-80d83e6c0b53";
+const HLS = `${PUBLIC}/${SESSION}/master.m3u8`;
+
+const prefixOf = (r: ReturnType<typeof resolveCollectionPrefix>) =>
+    "prefix" in r ? r.prefix : undefined;
+const refusalOf = (r: ReturnType<typeof resolveCollectionPrefix>) =>
+    "refusal" in r ? r.refusal : undefined;
+
+describe("resolveCollectionPrefix", () => {
+    describe("resolves a collection this API wrote", () => {
+        it("strips the bucket's public base and the master filename", () => {
+            expect(prefixOf(resolveCollectionPrefix(HLS, PUBLIC))).toBe(SESSION);
+        });
+
+        it("tolerates a trailing slash on the configured public URL", () => {
+            expect(prefixOf(resolveCollectionPrefix(HLS, `${PUBLIC}/`))).toBe(SESSION);
+            expect(prefixOf(resolveCollectionPrefix(HLS, `${PUBLIC}///`))).toBe(SESSION);
+        });
+
+        it("keeps a nested path prefix intact", () => {
+            // pathPrefix on the session puts the collection in a subfolder.
+            const url = `${PUBLIC}/tenant-a/videos/${SESSION}/master.m3u8`;
+            expect(prefixOf(resolveCollectionPrefix(url, PUBLIC))).toBe(
+                `tenant-a/videos/${SESSION}`,
+            );
+        });
+
+        it("ignores a query string or fragment", () => {
+            expect(prefixOf(resolveCollectionPrefix(`${HLS}?v=2`, PUBLIC))).toBe(SESSION);
+            expect(prefixOf(resolveCollectionPrefix(`${HLS}#top`, PUBLIC))).toBe(SESSION);
+        });
+
+        it("handles a bucket published at a bare host", () => {
+            const base = "https://cdn.example.com";
+            expect(
+                prefixOf(resolveCollectionPrefix(`${base}/${SESSION}/master.m3u8`, base)),
+            ).toBe(SESSION);
+        });
+    });
+
+    describe("refuses anything it cannot prove it wrote", () => {
+        it("refuses a URL in a different bucket", () => {
+            const other = "https://someone-elses-cdn.example.com/media";
+            expect(refusalOf(resolveCollectionPrefix(`${other}/${SESSION}/master.m3u8`, PUBLIC)))
+                .toMatch(/not in this bucket/);
+        });
+
+        it("refuses a bucket whose name merely prefixes another", () => {
+            // The separator is part of the match, or `…/media` would claim
+            // `…/media-archive/<session>/master.m3u8`.
+            const url = `${PUBLIC}-archive/${SESSION}/master.m3u8`;
+            expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/not in this bucket/);
+        });
+
+        it("refuses a URL that is not a master playlist", () => {
+            expect(refusalOf(resolveCollectionPrefix(`${PUBLIC}/${SESSION}/`, PUBLIC)))
+                .toMatch(/master playlist/);
+            expect(
+                refusalOf(
+                    resolveCollectionPrefix(`${PUBLIC}/${SESSION}/stream/playlist.m3u8`, PUBLIC),
+                ),
+            ).toMatch(/master playlist/);
+        });
+
+        it("refuses the bucket root", () => {
+            expect(refusalOf(resolveCollectionPrefix(`${PUBLIC}/master.m3u8`, PUBLIC)))
+                .toMatch(/bucket root/);
+        });
+
+        it("refuses a path that tries to climb out", () => {
+            const url = `${PUBLIC}/../other-tenant/${SESSION}/master.m3u8`;
+            expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/suspicious path/);
+        });
+
+        it("refuses a folder that is not a session id", () => {
+            // The case the tick box makes possible: hlsUrl is editable, so someone
+            // can paste a URL naming a folder shared with other content.
+            const url = `${PUBLIC}/shared-videos/master.m3u8`;
+            expect(refusalOf(resolveCollectionPrefix(url, PUBLIC))).toMatch(/not a session id/);
+        });
+
+        it("refuses when either side is missing", () => {
+            expect(refusalOf(resolveCollectionPrefix(undefined, PUBLIC))).toMatch(/no media URL/);
+            expect(refusalOf(resolveCollectionPrefix(HLS, undefined))).toMatch(/no public URL/);
+            expect(refusalOf(resolveCollectionPrefix("", PUBLIC))).toMatch(/no media URL/);
+        });
+    });
+
+    it("never returns a prefix with a leading or trailing slash", () => {
+        // The caller appends '/' to scope the listing; a stray slash would widen it.
+        const prefix = prefixOf(resolveCollectionPrefix(HLS, PUBLIC))!;
+        expect(prefix.startsWith("/")).toBe(false);
+        expect(prefix.endsWith("/")).toBe(false);
+    });
+});

api/src/changeRequests/documentProcessing/deleteMediaCollection.ts

@@ -0,0 +1,169 @@
+import { MediaDto } from "../../dto/MediaDto";
+import { DbService } from "../../db/db.service";
+import { S3Service } from "../../s3/s3.service";
+
+/**
+ * Where a collection lives in its bucket, or why we will not touch it.
+ *
+ * A refusal is not an error: it is the safe answer for a URL we cannot prove we
+ * wrote, and the caller reports it as a warning rather than failing the request.
+ */
+export type PrefixResolution = { prefix: string } | { refusal: string };
+
+/**
+ * The encoder names every collection prefix after its session id.
+ *
+ * Checked because `hlsUrl` is an editable field: someone can paste a URL naming a
+ * shared folder, and "delete everything under it" would then be a data-loss bug
+ * wearing a tick box. A collection this API did not produce is one it must not
+ * remove — and every collection the encoder has ever written satisfies this.
+ */
+const SESSION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/** What the encoder publishes at the root of a collection. */
+const MASTER = "/master.m3u8";
+
+/**
+ * Turn a published `hlsUrl` into the object prefix holding that collection.
+ *
+ * The guard falls out of the arithmetic rather than being bolted on: the only way
+ * to get an object key from a public URL is to strip the bucket's own public base
+ * from it, so a URL that does not start with that base cannot be resolved at all.
+ * That is precisely the "never delete a prefix we did not create" rule, and unlike
+ * a marker object it also protects every collection already in a bucket.
+ */
+export function resolveCollectionPrefix(
+    hlsUrl: string | undefined,
+    publicUrl: string | undefined,
+): PrefixResolution {
+    if (!hlsUrl) return { refusal: "the document has no media URL" };
+    if (!publicUrl) return { refusal: "the bucket has no public URL configured" };
+
+    // Query strings and fragments are addressing, not location.
+    const url = hlsUrl.split(/[?#]/)[0];
+    const base = publicUrl.replace(/\/+$/, "");
+
+    // The separator has to be part of the match, or a bucket published at
+    // `https://cdn/media` would claim URLs belonging to `https://cdn/media-archive`.
+    if (!url.startsWith(`${base}/`)) {
+        return {
+            refusal: `the media URL is not in this bucket (expected it to start with ${base}/)`,
+        };
+    }
+
+    const key = url.slice(base.length + 1);
+
+    // Named before the suffix check below, which would otherwise report a master
+    // at the bucket root as "not a master playlist" — true but unhelpful for the
+    // one input where being clear matters most.
+    if (key === MASTER.slice(1)) return { refusal: "the media URL names the bucket root" };
+
+    if (!key.endsWith(MASTER)) {
+        return {
+            refusal: `the media URL does not name a master playlist (expected it to end with ${MASTER})`,
+        };
+    }
+
+    const prefix = key.slice(0, -MASTER.length);
+    if (!prefix) return { refusal: "the media URL names the bucket root" };
+
+    // A traversal cannot reach outside the bucket, but it can certainly reach a
+    // sibling prefix, and there is no legitimate reason for one to be here.
+    const segments = prefix.split("/");
+    if (segments.some((s) => s === "" || s === "." || s === "..")) {
+        return { refusal: `the media URL has a suspicious path (${prefix})` };
+    }
+
+    const last = segments[segments.length - 1];
+    if (!SESSION_ID.test(last)) {
+        return {
+            refusal:
+                `the media URL was not written by the encoder — its last folder ` +
+                `(${last}) is not a session id, so this API did not create it`,
+        };
+    }
+
+    return { prefix };
+}
+
+/**
+ * Delete the collection a document points at, if we can prove we wrote it.
+ *
+ * Best-effort by design, matching how images are handled: the caller is deleting a
+ * document, and refusing to do that because a bucket was unreachable would be
+ * worse than leaving objects behind. Everything that goes wrong comes back as a
+ * warning the CMS shows, and every key removed is logged first — the first real
+ * deletion in any bucket should be auditable after the fact.
+ */
+export async function deleteMediaCollection(
+    media: MediaDto | undefined,
+    bucketId: string | undefined,
+    db: DbService,
+): Promise<string[]> {
+    const warnings: string[] = [];
+
+    if (!media?.hlsUrl) return warnings;
+    if (!bucketId) {
+        warnings.push(
+            "Media files were not deleted: the document has no storage bucket. " +
+                "Please remove them on the storage provider.",
+        );
+        return warnings;
+    }
+
+    let bucket: { publicUrl?: string; name?: string };
+    try {
+        const result = await db.getDoc(bucketId);
+        if (!result.docs?.length) {
+            warnings.push(
+                `Media files were not deleted: bucket ${bucketId} no longer exists. ` +
+                    "Please remove them on the storage provider.",
+            );
+            return warnings;
+        }
+        bucket = result.docs[0];
+    } catch (error) {
+        warnings.push(`Media files were not deleted: ${error.message}`);
+        return warnings;
+    }
+
+    const resolved = resolveCollectionPrefix(media.hlsUrl, bucket.publicUrl);
+    if ("refusal" in resolved) {
+        warnings.push(
+            `Media files were not deleted because ${resolved.refusal}. ` +
+                "Please remove them on the storage provider if they are no longer needed.",
+        );
+        return warnings;
+    }
+
+    try {
+        const s3 = await S3Service.create(bucketId, db);
+        const keys = await s3.listObjectsUnder(`${resolved.prefix}/`);
+
+        if (keys.length === 0) {
+            // Already gone, or never uploaded. Not worth a warning: the caller
+            // asked for the files to be absent and they are.
+            return warnings;
+        }
+
+        // Named before removal, so a deletion that turns out to be wrong can be
+        // reconstructed from the log rather than guessed at.
+        console.log(
+            `Deleting ${keys.length} media object(s) under ${resolved.prefix}/ ` +
+                `in bucket ${bucket.name ?? bucketId}: ${keys.join(", ")}`,
+        );
+
+        // Batched: removeObjects takes a list, and a collection can run to
+        // hundreds of objects. 1000 is the S3 API's own limit per call.
+        for (let i = 0; i < keys.length; i += 1000) {
+            await s3.removeObjects(keys.slice(i, i + 1000));
+        }
+    } catch (error) {
+        warnings.push(
+            `Some media files could not be deleted from storage: ${error.message}. ` +
+                "Please check the storage provider.",
+        );
+    }
+
+    return warnings;
+}