feat: stream images from s3 instead of downloading the entire bucket

db/migrations.sql

@@ -3,3 +3,4 @@ INSERT INTO Users (Username, Password, FirstName, LastName, Email) VALUES ('Zero
 CREATE TABLE Projects (PName VARCHAR NOT NULL, PDescription VARCHAR NOT NULL, AutoSave INTEGER NOT NULL DEFAULT 0, Admin TEXT NOT NULL DEFAULT 'ZeroUser', Validate VARCHAR NOT NULL DEFAULT 0, FOREIGN KEY(Admin) REFERENCES Users(Username), PRIMARY KEY(PName, Admin));
 CREATE TABLE Access (Username TEXT NOT NULL, PName VARCHAR NOT NULL,  Admin TEXT NOT NULL, FOREIGN KEY(Username) REFERENCES Users(Username), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin));
 CREATE TABLE S3Buckets (PName VARCHAR NOT NULL, Admin TEXT NOT NULL, BucketName VARCHAR NOT NULL, Region VARCHAR NOT NULL, Prefix VARCHAR NOT NULL DEFAULT '', AccessKeyId VARCHAR, SecretAccessKey VARCHAR, LastSyncedAt TEXT, Endpoint TEXT, PRIMARY KEY(PName, Admin), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin));
+ALTER TABLE S3Buckets ADD COLUMN SyncMode VARCHAR NOT NULL DEFAULT 'download';

docker-compose.yml

@@ -6,6 +6,14 @@ services:
     image: ghcr.io/${GITHUB_REPOSITORY:-njobvu-ai}/app:latest
     container_name: njobvu-ai
     restart: unless-stopped
+    # The image has no USER directive, so without this the container writes
+    # to the bind-mounted host directories below as root. Anything it
+    # creates (e.g. project databases/images) then can't be written by a
+    # normal host user running the app outside Docker (`node .`), failing
+    # with SQLITE_READONLY. Export UID/GID before `docker compose up`
+    # (most shells: `export UID GID` - `UID` is a shell built-in that isn't
+    # exported by default) so files it creates are owned by you instead.
+    user: "${UID:-1000}:${GID:-1000}"
     ports:
       - "3000:3000"
     environment:

queries/projects/projects.js

@@ -202,28 +202,54 @@ module.exports = {
                 "CREATE TABLE IF NOT EXISTS Classes (CName VARCHAR NOT NULL PRIMARY KEY)",
             );
             await db.run(
-                "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0)",
+                "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0, Source VARCHAR DEFAULT NULL, SourceKey VARCHAR DEFAULT NULL)",
             );
             await db.run(
                 "CREATE TABLE IF NOT EXISTS Labels (LID INTEGER PRIMARY KEY, CName VARCHAR NOT NULL, X VARCHAR NOT NULL, Y VARCHAR NOT NULL, W INTEGER NOT NULL, H INTEGER NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(CName) REFERENCES Classes(CName), FOREIGN KEY(IName) REFERENCES Images(IName))",
             );
             await db.run(
                 "CREATE TABLE IF NOT EXISTS Validation (Confidence INTEGER NOT NULL, LID INTEGER NOT NULL PRIMARY KEY, CName VARCHAR NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(LID) REFERENCES Labels(LID), FOREIGN KEY(IName) REFERENCES Images(IName), FOREIGN KEY(CName) REFERENCES Classes(CName))",
             );
+
+            // Images predates the Source/SourceKey columns, so CREATE TABLE IF NOT EXISTS
+            // above is a no-op on any project database created before this change.
+            // Back-fill them here, guarded by a PRAGMA check since SQLite has no
+            // ADD COLUMN IF NOT EXISTS. SourceKey holds the literal S3 object key (which
+            // may differ from IName once collisions are disambiguated), decoupled from the
+            // display name.
+            const imageColumns = await db.all("PRAGMA table_info(Images)");
+            const existingColumnNames = new Set(
+                (imageColumns.rows || []).map((column) => column.name),
+            );
+
+            const backfillColumns = [
+                { name: "Source", ddl: "Source VARCHAR DEFAULT NULL" },
+                { name: "SourceKey", ddl: "SourceKey VARCHAR DEFAULT NULL" },
+            ];
+
+            for (const column of backfillColumns) {
+                if (!existingColumnNames.has(column.name)) {
+                    await db.run(`ALTER TABLE Images ADD COLUMN ${column.ddl}`);
+                }
+            }
         },
         addImages: async function(
             projectPath,
             imageName,
             reviewImage,
             validateImage,
+            source = null,
+            sourceKey = null,
         ) {
             const db = getDbClient(projectPath);
             const query =
-                "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, ?, ?)";
+                "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage, Source, SourceKey) VALUES (?, ?, ?, ?, ?)";
             const results = await db.run(query, [
                 imageName,
                 reviewImage,
                 validateImage,
+                source,
+                sourceKey,
             ]);
 
             return results;

queries/s3/s3.js

@@ -8,14 +8,16 @@ module.exports = {
             prefix,
             accessKeyId,
             secretAccessKey,
-            endpoint
+            endpoint,
+            syncMode = "download",
         ) {
             const query =
-                "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint) " +
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " +
+                "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint, SyncMode) " +
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " +
                 "ON CONFLICT(PName, Admin) DO UPDATE SET " +
                 "BucketName = excluded.BucketName, Region = excluded.Region, Prefix = excluded.Prefix, " +
-                "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint";
+                "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint, " +
+                "SyncMode = excluded.SyncMode";
 
             const result = await global.managedDbClient.run(query, [
                 projectName,
@@ -26,6 +28,7 @@ module.exports = {
                 accessKeyId || null,
                 secretAccessKey || null,
                 endpoint || "",
+                syncMode,
             ]);
 
             return result;

routes/api.js

@@ -59,6 +59,7 @@ const importDataset = require("./projects/importDataset");
 const importYolo = require("./projects/importYolo");
 const importKwCoco = require("./projects/importKwCoco");
 const importIfcb = require("./projects/importIfcb");
+const mapKwCocoCsv = require("./projects/mapKwCocoCsv");
 const mergeLocal = require("./projects/mergeLocal");
 const removeAccess = require("./projects/removeAccess");
 const transferAdmin = require("./projects/transferAdmin");
@@ -70,6 +71,7 @@ const {
     getS3Bucket,
     deleteS3Bucket,
     syncS3Bucket,
+    getProjectImage,
 } = require("./api/v2/s3Buckets");
 
 const updateLabels = require("./labelling/updateLabels");
@@ -169,6 +171,8 @@ api.post("/import", importProject);
 api.post("/api/projects/import-dataset", importDataset);
 api.post("/api/projects/import-yolo", importYolo);
 api.post("/api/projects/import-kwcoco", importKwCoco);
+api.post("/api/projects/map-kwcoco-csv", mapKwCocoCsv);
+api.post("/mapKwCocoCsv", mapKwCocoCsv);
 api.post("/api/projects/import-ifcb", importIfcb);
 api.post("/mergeLocal", mergeLocal);
 api.post("/removeAccess", removeAccess);
@@ -185,6 +189,7 @@ api.post("/api/v2/projects/:admin/:projectName/s3-bucket", attachS3Bucket);
 api.get("/api/v2/projects/:admin/:projectName/s3-bucket", getS3Bucket);
 api.delete("/api/v2/projects/:admin/:projectName/s3-bucket", deleteS3Bucket);
 api.post("/api/v2/projects/:admin/:projectName/s3-bucket/sync", syncS3Bucket);
+api.get("/api/v2/projects/:admin/:projectName/images/:imageName", getProjectImage);
 
 // LABELLING ROUTES
 api.post("/updateLabels", updateLabels);

routes/api/v2/s3Buckets.js

@@ -1,13 +1,17 @@
 const fs = require("fs");
 const path = require("path");
+const crypto = require("crypto");
 const queries = require("../../../queries/queries");
 const {
     buildS3Client,
     verifyBucketAccess,
     listImageObjects,
     downloadObjectToFile,
+    getObjectStream,
 } = require("../../../utils/s3Client");
 
+const SYNC_MODES = new Set(["download", "stream"]);
+
 function getProjectPath(admin, projectName) {
     return path.join(currentPath, "public", "projects", `${admin}-${projectName}`);
 }
@@ -25,6 +29,18 @@ function sanitizeFileName(name) {
         .join("_");
 }
 
+// Two distinct S3 keys can share a basename (e.g. "2024/img.jpg" and
+// "2025/img.jpg"), so a plain basename can't be assumed unique across a
+// bucket. Disambiguate deterministically from the full key so re-running a
+// sync never renames or duplicates a file that was already given a suffix.
+function disambiguateFileName(fileName, key) {
+    const ext = path.extname(fileName);
+    const base = fileName.slice(0, fileName.length - ext.length);
+    const hash = crypto.createHash("sha1").update(key).digest("hex").slice(0, 8);
+
+    return `${base}_${hash}${ext}`;
+}
+
 function trimOrUndefined(value) {
     return typeof value === "string" ? value.trim() : value;
 }
@@ -38,6 +54,7 @@ async function attachS3Bucket(req, res) {
     const AccessKeyId = trimOrUndefined(body.AccessKeyId);
     const SecretAccessKey = trimOrUndefined(body.SecretAccessKey);
     const Endpoint = trimOrUndefined(body.Endpoint);
+    const SyncMode = SYNC_MODES.has(body.SyncMode) ? body.SyncMode : "download";
 
     if (!isOwner(req, admin)) {
         return res.status(403).json({ success: false, error: "Not authorized for this project" });
@@ -70,7 +87,8 @@ async function attachS3Bucket(req, res) {
             Prefix || "",
             AccessKeyId,
             SecretAccessKey,
-            ...(Endpoint ? [Endpoint] : []),
+            Endpoint || "",
+            SyncMode,
         );
 
         return res.status(200).json({ success: true });
@@ -110,6 +128,7 @@ async function getS3Bucket(req, res) {
                 Prefix: row.Prefix,
                 Endpoint: row.Endpoint,
                 LastSyncedAt: row.LastSyncedAt,
+                SyncMode: row.SyncMode || "download",
                 hasCredentials: !!row.AccessKeyId,
             },
         });
@@ -167,24 +186,65 @@ async function syncS3Bucket(req, res) {
         });
 
         const objectKeys = await listImageObjects(s3Client, bucket.BucketName, bucket.Prefix);
-        const existingImages = new Set(await global.readdirAsync(imagesPath));
+
+        // Snapshot of what's on disk *before* this run - unrelated to what this run
+        // itself assigns. A basename already present here (a prior sync from before
+        // SourceKey existed, or an unrelated local file) is treated as already covered,
+        // same as before. `assignedNames` starts from this snapshot and grows as this
+        // run hands out names, so it also catches two keys *from this same bucket
+        // listing* colliding with each other.
+        const preExistingImages = new Set(await global.readdirAsync(imagesPath));
+        const assignedNames = new Set(preExistingImages);
+
+        const existingImageRows = await queries.project.getAllImages(projectPath);
+        const existingSourceKeys = new Set(
+            (existingImageRows.rows || [])
+                .map((row) => row.SourceKey)
+                .filter(Boolean),
+        );
 
         const syncedImages = [];
         let skippedCount = 0;
 
         for (const key of objectKeys) {
-            const fileName = sanitizeFileName(path.basename(key));
+            // Already synced this exact object in a prior run - never re-download or
+            // re-disambiguate a key we've already assigned a name to.
+            if (existingSourceKeys.has(key)) {
+                skippedCount += 1;
+                continue;
+            }
+
+            const baseName = sanitizeFileName(path.basename(key));
+
+            if (!baseName) {
+                skippedCount += 1;
+                continue;
+            }
 
-            if (!fileName || existingImages.has(fileName)) {
+            if (preExistingImages.has(baseName)) {
                 skippedCount += 1;
                 continue;
             }
 
-            const destPath = path.join(imagesPath, fileName);
-            await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath);
-            await queries.project.addImages(projectPath, fileName, 0, 0);
+            // A different key from this same bucket listing already claimed this
+            // basename - these are genuinely distinct objects, so disambiguate instead
+            // of dropping this one.
+            const fileName = assignedNames.has(baseName)
+                ? disambiguateFileName(baseName, key)
+                : baseName;
+
+            // "stream" mode registers the image (so it's browsable/labelable) without
+            // pulling its bytes to disk - GET .../images/:imageName fetches them from
+            // the bucket live, on the rare request that actually needs them.
+            if (bucket.SyncMode !== "stream") {
+                const destPath = path.join(imagesPath, fileName);
+                await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath);
+            }
+
+            await queries.project.addImages(projectPath, fileName, 0, 0, "s3", key);
 
-            existingImages.add(fileName);
+            assignedNames.add(fileName);
+            existingSourceKeys.add(key);
             syncedImages.push(fileName);
         }
 
@@ -206,9 +266,73 @@ async function syncS3Bucket(req, res) {
     }
 }
 
+// Serves a single project image, transparently covering both storage modes:
+// a locally-present file (the common case - local imports, or "download"-mode
+// S3 sync) is sent straight from disk; an image registered from S3 that was
+// never downloaded ("stream" mode) is fetched from the bucket live, for this
+// request only, and never written to disk. Any logged-in user can view it -
+// this mirrors the existing (also unauthenticated) static file serving for
+// local images, rather than introducing a stricter, inconsistent check here.
+async function getProjectImage(req, res) {
+    const { admin, projectName, imageName } = req.params;
+
+    if (!req.cookies || !req.cookies.Username) {
+        return res.status(403).json({ success: false, error: "Not authorized" });
+    }
+
+    const projectPath = getProjectPath(admin, projectName);
+    const localPath = path.join(projectPath, "images", imageName);
+
+    if (fs.existsSync(localPath)) {
+        return res.sendFile(localPath);
+    }
+
+    try {
+        const imageResult = await queries.project.getImage(projectPath, imageName);
+        const image = imageResult && imageResult.row;
+
+        if (!image || image.Source !== "s3" || !image.SourceKey) {
+            return res.status(404).json({ success: false, error: "Image not found" });
+        }
+
+        const bucketResult = await queries.managed.getBucket(projectName, admin);
+        const bucket = bucketResult && bucketResult.row;
+
+        if (!bucket) {
+            return res.status(404).json({ success: false, error: "No S3 bucket attached to this project" });
+        }
+
+        const s3Client = buildS3Client({
+            region: bucket.Region,
+            accessKeyId: bucket.AccessKeyId,
+            secretAccessKey: bucket.SecretAccessKey,
+            endpoint: bucket.Endpoint,
+        });
+
+        const { body, contentType } = await getObjectStream(s3Client, bucket.BucketName, image.SourceKey);
+
+        res.setHeader("Content-Type", contentType || "application/octet-stream");
+        body.on("error", (err) => {
+            global.logger.error(err);
+            if (!res.headersSent) {
+                res.status(500).end();
+            }
+        });
+        body.pipe(res);
+    } catch (err) {
+        global.logger.error(err, {
+            httpStatusCode: err.$metadata?.httpStatusCode,
+            code: err.Code || err.name,
+            requestId: err.$metadata?.requestId,
+        });
+        return res.status(500).json({ success: false, error: "Error fetching image" });
+    }
+}
+
 module.exports = {
     attachS3Bucket,
     getS3Bucket,
     deleteS3Bucket,
     syncS3Bucket,
+    getProjectImage,
 };