feat: stream s3 images and allow new dataset options for inference

controllers/inference/datatovalues.py

@@ -275,9 +275,12 @@ def copy_raw_image(original_src: str, dest_path: str):
         err = "You need more options to run the tool"
         print(err)
 
-    if image_path == "":
-        err = "You need more options to run the tool"
-        print(err)
+    if image_path.lower().endswith(".zip") or image_path.lower().endswith(".7z") or (os.path.isfile(image_path) and zipfile.is_zipfile(image_path)):
+        import tempfile
+        temp_dir = tempfile.mkdtemp(prefix="yolo_zip_")
+        with zipfile.ZipFile(image_path, 'r') as zip_ref:
+            zip_ref.extractall(temp_dir)
+        image_path = temp_dir
 
     print("Ultralytics Version of YOLO Requested:")
 

controllers/inference/inception.py

@@ -112,10 +112,26 @@ def preprocess_image(img_path):
 # Inference
 # ---------------------------------------------------
 
-image_files = [
-    f for f in os.listdir(args.image_path)
-    if f.lower().endswith((".jpg", ".jpeg", ".png", ".bmp"))
-]
+target_dir = args.image_path
+temp_dir = None
+
+if target_dir.lower().endswith(".zip") or target_dir.lower().endswith(".7z") or zipfile.is_zipfile(target_dir):
+    import tempfile
+    import shutil
+    temp_dir = tempfile.mkdtemp(prefix="inception_zip_")
+    with zipfile.ZipFile(target_dir, 'r') as zip_ref:
+        zip_ref.extractall(temp_dir)
+    target_dir = temp_dir
+
+image_files = []
+if os.path.isdir(target_dir):
+    for root, dirs, files in os.walk(target_dir):
+        for f in files:
+            if f.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".gif", ".webp")):
+                image_files.append(os.path.relpath(os.path.join(root, f), target_dir))
+elif os.path.isfile(target_dir) and target_dir.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".gif", ".webp")):
+    image_files.append(os.path.basename(target_dir))
+    target_dir = os.path.dirname(target_dir)
 
 os.makedirs(args.output_path, exist_ok=True)
 

controllers/inference/megadetector.py

@@ -266,8 +266,36 @@ def write_csvs(result, image_dir, output_dir):
 temp_dir = None
 image_dir = input_path
 
-if os.path.isdir(input_path):
+def process_directory_for_videos_and_subdirs(target_dir, fps):
+    for root, dirs, files in os.walk(target_dir):
+        for f in files:
+            full_f = os.path.join(root, f)
+            if is_video(f):
+                vid_name = os.path.splitext(f)[0]
+                frame_out = os.path.join(target_dir, f"{vid_name}_frames")
+                extract_frames(full_f, frame_out, fps)
+                for frame_f in os.listdir(frame_out):
+                    if is_image(frame_f):
+                        shutil.copy2(os.path.join(frame_out, frame_f), os.path.join(target_dir, f"{vid_name}_{frame_f}"))
+                shutil.rmtree(frame_out, ignore_errors=True)
+            elif is_image(f) and root != target_dir:
+                shutil.copy2(full_f, os.path.join(target_dir, f))
+
+if input_path.lower().endswith(".zip") or input_path.lower().endswith(".7z") or zipfile.is_zipfile(input_path):
+    if not os.path.exists(input_path):
+        print(f"Error: input archive not found: {input_path}")
+        sys.exit(1)
+    temp_dir = tempfile.mkdtemp(prefix="megadetector_zip_")
+    image_dir = os.path.join(temp_dir, "extracted")
+    os.makedirs(image_dir, exist_ok=True)
+    with zipfile.ZipFile(input_path, 'r') as zip_ref:
+        zip_ref.extractall(image_dir)
+    process_directory_for_videos_and_subdirs(image_dir, args.fps)
+
+elif os.path.isdir(input_path):
     image_dir = input_path
+    process_directory_for_videos_and_subdirs(image_dir, args.fps)
+
 elif is_video(input_path):
     if not os.path.exists(input_path):
         print(f"Error: input video not found: {input_path}")
@@ -277,6 +305,7 @@ def write_csvs(result, image_dir, output_dir):
     print(f"Extracting frames from video at {args.fps} fps...")
     extract_frames(input_path, image_dir, args.fps)
     print(f"Frames extracted to {image_dir}")
+
 elif is_image(input_path):
     if not os.path.exists(input_path):
         print(f"Error: input image not found: {input_path}")
@@ -285,6 +314,7 @@ def write_csvs(result, image_dir, output_dir):
     image_dir = os.path.join(temp_dir, "single")
     os.makedirs(image_dir, exist_ok=True)
     shutil.copy2(input_path, os.path.join(image_dir, os.path.basename(input_path)))
+
 else:
     print(f"Error: unsupported input type: {input_path}")
     sys.exit(1)

db/migrations.sql

@@ -3,3 +3,4 @@ INSERT INTO Users (Username, Password, FirstName, LastName, Email) VALUES ('Zero
 CREATE TABLE Projects (PName VARCHAR NOT NULL, PDescription VARCHAR NOT NULL, AutoSave INTEGER NOT NULL DEFAULT 0, Admin TEXT NOT NULL DEFAULT 'ZeroUser', Validate VARCHAR NOT NULL DEFAULT 0, FOREIGN KEY(Admin) REFERENCES Users(Username), PRIMARY KEY(PName, Admin));
 CREATE TABLE Access (Username TEXT NOT NULL, PName VARCHAR NOT NULL,  Admin TEXT NOT NULL, FOREIGN KEY(Username) REFERENCES Users(Username), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin));
 CREATE TABLE S3Buckets (PName VARCHAR NOT NULL, Admin TEXT NOT NULL, BucketName VARCHAR NOT NULL, Region VARCHAR NOT NULL, Prefix VARCHAR NOT NULL DEFAULT '', AccessKeyId VARCHAR, SecretAccessKey VARCHAR, LastSyncedAt TEXT, Endpoint TEXT, PRIMARY KEY(PName, Admin), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin));
+ALTER TABLE S3Buckets ADD COLUMN SyncMode VARCHAR NOT NULL DEFAULT 'download';

docker-compose.yml

@@ -6,6 +6,14 @@ services:
     image: ghcr.io/${GITHUB_REPOSITORY:-njobvu-ai}/app:latest
     container_name: njobvu-ai
     restart: unless-stopped
+    # The image has no USER directive, so without this the container writes
+    # to the bind-mounted host directories below as root. Anything it
+    # creates (e.g. project databases/images) then can't be written by a
+    # normal host user running the app outside Docker (`node .`), failing
+    # with SQLITE_READONLY. Export UID/GID before `docker compose up`
+    # (most shells: `export UID GID` - `UID` is a shell built-in that isn't
+    # exported by default) so files it creates are owned by you instead.
+    user: "${UID:-1000}:${GID:-1000}"
     ports:
       - "3000:3000"
     environment:

queries/projects/projects.js

@@ -202,42 +202,59 @@ module.exports = {
                 "CREATE TABLE IF NOT EXISTS Classes (CName VARCHAR NOT NULL PRIMARY KEY)",
             );
             await db.run(
-                "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0)",
+                "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0, Source VARCHAR DEFAULT NULL, SourceKey VARCHAR DEFAULT NULL)",
             );
-            try {
-                await db.run(
-                    "ALTER TABLE Images ADD COLUMN reviewImage INTEGER NOT NULL DEFAULT 0",
-                );
-            } catch (e) {
-                // Column already exists
-            }
-            try {
-                await db.run(
-                    "ALTER TABLE Images ADD COLUMN validateImage INTEGER NOT NULL DEFAULT 0",
-                );
-            } catch (e) {
-                // Column already exists
-            }
             await db.run(
                 "CREATE TABLE IF NOT EXISTS Labels (LID INTEGER PRIMARY KEY, CName VARCHAR NOT NULL, X VARCHAR NOT NULL, Y VARCHAR NOT NULL, W INTEGER NOT NULL, H INTEGER NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(CName) REFERENCES Classes(CName), FOREIGN KEY(IName) REFERENCES Images(IName))",
             );
             await db.run(
                 "CREATE TABLE IF NOT EXISTS Validation (Confidence INTEGER NOT NULL, LID INTEGER NOT NULL PRIMARY KEY, CName VARCHAR NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(LID) REFERENCES Labels(LID), FOREIGN KEY(IName) REFERENCES Images(IName), FOREIGN KEY(CName) REFERENCES Classes(CName))",
             );
+
+            // Images predates the reviewImage/validateImage/Source/SourceKey columns, so
+            // CREATE TABLE IF NOT EXISTS above is a no-op on any project database created
+            // before this change. Back-fill them here, guarded by a PRAGMA check since
+            // SQLite has no ADD COLUMN IF NOT EXISTS. SourceKey holds the literal S3 object
+            // key (which may differ from IName once collisions are disambiguated), decoupled
+            // from the display name.
+            const imageColumnsResult = await db.all("PRAGMA table_info(Images)");
+            const imageColumns = Array.isArray(imageColumnsResult)
+                ? imageColumnsResult
+                : (imageColumnsResult && imageColumnsResult.rows) || [];
+            const existingColumnNames = new Set(
+                imageColumns.map((column) => column.name),
+            );
+
+            const backfillColumns = [
+                { name: "reviewImage", ddl: "reviewImage INTEGER NOT NULL DEFAULT 0" },
+                { name: "validateImage", ddl: "validateImage INTEGER NOT NULL DEFAULT 0" },
+                { name: "Source", ddl: "Source VARCHAR DEFAULT NULL" },
+                { name: "SourceKey", ddl: "SourceKey VARCHAR DEFAULT NULL" },
+            ];
+
+            for (const column of backfillColumns) {
+                if (!existingColumnNames.has(column.name)) {
+                    await db.run(`ALTER TABLE Images ADD COLUMN ${column.ddl}`);
+                }
+            }
         },
         addImages: async function(
             projectPath,
             imageName,
             reviewImage,
             validateImage,
+            source = null,
+            sourceKey = null,
         ) {
             const db = getDbClient(projectPath);
             const query =
-                "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, ?, ?)";
+                "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage, Source, SourceKey) VALUES (?, ?, ?, ?, ?)";
             const results = await db.run(query, [
                 imageName,
                 reviewImage,
                 validateImage,
+                source,
+                sourceKey,
             ]);
 
             return results;

queries/s3/s3.js

@@ -8,16 +8,18 @@ module.exports = {
             prefix,
             accessKeyId,
             secretAccessKey,
-            endpoint
+            endpoint,
+            syncMode = "download",
         ) {
             const query =
-                "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint) " +
-                "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " +
+                "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint, SyncMode) " +
+                "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " +
                 "ON CONFLICT(PName, Admin) DO UPDATE SET " +
                 "BucketName = excluded.BucketName, Region = excluded.Region, Prefix = excluded.Prefix, " +
-                "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint";
+                "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint, " +
+                "SyncMode = excluded.SyncMode";
 
-            const result = await global.managedDbClient.run(query, [
+            return await global.managedDbClient.run(query, [
                 projectName,
                 admin,
                 bucketName,
@@ -26,9 +28,8 @@ module.exports = {
                 accessKeyId || null,
                 secretAccessKey || null,
                 endpoint || "",
+                syncMode,
             ]);
-
-            return result;
         },
         getBucket: async function(projectName, admin) {
             const query =

routes/api.js

@@ -72,6 +72,7 @@ const {
     getS3Bucket,
     deleteS3Bucket,
     syncS3Bucket,
+    getProjectImage,
 } = require("./api/v2/s3Buckets");
 
 const updateLabels = require("./labelling/updateLabels");
@@ -191,6 +192,7 @@ api.post("/api/v2/projects/:admin/:projectName/s3-bucket", attachS3Bucket);
 api.get("/api/v2/projects/:admin/:projectName/s3-bucket", getS3Bucket);
 api.delete("/api/v2/projects/:admin/:projectName/s3-bucket", deleteS3Bucket);
 api.post("/api/v2/projects/:admin/:projectName/s3-bucket/sync", syncS3Bucket);
+api.get("/api/v2/projects/:admin/:projectName/images/:imageName", getProjectImage);
 
 // LABELLING ROUTES
 api.post("/updateLabels", updateLabels);