diff --git a/controllers/inference/datatovalues.py b/controllers/inference/datatovalues.py index bb1e4e68..51ae5d6a 100755 --- a/controllers/inference/datatovalues.py +++ b/controllers/inference/datatovalues.py @@ -275,9 +275,12 @@ def copy_raw_image(original_src: str, dest_path: str): err = "You need more options to run the tool" print(err) - if image_path == "": - err = "You need more options to run the tool" - print(err) + if image_path.lower().endswith(".zip") or image_path.lower().endswith(".7z") or (os.path.isfile(image_path) and zipfile.is_zipfile(image_path)): + import tempfile + temp_dir = tempfile.mkdtemp(prefix="yolo_zip_") + with zipfile.ZipFile(image_path, 'r') as zip_ref: + zip_ref.extractall(temp_dir) + image_path = temp_dir print("Ultralytics Version of YOLO Requested:") diff --git a/controllers/inference/inception.py b/controllers/inference/inception.py index 13fab923..2e4af891 100644 --- a/controllers/inference/inception.py +++ b/controllers/inference/inception.py @@ -112,10 +112,26 @@ def preprocess_image(img_path): # Inference # --------------------------------------------------- -image_files = [ - f for f in os.listdir(args.image_path) - if f.lower().endswith((".jpg", ".jpeg", ".png", ".bmp")) -] +target_dir = args.image_path +temp_dir = None + +if target_dir.lower().endswith(".zip") or target_dir.lower().endswith(".7z") or zipfile.is_zipfile(target_dir): + import tempfile + import shutil + temp_dir = tempfile.mkdtemp(prefix="inception_zip_") + with zipfile.ZipFile(target_dir, 'r') as zip_ref: + zip_ref.extractall(temp_dir) + target_dir = temp_dir + +image_files = [] +if os.path.isdir(target_dir): + for root, dirs, files in os.walk(target_dir): + for f in files: + if f.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".gif", ".webp")): + image_files.append(os.path.relpath(os.path.join(root, f), target_dir)) +elif os.path.isfile(target_dir) and target_dir.lower().endswith((".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".gif", ".webp")): + image_files.append(os.path.basename(target_dir)) + target_dir = os.path.dirname(target_dir) os.makedirs(args.output_path, exist_ok=True) diff --git a/controllers/inference/megadetector.py b/controllers/inference/megadetector.py index 7e718903..bf14afb6 100644 --- a/controllers/inference/megadetector.py +++ b/controllers/inference/megadetector.py @@ -266,8 +266,36 @@ def write_csvs(result, image_dir, output_dir): temp_dir = None image_dir = input_path -if os.path.isdir(input_path): +def process_directory_for_videos_and_subdirs(target_dir, fps): + for root, dirs, files in os.walk(target_dir): + for f in files: + full_f = os.path.join(root, f) + if is_video(f): + vid_name = os.path.splitext(f)[0] + frame_out = os.path.join(target_dir, f"{vid_name}_frames") + extract_frames(full_f, frame_out, fps) + for frame_f in os.listdir(frame_out): + if is_image(frame_f): + shutil.copy2(os.path.join(frame_out, frame_f), os.path.join(target_dir, f"{vid_name}_{frame_f}")) + shutil.rmtree(frame_out, ignore_errors=True) + elif is_image(f) and root != target_dir: + shutil.copy2(full_f, os.path.join(target_dir, f)) + +if input_path.lower().endswith(".zip") or input_path.lower().endswith(".7z") or zipfile.is_zipfile(input_path): + if not os.path.exists(input_path): + print(f"Error: input archive not found: {input_path}") + sys.exit(1) + temp_dir = tempfile.mkdtemp(prefix="megadetector_zip_") + image_dir = os.path.join(temp_dir, "extracted") + os.makedirs(image_dir, exist_ok=True) + with zipfile.ZipFile(input_path, 'r') as zip_ref: + zip_ref.extractall(image_dir) + process_directory_for_videos_and_subdirs(image_dir, args.fps) + +elif os.path.isdir(input_path): image_dir = input_path + process_directory_for_videos_and_subdirs(image_dir, args.fps) + elif is_video(input_path): if not os.path.exists(input_path): print(f"Error: input video not found: {input_path}") @@ -277,6 +305,7 @@ def write_csvs(result, image_dir, output_dir): print(f"Extracting frames from video at {args.fps} fps...") extract_frames(input_path, image_dir, args.fps) print(f"Frames extracted to {image_dir}") + elif is_image(input_path): if not os.path.exists(input_path): print(f"Error: input image not found: {input_path}") @@ -285,6 +314,7 @@ def write_csvs(result, image_dir, output_dir): image_dir = os.path.join(temp_dir, "single") os.makedirs(image_dir, exist_ok=True) shutil.copy2(input_path, os.path.join(image_dir, os.path.basename(input_path))) + else: print(f"Error: unsupported input type: {input_path}") sys.exit(1) diff --git a/db/migrations.sql b/db/migrations.sql index 864395ac..8a0431a5 100644 --- a/db/migrations.sql +++ b/db/migrations.sql @@ -3,3 +3,4 @@ INSERT INTO Users (Username, Password, FirstName, LastName, Email) VALUES ('Zero CREATE TABLE Projects (PName VARCHAR NOT NULL, PDescription VARCHAR NOT NULL, AutoSave INTEGER NOT NULL DEFAULT 0, Admin TEXT NOT NULL DEFAULT 'ZeroUser', Validate VARCHAR NOT NULL DEFAULT 0, FOREIGN KEY(Admin) REFERENCES Users(Username), PRIMARY KEY(PName, Admin)); CREATE TABLE Access (Username TEXT NOT NULL, PName VARCHAR NOT NULL, Admin TEXT NOT NULL, FOREIGN KEY(Username) REFERENCES Users(Username), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin)); CREATE TABLE S3Buckets (PName VARCHAR NOT NULL, Admin TEXT NOT NULL, BucketName VARCHAR NOT NULL, Region VARCHAR NOT NULL, Prefix VARCHAR NOT NULL DEFAULT '', AccessKeyId VARCHAR, SecretAccessKey VARCHAR, LastSyncedAt TEXT, Endpoint TEXT, PRIMARY KEY(PName, Admin), FOREIGN KEY(PName) REFERENCES Projects(PName), FOREIGN KEY(Admin) REFERENCES Projects(Admin)); +ALTER TABLE S3Buckets ADD COLUMN SyncMode VARCHAR NOT NULL DEFAULT 'download'; diff --git a/docker-compose.yml b/docker-compose.yml index b32c102f..220ce1ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,6 +6,14 @@ services: image: ghcr.io/${GITHUB_REPOSITORY:-njobvu-ai}/app:latest container_name: njobvu-ai restart: unless-stopped + # The image has no USER directive, so without this the container writes + # to the bind-mounted host directories below as root. Anything it + # creates (e.g. project databases/images) then can't be written by a + # normal host user running the app outside Docker (`node .`), failing + # with SQLITE_READONLY. Export UID/GID before `docker compose up` + # (most shells: `export UID GID` - `UID` is a shell built-in that isn't + # exported by default) so files it creates are owned by you instead. + user: "${UID:-1000}:${GID:-1000}" ports: - "3000:3000" environment: diff --git a/package-lock.json b/package-lock.json index 3ddbfdbe..81e2e1bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -407,7 +407,6 @@ "integrity": "sha512-bXYxrXFubeYdvB0NhD/NBB3Qi6aZeV20GOWVI47t2dkecCEoneR4NPVcb7abpXDEvejgrUfFtG6vG/zxAKmg+g==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -2382,7 +2381,6 @@ "version": "19.0.6", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.6.tgz", "integrity": "sha512-gIlMztcTeDgXCUj0vCBOqEuSEhX//63fW9SZtCJ+agxoQTOklwDfiEMlTWn4mR/C/UK5VHlpwsCsOyf7/hc4lw==", - "peer": true, "dependencies": { "csstype": "^3.0.2" } @@ -3205,7 +3203,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001718", "electron-to-chromium": "^1.5.160", @@ -4392,6 +4389,29 @@ "node": ">= 0.8" } }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "license": "MIT", + "optional": true, + "dependencies": { + "iconv-lite": "^0.6.2" + } + }, + "node_modules/encoding/node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", diff --git a/queries/projects/projects.js b/queries/projects/projects.js index ef7bcf11..dccbea91 100644 --- a/queries/projects/projects.js +++ b/queries/projects/projects.js @@ -202,42 +202,59 @@ module.exports = { "CREATE TABLE IF NOT EXISTS Classes (CName VARCHAR NOT NULL PRIMARY KEY)", ); await db.run( - "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0)", + "CREATE TABLE IF NOT EXISTS Images (IName VARCHAR NOT NULL PRIMARY KEY, reviewImage INTEGER NOT NULL DEFAULT 0, validateImage INTEGER NOT NULL DEFAULT 0, Source VARCHAR DEFAULT NULL, SourceKey VARCHAR DEFAULT NULL)", ); - try { - await db.run( - "ALTER TABLE Images ADD COLUMN reviewImage INTEGER NOT NULL DEFAULT 0", - ); - } catch (e) { - // Column already exists - } - try { - await db.run( - "ALTER TABLE Images ADD COLUMN validateImage INTEGER NOT NULL DEFAULT 0", - ); - } catch (e) { - // Column already exists - } await db.run( "CREATE TABLE IF NOT EXISTS Labels (LID INTEGER PRIMARY KEY, CName VARCHAR NOT NULL, X VARCHAR NOT NULL, Y VARCHAR NOT NULL, W INTEGER NOT NULL, H INTEGER NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(CName) REFERENCES Classes(CName), FOREIGN KEY(IName) REFERENCES Images(IName))", ); await db.run( "CREATE TABLE IF NOT EXISTS Validation (Confidence INTEGER NOT NULL, LID INTEGER NOT NULL PRIMARY KEY, CName VARCHAR NOT NULL, IName VARCHAR NOT NULL, FOREIGN KEY(LID) REFERENCES Labels(LID), FOREIGN KEY(IName) REFERENCES Images(IName), FOREIGN KEY(CName) REFERENCES Classes(CName))", ); + + // Images predates the reviewImage/validateImage/Source/SourceKey columns, so + // CREATE TABLE IF NOT EXISTS above is a no-op on any project database created + // before this change. Back-fill them here, guarded by a PRAGMA check since + // SQLite has no ADD COLUMN IF NOT EXISTS. SourceKey holds the literal S3 object + // key (which may differ from IName once collisions are disambiguated), decoupled + // from the display name. + const imageColumnsResult = await db.all("PRAGMA table_info(Images)"); + const imageColumns = Array.isArray(imageColumnsResult) + ? imageColumnsResult + : (imageColumnsResult && imageColumnsResult.rows) || []; + const existingColumnNames = new Set( + imageColumns.map((column) => column.name), + ); + + const backfillColumns = [ + { name: "reviewImage", ddl: "reviewImage INTEGER NOT NULL DEFAULT 0" }, + { name: "validateImage", ddl: "validateImage INTEGER NOT NULL DEFAULT 0" }, + { name: "Source", ddl: "Source VARCHAR DEFAULT NULL" }, + { name: "SourceKey", ddl: "SourceKey VARCHAR DEFAULT NULL" }, + ]; + + for (const column of backfillColumns) { + if (!existingColumnNames.has(column.name)) { + await db.run(`ALTER TABLE Images ADD COLUMN ${column.ddl}`); + } + } }, addImages: async function( projectPath, imageName, reviewImage, validateImage, + source = null, + sourceKey = null, ) { const db = getDbClient(projectPath); const query = - "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, ?, ?)"; + "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage, Source, SourceKey) VALUES (?, ?, ?, ?, ?)"; const results = await db.run(query, [ imageName, reviewImage, validateImage, + source, + sourceKey, ]); return results; diff --git a/queries/s3/s3.js b/queries/s3/s3.js index 5c5fe857..fde48233 100644 --- a/queries/s3/s3.js +++ b/queries/s3/s3.js @@ -8,16 +8,18 @@ module.exports = { prefix, accessKeyId, secretAccessKey, - endpoint + endpoint, + syncMode = "download", ) { const query = - "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint) " + - "VALUES (?, ?, ?, ?, ?, ?, ?, ?) " + + "INSERT INTO S3Buckets (PName, Admin, BucketName, Region, Prefix, AccessKeyId, SecretAccessKey, Endpoint, SyncMode) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) " + "ON CONFLICT(PName, Admin) DO UPDATE SET " + "BucketName = excluded.BucketName, Region = excluded.Region, Prefix = excluded.Prefix, " + - "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint"; + "AccessKeyId = excluded.AccessKeyId, SecretAccessKey = excluded.SecretAccessKey, Endpoint = excluded.Endpoint, " + + "SyncMode = excluded.SyncMode"; - const result = await global.managedDbClient.run(query, [ + return await global.managedDbClient.run(query, [ projectName, admin, bucketName, @@ -26,9 +28,8 @@ module.exports = { accessKeyId || null, secretAccessKey || null, endpoint || "", + syncMode, ]); - - return result; }, getBucket: async function(projectName, admin) { const query = diff --git a/routes/api.js b/routes/api.js index eb75981f..9a811921 100755 --- a/routes/api.js +++ b/routes/api.js @@ -72,6 +72,7 @@ const { getS3Bucket, deleteS3Bucket, syncS3Bucket, + getProjectImage, } = require("./api/v2/s3Buckets"); const updateLabels = require("./labelling/updateLabels"); @@ -191,6 +192,7 @@ api.post("/api/v2/projects/:admin/:projectName/s3-bucket", attachS3Bucket); api.get("/api/v2/projects/:admin/:projectName/s3-bucket", getS3Bucket); api.delete("/api/v2/projects/:admin/:projectName/s3-bucket", deleteS3Bucket); api.post("/api/v2/projects/:admin/:projectName/s3-bucket/sync", syncS3Bucket); +api.get("/api/v2/projects/:admin/:projectName/images/:imageName", getProjectImage); // LABELLING ROUTES api.post("/updateLabels", updateLabels); diff --git a/routes/api/v2/s3Buckets.js b/routes/api/v2/s3Buckets.js index 525fa438..f9a61fa6 100644 --- a/routes/api/v2/s3Buckets.js +++ b/routes/api/v2/s3Buckets.js @@ -1,13 +1,17 @@ const fs = require("fs"); const path = require("path"); +const crypto = require("crypto"); const queries = require("../../../queries/queries"); const { buildS3Client, verifyBucketAccess, listImageObjects, downloadObjectToFile, + getObjectStream, } = require("../../../utils/s3Client"); +const SYNC_MODES = new Set(["download", "stream"]); + function getProjectPath(admin, projectName) { return path.join(currentPath, "public", "projects", `${admin}-${projectName}`); } @@ -25,6 +29,18 @@ function sanitizeFileName(name) { .join("_"); } +// Two distinct S3 keys can share a basename (e.g. "2024/img.jpg" and +// "2025/img.jpg"), so a plain basename can't be assumed unique across a +// bucket. Disambiguate deterministically from the full key so re-running a +// sync never renames or duplicates a file that was already given a suffix. +function disambiguateFileName(fileName, key) { + const ext = path.extname(fileName); + const base = fileName.slice(0, fileName.length - ext.length); + const hash = crypto.createHash("sha1").update(key).digest("hex").slice(0, 8); + + return `${base}_${hash}${ext}`; +} + function trimOrUndefined(value) { return typeof value === "string" ? value.trim() : value; } @@ -38,6 +54,7 @@ async function attachS3Bucket(req, res) { const AccessKeyId = trimOrUndefined(body.AccessKeyId); const SecretAccessKey = trimOrUndefined(body.SecretAccessKey); const Endpoint = trimOrUndefined(body.Endpoint); + const SyncMode = SYNC_MODES.has(body.SyncMode) ? body.SyncMode : "download"; if (!isOwner(req, admin)) { return res.status(403).json({ success: false, error: "Not authorized for this project" }); @@ -70,7 +87,8 @@ async function attachS3Bucket(req, res) { Prefix || "", AccessKeyId, SecretAccessKey, - ...(Endpoint ? [Endpoint] : []), + Endpoint || "", + SyncMode, ); return res.status(200).json({ success: true }); @@ -110,6 +128,7 @@ async function getS3Bucket(req, res) { Prefix: row.Prefix, Endpoint: row.Endpoint, LastSyncedAt: row.LastSyncedAt, + SyncMode: row.SyncMode || "download", hasCredentials: !!row.AccessKeyId, }, }); @@ -157,7 +176,9 @@ async function syncS3Bucket(req, res) { return res.status(404).json({ success: false, error: "No S3 bucket attached to this project" }); } - console.log(bucket); + const rawMaxLimit = (req.body && (req.body.maxImages || req.body.MaxImages || req.body.max_images || req.body.limit)) || (req.query && req.query.limit); + const parsedMaxLimit = rawMaxLimit !== undefined && rawMaxLimit !== null && rawMaxLimit !== "" ? parseInt(rawMaxLimit, 10) : null; + const maxImages = parsedMaxLimit !== null ? parsedMaxLimit : null; const s3Client = buildS3Client({ region: bucket.Region, @@ -167,24 +188,69 @@ async function syncS3Bucket(req, res) { }); const objectKeys = await listImageObjects(s3Client, bucket.BucketName, bucket.Prefix); - const existingImages = new Set(await global.readdirAsync(imagesPath)); + + // Snapshot of what's on disk *before* this run - unrelated to what this run + // itself assigns. A basename already present here (a prior sync from before + // SourceKey existed, or an unrelated local file) is treated as already covered, + // same as before. `assignedNames` starts from this snapshot and grows as this + // run hands out names, so it also catches two keys *from this same bucket + // listing* colliding with each other. + const preExistingImages = new Set(await global.readdirAsync(imagesPath)); + const assignedNames = new Set(preExistingImages); + + const existingImageRows = await queries.project.getAllImages(projectPath); + const existingSourceKeys = new Set( + (existingImageRows.rows || []) + .map((row) => row.SourceKey) + .filter(Boolean), + ); const syncedImages = []; let skippedCount = 0; for (const key of objectKeys) { - const fileName = sanitizeFileName(path.basename(key)); + if (Number.isFinite(maxImages) && maxImages > 0 && syncedImages.length >= maxImages) { + break; + } - if (!fileName || existingImages.has(fileName)) { + // Already synced this exact object in a prior run - never re-download or + // re-disambiguate a key we've already assigned a name to. + if (existingSourceKeys.has(key)) { skippedCount += 1; continue; } - const destPath = path.join(imagesPath, fileName); - await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath); - await queries.project.addImages(projectPath, fileName, 0, 0); + const baseName = sanitizeFileName(path.basename(key)); - existingImages.add(fileName); + if (!baseName) { + skippedCount += 1; + continue; + } + + if (preExistingImages.has(baseName)) { + skippedCount += 1; + continue; + } + + // A different key from this same bucket listing already claimed this + // basename - these are genuinely distinct objects, so disambiguate instead + // of dropping this one. + const fileName = assignedNames.has(baseName) + ? disambiguateFileName(baseName, key) + : baseName; + + // "stream" mode registers the image (so it's browsable/labelable) without + // pulling its bytes to disk - GET .../images/:imageName fetches them from + // the bucket live, on the rare request that actually needs them. + if (bucket.SyncMode !== "stream") { + const destPath = path.join(imagesPath, fileName); + await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath); + } + + await queries.project.addImages(projectPath, fileName, 0, 0, "s3", key); + + assignedNames.add(fileName); + existingSourceKeys.add(key); syncedImages.push(fileName); } @@ -206,9 +272,73 @@ async function syncS3Bucket(req, res) { } } +// Serves a single project image, transparently covering both storage modes: +// a locally-present file (the common case - local imports, or "download"-mode +// S3 sync) is sent straight from disk; an image registered from S3 that was +// never downloaded ("stream" mode) is fetched from the bucket live, for this +// request only, and never written to disk. Any logged-in user can view it - +// this mirrors the existing (also unauthenticated) static file serving for +// local images, rather than introducing a stricter, inconsistent check here. +async function getProjectImage(req, res) { + const { admin, projectName, imageName } = req.params; + + if (!req.cookies || !req.cookies.Username) { + return res.status(403).json({ success: false, error: "Not authorized" }); + } + + const projectPath = getProjectPath(admin, projectName); + const localPath = path.join(projectPath, "images", imageName); + + if (fs.existsSync(localPath)) { + return res.sendFile(localPath); + } + + try { + const imageResult = await queries.project.getImage(projectPath, imageName); + const image = imageResult && imageResult.row; + + if (!image || image.Source !== "s3" || !image.SourceKey) { + return res.status(404).json({ success: false, error: "Image not found" }); + } + + const bucketResult = await queries.managed.getBucket(projectName, admin); + const bucket = bucketResult && bucketResult.row; + + if (!bucket) { + return res.status(404).json({ success: false, error: "No S3 bucket attached to this project" }); + } + + const s3Client = buildS3Client({ + region: bucket.Region, + accessKeyId: bucket.AccessKeyId, + secretAccessKey: bucket.SecretAccessKey, + endpoint: bucket.Endpoint, + }); + + const { body, contentType } = await getObjectStream(s3Client, bucket.BucketName, image.SourceKey); + + res.setHeader("Content-Type", contentType || "application/octet-stream"); + body.on("error", (err) => { + global.logger.error(err); + if (!res.headersSent) { + res.status(500).end(); + } + }); + body.pipe(res); + } catch (err) { + global.logger.error(err, { + httpStatusCode: err.$metadata?.httpStatusCode, + code: err.Code || err.name, + requestId: err.$metadata?.requestId, + }); + return res.status(500).json({ success: false, error: "Error fetching image" }); + } +} + module.exports = { attachS3Bucket, getS3Bucket, deleteS3Bucket, syncS3Bucket, + getProjectImage, }; diff --git a/routes/inference/inceptionInference.js b/routes/inference/inceptionInference.js index 2d2dd10d..8d85bae9 100644 --- a/routes/inference/inceptionInference.js +++ b/routes/inference/inceptionInference.js @@ -1,6 +1,7 @@ const queries = require("../../queries/queries"); const { exec } = require("child_process"); const formatRunOptionsHeader = require("../../utils/formatRunOptionsHeader"); +const { prepareInferenceDataset } = require("../../utils/inferenceDatasetPipeline"); async function inceptionInference(req, res) { try { @@ -50,6 +51,25 @@ async function inceptionInference(req, res) { fs.copyFileSync(inferenceScript, inferenceScriptCopyPath); } + const datasetResult = await prepareInferenceDataset({ + PName, + Admin, + inference_file: inferenceFile, + use_s3_bucket: req.body.use_s3_bucket || req.body.s3_bucket || req.body.inference_source === "s3", + max_images: req.body.max_images || req.body.maxImages || req.body.limit, + projectPath, + inferenceUploadPath, + }); + + inferenceFilePath = datasetResult.inferenceFilePath || inferenceFilePath; + + if (!fs.existsSync(inferenceFilePath)) { + const fallbackInferenceFilePath = path.join(inferenceUploadPath, inferenceFilePath); + if (fs.existsSync(fallbackInferenceFilePath)) { + inferenceFilePath = fallbackInferenceFilePath; + } + } + let existingClasses; try { diff --git a/routes/inference/megadetectorInference.js b/routes/inference/megadetectorInference.js index 611ad991..64eba5fc 100644 --- a/routes/inference/megadetectorInference.js +++ b/routes/inference/megadetectorInference.js @@ -1,6 +1,7 @@ const path = require("path"); const formatRunOptionsHeader = require("../../utils/formatRunOptionsHeader"); const config = require("../../utils/config"); +const { prepareInferenceDataset } = require("../../utils/inferenceDatasetPipeline"); async function megadetectorInference(req, res) { try { @@ -45,6 +46,18 @@ async function megadetectorInference(req, res) { fs.copyFileSync(megadetectorScript, megadetectorScriptCopyPath); } + const datasetResult = await prepareInferenceDataset({ + PName, + Admin, + inference_file: inferenceFile, + use_s3_bucket: req.body.use_s3_bucket || req.body.s3_bucket || req.body.inference_source === "s3", + max_images: req.body.max_images || req.body.maxImages || req.body.limit, + projectPath, + inferenceUploadPath, + }); + + inferenceFilePath = datasetResult.inferenceFilePath || inferenceFilePath; + if (!fs.existsSync(inferenceFilePath)) { const fallbackInferenceFilePath = path.join(inferenceUploadPath, inferenceFilePath); if (fs.existsSync(fallbackInferenceFilePath)) { diff --git a/routes/inference/uploadInferenceFile.js b/routes/inference/uploadInferenceFile.js index 0e894bcd..0548a8a3 100644 --- a/routes/inference/uploadInferenceFile.js +++ b/routes/inference/uploadInferenceFile.js @@ -1,34 +1,70 @@ +const path = require("path"); +const fs = require("fs"); +const unzipFile = require("../../utils/unzipFile"); + async function uploadInferenceFile(req, res) { var PName = req.body.PName, Admin = req.body.Admin, - user = req.cookies.Username, - inferenceFile = req.files.upload_inference; + user = req.cookies && req.cookies.Username, + inferenceFile = req.files && req.files.upload_inference; + + if (!inferenceFile) { + return res.status(400).send({ Success: "ERROR: No file uploaded" }); + } var publicPath = currentPath, - mainPath = publicPath + "public/projects/", // $LABELING_TOOL_PATH/public/projects/ - projectPath = mainPath + Admin + "-" + PName, // $LABELING_TOOL_PATH/public/projects/project_name - trainingPath = projectPath + "/training", + mainPath = publicPath + "public/projects/", + projectPath = mainPath + Admin + "-" + PName, inferencePath = projectPath + "/inference/", inferenceUploadPath = inferencePath + "/uploads/", inferenceFilePath = inferenceUploadPath + inferenceFile.name; - const validFileNames = ["png", "tif", "jpg", "jpeg", "gif", "mp4", "mov"]; + const validFileNames = ["png", "tif", "jpg", "jpeg", "gif", "mp4", "mov", "zip", "7z"]; + const ext = (inferenceFile.name.split(".").pop() || "").toLowerCase(); - if (!validFileNames.includes(inferenceFile.name.split(".").pop())) { + if (!validFileNames.includes(ext)) { res.send({ Success: - "ERROR: Wrong filetype. Must be type .png, .jpg, jpeg, tif or .gif", + "ERROR: Wrong filetype. Must be type .png, .jpg, .jpeg, .tif, .gif, .mp4, .mov, or .zip", }); } else { if (!fs.existsSync(inferencePath)) { - fs.mkdirSync(inferencePath); + fs.mkdirSync(inferencePath, { recursive: true }); } if (!fs.existsSync(inferenceUploadPath)) { - fs.mkdir(inferenceUploadPath); + fs.mkdirSync(inferenceUploadPath, { recursive: true }); } - await inferenceFile.mv(inferenceFilePath); + if (inferenceFile && typeof inferenceFile.mv === "function") { + await new Promise((resolve, reject) => { + try { + const ret = inferenceFile.mv(inferenceFilePath, (err) => { + if (err) return reject(err); + resolve(); + }); + if (ret && typeof ret.then === "function") { + ret.then(resolve).catch(reject); + } + } catch (e) { + reject(e); + } + }); + } + + let extractedPath = null; + if (ext === "zip" || ext === "7z") { + const folderName = path.parse(inferenceFile.name).name; + const outputDir = path.join(inferenceUploadPath, folderName); + try { + await unzipFile(inferenceFilePath, outputDir); + extractedPath = outputDir; + } catch (err) { + if (global.logger) { + global.logger.error(err); + } + } + } res.send({ Success: "Your inference file has been uploaded and saved", diff --git a/routes/inference/yoloInference.js b/routes/inference/yoloInference.js index 3e6c9ffc..f302805d 100644 --- a/routes/inference/yoloInference.js +++ b/routes/inference/yoloInference.js @@ -2,6 +2,7 @@ const queries = require("../../queries/queries"); const path = require("path"); const formatRunOptionsHeader = require("../../utils/formatRunOptionsHeader"); const config = require("../../utils/config"); +const { prepareInferenceDataset } = require("../../utils/inferenceDatasetPipeline"); async function yoloInference(req, res) { try { @@ -78,9 +79,12 @@ async function yoloInference(req, res) { var dictImagesLabels = {}; for (var i = 0; i < existingImages.rows.length; i++) { - var img = fs.readFileSync( - `${imagesPath}/${existingImages.rows[i].IName}`, - ), + const imgPath = path.join(imagesPath, existingImages.rows[i].IName); + if (!fs.existsSync(imgPath)) { + continue; + } + + var img = fs.readFileSync(imgPath), imgData = probe.sync(img), imgW = imgData.width, imgH = imgData.height; @@ -149,6 +153,18 @@ async function yoloInference(req, res) { ultralyticsProjectRun = runPath; + const datasetResult = await prepareInferenceDataset({ + PName, + Admin, + inference_file: inferenceFile, + use_s3_bucket: req.body.use_s3_bucket || req.body.s3_bucket || req.body.inference_source === "s3", + max_images: req.body.max_images || req.body.maxImages || req.body.limit, + projectPath, + inferenceUploadPath, + }); + + inferenceFilePath = datasetResult.inferenceFilePath || inferenceFilePath; + if (!fs.existsSync(inferenceFilePath)) { const fallbackInferenceFilePath = path.join(inferenceUploadPath, inferenceFilePath); if (fs.existsSync(fallbackInferenceFilePath)) { diff --git a/routes/pages/getAnnotatePage.js b/routes/pages/getAnnotatePage.js index 799b1a3a..cd694a77 100644 --- a/routes/pages/getAnnotatePage.js +++ b/routes/pages/getAnnotatePage.js @@ -1,9 +1,11 @@ -const path = require('path'); -const fs = require('fs'); -const sqlite3 = require('sqlite3').verbose(); -const probe = require('probe-image-size'); +const queries = require("../../queries/queries"); +const { buildS3Client, getObjectStream } = require("../../utils/s3Client"); async function getAnnotatePage(req, res) { + var path = global.path || require("path"); + var fs = global.fs || require("fs"); + var sqlite3 = global.sqlite3 || require("sqlite3").verbose(); + var probe = global.probe || require("probe-image-size"); var IDX = parseInt(req.query.IDX), IName = String(req.query.IName), curr_class = req.query.curr_class, @@ -133,22 +135,61 @@ async function getAnnotatePage(req, res) { } } - var abs_image_path = path.join(project_path, "images", IName); + var abs_image_path = project_path + "/images/" + IName; + var imageExistsLocally = fs.existsSync(abs_image_path); + var imageRow = results4 && results4[0]; - if (!results4 || results4.length === 0 || !fs.existsSync(abs_image_path)) { + // A "download"-mode (or local-import) image is a real file at + // abs_image_path, same as always. A "stream"-mode S3 image never has + // one - it's only ever fetched live, on view, via the on-demand proxy + // below - so only 404 here if neither a local file nor an S3-backed row + // exists for this name. + if (!imageRow || (!imageExistsLocally && imageRow.Source !== "s3")) { ldb.close(); res.render("404", { title: "404", user: req.cookies.Username, }); } else { - var rel_image_path = rel_project_path + "/images/" + results4[0].IName; - var img = fs.readFileSync(abs_image_path), - img_data = probe.sync(img), - img_w = img_data.width, - img_h = img_data.height, - image_ratio = img_h / img_w, - image_width = img_w, + var rel_image_path; + var img_w, img_h; + + if (imageExistsLocally) { + rel_image_path = rel_project_path + "/images/" + imageRow.IName; + var img = fs.readFileSync(project_path + "/images/" + imageRow.IName); + var img_data = probe.sync(img); + img_w = img_data.width; + img_h = img_data.height; + } else { + // Point the browser at the on-demand proxy instead of a static + // path that doesn't exist. Probe just enough of the object's + // header (over that same live fetch, aborted by probe() once it + // has what it needs) to lay out the page, rather than pulling + // the whole image server-side just to measure it. + rel_image_path = `api/v2/projects/${admin}/${PName}/images/${imageRow.IName}`; + + try { + var bucketResult = await queries.managed.getBucket(PName, admin); + var bucket = bucketResult && bucketResult.row; + var s3Client = buildS3Client({ + region: bucket.Region, + accessKeyId: bucket.AccessKeyId, + secretAccessKey: bucket.SecretAccessKey, + endpoint: bucket.Endpoint, + }); + var objectStream = await getObjectStream(s3Client, bucket.BucketName, imageRow.SourceKey); + var probed = await probe(objectStream.body); + img_w = probed.width; + img_h = probed.height; + } catch (err) { + global.logger.error(err); + img_w = 0; + img_h = 0; + } + } + + var image_ratio = img_w ? img_h / img_w : 1, + image_width = img_w || 0, image_height = image_ratio * image_width, prev_IName = -1, next_IName = -1; diff --git a/routes/pages/getInceptionSettingsPage.js b/routes/pages/getInceptionSettingsPage.js index 0690cda2..b37d0f79 100644 --- a/routes/pages/getInceptionSettingsPage.js +++ b/routes/pages/getInceptionSettingsPage.js @@ -131,6 +131,11 @@ async function getYoloXInferencePage(req, res) { var global_inference_upload = await readdirAsync(inference_upload_path); global_inference_upload.push(project_path + "/images"); + const attachedBucket = await queries.managed.getBucket(PName, admin); + if (attachedBucket && attachedBucket.row) { + global_inference_upload.push("s3"); + } + // get runs var runs = await readdirAsync(log_path); runs = runs.reverse(); diff --git a/routes/pages/getMegadetectorSettingsPage.js b/routes/pages/getMegadetectorSettingsPage.js index 40aee167..a1bef9d5 100644 --- a/routes/pages/getMegadetectorSettingsPage.js +++ b/routes/pages/getMegadetectorSettingsPage.js @@ -56,6 +56,11 @@ async function getMegadetectorSettingsPage(req, res) { var global_inference_upload = await readdirAsync(inference_upload_path); global_inference_upload.push(project_path + "/images"); + const attachedBucket = await queries.managed.getBucket(PName, admin); + if (attachedBucket && attachedBucket.row) { + global_inference_upload.push("s3"); + } + res.render("training/megadetectorSettings", { title: "megadetectorSettings", user: req.cookies.Username, diff --git a/routes/pages/getYoloXInferenceSettingsPage.js b/routes/pages/getYoloXInferenceSettingsPage.js index 7274947a..b6f8be15 100644 --- a/routes/pages/getYoloXInferenceSettingsPage.js +++ b/routes/pages/getYoloXInferenceSettingsPage.js @@ -1,3 +1,5 @@ +const queries = require("../../queries/queries"); + async function getYoloXInferencePage(req, res) { const readdir = util.promisify(fs.readdir); const readFile = util.promisify(fs.readFile); @@ -48,12 +50,12 @@ async function getYoloXInferencePage(req, res) { fs.mkdirSync(log_path); fs.mkdirSync(python_path); fs.mkdirSync(weights_path); - fs.writeFile(python_path_file, "", function (err) { + fs.writeFile(python_path_file, "", function(err) { if (err) { global.logger.error(err); } }); - fs.writeFile(yolovx_path_file, "", function (err) { + fs.writeFile(yolovx_path_file, "", function(err) { if (err) { global.logger.error(err); } @@ -61,7 +63,7 @@ async function getYoloXInferencePage(req, res) { } else if (!fs.existsSync(weights_path)) { fs.mkdirSync(weights_path); } else if (!fs.existsSync(yolovx_path_file)) { - fs.writeFile(yolovx_path_file, "", function (err) { + fs.writeFile(yolovx_path_file, "", function(err) { if (err) { global.logger.error(err); } @@ -77,10 +79,10 @@ async function getYoloXInferencePage(req, res) { }); // create async database object functions - tdb.getAsync = function (sql) { + tdb.getAsync = function(sql) { var that = this; - return new Promise(function (resolve, reject) { - that.get(sql, function (err, row) { + return new Promise(function(resolve, reject) { + that.get(sql, function(err, row) { if (err) { global.logger.error("runAsync ERROR!", err) reject(err); @@ -90,10 +92,10 @@ async function getYoloXInferencePage(req, res) { global.logger.error(err); }); }; - tdb.allAsync = function (sql) { + tdb.allAsync = function(sql) { var that = this; - return new Promise(function (resolve, reject) { - that.all(sql, function (err, row) { + return new Promise(function(resolve, reject) { + that.all(sql, function(err, row) { if (err) { global.logger.error("runAsync ERROR!", err) reject(err); @@ -106,19 +108,19 @@ async function getYoloXInferencePage(req, res) { var results1 = await db.getAsync( "SELECT * FROM `Projects` WHERE PName = '" + - PName + - "' AND Admin = '" + - admin + - "'", + PName + + "' AND Admin = '" + + admin + + "'", ); var results2 = await tdb.allAsync("SELECT * FROM `Classes`"); var acc = await db.allAsync( "SELECT * FROM `Access` WHERE PName = '" + - PName + - "' AND Admin = '" + - admin + - "'", + PName + + "' AND Admin = '" + + admin + + "'", ); var access = []; for (var i = 0; i < acc.length; i++) { @@ -131,6 +133,11 @@ async function getYoloXInferencePage(req, res) { var global_inference_upload = await readdirAsync(inference_upload_path); global_inference_upload.push(project_path + "/images"); + const attachedBucket = await queries.managed.getBucket(PName, admin); + if (attachedBucket && attachedBucket.row) { + global_inference_upload.push("s3"); + } + // get runs var runs = await readdirAsync(log_path); runs = runs.reverse(); @@ -218,7 +225,7 @@ async function getYoloXInferencePage(req, res) { } // close the database - tdb.close(function (err) { + tdb.close(function(err) { if (err) { global.logger.error(err); } else { diff --git a/routes/pages/getYoloXSettingsPage.js b/routes/pages/getYoloXSettingsPage.js index 5700481a..c113b5f6 100644 --- a/routes/pages/getYoloXSettingsPage.js +++ b/routes/pages/getYoloXSettingsPage.js @@ -50,12 +50,12 @@ async function getYoloXSettingsPage(req, res) { fs.mkdirSync(log_path); fs.mkdirSync(python_path); fs.mkdirSync(weights_path); - fs.writeFile(python_path_file, "", function (err) { + fs.writeFile(python_path_file, "", function(err) { if (err) { global.logger.error(err); } }); - fs.writeFile(yolovx_path_file, "", function (err) { + fs.writeFile(yolovx_path_file, "", function(err) { if (err) { global.logger.error(err); } @@ -63,7 +63,7 @@ async function getYoloXSettingsPage(req, res) { } else if (!fs.existsSync(weights_path)) { fs.mkdirSync(weights_path); } else if (!fs.existsSync(yolovx_path_file)) { - fs.writeFile(yolovx_path_file, "", function (err) { + fs.writeFile(yolovx_path_file, "", function(err) { if (err) { global.logger.error(err); } @@ -79,10 +79,10 @@ async function getYoloXSettingsPage(req, res) { }); // create async database object functions - tdb.getAsync = function (sql) { + tdb.getAsync = function(sql) { var that = this; - return new Promise(function (resolve, reject) { - that.get(sql, function (err, row) { + return new Promise(function(resolve, reject) { + that.get(sql, function(err, row) { if (err) { global.logger.error("runAsync ERROR!", err) reject(err); @@ -92,10 +92,10 @@ async function getYoloXSettingsPage(req, res) { global.logger.error(err); }); }; - tdb.allAsync = function (sql) { + tdb.allAsync = function(sql) { var that = this; - return new Promise(function (resolve, reject) { - that.all(sql, function (err, row) { + return new Promise(function(resolve, reject) { + that.all(sql, function(err, row) { if (err) { global.logger.error("runAsync ERROR!", err) reject(err); @@ -121,7 +121,7 @@ async function getYoloXSettingsPage(req, res) { var countsResult = await queries.project.getClassLabelCounts(project_path); if (countsResult && countsResult.rows) { - countsResult.rows.forEach(function (row) { + countsResult.rows.forEach(function(row) { classLabelCounts[row.CName] = row.labelCount; }); } @@ -129,7 +129,7 @@ async function getYoloXSettingsPage(req, res) { global.logger.error(err); } - results2 = results2.map(function (cls) { + results2 = results2.map(function(cls) { return Object.assign({}, cls, { labelCount: classLabelCounts[cls.CName] || 0, }); @@ -239,7 +239,7 @@ async function getYoloXSettingsPage(req, res) { } // close the database - tdb.close(function (err) { + tdb.close(function(err) { if (err) { global.logger.error(err); } else { diff --git a/routes/training/run.js b/routes/training/run.js index a830492a..3b4886da 100644 --- a/routes/training/run.js +++ b/routes/training/run.js @@ -1,5 +1,11 @@ +const path = require("path"); +const fs = require("fs"); const queries = require("../../queries/queries"); const formatRunOptionsHeader = require("../../utils/formatRunOptionsHeader"); +const { + ensureTrainingImagesLocal, + cleanupJitTrainingImages, +} = require("../../utils/jitTrainingImages"); const { generateModelCard } = require("../../utils/runSummaryGenerator"); async function run(req, res) { @@ -28,7 +34,7 @@ async function run(req, res) { options = "EMPTY"; } global.logger.debug("options: ", options); - var publicPath = currentPath, + var publicPath = global.currentPath || (typeof currentPath !== "undefined" ? currentPath : (process.cwd() + "/")), mainPath = publicPath + "public/projects/", // $LABELING_TOOL_PATH/public/projects/ projectPath = mainPath + Admin + "-" + PName, // $LABELING_TOOL_PATH/public/projects/project_name imagesPath = projectPath + "/images", // $LABELING_TOOL_PATH/public/projects/project_name/images @@ -60,11 +66,22 @@ async function run(req, res) { if (err) throw err; }); + let existingImages; + let jitDownloadedFiles = []; + try { + existingImages = await queries.project.getAllImages(projectPath); + jitDownloadedFiles = await ensureTrainingImagesLocal(PName, Admin, projectPath, existingImages.rows); + } catch (err) { + global.logger.error("Error fetching streamed S3 images JIT for run:", err); + return res.status(500).send("Error fetching streamed S3 images for training: " + err.message); + } + let existingLabels; try { existingLabels = await queries.project.getAllLabels(projectPath); } catch (err) { global.logger.error(err); + await cleanupJitTrainingImages(jitDownloadedFiles); return res.status(500).send("Error fetching labels"); } const labels = existingLabels.rows; @@ -169,8 +186,9 @@ async function run(req, res) { global.logger.debug("stdout: ", stdout); global.logger.debug("stderr: ", stderr); global.logger.debug("err: ", err); - fs.writeFile(`${runPath}/done.log`, success, (err) => { + fs.writeFile(`${runPath}/done.log`, success, async (err) => { if (err) throw err; + await cleanupJitTrainingImages(jitDownloadedFiles); }); if (!err) { diff --git a/routes/training/yoloRun.js b/routes/training/yoloRun.js index ffd61ef3..c96d6859 100644 --- a/routes/training/yoloRun.js +++ b/routes/training/yoloRun.js @@ -6,6 +6,10 @@ const probe = require("probe-image-size"); const os = require("os"); const sharp = require("sharp"); const formatRunOptionsHeader = require("../../utils/formatRunOptionsHeader"); +const { + ensureTrainingImagesLocal, + cleanupJitTrainingImages, +} = require("../../utils/jitTrainingImages"); const { generateModelCard } = require("../../utils/runSummaryGenerator"); // Function to detect the best available device for YOLO training @@ -487,6 +491,14 @@ async function yoloRun(req, res) { targetImages = targetImages.slice(0, maxImages); } + let jitDownloadedFiles = []; + try { + jitDownloadedFiles = await ensureTrainingImagesLocal(PName, Admin, projectPath, targetImages); + } catch (err) { + global.logger.error("Error ensuring local images for training JIT:", err); + return res.status(500).send("Error fetching streamed S3 images for training: " + err.message); + } + // Parse Train : Validate : Test split ratio let trainPct = parseFloat(req.body.TrainingPercent || req.body.train_percent || 70); let valPct = parseFloat(req.body.ValPercent || req.body.val_percent); @@ -941,6 +953,7 @@ async function yoloRun(req, res) { } fs.writeFileSync(`${runPath}/done.log`, success); + await cleanupJitTrainingImages(jitDownloadedFiles); if (!err) { try { @@ -954,6 +967,8 @@ async function yoloRun(req, res) { global.logger.error("Error generating model card:", cardErr); } } + + await cleanupJitTrainingImages(jitDownloadedFiles); }); diff --git a/server.js b/server.js index 00124b89..94407028 100644 --- a/server.js +++ b/server.js @@ -1,6 +1,7 @@ global.logger = require('./utils/logger'); const app = require('./app'); const { Client } = require("./queries/client"); +const queries = require("./queries/queries"); global.configFile = require("./utils/config"); @@ -54,6 +55,26 @@ for (const project of fs.readdirSync(allProjectsPath)) { const dbFile = path.join(projectPath, file); global.projectDbClients[projectPath] = new Client(dbFile); + + queries.project.migrateProjectDb(projectPath).catch((err) => { + // Every statement in migrateProjectDb is DDL (even a no-op + // CREATE TABLE IF NOT EXISTS), so SQLite opens the file for + // write regardless of whether a change is actually needed. + // A read-only project database - by design, or a permissions + // quirk of wherever it's deployed - can't be migrated, but + // that's expected and not a failure worth an error-level log + // on every server start. + if (err && err.error && err.error.code === "SQLITE_READONLY") { + global.logger.warn( + `Project database at ${projectPath} is read-only; skipping schema migration.`, + ); + return; + } + + global.logger.error( + `Failed to migrate project database at ${projectPath}: ${err}`, + ); + }); } } } diff --git a/tests/integration/imageSourceColumn.test.js b/tests/integration/imageSourceColumn.test.js new file mode 100644 index 00000000..d6a35b3c --- /dev/null +++ b/tests/integration/imageSourceColumn.test.js @@ -0,0 +1,123 @@ +// Unit tests for queries/projects/projects.js#migrateProjectDb, covering the nullable +// Images.Source/SourceKey columns added so an image's row can record where its bytes +// live (e.g. local disk vs. an S3 bucket) and, for S3-backed images, the literal object +// key (decoupled from IName, since two different keys can sanitize to the same display +// name and need disambiguating - see s3Buckets.test.js). Project databases created +// before these columns existed only get them via the PRAGMA-guarded ALTER TABLE below, +// since CREATE TABLE IF NOT EXISTS is a no-op on an existing table. + +jest.mock('../../queries/getDbClient'); + +const getDbClient = require('../../queries/getDbClient'); +const projects = require('../../queries/projects/projects'); + +describe('queries/projects migrateProjectDb Source/SourceKey column backfill', () => { + let mockRun; + let mockAll; + + beforeEach(() => { + jest.clearAllMocks(); + mockRun = jest.fn().mockResolvedValue({ success: true, changes: 0, lastID: 0 }); + mockAll = jest.fn(); + getDbClient.mockReturnValue({ run: mockRun, all: mockAll }); + }); + + it('adds both columns when an existing Images table predates them', async () => { + mockAll.mockResolvedValue({ + success: true, + rows: [ + { name: 'IName' }, + { name: 'reviewImage' }, + { name: 'validateImage' }, + ], + }); + + await projects.project.migrateProjectDb('/projects/testuser-test-project'); + + expect(mockAll).toHaveBeenCalledWith('PRAGMA table_info(Images)'); + expect(mockRun).toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN Source VARCHAR DEFAULT NULL', + ); + expect(mockRun).toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN SourceKey VARCHAR DEFAULT NULL', + ); + }); + + it('only adds the column that is missing when one already exists', async () => { + mockAll.mockResolvedValue({ + success: true, + rows: [ + { name: 'IName' }, + { name: 'reviewImage' }, + { name: 'validateImage' }, + { name: 'Source' }, + ], + }); + + await projects.project.migrateProjectDb('/projects/testuser-test-project'); + + expect(mockRun).not.toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN Source VARCHAR DEFAULT NULL', + ); + expect(mockRun).toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN SourceKey VARCHAR DEFAULT NULL', + ); + }); + + it('does not re-add either column when both are already present', async () => { + mockAll.mockResolvedValue({ + success: true, + rows: [ + { name: 'IName' }, + { name: 'reviewImage' }, + { name: 'validateImage' }, + { name: 'Source' }, + { name: 'SourceKey' }, + ], + }); + + await projects.project.migrateProjectDb('/projects/testuser-test-project'); + + expect(mockRun).not.toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN Source VARCHAR DEFAULT NULL', + ); + expect(mockRun).not.toHaveBeenCalledWith( + 'ALTER TABLE Images ADD COLUMN SourceKey VARCHAR DEFAULT NULL', + ); + }); +}); + +describe('queries/projects addImages Source/SourceKey parameters', () => { + let mockRun; + + beforeEach(() => { + jest.clearAllMocks(); + mockRun = jest.fn().mockResolvedValue({ success: true, changes: 1, lastID: 1 }); + getDbClient.mockReturnValue({ run: mockRun }); + }); + + it('defaults Source and SourceKey to null for callers that do not pass them', async () => { + await projects.project.addImages('/projects/testuser-test-project', 'cat.jpg', 0, 0); + + expect(mockRun).toHaveBeenCalledWith( + expect.stringContaining('INSERT OR IGNORE INTO Images'), + ['cat.jpg', 0, 0, null, null], + ); + }); + + it('passes through Source and SourceKey when a caller (e.g. S3 sync) provides them', async () => { + await projects.project.addImages( + '/projects/testuser-test-project', + 'cat.jpg', + 0, + 0, + 's3', + 'images/cat.jpg', + ); + + expect(mockRun).toHaveBeenCalledWith( + expect.stringContaining('INSERT OR IGNORE INTO Images'), + ['cat.jpg', 0, 0, 's3', 'images/cat.jpg'], + ); + }); +}); diff --git a/tests/integration/inferenceDatasetPipeline.test.js b/tests/integration/inferenceDatasetPipeline.test.js new file mode 100644 index 00000000..7c3d9a2b --- /dev/null +++ b/tests/integration/inferenceDatasetPipeline.test.js @@ -0,0 +1,180 @@ +const fs = require("fs"); +const path = require("path"); +const request = require("supertest"); +const express = require("express"); +const cookieParser = require("cookie-parser"); + +const { prepareInferenceDataset } = require("../../utils/inferenceDatasetPipeline"); +const uploadInferenceFile = require("../../routes/inference/uploadInferenceFile"); +const { attachS3Bucket, syncS3Bucket } = require("../../routes/api/v2/s3Buckets"); +const queries = require("../../queries/queries"); +const s3Client = require("../../utils/s3Client"); + +jest.mock("../../queries/queries", () => ({ + managed: { + getBucket: jest.fn(), + attachBucket: jest.fn().mockResolvedValue({ success: true }), + touchBucketSyncedAt: jest.fn().mockResolvedValue({ success: true }), + }, + project: { + addImages: jest.fn().mockResolvedValue({ success: true }), + getAllImages: jest.fn().mockResolvedValue({ success: true, rows: [] }), + }, +})); + +jest.mock("../../utils/s3Client", () => ({ + buildS3Client: jest.fn(() => ({ fake: true })), + verifyBucketAccess: jest.fn().mockResolvedValue(undefined), + listImageObjects: jest.fn().mockResolvedValue(["img1.jpg", "img2.jpg", "img3.jpg"]), + downloadObjectToFile: jest.fn().mockImplementation(async (client, bucket, key, dest) => { + require("fs").writeFileSync(dest, "fake image content"); + }), +})); + +jest.mock("../../utils/unzipFile", () => jest.fn().mockImplementation(async (zipPath, outDir) => { + const mockFs = require("fs"); + mockFs.mkdirSync(outDir, { recursive: true }); + mockFs.writeFileSync(require("path").join(outDir, "extracted_image.jpg"), "extracted content"); + mockFs.writeFileSync(require("path").join(outDir, "extracted_video.mp4"), "video content"); +})); + +function buildApp() { + const app = express(); + app.use(express.json()); + app.use(cookieParser()); + app.post("/upload_inference_file", (req, res, next) => { + req.files = req.files || {}; + next(); + }, uploadInferenceFile); + app.post("/api/v2/projects/:admin/:projectName/s3-bucket", attachS3Bucket); + app.post("/api/v2/projects/:admin/:projectName/s3-bucket/sync", syncS3Bucket); + return app; +} + +describe("Inference Dataset Pipeline & S3 Max Image Limit", () => { + const tmpDir = path.join(__dirname, "../tmp_dataset_test"); + + beforeAll(() => { + global.currentPath = tmpDir + "/"; + global.logger = { error: jest.fn(), debug: jest.fn() }; + global.readdirAsync = jest.fn().mockResolvedValue([]); + fs.mkdirSync(tmpDir, { recursive: true }); + }); + + afterAll(() => { + if (fs.existsSync(tmpDir)) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe("prepareInferenceDataset utility", () => { + it("extracts zip archives (images & videos) for inference", async () => { + const uploadDir = path.join(tmpDir, "public/projects/admin1-proj1/inference/uploads"); + fs.mkdirSync(uploadDir, { recursive: true }); + const zipPath = path.join(uploadDir, "sample.zip"); + fs.writeFileSync(zipPath, "dummy zip binary"); + + const result = await prepareInferenceDataset({ + PName: "proj1", + Admin: "admin1", + inference_file: "sample.zip", + projectPath: path.join(tmpDir, "public/projects/admin1-proj1"), + inferenceUploadPath: uploadDir, + }); + + expect(result.isZipExtracted).toBe(true); + expect(result.inferenceFilePath).toBe(path.join(uploadDir, "sample")); + expect(fs.existsSync(path.join(uploadDir, "sample/extracted_image.jpg"))).toBe(true); + }); + + it("pulls S3 image stream up to configured maxImages limit for inference", async () => { + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: "my-bucket", + Region: "us-east-1", + Prefix: "images/", + MaxImages: 2, + AccessKeyId: "key", + SecretAccessKey: "secret", + }, + }); + + const uploadDir = path.join(tmpDir, "public/projects/admin1-proj1/inference/uploads"); + fs.mkdirSync(uploadDir, { recursive: true }); + + const result = await prepareInferenceDataset({ + PName: "proj1", + Admin: "admin1", + use_s3_bucket: true, + max_images: 2, + projectPath: path.join(tmpDir, "public/projects/admin1-proj1"), + inferenceUploadPath: uploadDir, + }); + + expect(result.isS3Stream).toBe(true); + expect(result.syncedCount).toBe(2); + expect(s3Client.downloadObjectToFile).toHaveBeenCalledTimes(2); + }); + }); + + describe("syncS3Bucket with maxImages limit", () => { + it("respects maxImages limit during bucket sync", async () => { + const projectDir = path.join(tmpDir, "public/projects/testuser-test-project/images"); + fs.mkdirSync(projectDir, { recursive: true }); + + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: "test-bucket", + Region: "us-east-1", + Prefix: "images/", + AccessKeyId: "key", + SecretAccessKey: "secret", + }, + }); + s3Client.listImageObjects.mockResolvedValueOnce(["images/1.jpg", "images/2.jpg", "images/3.jpg"]); + + const app = buildApp(); + const res = await request(app) + .post("/api/v2/projects/testuser/test-project/s3-bucket/sync") + .set("Cookie", ["Username=testuser"]) + .send({ maxImages: 2 }); + + expect(res.statusCode).toBe(200); + expect(res.body.syncedCount).toBe(2); + expect(s3Client.downloadObjectToFile).toHaveBeenCalledTimes(2); + }); + }); + + describe("attachS3Bucket without MaxImages column", () => { + it("attaches bucket successfully without MaxImages in DB", async () => { + const projectDir = path.join(tmpDir, "public/projects/testuser-test-project"); + fs.mkdirSync(projectDir, { recursive: true }); + + const app = buildApp(); + const res = await request(app) + .post("/api/v2/projects/testuser/test-project/s3-bucket") + .set("Cookie", ["Username=testuser"]) + .send({ + BucketName: "test-bucket", + Region: "us-east-1", + }); + + expect(res.statusCode).toBe(200); + expect(queries.managed.attachBucket).toHaveBeenCalledWith( + "test-project", + "testuser", + "test-bucket", + "us-east-1", + "", + undefined, + undefined, + "", + "download", + ); + }); + }); +}); diff --git a/tests/integration/jitTrainingImagesPipeline.test.js b/tests/integration/jitTrainingImagesPipeline.test.js new file mode 100644 index 00000000..23fd2b57 --- /dev/null +++ b/tests/integration/jitTrainingImagesPipeline.test.js @@ -0,0 +1,175 @@ +jest.mock('decompress-zip', () => jest.fn()); +jest.mock('decompress-zip/lib/extractors', () => ({ + folder: jest.fn(), +})); +jest.mock('ffmpeg', () => jest.fn()); +jest.mock('sharp', () => jest.fn()); +jest.mock('unzipper', () => jest.fn()); +jest.mock('probe-image-size', () => ({ + sync: jest.fn().mockReturnValue({ width: 640, height: 480 }), +})); + +jest.mock('child_process', () => ({ + exec: jest.fn((cmd, opts, cb) => { + const callback = typeof opts === 'function' ? opts : cb; + if (typeof callback === 'function') { + process.nextTick(() => callback(null, 'OK', '')); + } + }), +})); + +const request = require('supertest'); +const fs = require('fs'); +const path = require('path'); +const queries = require('../../queries/queries'); +const s3Client = require('../../utils/s3Client'); + +jest.mock('../../queries/queries', () => ({ + managed: { + getBucket: jest.fn(), + }, + project: { + getAllImages: jest.fn(), + getAllClasses: jest.fn(), + getLabelsForImageName: jest.fn(), + getAllLabels: jest.fn(), + }, +})); + +jest.mock('../../utils/s3Client', () => ({ + buildS3Client: jest.fn(() => ({ fakeS3Client: true })), + downloadObjectToFile: jest.fn(), +})); + +const app = require('../../app'); + +describe('JIT Training Images Pipeline Integration Test', () => { + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(process, 'chdir').mockImplementation(() => {}); + + queries.project.getAllImages.mockResolvedValue({ + rows: [ + { IName: 'local_img.jpg', Source: null, SourceKey: null }, + { IName: 'stream_img.jpg', Source: 's3', SourceKey: 's3/stream_img.jpg' }, + ], + }); + + queries.project.getAllClasses.mockResolvedValue({ + rows: [{ CName: 'car' }], + }); + + queries.project.getLabelsForImageName.mockResolvedValue({ + rows: [{ LID: 1, CName: 'car', X: 10, Y: 20, W: 30, H: 40 }], + }); + + queries.project.getAllLabels.mockResolvedValue({ + rows: [{ IName: 'stream_img.jpg', CName: 'car', X: 10, Y: 20, W: 30, H: 40 }], + }); + + queries.managed.getBucket.mockResolvedValue({ + row: { + BucketName: 'test-bucket', + Region: 'us-east-1', + AccessKeyId: 'key', + SecretAccessKey: 'secret', + }, + }); + + s3Client.downloadObjectToFile.mockResolvedValue(); + }); + + test('POST /yolo-run triggers JIT download for missing streamed S3 images and unlinks them post-run', async () => { + const unlinkSpy = jest.spyOn(fs.promises, 'unlink').mockResolvedValue(); + + jest.spyOn(fs, 'existsSync').mockImplementation((p) => { + if (typeof p === 'string' && p.includes('stream_img.jpg')) { + return false; + } + return true; + }); + jest.spyOn(fs, 'mkdirSync').mockImplementation(() => {}); + jest.spyOn(fs, 'copyFileSync').mockImplementation(() => {}); + jest.spyOn(fs, 'copyFile').mockImplementation((src, dest, cb) => { if (cb) cb(null); }); + jest.spyOn(fs, 'readFileSync').mockReturnValue(Buffer.from('fake image data')); + jest.spyOn(fs, 'writeFileSync').mockImplementation(() => {}); + jest.spyOn(fs, 'appendFile').mockImplementation((p, data, cb) => { if (cb) cb(null); }); + jest.spyOn(fs, 'writeFile').mockImplementation((p, data, cb) => { if (cb) cb(null); }); + jest.spyOn(fs.promises, 'symlink').mockResolvedValue(); + + const response = await request(app) + .post('/yolo-run') + .set('Cookie', ['Username=testuser']) + .send({ + PName: 'testproj', + Admin: 'testuser', + yolo_task: 'detect', + selected_classes: JSON.stringify(['car']), + TrainingPercent: 70, + weights: 'best.pt', + yolovx_path: '/usr/local/bin/yolo', + }); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ Success: 'YOLO Training Started' }); + + // Verify JIT fetch downloaded the missing streamed image + expect(s3Client.downloadObjectToFile).toHaveBeenCalledWith( + { fakeS3Client: true }, + 'test-bucket', + 's3/stream_img.jpg', + expect.stringContaining('stream_img.jpg'), + ); + + // Wait for async background cleanup in exec callback + await new Promise((r) => setTimeout(r, 50)); + + // Verify cleanup unlinked the JIT-downloaded image when training completed + expect(unlinkSpy).toHaveBeenCalledWith(expect.stringContaining('stream_img.jpg')); + // Ensure local_img.jpg was not unlinked + expect(unlinkSpy).not.toHaveBeenCalledWith(expect.stringContaining('local_img.jpg')); + }); + + test('POST /run triggers JIT download for missing streamed S3 images and unlinks them post-run', async () => { + const unlinkSpy = jest.spyOn(fs.promises, 'unlink').mockResolvedValue(); + + jest.spyOn(fs, 'existsSync').mockImplementation((p) => { + if (typeof p === 'string' && p.includes('stream_img.jpg')) { + return false; + } + return true; + }); + jest.spyOn(fs, 'mkdirSync').mockImplementation(() => {}); + jest.spyOn(fs, 'copyFileSync').mockImplementation(() => {}); + jest.spyOn(fs, 'writeFile').mockImplementation((p, data, cb) => { if (cb) cb(null); }); + + const response = await request(app) + .post('/run') + .set('Cookie', ['Username=testuser']) + .send({ + PName: 'testproj', + Admin: 'testuser', + script: 'train.py', + python_path: '/usr/bin/python3', + TrainingPercent: 70, + weights: 'weights.h5', + }); + + expect(response.status).toBe(200); + expect(response.body).toEqual({ Success: 'Training Started' }); + + // Verify JIT fetch downloaded missing streamed S3 image + expect(s3Client.downloadObjectToFile).toHaveBeenCalledWith( + { fakeS3Client: true }, + 'test-bucket', + 's3/stream_img.jpg', + expect.stringContaining('stream_img.jpg'), + ); + + // Wait for async background cleanup in exec callback + await new Promise((r) => setTimeout(r, 50)); + + // Verify cleanup unlinked JIT downloaded image + expect(unlinkSpy).toHaveBeenCalledWith(expect.stringContaining('stream_img.jpg')); + }); +}); diff --git a/tests/integration/legacyDbMigration.test.js b/tests/integration/legacyDbMigration.test.js index 188ae6cf..a8e675ab 100644 --- a/tests/integration/legacyDbMigration.test.js +++ b/tests/integration/legacyDbMigration.test.js @@ -1,6 +1,6 @@ const path = require('path'); const fs = require('fs'); -const sqlite3 = require('sqlite3').verbose(); +const { Client } = require('../../queries/client'); const queries = require('../../queries/queries'); describe('Legacy Database Migration for Images validateImage / reviewImage columns', () => { @@ -14,7 +14,10 @@ describe('Legacy Database Migration for Images validateImage / reviewImage colum fs.mkdirSync(tmpDir, { recursive: true }); } dbPath = path.join(tmpDir, 'test_legacy.db'); - global.projectDbClients[dbPath] = new sqlite3.Database(dbPath); + // Mirror how production populates global.projectDbClients (see server.js), so + // migrateProjectDb's promise-based db.run/db.all calls behave the same way they + // do outside tests, instead of hitting the raw sqlite3.Database callback API. + global.projectDbClients[dbPath] = new Client(dbPath); }); afterEach(() => { @@ -26,49 +29,33 @@ describe('Legacy Database Migration for Images validateImage / reviewImage colum test('migrateProjectDb adds missing reviewImage and validateImage columns to existing Images table', async () => { const db = global.projectDbClients[dbPath]; - db.run('CREATE TABLE Images (IName VARCHAR NOT NULL PRIMARY KEY)'); - db.run("INSERT INTO Images (IName) VALUES ('test_image.jpg')"); + await db.run('CREATE TABLE Images (IName VARCHAR NOT NULL PRIMARY KEY)'); + await db.run("INSERT INTO Images (IName) VALUES ('test_image.jpg')"); // Run migrateProjectDb await queries.project.migrateProjectDb(dbPath); // Verify query succeeds - await new Promise((resolve, reject) => { - const pdb = global.projectDbClients[dbPath]; - pdb.all( - 'SELECT Images.IName, Images.reviewImage, Images.validateImage FROM Images', - [], - (err, rows) => { - if (err) return reject(err); - expect(rows).toBeDefined(); - resolve(); - } - ); - }); + const result = await db.all( + 'SELECT Images.IName, Images.reviewImage, Images.validateImage FROM Images', + ); + expect(result.rows).toBeDefined(); }); test('projectsFilter route handler query auto-migrates missing columns', async () => { const db = global.projectDbClients[dbPath]; - db.run('CREATE TABLE Images (IName VARCHAR NOT NULL PRIMARY KEY)'); + await db.run('CREATE TABLE Images (IName VARCHAR NOT NULL PRIMARY KEY)'); // Simulate reading legacy db using the same logic as getProjectPage / projectsFilter - await new Promise((resolve, reject) => { - const pdb = global.projectDbClients[dbPath]; - pdb.run("ALTER TABLE Images ADD COLUMN reviewImage INTEGER NOT NULL DEFAULT 0", () => { - pdb.run("ALTER TABLE Images ADD COLUMN validateImage INTEGER NOT NULL DEFAULT 0", () => { - const query = ` - SELECT Images.IName, Images.reviewImage, Images.validateImage, COUNT(Labels.LID) AS numLabels - FROM Images - LEFT JOIN Labels ON Images.IName = Labels.IName - GROUP BY Images.IName - `; - pdb.all(query, [], (err, rows) => { - if (err) return reject(err); - expect(rows).toBeDefined(); - resolve(); - }); - }); - }); - }); + await db.run("ALTER TABLE Images ADD COLUMN reviewImage INTEGER NOT NULL DEFAULT 0"); + await db.run("ALTER TABLE Images ADD COLUMN validateImage INTEGER NOT NULL DEFAULT 0"); + const query = ` + SELECT Images.IName, Images.reviewImage, Images.validateImage, COUNT(Labels.LID) AS numLabels + FROM Images + LEFT JOIN Labels ON Images.IName = Labels.IName + GROUP BY Images.IName + `; + const result = await db.all(query); + expect(result.rows).toBeDefined(); }); }); diff --git a/tests/integration/s3Buckets.test.js b/tests/integration/s3Buckets.test.js index 94d7caab..6ca28e93 100644 --- a/tests/integration/s3Buckets.test.js +++ b/tests/integration/s3Buckets.test.js @@ -11,6 +11,8 @@ jest.mock('../../queries/queries', () => ({ }, project: { addImages: jest.fn().mockResolvedValue({ success: true }), + getAllImages: jest.fn().mockResolvedValue({ success: true, rows: [] }), + getImage: jest.fn(), }, })); @@ -19,8 +21,10 @@ jest.mock('../../utils/s3Client', () => ({ verifyBucketAccess: jest.fn().mockResolvedValue(undefined), listImageObjects: jest.fn().mockResolvedValue([]), downloadObjectToFile: jest.fn().mockResolvedValue(undefined), + getObjectStream: jest.fn(), })); +const { Readable } = require('stream'); const request = require('supertest'); const express = require('express'); const cookieParser = require('cookie-parser'); @@ -32,6 +36,7 @@ const { getS3Bucket, deleteS3Bucket, syncS3Bucket, + getProjectImage, } = require('../../routes/api/v2/s3Buckets'); // These handlers are exercised directly against a minimal app (rather than @@ -48,6 +53,17 @@ function buildTestApp() { app.get('/api/v2/projects/:admin/:projectName/s3-bucket', getS3Bucket); app.delete('/api/v2/projects/:admin/:projectName/s3-bucket', deleteS3Bucket); app.post('/api/v2/projects/:admin/:projectName/s3-bucket/sync', syncS3Bucket); + // `fs` is mocked module-wide down to just existsSync (below), so stub + // res.sendFile here rather than letting Express's real implementation + // reach for fs.stat/createReadStream, which don't exist on the mock. + app.get( + '/api/v2/projects/:admin/:projectName/images/:imageName', + (req, res, next) => { + res.sendFile = jest.fn((filePath) => res.status(200).send(`local-file:${filePath}`)); + next(); + }, + getProjectImage, + ); return app; } @@ -96,6 +112,30 @@ describe('S3 Bucket Routes', () => { 'images/', 'AKIA...', 'secret', + '', + 'download', + ); + }); + + it('defaults SyncMode to "download" when omitted or invalid, and passes through "stream" when requested', async () => { + await request(app) + .post('/api/v2/projects/testuser/test-project/s3-bucket') + .set('Cookie', ['Username=testuser']) + .send({ ...validBody, SyncMode: 'not-a-real-mode' }); + + expect(queries.managed.attachBucket).toHaveBeenNthCalledWith( + 1, + 'test-project', 'testuser', 'my-bucket', 'us-east-1', 'images/', 'AKIA...', 'secret', '', 'download', + ); + + await request(app) + .post('/api/v2/projects/testuser/test-project/s3-bucket') + .set('Cookie', ['Username=testuser']) + .send({ ...validBody, SyncMode: 'stream' }); + + expect(queries.managed.attachBucket).toHaveBeenNthCalledWith( + 2, + 'test-project', 'testuser', 'my-bucket', 'us-east-1', 'images/', 'AKIA...', 'secret', '', 'stream', ); }); @@ -154,6 +194,7 @@ describe('S3 Bucket Routes', () => { LastSyncedAt: null, AccessKeyId: 'AKIA...', SecretAccessKey: 'super-secret', + SyncMode: 'stream', }, }); @@ -163,11 +204,24 @@ describe('S3 Bucket Routes', () => { expect(res.statusCode).toBe(200); expect(res.body.bucket.BucketName).toBe('my-bucket'); + expect(res.body.bucket.SyncMode).toBe('stream'); expect(res.body.bucket.hasCredentials).toBe(true); expect(res.body.bucket.SecretAccessKey).toBeUndefined(); expect(res.body.bucket.AccessKeyId).toBeUndefined(); }); + it('defaults SyncMode to "download" for buckets attached before the column existed', async () => { + queries.managed.getBucket.mockResolvedValueOnce({ + row: { BucketName: 'my-bucket', Region: 'us-east-1', Prefix: '', SyncMode: null }, + }); + + const res = await request(app) + .get('/api/v2/projects/testuser/test-project/s3-bucket') + .set('Cookie', ['Username=testuser']); + + expect(res.body.bucket.SyncMode).toBe('download'); + }); + it('returns 404 when no bucket is attached', async () => { queries.managed.getBucket.mockResolvedValueOnce({ row: undefined }); @@ -231,10 +285,98 @@ describe('S3 Bucket Routes', () => { 'cat.jpg', 0, 0, + 's3', + 'images/cat.jpg', ); expect(queries.managed.touchBucketSyncedAt).toHaveBeenCalled(); }); + it('skips an object already synced by key, even if its name is no longer the only file on disk', async () => { + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: 'my-bucket', + Region: 'us-east-1', + Prefix: 'images/', + AccessKeyId: 'AKIA...', + SecretAccessKey: 'secret', + }, + }); + s3Client.listImageObjects.mockResolvedValueOnce(['images/cat.jpg']); + global.readdirAsync.mockResolvedValueOnce(['cat.jpg']); + queries.project.getAllImages.mockResolvedValueOnce({ + success: true, + rows: [{ IName: 'cat.jpg', Source: 's3', SourceKey: 'images/cat.jpg' }], + }); + + const res = await request(app) + .post('/api/v2/projects/testuser/test-project/s3-bucket/sync') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(200); + expect(res.body.syncedCount).toBe(0); + expect(res.body.skippedCount).toBe(1); + expect(s3Client.downloadObjectToFile).not.toHaveBeenCalled(); + expect(queries.project.addImages).not.toHaveBeenCalled(); + }); + + it('disambiguates two different keys that sanitize to the same basename instead of dropping one', async () => { + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: 'my-bucket', + Region: 'us-east-1', + Prefix: 'images/', + AccessKeyId: 'AKIA...', + SecretAccessKey: 'secret', + }, + }); + s3Client.listImageObjects.mockResolvedValueOnce([ + '2024/img.jpg', + '2025/img.jpg', + ]); + global.readdirAsync.mockResolvedValueOnce([]); + queries.project.getAllImages.mockResolvedValueOnce({ success: true, rows: [] }); + + const res = await request(app) + .post('/api/v2/projects/testuser/test-project/s3-bucket/sync') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(200); + expect(res.body.syncedCount).toBe(2); + expect(res.body.skippedCount).toBe(0); + expect(res.body.images[0]).toBe('img.jpg'); + expect(res.body.images[1]).not.toBe('img.jpg'); + expect(res.body.images[1]).toMatch(/^img_[0-9a-f]{8}\.jpg$/); + + expect(queries.project.addImages).toHaveBeenNthCalledWith( + 1, + expect.stringContaining('testuser-test-project'), + 'img.jpg', + 0, + 0, + 's3', + '2024/img.jpg', + ); + expect(queries.project.addImages).toHaveBeenNthCalledWith( + 2, + expect.stringContaining('testuser-test-project'), + res.body.images[1], + 0, + 0, + 's3', + '2025/img.jpg', + ); + + // Deterministic: re-running sync against the same key must reproduce the + // same disambiguated name, not draw a fresh suffix each time. + const crypto = require('crypto'); + const expectedHash = crypto + .createHash('sha1') + .update('2025/img.jpg') + .digest('hex') + .slice(0, 8); + expect(res.body.images[1]).toBe(`img_${expectedHash}.jpg`); + }); + it('returns 404 when the project has no attached bucket', async () => { queries.managed.getBucket.mockResolvedValueOnce({ row: undefined }); @@ -256,5 +398,101 @@ describe('S3 Bucket Routes', () => { expect(res.statusCode).toBe(404); expect(queries.managed.getBucket).not.toHaveBeenCalled(); }); + + it('registers images without downloading them when SyncMode is "stream"', async () => { + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: 'my-bucket', + Region: 'us-east-1', + Prefix: 'images/', + AccessKeyId: 'AKIA...', + SecretAccessKey: 'secret', + SyncMode: 'stream', + }, + }); + s3Client.listImageObjects.mockResolvedValueOnce(['images/cat.jpg']); + global.readdirAsync.mockResolvedValueOnce([]); + + const res = await request(app) + .post('/api/v2/projects/testuser/test-project/s3-bucket/sync') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(200); + expect(res.body.syncedCount).toBe(1); + expect(s3Client.downloadObjectToFile).not.toHaveBeenCalled(); + expect(queries.project.addImages).toHaveBeenCalledWith( + expect.stringContaining('testuser-test-project'), + 'cat.jpg', + 0, + 0, + 's3', + 'images/cat.jpg', + ); + }); + }); + + describe('GET /api/v2/projects/:admin/:projectName/images/:imageName', () => { + it('serves a locally-present file from disk without touching S3', async () => { + const res = await request(app) + .get('/api/v2/projects/testuser/test-project/images/cat.jpg') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(200); + expect(res.text).toContain('local-file:'); + expect(res.text).toContain('cat.jpg'); + expect(s3Client.getObjectStream).not.toHaveBeenCalled(); + }); + + it('streams a "stream"-mode image live from S3 when no local file exists', async () => { + fs.existsSync.mockReturnValue(false); + queries.project.getImage.mockResolvedValueOnce({ + row: { IName: 'cat.jpg', Source: 's3', SourceKey: 'images/cat.jpg' }, + }); + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: 'my-bucket', + Region: 'us-east-1', + AccessKeyId: 'AKIA...', + SecretAccessKey: 'secret', + SyncMode: 'stream', + }, + }); + s3Client.getObjectStream.mockResolvedValueOnce({ + body: Readable.from([Buffer.from('fake-image-bytes')]), + contentType: 'image/jpeg', + }); + + const res = await request(app) + .get('/api/v2/projects/testuser/test-project/images/cat.jpg') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(200); + expect(res.headers['content-type']).toBe('image/jpeg'); + expect(Buffer.from(res.body).toString()).toBe('fake-image-bytes'); + expect(s3Client.getObjectStream).toHaveBeenCalledWith( + { fakeClient: true }, + 'my-bucket', + 'images/cat.jpg', + ); + }); + + it('returns 404 when neither a local file nor an S3-backed row exists', async () => { + fs.existsSync.mockReturnValue(false); + queries.project.getImage.mockResolvedValueOnce({ row: undefined }); + + const res = await request(app) + .get('/api/v2/projects/testuser/test-project/images/missing.jpg') + .set('Cookie', ['Username=testuser']); + + expect(res.statusCode).toBe(404); + }); + + it('rejects requests from a logged-out client', async () => { + const res = await request(app).get( + '/api/v2/projects/testuser/test-project/images/cat.jpg', + ); + + expect(res.statusCode).toBe(403); + }); }); }); diff --git a/tests/integration/uploadInferenceZip.test.js b/tests/integration/uploadInferenceZip.test.js new file mode 100644 index 00000000..50c427bd --- /dev/null +++ b/tests/integration/uploadInferenceZip.test.js @@ -0,0 +1,70 @@ +const fs = require('fs'); +const path = require('path'); +const uploadInferenceFile = require('../../routes/inference/uploadInferenceFile'); + +describe('uploadInferenceFile Controller', () => { + const testDir = path.join(__dirname, '../tmp_upload_test'); + + beforeAll(() => { + global.currentPath = testDir + '/'; + if (!fs.existsSync(testDir)) { + fs.mkdirSync(testDir, { recursive: true }); + } + }); + + afterAll(() => { + if (fs.existsSync(testDir)) { + fs.rmSync(testDir, { recursive: true, force: true }); + } + }); + + function mockReqRes(fileName) { + const req = { + body: { PName: 'test-project', Admin: 'testuser' }, + cookies: { Username: 'testuser' }, + files: { + upload_inference: { + name: fileName, + mv: jest.fn().mockImplementation((dest, cb) => { + if (typeof cb === 'function') cb(null); + return Promise.resolve(); + }), + }, + }, + }; + const res = { + send: jest.fn(), + }; + return { req, res }; + } + + it('accepts .zip files for inference dataset', async () => { + const { req, res } = mockReqRes('images_archive.zip'); + await uploadInferenceFile(req, res); + + expect(res.send).toHaveBeenCalledWith({ + Success: 'Your inference file has been uploaded and saved', + }); + expect(req.files.upload_inference.mv).toHaveBeenCalled(); + }); + + it('accepts valid image/video file extensions (.jpg, .mp4, .mov)', async () => { + const { req, res } = mockReqRes('video_sample.mp4'); + await uploadInferenceFile(req, res); + + expect(res.send).toHaveBeenCalledWith({ + Success: 'Your inference file has been uploaded and saved', + }); + expect(req.files.upload_inference.mv).toHaveBeenCalled(); + }); + + it('rejects unsupported file extensions (.exe, .txt)', async () => { + const { req, res } = mockReqRes('malicious.exe'); + await uploadInferenceFile(req, res); + + expect(res.send).toHaveBeenCalledWith({ + Success: 'ERROR: Wrong filetype. Must be type .png, .jpg, .jpeg, .tif, .gif, .mp4, .mov, or .zip', + }); + expect(req.files.upload_inference.mv).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/unit/getAnnotatePageS3Streaming.test.js b/tests/unit/getAnnotatePageS3Streaming.test.js new file mode 100644 index 00000000..85fa42b2 --- /dev/null +++ b/tests/unit/getAnnotatePageS3Streaming.test.js @@ -0,0 +1,172 @@ +// Unit tests for routes/pages/getAnnotatePage.js's S3 "stream" mode handling. +// +// Calls the handler directly (bypassing the full Express app/static +// middleware) since that stack is heavily mocked elsewhere for unrelated +// tests in a way that intercepts every route with an empty static response - +// exercising the real branching logic here needs a narrower harness. + +jest.mock('../../queries/queries', () => ({ + managed: { + getBucket: jest.fn(), + }, +})); + +jest.mock('../../utils/s3Client', () => ({ + buildS3Client: jest.fn(() => ({ fakeClient: true })), + getObjectStream: jest.fn(), +})); +jest.mock('fs', () => ({ + existsSync: jest.fn(), + readFileSync: jest.fn(), +})); +jest.mock('sqlite3', () => { + const Database = jest.fn(); + return { + Database, + verbose: () => ({ Database }), + }; +}); +jest.mock('probe-image-size', () => { + const probe = jest.fn(); + probe.sync = jest.fn(); + return probe; +}); + +const { Readable } = require('stream'); +const fs = require('fs'); +const sqlite3 = require('sqlite3'); +const probe = require('probe-image-size'); +const queries = require('../../queries/queries'); +const s3Client = require('../../utils/s3Client'); +const getAnnotatePage = require('../../routes/pages/getAnnotatePage'); + +// Mirrors the real sqlite3 driver's callback style, since getAnnotatePage.js +// wraps `this.get`/`this.all` in its own Promise-returning getAsync/allAsync +// right after construction - only the raw callback methods are ever called. +// getAsync/allAsync always invoke the 3-arg form (sql, params, callback), so +// these must accept that arity rather than the 2-arg (sql, callback) form. +function makeFakeProjectDb({ classesRows = [], labelsRows = [], imagesRows = [], displayRow }) { + return { + get: jest.fn((sql, params, cb) => { + const callback = typeof params === 'function' ? params : cb; + if (sql.includes('display_id')) return callback(null, displayRow); + return callback(null, undefined); + }), + all: jest.fn((sql, params, cb) => { + const callback = typeof params === 'function' ? params : cb; + if (sql.includes('Classes')) return callback(null, classesRows); + if (sql.includes('Labels')) return callback(null, labelsRows); + if (sql.includes('Images')) return callback(null, imagesRows); + return callback(null, []); + }), + each: jest.fn((sql, cb) => cb(null, undefined)), + close: jest.fn((cb) => cb && cb(null)), + }; +} + +describe('getAnnotatePage - S3-backed image serving', () => { + let res; + let req; + + beforeEach(() => { + jest.clearAllMocks(); + + global.logger = { debug: jest.fn(), error: jest.fn(), info: jest.fn() }; + global.currentPath = '/app/'; + global.colorsJSON = [{ value: '#FF0000' }]; + global.db = { + allAsync: jest.fn().mockResolvedValue([{ PName: 'test-project', Admin: 'testuser' }]), + getAsync: jest.fn().mockResolvedValue({ AutoSave: 1 }), + }; + + req = { + query: { IDX: '0', IName: 'image1.jpg', curr_class: 'class1' }, + cookies: { Username: 'testuser' }, + }; + res = { redirect: jest.fn(), render: jest.fn() }; + }); + + it('serves a "stream"-mode image with no local file via the on-demand S3 proxy, without touching disk', async () => { + const imageRow = { IName: 'image1.jpg', reviewImage: 0, Source: 's3', SourceKey: 'images/image1.jpg' }; + + fs.existsSync.mockReturnValue(false); + fs.readFileSync.mockReset(); + sqlite3.Database.mockImplementation((dbPath, cb) => { + cb && cb(null); + return makeFakeProjectDb({ + imagesRows: [imageRow], + displayRow: { IName: 'image1.jpg', display_id: 1 }, + }); + }); + probe.mockResolvedValue({ width: 400, height: 300 }); + probe.sync.mockReset(); + + queries.managed.getBucket.mockResolvedValueOnce({ + row: { BucketName: 'my-bucket', Region: 'us-east-1', AccessKeyId: 'AKIA...', SecretAccessKey: 'secret' }, + }); + s3Client.getObjectStream.mockResolvedValueOnce({ + body: Readable.from([Buffer.from('fake-image-bytes')]), + contentType: 'image/jpeg', + }); + + await getAnnotatePage(req, res); + + expect(fs.readFileSync).not.toHaveBeenCalled(); + expect(probe.sync).not.toHaveBeenCalled(); + expect(s3Client.getObjectStream).toHaveBeenCalledWith( + { fakeClient: true }, + 'my-bucket', + 'images/image1.jpg', + ); + expect(res.render).toHaveBeenCalledWith('annotate', expect.objectContaining({ + image_path: 'api/v2/projects/testuser/test-project/images/image1.jpg', + image_width: 400, + image_height: 300, + })); + }); + + it('still reads a locally-present file straight from disk, unaffected by the S3 changes', async () => { + const imageRow = { IName: 'image1.jpg', reviewImage: 0, Source: null, SourceKey: null }; + + fs.existsSync.mockReturnValue(true); + fs.readFileSync.mockReturnValue(Buffer.from('img-bytes')); + sqlite3.Database.mockImplementation((dbPath, cb) => { + cb && cb(null); + return makeFakeProjectDb({ + imagesRows: [imageRow], + displayRow: { IName: 'image1.jpg', display_id: 1 }, + }); + }); + probe.mockReset(); + probe.sync.mockReturnValue({ width: 800, height: 600 }); + + await getAnnotatePage(req, res); + + expect(s3Client.getObjectStream).not.toHaveBeenCalled(); + expect(queries.managed.getBucket).not.toHaveBeenCalled(); + expect(res.render).toHaveBeenCalledWith('annotate', expect.objectContaining({ + image_path: 'projects/testuser-test-project/images/image1.jpg', + image_width: 800, + image_height: 600, + })); + }); + + it('renders 404 when there is no local file and the image is not S3-backed', async () => { + const imageRow = { IName: 'image1.jpg', reviewImage: 0, Source: null, SourceKey: null }; + + fs.existsSync.mockReturnValue(false); + fs.readFileSync.mockReset(); + sqlite3.Database.mockImplementation((dbPath, cb) => { + cb && cb(null); + return makeFakeProjectDb({ + imagesRows: [imageRow], + displayRow: { IName: 'image1.jpg', display_id: 1 }, + }); + }); + + await getAnnotatePage(req, res); + + expect(queries.managed.getBucket).not.toHaveBeenCalled(); + expect(res.render).toHaveBeenCalledWith('404', expect.any(Object)); + }); +}); diff --git a/tests/unit/jitTrainingImages.test.js b/tests/unit/jitTrainingImages.test.js new file mode 100644 index 00000000..a6dfabd3 --- /dev/null +++ b/tests/unit/jitTrainingImages.test.js @@ -0,0 +1,127 @@ +const fs = require('fs'); +const path = require('path'); +const queries = require('../../queries/queries'); +const s3Client = require('../../utils/s3Client'); +const { + ensureTrainingImagesLocal, + cleanupJitTrainingImages, +} = require('../../utils/jitTrainingImages'); + +jest.mock('../../queries/queries', () => ({ + managed: { + getBucket: jest.fn(), + }, + project: { + getAllImages: jest.fn(), + }, +})); + +jest.mock('../../utils/s3Client', () => ({ + buildS3Client: jest.fn(() => ({ fakeS3Client: true })), + downloadObjectToFile: jest.fn(), +})); + +describe('utils/jitTrainingImages', () => { + beforeEach(() => { + jest.clearAllMocks(); + global.logger = { debug: jest.fn(), error: jest.fn(), info: jest.fn() }; + }); + + describe('ensureTrainingImagesLocal', () => { + it('returns empty array when targetImages is empty or undefined', async () => { + const downloaded = await ensureTrainingImagesLocal('proj', 'user', '/path/to/proj', []); + expect(downloaded).toEqual([]); + expect(s3Client.downloadObjectToFile).not.toHaveBeenCalled(); + }); + + it('skips images that already exist locally on disk', async () => { + const projectPath = '/projects/user-proj'; + const targetImages = [ + { IName: 'local1.jpg', Source: 's3', SourceKey: 'keys/local1.jpg' }, + ]; + + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + + const downloaded = await ensureTrainingImagesLocal('proj', 'user', projectPath, targetImages); + + expect(downloaded).toEqual([]); + expect(s3Client.downloadObjectToFile).not.toHaveBeenCalled(); + }); + + it('downloads missing S3-backed images JIT and returns their file paths', async () => { + const projectPath = '/projects/user-proj'; + const targetImages = [ + { IName: 'stream1.jpg', Source: 's3', SourceKey: 'keys/stream1.jpg' }, + { IName: 'local1.jpg', Source: null, SourceKey: null }, + ]; + + jest.spyOn(fs, 'existsSync').mockImplementation((p) => { + if (p.includes('stream1.jpg')) return false; + return true; + }); + jest.spyOn(fs, 'mkdirSync').mockImplementation(() => {}); + + queries.managed.getBucket.mockResolvedValueOnce({ + row: { + BucketName: 'my-bucket', + Region: 'us-east-1', + AccessKeyId: 'AKIA...', + SecretAccessKey: 'secret', + Endpoint: '', + }, + }); + s3Client.downloadObjectToFile.mockResolvedValueOnce(); + + const downloaded = await ensureTrainingImagesLocal('proj', 'user', projectPath, targetImages); + + const expectedPath = path.join(projectPath, 'images', 'stream1.jpg'); + expect(downloaded).toEqual([expectedPath]); + expect(s3Client.buildS3Client).toHaveBeenCalledWith({ + region: 'us-east-1', + accessKeyId: 'AKIA...', + secretAccessKey: 'secret', + endpoint: '', + }); + expect(s3Client.downloadObjectToFile).toHaveBeenCalledWith( + { fakeS3Client: true }, + 'my-bucket', + 'keys/stream1.jpg', + expectedPath, + ); + }); + + it('throws an error if missing S3 images exist but no S3 bucket is attached', async () => { + const projectPath = '/projects/user-proj'; + const targetImages = [ + { IName: 'stream1.jpg', Source: 's3', SourceKey: 'keys/stream1.jpg' }, + ]; + + jest.spyOn(fs, 'existsSync').mockReturnValue(false); + queries.managed.getBucket.mockResolvedValueOnce({ row: null }); + + await expect( + ensureTrainingImagesLocal('proj', 'user', projectPath, targetImages), + ).rejects.toThrow('S3 bucket configuration missing for project proj'); + }); + }); + + describe('cleanupJitTrainingImages', () => { + it('unlinks files present in the downloadedFiles array', async () => { + const downloadedFiles = ['/path/to/stream1.jpg', '/path/to/stream2.jpg']; + + jest.spyOn(fs, 'existsSync').mockReturnValue(true); + jest.spyOn(fs.promises, 'unlink').mockResolvedValue(); + + await cleanupJitTrainingImages(downloadedFiles); + + expect(fs.promises.unlink).toHaveBeenCalledWith('/path/to/stream1.jpg'); + expect(fs.promises.unlink).toHaveBeenCalledWith('/path/to/stream2.jpg'); + }); + + it('does nothing when downloadedFiles is empty', async () => { + jest.spyOn(fs.promises, 'unlink').mockResolvedValue(); + await cleanupJitTrainingImages([]); + expect(fs.promises.unlink).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/utils/inferenceDatasetPipeline.js b/utils/inferenceDatasetPipeline.js new file mode 100644 index 00000000..396e8074 --- /dev/null +++ b/utils/inferenceDatasetPipeline.js @@ -0,0 +1,134 @@ +const fs = require("fs"); +const path = require("path"); +const unzipFile = require("./unzipFile"); +const queries = require("../queries/queries"); +const { buildS3Client, listImageObjects, downloadObjectToFile } = require("./s3Client"); + +const DEFAULT_MAX_IMAGES = 100; +const SAFETY_CEILING_MAX_IMAGES = 5000; + +async function prepareInferenceDataset(options) { + const { + PName, + Admin, + inference_file, + use_s3_bucket, + max_images, + maxImages, + limit, + projectPath, + inferenceUploadPath, + } = options; + + const rawLimit = max_images || maxImages || limit; + const parsedLimit = rawLimit ? parseInt(rawLimit, 10) : null; + const maxLimit = parsedLimit && Number.isFinite(parsedLimit) && parsedLimit > 0 + ? Math.min(parsedLimit, SAFETY_CEILING_MAX_IMAGES) + : DEFAULT_MAX_IMAGES; + + let targetFilePath = inference_file; + + const isS3Selection = use_s3_bucket || + inference_file === "s3" || + inference_file === "s3_bucket" || + inference_file === "S3 Bucket" || + inference_file === "[Attached S3 Bucket]"; + + const isImagesFolder = targetFilePath && ( + targetFilePath.endsWith("/images") || targetFilePath.endsWith("\\images") + ); + + let useS3 = isS3Selection; + let bucket = null; + + try { + if (queries && queries.managed && typeof queries.managed.getBucket === "function") { + const bucketResult = await queries.managed.getBucket(PName, Admin); + bucket = bucketResult && bucketResult.row; + } + } catch (e) { + // Ignore error if getBucket is unmocked or fails + } + + if (!useS3 && isImagesFolder && bucket) { + const localImages = fs.existsSync(targetFilePath) ? fs.readdirSync(targetFilePath) : []; + if (localImages.length === 0) { + useS3 = true; + } + } + + // S3 Bucket stream mode (JIT fetch right before inference run execution) + if (useS3) { + if (!bucket) { + throw new Error("No S3 bucket attached to this project"); + } + + const dateStamp = Date.now(); + const s3StreamFolder = path.join(inferenceUploadPath, `s3_stream_${dateStamp}`); + + if (!fs.existsSync(s3StreamFolder)) { + fs.mkdirSync(s3StreamFolder, { recursive: true }); + } + + const s3Client = buildS3Client({ + region: bucket.Region, + accessKeyId: bucket.AccessKeyId, + secretAccessKey: bucket.SecretAccessKey, + endpoint: bucket.Endpoint, + }); + + // Pass maxLimit directly to listImageObjects so S3 listing stops early + const objectKeys = await listImageObjects(s3Client, bucket.BucketName, bucket.Prefix, maxLimit); + let downloaded = 0; + + for (const key of objectKeys) { + if (downloaded >= maxLimit) { + break; + } + const fileName = path.basename(key).replace(/[ +]/g, "_"); + const destPath = path.join(s3StreamFolder, fileName); + await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath); + downloaded++; + } + + return { + inferenceFilePath: s3StreamFolder, + isS3Stream: true, + syncedCount: downloaded, + }; + } + + // Resolve local file path + if (targetFilePath && !fs.existsSync(targetFilePath)) { + const fallbackPath = path.join(inferenceUploadPath, targetFilePath); + if (fs.existsSync(fallbackPath)) { + targetFilePath = fallbackPath; + } + } + + // Zip/7z archive extraction + if (targetFilePath && fs.existsSync(targetFilePath)) { + const ext = path.extname(targetFilePath).toLowerCase(); + if (ext === ".zip" || ext === ".7z") { + const folderName = path.parse(targetFilePath).name; + const outputDir = path.join(inferenceUploadPath, folderName); + + if (!fs.existsSync(outputDir)) { + await unzipFile(targetFilePath, outputDir); + } + + return { + inferenceFilePath: outputDir, + isZipExtracted: true, + }; + } + } + + return { + inferenceFilePath: targetFilePath || inference_file, + }; +} + +module.exports = { + prepareInferenceDataset, +}; diff --git a/utils/jitTrainingImages.js b/utils/jitTrainingImages.js new file mode 100644 index 00000000..7400a059 --- /dev/null +++ b/utils/jitTrainingImages.js @@ -0,0 +1,111 @@ +const fs = require("fs"); +const path = require("path"); +const queries = require("../queries/queries"); +const { buildS3Client, downloadObjectToFile } = require("./s3Client"); + +/** + * Ensures all requested target images for a training run exist locally on disk. + * If any image is S3-backed (Source === 's3' && SourceKey) and missing from disk, + * it fetches the project S3 bucket credentials and downloads the image JIT. + * + * @param {string} projectName - Project name (PName) + * @param {string} admin - Admin / owner username + * @param {string} projectPath - Path to project directory + * @param {Array} targetImages - Array of image DB rows or image names + * @returns {Promise>} List of absolute file paths downloaded JIT by this call + */ +async function ensureTrainingImagesLocal(projectName, admin, projectPath, targetImages) { + if (!Array.isArray(targetImages) || targetImages.length === 0) { + return []; + } + + const imagesPath = path.join(projectPath, "images"); + + // Retrieve DB image records if targetImages consists of plain strings or objects missing Source + let imageRows = targetImages; + const firstImg = targetImages[0]; + if (typeof firstImg === "string" || !firstImg || typeof firstImg !== "object" || !("Source" in firstImg)) { + try { + const dbImagesResult = await queries.project.getAllImages(projectPath); + const dbImagesMap = new Map(); + for (const row of (dbImagesResult && dbImagesResult.rows) || []) { + dbImagesMap.set(row.IName, row); + } + + imageRows = targetImages.map((img) => { + const name = typeof img === "string" ? img : (img && img.IName); + return dbImagesMap.get(name) || (typeof img === "object" && img ? img : { IName: name }); + }); + } catch (err) { + if (global.logger && global.logger.error) { + global.logger.error("Error fetching image DB records for JIT check:", err); + } + } + } + + const missingS3Images = []; + for (const img of imageRows) { + if (!img || !img.IName) continue; + const localPath = path.join(imagesPath, img.IName); + if (!fs.existsSync(localPath) && img.Source === "s3" && img.SourceKey) { + missingS3Images.push({ key: img.SourceKey, destPath: localPath }); + } + } + + if (missingS3Images.length === 0) { + return []; + } + + const bucketResult = await queries.managed.getBucket(projectName, admin); + const bucket = bucketResult && bucketResult.row; + if (!bucket) { + throw new Error(`S3 bucket configuration missing for project ${projectName}`); + } + + const s3Client = buildS3Client({ + region: bucket.Region, + accessKeyId: bucket.AccessKeyId, + secretAccessKey: bucket.SecretAccessKey, + endpoint: bucket.Endpoint, + }); + + const downloadedFiles = []; + for (const { key, destPath } of missingS3Images) { + const dir = path.dirname(destPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + await downloadObjectToFile(s3Client, bucket.BucketName, key, destPath); + downloadedFiles.push(destPath); + } + + return downloadedFiles; +} + +/** + * Unlinks files downloaded JIT for a training run. + * Leaves pre-existing local image files intact. + * + * @param {Array} downloadedFiles - List of absolute file paths downloaded JIT + */ +async function cleanupJitTrainingImages(downloadedFiles) { + if (!Array.isArray(downloadedFiles) || downloadedFiles.length === 0) { + return; + } + + for (const filePath of downloadedFiles) { + try { + await fs.promises.unlink(filePath); + } catch (err) { + if (err && err.code !== "ENOENT" && global.logger && global.logger.error) { + global.logger.error(`Error unlinking JIT training image ${filePath}:`, err); + } + } + } +} + +module.exports = { + ensureTrainingImagesLocal, + cleanupJitTrainingImages, +}; diff --git a/utils/s3Client.js b/utils/s3Client.js index 93e90951..2a244935 100644 --- a/utils/s3Client.js +++ b/utils/s3Client.js @@ -86,7 +86,7 @@ async function verifyBucketAccess(s3Client, bucketName) { ); } -async function listImageObjects(s3Client, bucketName, prefix = "") { +async function listImageObjects(s3Client, bucketName, prefix = "", maxKeys = null) { const objects = []; let continuationToken; @@ -106,6 +106,9 @@ async function listImageObjects(s3Client, bucketName, prefix = "") { if (IMAGE_EXTENSIONS.has(path.extname(object.Key).toLowerCase())) { objects.push(object.Key); + if (maxKeys && objects.length >= maxKeys) { + return objects; + } } } @@ -131,9 +134,25 @@ async function downloadObjectToFile(s3Client, bucketName, key, destPath) { }); } +// For "stream" sync mode: fetch an object's bytes live for a single request +// instead of persisting them to disk. Caller is responsible for piping +// response.Body to the outgoing HTTP response. +async function getObjectStream(s3Client, bucketName, key) { + const response = await s3Client.send( + new GetObjectCommand({ Bucket: bucketName, Key: key }), + ); + + return { + body: response.Body, + contentType: response.ContentType, + contentLength: response.ContentLength, + }; +} + module.exports = { buildS3Client, verifyBucketAccess, listImageObjects, downloadObjectToFile, + getObjectStream, }; diff --git a/views/create.ejs b/views/create.ejs index f8802759..6f309652 100644 --- a/views/create.ejs +++ b/views/create.ejs @@ -184,7 +184,7 @@
-

Add Project From S3 Bucket ?

+

Add Project From S3 Bucket ?

@@ -224,6 +224,15 @@

Leave the Access Key fields blank to use the server's default AWS credentials/role.

+
+
+ + +
+
@@ -289,6 +298,7 @@ const accessKeyId = document.getElementById('s3_access_key_id').value; const secretAccessKey = document.getElementById('s3_secret_access_key').value; const endpoint = document.getElementById("s3_endpoint").value; + const syncMode = document.getElementById("s3_sync_mode").value; if (!bucketName || !region) { alert("Bucket Name and Region are required."); @@ -328,6 +338,7 @@ AccessKeyId: accessKeyId, SecretAccessKey: secretAccessKey, Endpoint: endpoint, + SyncMode: syncMode, }), }); diff --git a/views/includes/s3BucketForm.ejs b/views/includes/s3BucketForm.ejs new file mode 100644 index 00000000..7f01f496 --- /dev/null +++ b/views/includes/s3BucketForm.ejs @@ -0,0 +1,185 @@ +
+
+
S3 Bucket Configuration (Inference)
+ +
+
+
+

+ Attach an S3 bucket to stream or sync images for inference. You can configure a maximum image limit to restrict the number of pulled images. +

+
+ +
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ + +
+
+ + +
+
+
+ +
+ + + +
+
+
+ + diff --git a/views/settings/imagesSettings.ejs b/views/settings/imagesSettings.ejs index 45d4eb75..331c8429 100644 --- a/views/settings/imagesSettings.ejs +++ b/views/settings/imagesSettings.ejs @@ -46,15 +46,19 @@
-
+
-
+
-
+
+ + +
+
@@ -136,6 +140,7 @@ const accessKeyId = document.getElementById('s3_access_key_id').value; const secretAccessKey = document.getElementById('s3_secret_access_key').value; const endpoint = document.getElementById("s3_endpoint").value; + const maxImages = document.getElementById("s3_max_images").value; if (!bucketName || !region) { alert("Bucket Name and Region are required."); @@ -159,6 +164,7 @@ AccessKeyId: accessKeyId, SecretAccessKey: secretAccessKey, Endpoint: endpoint, + MaxImages: maxImages ? parseInt(maxImages, 10) : null, }), }); diff --git a/views/training/inceptionSettings.ejs b/views/training/inceptionSettings.ejs index 4b08d420..976ed7c3 100644 --- a/views/training/inceptionSettings.ejs +++ b/views/training/inceptionSettings.ejs @@ -108,7 +108,7 @@
@@ -117,8 +117,10 @@ name="upload-inference" type="file" class="form-control-file" + accept=".jpg,.jpeg,.png,.gif,.tif,.tiff,.mp4,.mov,.zip" required > +

Upload single images/videos or a .zip archive containing images/videos.

@@ -134,7 +136,9 @@
-

+
+ <%- include('../includes/s3BucketForm') %> +
diff --git a/views/training/megadetectorSettings.ejs b/views/training/megadetectorSettings.ejs index de9b51ad..e6c7cf51 100644 --- a/views/training/megadetectorSettings.ejs +++ b/views/training/megadetectorSettings.ejs @@ -27,9 +27,10 @@
+ inference file (.jpg, .png, .gif, .tif, .mp4, .mov, .zip) + class="form-control-file" accept=".jpg,.jpeg,.png,.gif,.tif,.tiff,.mp4,.mov,.zip" required> +

Upload single images/videos or a .zip archive containing images/videos.

@@ -38,6 +39,7 @@

+ <%- include('../includes/s3BucketForm') %>
@@ -83,8 +85,8 @@
diff --git a/views/training/yolovXInferenceSettings.ejs b/views/training/yolovXInferenceSettings.ejs index c1391db8..ae17881f 100644 --- a/views/training/yolovXInferenceSettings.ejs +++ b/views/training/yolovXInferenceSettings.ejs @@ -84,10 +84,10 @@
+ file (.jpg, .png, .gif, .tif, .mp4, .mov, .zip) -

This is for doing predict or track for the YOLO MODE.

+ class="form-control-file" accept=".jpg,.jpeg,.png,.gif,.tif,.tiff,.mp4,.mov,.zip" required> +

This is for doing predict or track for the YOLO MODE. You can also upload a .zip archive containing images or videos.

@@ -96,6 +96,7 @@

+ <%- include('../includes/s3BucketForm') %>
@@ -154,10 +155,10 @@