fix: flatten nested image subfolders in zip archive uploads and imports

controllers/imports/importNJ.py

@@ -123,12 +123,18 @@ def create_project(db_name, txt_file, nj_path, class_label, img_dir, classificat
 
     if classification == '':
         print("Copying images for non-classification mode...")
-        for img_name in os.listdir(img_dir):
-            src = os.path.join(img_dir, img_name)
-            dst = os.path.join(project_path, 'images', img_name)
-            if os.path.isfile(src):
-                shutil.copy(src,dst)
-                print(f"Copied: {src} -> {dst}")
+        image_exts = ('.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff', '.gif', '.webp')
+        for root, dirs, files in os.walk(img_dir):
+            if '__MACOSX' in root:
+                continue
+            for file in files:
+                if file.lower().endswith(image_exts):
+                    src = os.path.join(root, file)
+                    rel_path = os.path.relpath(src, img_dir)
+                    new_img_name = rel_path.replace(os.sep, '_').replace(' ', '_').replace('+', '_')
+                    dst = os.path.join(project_path, 'images', new_img_name)
+                    shutil.copy(src, dst)
+                    print(f"Copied: {src} -> {dst}")
     else:
         print("Copying images for classification mode...")
         for dir_name in os.listdir(img_dir):

routes/api.js

@@ -59,6 +59,7 @@ const importDataset = require("./projects/importDataset");
 const importYolo = require("./projects/importYolo");
 const importKwCoco = require("./projects/importKwCoco");
 const importIfcb = require("./projects/importIfcb");
+const mapKwCocoCsv = require("./projects/mapKwCocoCsv");
 const mergeLocal = require("./projects/mergeLocal");
 const removeAccess = require("./projects/removeAccess");
 const transferAdmin = require("./projects/transferAdmin");
@@ -169,6 +170,8 @@ api.post("/import", importProject);
 api.post("/api/projects/import-dataset", importDataset);
 api.post("/api/projects/import-yolo", importYolo);
 api.post("/api/projects/import-kwcoco", importKwCoco);
+api.post("/api/projects/map-kwcoco-csv", mapKwCocoCsv);
+api.post("/mapKwCocoCsv", mapKwCocoCsv);
 api.post("/api/projects/import-ifcb", importIfcb);
 api.post("/mergeLocal", mergeLocal);
 api.post("/removeAccess", removeAccess);

routes/bootstrap/bootstrapController.js

@@ -1,4 +1,6 @@
+const path = require("path");
 const queries = require("../../queries/queries");
+const flattenDirectory = require("../../utils/flattenDirectory");
 
 async function boostrap(req, res) {
     var projectName = req.body.PName,
@@ -56,7 +58,7 @@ async function boostrap(req, res) {
 
     var zip = new StreamZip.async({ file: zipPath });
 
-    await zip.extract(mergeImages);
+    await zip.extract(null, mergeImages);
     await zip.close();
 
     rimraf(zipPath, (err) => {
@@ -67,30 +69,24 @@ async function boostrap(req, res) {
     });
 
     let files = await readdirAsync(imagesPath);
-    let newFiles = await readdirAsync(mergeImages);
+    let newFiles = await flattenDirectory(mergeImages);
 
     for (var i = 0; i < newFiles.length; i++) {
-        var temp = mergeImages + "/" + newFiles[i];
-        newFiles[i] = newFiles[i].trim();
-        newFiles[i] = newFiles[i].split(" ").join("_");
-        newFiles[i] = newFiles[i].split("+").join("_");
-        fs.rename(temp, mergeImages + "/" + newFiles[i], () => {});
-        if (newFiles[i] == "__MACOSX") {
-            continue;
-        } else if (!files.includes(newFiles[i])) {
+        const imageName = newFiles[i];
+        if (!files.includes(imageName)) {
             try {
                 fs.renameSync(
-                    mergeImages + "/" + newFiles[i],
-                    imagesPath + "/" + newFiles[i],
+                    path.join(mergeImages, imageName),
+                    path.join(imagesPath, imageName),
                 );
 
-                await queries.project.addImages(projectPath, newFiles[i], 0, 1);
+                await queries.project.addImages(projectPath, imageName, 0, 1);
 
-                newImages.push(newFiles[i]);
-                bootstrapString += newFiles[i] + "\n";
+                newImages.push(imageName);
+                bootstrapString += imageName + "\n";
             } catch (err) {
                 global.logger.error(err);
-                return res.stauts(500).send("Error inserting images");
+                return res.status(500).send("Error inserting images");
             }
         }
     }

routes/projects/addImages.js

@@ -1,7 +1,9 @@
 const fs = require("fs");
+const path = require("path");
 const rimraf = require("../../public/libraries/rimraf");
 const StreamZip = require("node-stream-zip");
 const queries = require("../../queries/queries");
+const flattenDirectory = require("../../utils/flattenDirectory");
 
 async function addImages(req, res) {
     var uploadImages = req.files["upload_images"],
@@ -65,43 +67,29 @@ async function addImages(req, res) {
             });
 
             files = await readdirAsync(imagesPath);
-            newFiles = await readdirAsync(mergeImages);
+            const newFiles = await flattenDirectory(mergeImages);
 
             for (var i = 0; i < newFiles.length; i++) {
-                var temp = mergeImages + "/" + newFiles[i];
-                newFiles[i] = newFiles[i].trim();
-                newFiles[i] = newFiles[i].split(" ").join("_");
-                newFiles[i] = newFiles[i].split("+").join("_");
-
-                fs.rename(temp, mergeImages + "/" + newFiles[i], () => {});
-
-                if (newFiles[i] == "__MACOSX") {
-                    continue;
-                } else if (!files.includes(newFiles[i])) {
-                    fs.rename(
-                        mergeImages + "/" + newFiles[i],
-                        imagesPath + "/" + newFiles[i],
-                        function (err) {
-                            if (err) {
-                                global.logger.error(err);
-                                return res.send("ERROR! " + err);
-                            }
-                        },
-                    );
-
+                const imageName = newFiles[i];
+                if (!files.includes(imageName)) {
                     try {
+                        fs.renameSync(
+                            path.join(mergeImages, imageName),
+                            path.join(imagesPath, imageName),
+                        );
+
                         await queries.project.addImages(
                             projectPath,
-                            newFiles[i],
+                            imageName,
                             0,
                             0,
                         );
+
+                        newImages.push(imageName);
                     } catch (err) {
                         global.logger.error(err);
                         return res.send("Error adding images");
                     }
-
-                    newImages.push(newFiles[i]);
                 }
             }
 

routes/projects/createProject.js

@@ -5,6 +5,7 @@ const StreamZip = require("node-stream-zip");
 const queries = require("../../queries/queries");
 const rimraf = require("../../public/libraries/rimraf");
 const { Client } = require("../../queries/client");
+const flattenDirectory = require("../../utils/flattenDirectory");
 
 async function createProject(req, res) {
     const files = req.files || {};
@@ -113,38 +114,16 @@ async function createProject(req, res) {
             rimraf(zipPath, (err) => {
                 if (err) {
                     global.logger.error(err);
-                    res.status(500).send("Error removing zip file");
                 }
             });
 
-            const files = fs.readdirSync(imagesPath);
-
-            for (var i = 0; i < files.length; i++) {
-                if (files[i] == "__MACOSX") {
-                    continue;
-                }
-
-                if (files[i].endsWith(".zip")) {
-                    fs.unlink(imagesPath + "/" + files[i], () => { });
-                    continue;
-                }
-
-                if (files[i].endsWith(".zip") || files[i] === "blob") {
-                    continue;
-                }
-
-                var temp = imagesPath + "/" + files[i];
-
-                files[i] = files[i].trim();
-                files[i] = files[i].split(" ").join("_");
-                files[i] = files[i].split("+").join("_");
-
-                fs.rename(temp, imagesPath + "/" + files[i], () => { });
+            const imageFiles = await flattenDirectory(imagesPath);
 
+            for (var i = 0; i < imageFiles.length; i++) {
                 try {
                     await queries.project.addImages(
                         projectPath,
-                        files[i],
+                        imageFiles[i],
                         0,
                         0,
                     );

routes/projects/importProject.js

@@ -2,6 +2,7 @@ const StreamZip = require("node-stream-zip");
 const rimraf = require("../../public/libraries/rimraf");
 const queries = require("../../queries/queries");
 const { Client } = require("../../queries/client");
+const flattenDirectory = require("../../utils/flattenDirectory");
 
 async function importProject(req, res) {
     req.setTimeout(600000);
@@ -108,6 +109,9 @@ async function importProject(req, res) {
                 return res.status(500).send("Error removing .zip file");
             }
 
+            if (fs.existsSync(imagesPath)) {
+                await flattenDirectory(imagesPath);
+            }
             let images = await readdirAsync(imagesPath);
             let oldImages;
 

routes/projects/mapKwCocoCsv.js

@@ -0,0 +1,127 @@
+const path = require('path');
+const fs = require('fs');
+const parseKwCocoCsv = require('../../utils/parseKwCocoCsv');
+const parseKwCocoJson = require('../../utils/parseKwCocoJson');
+const queries = require('../../queries/queries');
+const { Client } = require('../../queries/client');
+
+async function mapKwCocoCsv(req, res) {
+    try {
+        const projectName = req.body.PName || req.body.project_name || req.body.projectName;
+        const admin = req.body.Admin || req.cookies?.Username || 'admin';
+
+        if (!projectName) {
+            return res.status(400).json({ success: false, message: 'Project name is required.' });
+        }
+
+        if (!req.files || Object.keys(req.files).length === 0) {
+            return res.status(400).json({ success: false, message: 'No annotation file was uploaded.' });
+        }
+
+        const uploadedFile = req.files.kwcoco_csv || req.files.kwcoco_json || req.files.csv_file
+            || req.files.json_file || req.files.upload_csv || req.files.upload_json || Object.values(req.files)[0];
+        if (!uploadedFile) {
+            return res.status(400).json({ success: false, message: 'Invalid file upload payload.' });
+        }
+
+        const fileContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8');
+
+        const ext = path.extname(uploadedFile.name || '').toLowerCase();
+        const trimmedContent = fileContent.trim();
+        const isJson = ext === '.json' || (ext !== '.csv' && (trimmedContent.startsWith('{') || trimmedContent.startsWith('[')));
+
+        const parsedAnnotations = isJson ? parseKwCocoJson(fileContent) : parseKwCocoCsv(fileContent);
+        if (parsedAnnotations.length === 0) {
+            return res.status(400).json({ success: false, message: 'No valid KW COCO annotations found in file.' });
+        }
+
+        const mainPath = path.join(__dirname, '..', '..', 'public', 'projects');
+        const projectPath = path.join(mainPath, `${admin}-${projectName}`);
+
+        if (!fs.existsSync(projectPath)) {
+            return res.status(404).json({ success: false, message: `Project path not found: ${admin}-${projectName}` });
+        }
+
+        const dbPath = path.join(projectPath, `${projectName}.db`);
+        if (!global.projectDbClients[projectPath]) {
+            global.projectDbClients[projectPath] = new Client(dbPath);
+            const client = global.projectDbClients[projectPath];
+            if (typeof client.open === 'function') {
+                client.open();
+            }
+        }
+
+        // Migrate DB if needed
+        await queries.project.migrateProjectDb(projectPath);
+
+        // 1. Ensure all referenced classes exist in Classes table
+        const existingClassResult = await queries.project.getAllClasses(projectPath);
+        const existingClassRows = existingClassResult?.rows || [];
+        const existingClassSet = new Set(existingClassRows.map(c => c.CName));
+
+        const uniqueClasses = new Set(parsedAnnotations.map(a => a.className));
+        let classesAdded = 0;
+
+        for (const cname of uniqueClasses) {
+            if (!existingClassSet.has(cname)) {
+                await queries.project.createClass(projectPath, cname);
+                existingClassSet.add(cname);
+                classesAdded++;
+            }
+        }
+
+        // 2. Ensure referenced images exist in Images table
+        const existingImageResult = await queries.project.getAllImages(projectPath);
+        const existingImageRows = existingImageResult?.rows || [];
+        const existingImageSet = new Set(existingImageRows.map(i => i.IName));
+
+        const uniqueImages = new Set(parsedAnnotations.map(a => a.filename));
+        let imagesRegistered = 0;
+
+        for (const iname of uniqueImages) {
+            if (!existingImageSet.has(iname)) {
+                await queries.project.sql(projectPath, "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, 0, 0)", [iname]);
+                existingImageSet.add(iname);
+                imagesRegistered++;
+            }
+        }
+
+        // 3. Get current max LID in Labels table
+        const maxLidResult = await queries.project.getMaxLabelId(projectPath);
+        const maxLidRows = maxLidResult?.rows || [];
+        let nextLid = 1;
+        if (maxLidRows.length > 0 && maxLidRows[0].LID) {
+            nextLid = maxLidRows[0].LID + 1;
+        }
+
+        // 4. Insert labels
+        let labelsInserted = 0;
+        for (const ann of parsedAnnotations) {
+            await queries.project.createLabel(
+                projectPath,
+                nextLid++,
+                ann.className,
+                ann.x,
+                ann.y,
+                ann.w,
+                ann.h,
+                ann.filename
+            );
+            labelsInserted++;
+        }
+
+        return res.json({
+            success: true,
+            message: `Successfully mapped ${labelsInserted} KW COCO annotations.`,
+            labelsInserted,
+            classesAdded,
+            imagesRegistered
+        });
+
+    } catch (err) {
+        console.error('Error mapping KW COCO CSV annotations:', err);
+        return res.status(500).json({ success: false, message: err.message || 'Internal server error mapping KW COCO CSV.' });
+    }
+}
+
+module.exports = mapKwCocoCsv;