diff --git a/controllers/imports/importNJ.py b/controllers/imports/importNJ.py
index 02937ce5..674b90c3 100644
--- a/controllers/imports/importNJ.py
+++ b/controllers/imports/importNJ.py
@@ -123,12 +123,18 @@ def create_project(db_name, txt_file, nj_path, class_label, img_dir, classificat
if classification == '':
print("Copying images for non-classification mode...")
- for img_name in os.listdir(img_dir):
- src = os.path.join(img_dir, img_name)
- dst = os.path.join(project_path, 'images', img_name)
- if os.path.isfile(src):
- shutil.copy(src,dst)
- print(f"Copied: {src} -> {dst}")
+ image_exts = ('.jpg', '.jpeg', '.png', '.bmp', '.tif', '.tiff', '.gif', '.webp')
+ for root, dirs, files in os.walk(img_dir):
+ if '__MACOSX' in root:
+ continue
+ for file in files:
+ if file.lower().endswith(image_exts):
+ src = os.path.join(root, file)
+ rel_path = os.path.relpath(src, img_dir)
+ new_img_name = rel_path.replace(os.sep, '_').replace(' ', '_').replace('+', '_')
+ dst = os.path.join(project_path, 'images', new_img_name)
+ shutil.copy(src, dst)
+ print(f"Copied: {src} -> {dst}")
else:
print("Copying images for classification mode...")
for dir_name in os.listdir(img_dir):
diff --git a/routes/api.js b/routes/api.js
index 111e6915..72f9bd42 100755
--- a/routes/api.js
+++ b/routes/api.js
@@ -59,6 +59,7 @@ const importDataset = require("./projects/importDataset");
const importYolo = require("./projects/importYolo");
const importKwCoco = require("./projects/importKwCoco");
const importIfcb = require("./projects/importIfcb");
+const mapKwCocoCsv = require("./projects/mapKwCocoCsv");
const mergeLocal = require("./projects/mergeLocal");
const removeAccess = require("./projects/removeAccess");
const transferAdmin = require("./projects/transferAdmin");
@@ -169,6 +170,8 @@ api.post("/import", importProject);
api.post("/api/projects/import-dataset", importDataset);
api.post("/api/projects/import-yolo", importYolo);
api.post("/api/projects/import-kwcoco", importKwCoco);
+api.post("/api/projects/map-kwcoco-csv", mapKwCocoCsv);
+api.post("/mapKwCocoCsv", mapKwCocoCsv);
api.post("/api/projects/import-ifcb", importIfcb);
api.post("/mergeLocal", mergeLocal);
api.post("/removeAccess", removeAccess);
diff --git a/routes/bootstrap/bootstrapController.js b/routes/bootstrap/bootstrapController.js
index 5319f8ce..fa24b415 100644
--- a/routes/bootstrap/bootstrapController.js
+++ b/routes/bootstrap/bootstrapController.js
@@ -1,4 +1,6 @@
+const path = require("path");
const queries = require("../../queries/queries");
+const flattenDirectory = require("../../utils/flattenDirectory");
async function boostrap(req, res) {
var projectName = req.body.PName,
@@ -56,7 +58,7 @@ async function boostrap(req, res) {
var zip = new StreamZip.async({ file: zipPath });
- await zip.extract(mergeImages);
+ await zip.extract(null, mergeImages);
await zip.close();
rimraf(zipPath, (err) => {
@@ -67,30 +69,24 @@ async function boostrap(req, res) {
});
let files = await readdirAsync(imagesPath);
- let newFiles = await readdirAsync(mergeImages);
+ let newFiles = await flattenDirectory(mergeImages);
for (var i = 0; i < newFiles.length; i++) {
- var temp = mergeImages + "/" + newFiles[i];
- newFiles[i] = newFiles[i].trim();
- newFiles[i] = newFiles[i].split(" ").join("_");
- newFiles[i] = newFiles[i].split("+").join("_");
- fs.rename(temp, mergeImages + "/" + newFiles[i], () => {});
- if (newFiles[i] == "__MACOSX") {
- continue;
- } else if (!files.includes(newFiles[i])) {
+ const imageName = newFiles[i];
+ if (!files.includes(imageName)) {
try {
fs.renameSync(
- mergeImages + "/" + newFiles[i],
- imagesPath + "/" + newFiles[i],
+ path.join(mergeImages, imageName),
+ path.join(imagesPath, imageName),
);
- await queries.project.addImages(projectPath, newFiles[i], 0, 1);
+ await queries.project.addImages(projectPath, imageName, 0, 1);
- newImages.push(newFiles[i]);
- bootstrapString += newFiles[i] + "\n";
+ newImages.push(imageName);
+ bootstrapString += imageName + "\n";
} catch (err) {
global.logger.error(err);
- return res.stauts(500).send("Error inserting images");
+ return res.status(500).send("Error inserting images");
}
}
}
diff --git a/routes/projects/addImages.js b/routes/projects/addImages.js
index ba8d302f..df55f6da 100644
--- a/routes/projects/addImages.js
+++ b/routes/projects/addImages.js
@@ -1,7 +1,9 @@
const fs = require("fs");
+const path = require("path");
const rimraf = require("../../public/libraries/rimraf");
const StreamZip = require("node-stream-zip");
const queries = require("../../queries/queries");
+const flattenDirectory = require("../../utils/flattenDirectory");
async function addImages(req, res) {
var uploadImages = req.files["upload_images"],
@@ -65,43 +67,29 @@ async function addImages(req, res) {
});
files = await readdirAsync(imagesPath);
- newFiles = await readdirAsync(mergeImages);
+ const newFiles = await flattenDirectory(mergeImages);
for (var i = 0; i < newFiles.length; i++) {
- var temp = mergeImages + "/" + newFiles[i];
- newFiles[i] = newFiles[i].trim();
- newFiles[i] = newFiles[i].split(" ").join("_");
- newFiles[i] = newFiles[i].split("+").join("_");
-
- fs.rename(temp, mergeImages + "/" + newFiles[i], () => {});
-
- if (newFiles[i] == "__MACOSX") {
- continue;
- } else if (!files.includes(newFiles[i])) {
- fs.rename(
- mergeImages + "/" + newFiles[i],
- imagesPath + "/" + newFiles[i],
- function (err) {
- if (err) {
- global.logger.error(err);
- return res.send("ERROR! " + err);
- }
- },
- );
-
+ const imageName = newFiles[i];
+ if (!files.includes(imageName)) {
try {
+ fs.renameSync(
+ path.join(mergeImages, imageName),
+ path.join(imagesPath, imageName),
+ );
+
await queries.project.addImages(
projectPath,
- newFiles[i],
+ imageName,
0,
0,
);
+
+ newImages.push(imageName);
} catch (err) {
global.logger.error(err);
return res.send("Error adding images");
}
-
- newImages.push(newFiles[i]);
}
}
diff --git a/routes/projects/createProject.js b/routes/projects/createProject.js
index 708e5219..60a9b688 100644
--- a/routes/projects/createProject.js
+++ b/routes/projects/createProject.js
@@ -5,6 +5,7 @@ const StreamZip = require("node-stream-zip");
const queries = require("../../queries/queries");
const rimraf = require("../../public/libraries/rimraf");
const { Client } = require("../../queries/client");
+const flattenDirectory = require("../../utils/flattenDirectory");
async function createProject(req, res) {
const files = req.files || {};
@@ -113,38 +114,16 @@ async function createProject(req, res) {
rimraf(zipPath, (err) => {
if (err) {
global.logger.error(err);
- res.status(500).send("Error removing zip file");
}
});
- const files = fs.readdirSync(imagesPath);
-
- for (var i = 0; i < files.length; i++) {
- if (files[i] == "__MACOSX") {
- continue;
- }
-
- if (files[i].endsWith(".zip")) {
- fs.unlink(imagesPath + "/" + files[i], () => { });
- continue;
- }
-
- if (files[i].endsWith(".zip") || files[i] === "blob") {
- continue;
- }
-
- var temp = imagesPath + "/" + files[i];
-
- files[i] = files[i].trim();
- files[i] = files[i].split(" ").join("_");
- files[i] = files[i].split("+").join("_");
-
- fs.rename(temp, imagesPath + "/" + files[i], () => { });
+ const imageFiles = await flattenDirectory(imagesPath);
+ for (var i = 0; i < imageFiles.length; i++) {
try {
await queries.project.addImages(
projectPath,
- files[i],
+ imageFiles[i],
0,
0,
);
diff --git a/routes/projects/importProject.js b/routes/projects/importProject.js
index ab6e7e60..bb18793b 100644
--- a/routes/projects/importProject.js
+++ b/routes/projects/importProject.js
@@ -2,6 +2,7 @@ const StreamZip = require("node-stream-zip");
const rimraf = require("../../public/libraries/rimraf");
const queries = require("../../queries/queries");
const { Client } = require("../../queries/client");
+const flattenDirectory = require("../../utils/flattenDirectory");
async function importProject(req, res) {
req.setTimeout(600000);
@@ -108,6 +109,9 @@ async function importProject(req, res) {
return res.status(500).send("Error removing .zip file");
}
+ if (fs.existsSync(imagesPath)) {
+ await flattenDirectory(imagesPath);
+ }
let images = await readdirAsync(imagesPath);
let oldImages;
diff --git a/routes/projects/mapKwCocoCsv.js b/routes/projects/mapKwCocoCsv.js
new file mode 100644
index 00000000..6f09b827
--- /dev/null
+++ b/routes/projects/mapKwCocoCsv.js
@@ -0,0 +1,127 @@
+const path = require('path');
+const fs = require('fs');
+const parseKwCocoCsv = require('../../utils/parseKwCocoCsv');
+const parseKwCocoJson = require('../../utils/parseKwCocoJson');
+const queries = require('../../queries/queries');
+const { Client } = require('../../queries/client');
+
+async function mapKwCocoCsv(req, res) {
+ try {
+ const projectName = req.body.PName || req.body.project_name || req.body.projectName;
+ const admin = req.body.Admin || req.cookies?.Username || 'admin';
+
+ if (!projectName) {
+ return res.status(400).json({ success: false, message: 'Project name is required.' });
+ }
+
+ if (!req.files || Object.keys(req.files).length === 0) {
+ return res.status(400).json({ success: false, message: 'No annotation file was uploaded.' });
+ }
+
+ const uploadedFile = req.files.kwcoco_csv || req.files.kwcoco_json || req.files.csv_file
+ || req.files.json_file || req.files.upload_csv || req.files.upload_json || Object.values(req.files)[0];
+ if (!uploadedFile) {
+ return res.status(400).json({ success: false, message: 'Invalid file upload payload.' });
+ }
+
+ const fileContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8');
+
+ const ext = path.extname(uploadedFile.name || '').toLowerCase();
+ const trimmedContent = fileContent.trim();
+ const isJson = ext === '.json' || (ext !== '.csv' && (trimmedContent.startsWith('{') || trimmedContent.startsWith('[')));
+
+ const parsedAnnotations = isJson ? parseKwCocoJson(fileContent) : parseKwCocoCsv(fileContent);
+ if (parsedAnnotations.length === 0) {
+ return res.status(400).json({ success: false, message: 'No valid KW COCO annotations found in file.' });
+ }
+
+ const mainPath = path.join(__dirname, '..', '..', 'public', 'projects');
+ const projectPath = path.join(mainPath, `${admin}-${projectName}`);
+
+ if (!fs.existsSync(projectPath)) {
+ return res.status(404).json({ success: false, message: `Project path not found: ${admin}-${projectName}` });
+ }
+
+ const dbPath = path.join(projectPath, `${projectName}.db`);
+ if (!global.projectDbClients[projectPath]) {
+ global.projectDbClients[projectPath] = new Client(dbPath);
+ const client = global.projectDbClients[projectPath];
+ if (typeof client.open === 'function') {
+ client.open();
+ }
+ }
+
+ // Migrate DB if needed
+ await queries.project.migrateProjectDb(projectPath);
+
+ // 1. Ensure all referenced classes exist in Classes table
+ const existingClassResult = await queries.project.getAllClasses(projectPath);
+ const existingClassRows = existingClassResult?.rows || [];
+ const existingClassSet = new Set(existingClassRows.map(c => c.CName));
+
+ const uniqueClasses = new Set(parsedAnnotations.map(a => a.className));
+ let classesAdded = 0;
+
+ for (const cname of uniqueClasses) {
+ if (!existingClassSet.has(cname)) {
+ await queries.project.createClass(projectPath, cname);
+ existingClassSet.add(cname);
+ classesAdded++;
+ }
+ }
+
+ // 2. Ensure referenced images exist in Images table
+ const existingImageResult = await queries.project.getAllImages(projectPath);
+ const existingImageRows = existingImageResult?.rows || [];
+ const existingImageSet = new Set(existingImageRows.map(i => i.IName));
+
+ const uniqueImages = new Set(parsedAnnotations.map(a => a.filename));
+ let imagesRegistered = 0;
+
+ for (const iname of uniqueImages) {
+ if (!existingImageSet.has(iname)) {
+ await queries.project.sql(projectPath, "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, 0, 0)", [iname]);
+ existingImageSet.add(iname);
+ imagesRegistered++;
+ }
+ }
+
+ // 3. Get current max LID in Labels table
+ const maxLidResult = await queries.project.getMaxLabelId(projectPath);
+ const maxLidRows = maxLidResult?.rows || [];
+ let nextLid = 1;
+ if (maxLidRows.length > 0 && maxLidRows[0].LID) {
+ nextLid = maxLidRows[0].LID + 1;
+ }
+
+ // 4. Insert labels
+ let labelsInserted = 0;
+ for (const ann of parsedAnnotations) {
+ await queries.project.createLabel(
+ projectPath,
+ nextLid++,
+ ann.className,
+ ann.x,
+ ann.y,
+ ann.w,
+ ann.h,
+ ann.filename
+ );
+ labelsInserted++;
+ }
+
+ return res.json({
+ success: true,
+ message: `Successfully mapped ${labelsInserted} KW COCO annotations.`,
+ labelsInserted,
+ classesAdded,
+ imagesRegistered
+ });
+
+ } catch (err) {
+ console.error('Error mapping KW COCO CSV annotations:', err);
+ return res.status(500).json({ success: false, message: err.message || 'Internal server error mapping KW COCO CSV.' });
+ }
+}
+
+module.exports = mapKwCocoCsv;
diff --git a/tests/integration/mapKwCocoCsv.test.js b/tests/integration/mapKwCocoCsv.test.js
new file mode 100644
index 00000000..5005498f
--- /dev/null
+++ b/tests/integration/mapKwCocoCsv.test.js
@@ -0,0 +1,123 @@
+const request = require('supertest');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const app = require('../../app');
+const queries = require('../../queries/queries');
+const { Client } = require('../../queries/client');
+
+describe('POST /api/projects/map-kwcoco-csv', () => {
+ let tmpDir;
+ let projectDir;
+ let originalProjectsPath;
+
+ beforeEach(() => {
+ tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'map-kwcoco-test-'));
+ projectDir = path.join(__dirname, '..', '..', 'public', 'projects', 'admin-testproj');
+ fs.mkdirSync(projectDir, { recursive: true });
+
+ const mockClient = {
+ open: jest.fn(),
+ all: jest.fn().mockImplementation((sql) => {
+ if (sql.includes('Classes')) return Promise.resolve({ success: true, rows: [] });
+ if (sql.includes('Images')) return Promise.resolve({ success: true, rows: [] });
+ if (sql.includes('Labels')) return Promise.resolve({ success: true, rows: [] });
+ return Promise.resolve({ success: true, rows: [] });
+ }),
+ get: jest.fn().mockResolvedValue({ success: true, row: null }),
+ run: jest.fn().mockResolvedValue({ success: true, changes: 1, lastID: 1 }),
+ };
+
+ global.projectDbClients = {
+ [projectDir]: mockClient
+ };
+ });
+
+ afterEach(() => {
+ fs.rmSync(projectDir, { recursive: true, force: true });
+ fs.rmSync(tmpDir, { recursive: true, force: true });
+ });
+
+ test('returns 400 when project name is missing', async () => {
+ const res = await request(app)
+ .post('/api/projects/map-kwcoco-csv')
+ .set('Cookie', ['Username=admin']);
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.message).toMatch(/Project name is required/i);
+ });
+
+ test('returns 400 when no annotation file is uploaded', async () => {
+ const res = await request(app)
+ .post('/api/projects/map-kwcoco-csv')
+ .field('PName', 'testproj')
+ .field('Admin', 'admin');
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.message).toMatch(/No annotation file was uploaded/i);
+ });
+
+ test('returns 404 when project path does not exist', async () => {
+ const csvContent = 'filename,class,xmin,ymin,xmax,ymax\nimg1.jpg,dolphin,10,10,50,50';
+ const csvPath = path.join(tmpDir, 'test.csv');
+ fs.writeFileSync(csvPath, csvContent);
+
+ const res = await request(app)
+ .post('/api/projects/map-kwcoco-csv')
+ .field('PName', 'nonexistent_project')
+ .field('Admin', 'admin')
+ .attach('kwcoco_csv', csvPath);
+
+ expect(res.statusCode).toBe(404);
+ expect(res.body.success).toBe(false);
+ });
+
+ test('successfully maps KW COCO CSV annotations', async () => {
+ const csvContent = `filename,class,xmin,ymin,xmax,ymax
+img1.jpg,dolphin,10,20,100,150
+img2.jpg,shark,30,40,80,120`;
+ const csvPath = path.join(tmpDir, 'test.csv');
+ fs.writeFileSync(csvPath, csvContent);
+
+ const res = await request(app)
+ .post('/api/projects/map-kwcoco-csv')
+ .field('PName', 'testproj')
+ .field('Admin', 'admin')
+ .attach('kwcoco_csv', csvPath);
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ expect(res.body.labelsInserted).toBe(2);
+ });
+
+ test('successfully maps KW COCO JSON annotations', async () => {
+ const jsonContent = JSON.stringify({
+ images: [
+ { id: 1, file_name: 'img1.jpg' },
+ { id: 2, file_name: 'img2.jpg' }
+ ],
+ annotations: [
+ { id: 1, image_id: 1, category_id: 1, bbox: [10, 20, 90, 130] },
+ { id: 2, image_id: 2, category_id: 2, bbox: [30, 40, 50, 80] }
+ ],
+ categories: [
+ { id: 1, name: 'dolphin' },
+ { id: 2, name: 'shark' }
+ ]
+ });
+ const jsonPath = path.join(tmpDir, 'test.json');
+ fs.writeFileSync(jsonPath, jsonContent);
+
+ const res = await request(app)
+ .post('/api/projects/map-kwcoco-csv')
+ .field('PName', 'testproj')
+ .field('Admin', 'admin')
+ .attach('kwcoco_json', jsonPath);
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ expect(res.body.labelsInserted).toBe(2);
+ });
+});
diff --git a/tests/unit/flattenDirectory.test.js b/tests/unit/flattenDirectory.test.js
new file mode 100644
index 00000000..f12c9c10
--- /dev/null
+++ b/tests/unit/flattenDirectory.test.js
@@ -0,0 +1,90 @@
+const path = require("path");
+const fs = require("fs");
+const flattenDirectory = require("../../utils/flattenDirectory");
+
+describe("flattenDirectory utility", () => {
+ const testDir = path.join(__dirname, "tmp_flatten_test");
+
+ beforeEach(() => {
+ if (fs.existsSync(testDir)) {
+ fs.rmSync(testDir, { recursive: true, force: true });
+ }
+ fs.mkdirSync(testDir, { recursive: true });
+ });
+
+ afterEach(() => {
+ if (fs.existsSync(testDir)) {
+ fs.rmSync(testDir, { recursive: true, force: true });
+ }
+ });
+
+ it("should flatten images in nested subfolders and remove empty subdirectories", async () => {
+ // Setup folder structure:
+ // testDir/
+ // ├── root_img.jpg
+ // ├── folderA/
+ // │ ├── sub1.png
+ // │ └── nested/
+ // │ └── sub2.jpeg
+ // └── __MACOSX/
+ // └── ._junk
+ const folderA = path.join(testDir, "folderA");
+ const nested = path.join(folderA, "nested");
+ const macos = path.join(testDir, "__MACOSX");
+
+ fs.mkdirSync(nested, { recursive: true });
+ fs.mkdirSync(macos, { recursive: true });
+
+ fs.writeFileSync(path.join(testDir, "root_img.jpg"), "fake jpg");
+ fs.writeFileSync(path.join(folderA, "sub 1.png"), "fake png");
+ fs.writeFileSync(path.join(nested, "sub2.jpeg"), "fake jpeg");
+ fs.writeFileSync(path.join(macos, "._junk"), "macos junk");
+ fs.writeFileSync(path.join(folderA, ".DS_Store"), "ds store");
+
+ const result = await flattenDirectory(testDir);
+
+ expect(result.sort()).toEqual(["root_img.jpg", "sub_1.png", "sub2.jpeg"].sort());
+ expect(fs.existsSync(path.join(testDir, "root_img.jpg"))).toBe(true);
+ expect(fs.existsSync(path.join(testDir, "sub_1.png"))).toBe(true);
+ expect(fs.existsSync(path.join(testDir, "sub2.jpeg"))).toBe(true);
+
+ // Subdirectories should be removed
+ expect(fs.existsSync(folderA)).toBe(false);
+ expect(fs.existsSync(macos)).toBe(false);
+ });
+
+ it("should handle filename collisions when images in different subfolders have the same name", async () => {
+ const folder1 = path.join(testDir, "sub1");
+ const folder2 = path.join(testDir, "sub2");
+
+ fs.mkdirSync(folder1, { recursive: true });
+ fs.mkdirSync(folder2, { recursive: true });
+
+ fs.writeFileSync(path.join(folder1, "image.jpg"), "content 1");
+ fs.writeFileSync(path.join(folder2, "image.jpg"), "content 2");
+
+ const result = await flattenDirectory(testDir);
+
+ expect(result.length).toBe(2);
+ expect(result).toContain("image.jpg");
+ expect(fs.existsSync(path.join(testDir, "image.jpg"))).toBe(true);
+
+ // Subdirectories should be cleaned up
+ expect(fs.existsSync(folder1)).toBe(false);
+ expect(fs.existsSync(folder2)).toBe(false);
+ });
+
+ it("should remove non-image files inside subfolders", async () => {
+ const sub = path.join(testDir, "sub");
+ fs.mkdirSync(sub, { recursive: true });
+
+ fs.writeFileSync(path.join(sub, "valid.png"), "valid image");
+ fs.writeFileSync(path.join(sub, "readme.txt"), "text file");
+
+ const result = await flattenDirectory(testDir);
+
+ expect(result).toEqual(["valid.png"]);
+ expect(fs.existsSync(path.join(testDir, "valid.png"))).toBe(true);
+ expect(fs.existsSync(path.join(testDir, "readme.txt"))).toBe(false);
+ });
+});
diff --git a/tests/unit/parseKwCocoCsv.test.js b/tests/unit/parseKwCocoCsv.test.js
new file mode 100644
index 00000000..3f91e614
--- /dev/null
+++ b/tests/unit/parseKwCocoCsv.test.js
@@ -0,0 +1,52 @@
+const parseKwCocoCsv = require('../../utils/parseKwCocoCsv');
+
+describe('parseKwCocoCsv', () => {
+ test('returns empty array for invalid or empty input', () => {
+ expect(parseKwCocoCsv(null)).toEqual([]);
+ expect(parseKwCocoCsv('')).toEqual([]);
+ expect(parseKwCocoCsv(' ')).toEqual([]);
+ });
+
+ test('parses CSV with header containing filename, class, xmin, ymin, xmax, ymax', () => {
+ const csv = `filename,class,xmin,ymin,xmax,ymax
+image1.jpg,dolphin,10,20,100,150
+image2.png,blue whale,50,60,200,260`;
+
+ const result = parseKwCocoCsv(csv);
+ expect(result).toEqual([
+ { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 },
+ { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 }
+ ]);
+ });
+
+ test('parses CSV with header containing x, y, w, h', () => {
+ const csv = `file_name,category,x,y,w,h
+path/to/img3.jpg,sea turtle,15,25,80,95`;
+
+ const result = parseKwCocoCsv(csv);
+ expect(result).toEqual([
+ { filename: 'img3.jpg', className: 'sea_turtle', x: 15, y: 25, w: 80, h: 95 }
+ ]);
+ });
+
+ test('parses positional CSV without header', () => {
+ const csv = `img4.jpg,fish,5,10,45,60`;
+
+ const result = parseKwCocoCsv(csv);
+ expect(result).toEqual([
+ { filename: 'img4.jpg', className: 'fish', x: 5, y: 10, w: 40, h: 50 }
+ ]);
+ });
+
+ test('ignores invalid rows and negative width/height', () => {
+ const csv = `filename,class,xmin,ymin,xmax,ymax
+bad1.jpg,fish,100,100,50,50
+bad2.jpg,fish,invalid,10,20,30
+good.jpg,shark,10,10,30,30`;
+
+ const result = parseKwCocoCsv(csv);
+ expect(result).toEqual([
+ { filename: 'good.jpg', className: 'shark', x: 10, y: 10, w: 20, h: 20 }
+ ]);
+ });
+});
diff --git a/tests/unit/parseKwCocoJson.test.js b/tests/unit/parseKwCocoJson.test.js
new file mode 100644
index 00000000..cbb6b0de
--- /dev/null
+++ b/tests/unit/parseKwCocoJson.test.js
@@ -0,0 +1,64 @@
+const parseKwCocoJson = require('../../utils/parseKwCocoJson');
+
+describe('parseKwCocoJson', () => {
+ test('returns empty array for invalid or empty input', () => {
+ expect(parseKwCocoJson(null)).toEqual([]);
+ expect(parseKwCocoJson('')).toEqual([]);
+ expect(parseKwCocoJson('not json')).toEqual([]);
+ expect(parseKwCocoJson('{}')).toEqual([]);
+ });
+
+ test('parses standard COCO-style JSON with images/annotations/categories', () => {
+ const json = JSON.stringify({
+ images: [
+ { id: 1, file_name: 'image1.jpg' },
+ { id: 2, file_name: 'path/to/image2.png' }
+ ],
+ annotations: [
+ { id: 1, image_id: 1, category_id: 10, bbox: [10, 20, 90, 130] },
+ { id: 2, image_id: 2, category_id: 11, bbox: [50, 60, 150, 200] }
+ ],
+ categories: [
+ { id: 10, name: 'dolphin' },
+ { id: 11, name: 'blue whale' }
+ ]
+ });
+
+ const result = parseKwCocoJson(json);
+ expect(result).toEqual([
+ { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 },
+ { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 }
+ ]);
+ });
+
+ test('skips annotations referencing unknown images or categories', () => {
+ const json = JSON.stringify({
+ images: [{ id: 1, file_name: 'good.jpg' }],
+ annotations: [
+ { id: 1, image_id: 1, category_id: 5, bbox: [10, 10, 20, 20] },
+ { id: 2, image_id: 999, category_id: 5, bbox: [10, 10, 20, 20] },
+ { id: 3, image_id: 1, category_id: 999, bbox: [10, 10, 20, 20] }
+ ],
+ categories: [{ id: 5, name: 'fish' }]
+ });
+
+ const result = parseKwCocoJson(json);
+ expect(result).toEqual([
+ { filename: 'good.jpg', className: 'fish', x: 10, y: 10, w: 20, h: 20 }
+ ]);
+ });
+
+ test('ignores annotations with invalid or non-positive bbox dimensions', () => {
+ const json = JSON.stringify({
+ images: [{ id: 1, file_name: 'img.jpg' }],
+ annotations: [
+ { id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 0, 0] },
+ { id: 2, image_id: 1, category_id: 1, bbox: ['a', 'b', 'c', 'd'] },
+ { id: 3, image_id: 1, category_id: 1, bbox: [1, 2, 3] }
+ ],
+ categories: [{ id: 1, name: 'shark' }]
+ });
+
+ expect(parseKwCocoJson(json)).toEqual([]);
+ });
+});
diff --git a/utils/cleanDirectory.js b/utils/cleanDirectory.js
index e4ff3454..5d475b86 100644
--- a/utils/cleanDirectory.js
+++ b/utils/cleanDirectory.js
@@ -1,60 +1,10 @@
const path = require("path");
const fs = require("fs");
+const flattenDirectory = require("./flattenDirectory");
async function cleanDirectory(directory) {
try {
- const files = await fs.promises.readdir(directory);
-
- for (const file of files) {
- const filePath = path.join(directory, file);
- const stats = await fs.promises.stat(filePath);
-
- // Process subdirectories
- if (stats.isDirectory()) {
- if (file === "__MACOSX") continue;
-
- // Clean subdirectory name first
- if (
- file !== file.trim() ||
- file.includes(" ") ||
- file.includes("+")
- ) {
- const newDirName = file.trim().replace(/[ +]/g, "_");
- const newDirPath = path.join(directory, newDirName);
- await fs.promises.rename(filePath, newDirPath);
- await cleanDirectory(newDirPath);
- } else {
- await cleanDirectory(filePath);
- }
- continue;
- }
-
- if (
- file === ".DS_Store" ||
- file === "._.DS_Store" ||
- file.startsWith("._") ||
- file === "Thumbs.db" ||
- file === "desktop.ini"
- ) {
- await fs.promises.unlink(filePath);
- // console.log(`Removed system file: ${filePath}`);
- continue;
- }
-
- // Clean filename: Remove trailing/leading spaces and replace spaces and + with _
- if (
- file !== file.trim() ||
- file.includes(" ") ||
- file.includes("+")
- ) {
- const newFileName = file.trim().replace(/[ +]/g, "_");
- const newFilePath = path.join(directory, newFileName);
-
- await fs.promises.rename(filePath, newFilePath);
- }
- }
-
- // console.log(`Directory cleaned: ${directory}`);
+ await flattenDirectory(directory);
return directory;
} catch (error) {
console.error(`Error cleaning directory ${directory}:`, error);
diff --git a/utils/flattenDirectory.js b/utils/flattenDirectory.js
new file mode 100644
index 00000000..e660939a
--- /dev/null
+++ b/utils/flattenDirectory.js
@@ -0,0 +1,151 @@
+const path = require("path");
+const fs = require("fs");
+
+const IMAGE_EXTENSIONS = new Set([
+ ".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff", ".gif", ".webp"
+]);
+
+const JUNK_FILES = new Set([
+ ".DS_Store", "._.DS_Store", "Thumbs.db", "desktop.ini", "blob"
+]);
+
+/**
+ * Recursively flattens subdirectories in `directory`.
+ * All valid image files found in subdirectories are moved to the root of `directory`.
+ * System junk and macOS metadata files are removed.
+ * Filenames are sanitized (trimmed, spaces & '+' replaced with '_').
+ * Filename collisions are resolved by prepending relative directory names or numeric suffixes.
+ * Empty subdirectories are removed.
+ * Returns an array of relative image filenames located in root of `directory`.
+ */
+async function flattenDirectory(directory) {
+ if (!directory || !fs.existsSync(directory)) {
+ return [];
+ }
+
+ // Helper to recursively collect all file paths
+ async function getFilesRecursively(dir) {
+ let results = [];
+ try {
+ const list = await fs.promises.readdir(dir, { withFileTypes: true });
+ for (const entry of list) {
+ const fullPath = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ if (entry.name === "__MACOSX" || entry.name.startsWith("._")) {
+ await fs.promises.rm(fullPath, { recursive: true, force: true }).catch(() => {});
+ } else {
+ const subFiles = await getFilesRecursively(fullPath);
+ results = results.concat(subFiles);
+ }
+ } else if (entry.isFile()) {
+ results.push(fullPath);
+ }
+ }
+ } catch (err) {
+ // Ignore errors reading missing dirs
+ }
+ return results;
+ }
+
+ const allFiles = await getFilesRecursively(directory);
+
+ for (const filePath of allFiles) {
+ const fileName = path.basename(filePath);
+ const relPath = path.relative(directory, filePath);
+ const ext = path.extname(fileName).toLowerCase();
+
+ // Remove junk files, hidden system files, or archive files inside the target dir
+ if (
+ JUNK_FILES.has(fileName) ||
+ fileName.startsWith("._") ||
+ ext === ".zip" ||
+ ext === ".7z"
+ ) {
+ await fs.promises.unlink(filePath).catch(() => {});
+ continue;
+ }
+
+ const isImage = IMAGE_EXTENSIONS.has(ext);
+ if (!isImage) {
+ // If it's a non-image file inside a subdirectory, remove it
+ if (relPath !== fileName) {
+ await fs.promises.unlink(filePath).catch(() => {});
+ }
+ continue;
+ }
+
+ // Clean filename: remove leading/trailing spaces and replace spaces and '+' with '_'
+ let cleanName = fileName.trim().replace(/[ +]/g, "_");
+
+ const targetPath = path.join(directory, cleanName);
+
+ if (filePath === targetPath) {
+ // File is already in directory root with clean name
+ continue;
+ }
+
+ // Handle filename collisions if target file already exists and is a different file
+ let finalTargetName = cleanName;
+ let finalTargetPath = targetPath;
+
+ if (fs.existsSync(finalTargetPath) && finalTargetPath !== filePath) {
+ const relDir = path.dirname(relPath).replace(/[\\/ +]/g, "_").trim();
+ const extName = path.extname(cleanName);
+ const baseName = path.basename(cleanName, extName);
+ finalTargetName = `${relDir}_${baseName}${extName}`;
+ finalTargetPath = path.join(directory, finalTargetName);
+
+ let counter = 1;
+ while (fs.existsSync(finalTargetPath) && finalTargetPath !== filePath) {
+ finalTargetName = `${relDir}_${baseName}_${counter}${extName}`;
+ finalTargetPath = path.join(directory, finalTargetName);
+ counter++;
+ }
+ }
+
+ await fs.promises.rename(filePath, finalTargetPath).catch(() => {});
+ }
+
+ // Clean up empty subdirectories
+ async function removeEmptyDirs(dir) {
+ try {
+ const entries = await fs.promises.readdir(dir, { withFileTypes: true });
+ for (const entry of entries) {
+ if (entry.isDirectory()) {
+ const subDir = path.join(dir, entry.name);
+ await removeEmptyDirs(subDir);
+ const subEntries = await fs.promises.readdir(subDir).catch(() => []);
+ if (subEntries.length === 0) {
+ await fs.promises.rmdir(subDir).catch(() => {});
+ }
+ }
+ }
+ } catch (err) {}
+ }
+
+ await removeEmptyDirs(directory);
+
+ // Return list of image files in root directory
+ let rootEntries = [];
+ try {
+ rootEntries = await fs.promises.readdir(directory);
+ } catch (err) {
+ return [];
+ }
+
+ const resultFiles = [];
+ for (const entry of rootEntries) {
+ const full = path.join(directory, entry);
+ const stat = await fs.promises.stat(full).catch(() => null);
+ if (stat && stat.isFile()) {
+ const ext = path.extname(entry).toLowerCase();
+ if (IMAGE_EXTENSIONS.has(ext)) {
+ resultFiles.push(entry);
+ }
+ }
+ }
+
+ return resultFiles;
+}
+
+module.exports = flattenDirectory;
diff --git a/utils/parseKwCocoCsv.js b/utils/parseKwCocoCsv.js
new file mode 100644
index 00000000..b09bf5a3
--- /dev/null
+++ b/utils/parseKwCocoCsv.js
@@ -0,0 +1,141 @@
+const path = require('path');
+
+/**
+ * Parses KW COCO CSV annotation strings or file contents into structured bounding box records.
+ *
+ * Supports header-based CSVs (filename/file_name, class/category/label, xmin/x/bbox_x, ymin/y/bbox_y, xmax/w/bbox_w, ymax/h/bbox_h)
+ * and positional/VIAME CSVs (filename, class, xmin/x, ymin/y, xmax/w, ymax/h).
+ *
+ * @param {string} csvContent - Raw CSV string content
+ * @returns {Array<{filename: string, className: string, x: number, y: number, w: number, h: number}>} Parsed annotation objects
+ */
+function parseKwCocoCsv(csvContent) {
+ if (!csvContent || typeof csvContent !== 'string') {
+ return [];
+ }
+
+ const lines = csvContent
+ .split(/\r?\n/)
+ .map(line => line.trim())
+ .filter(line => line.length > 0 && !line.startsWith('#'));
+
+ if (lines.length === 0) {
+ return [];
+ }
+
+ const firstLine = lines[0];
+ const rawTokens = firstLine.split(',').map(t => t.trim());
+ const lowerTokens = rawTokens.map(t => t.toLowerCase());
+
+ const hasHeader = lowerTokens.some(t =>
+ ['filename', 'file_name', 'file', 'image', 'class', 'category', 'label', 'xmin', 'x', 'ymin', 'y', 'xmax', 'w', 'ymax', 'h', 'bbox_x'].includes(t)
+ );
+
+ let filenameIdx = 0;
+ let classIdx = 1;
+ let xIdx = 2;
+ let yIdx = 3;
+ let wOrXmaxIdx = 4;
+ let hOrYmaxIdx = 5;
+ let isWidthHeight = false;
+
+ let startLine = 0;
+
+ if (hasHeader) {
+ startLine = 1;
+
+ filenameIdx = lowerTokens.findIndex(t => ['filename', 'file_name', 'file', 'image', 'iname', 'image_name'].includes(t));
+ if (filenameIdx === -1) filenameIdx = 0;
+
+ classIdx = lowerTokens.findIndex(t => ['class', 'category', 'label', 'cname', 'class_name'].includes(t));
+ if (classIdx === -1) classIdx = 1;
+
+ xIdx = lowerTokens.findIndex(t => ['xmin', 'x', 'left_x', 'left', 'bbox_x', 'tl_x'].includes(t));
+ if (xIdx === -1) xIdx = 2;
+
+ yIdx = lowerTokens.findIndex(t => ['ymin', 'y', 'top_y', 'top', 'bbox_y', 'tl_y'].includes(t));
+ if (yIdx === -1) yIdx = 3;
+
+ const wIdx = lowerTokens.findIndex(t => ['w', 'width', 'box_w', 'bbox_w'].includes(t));
+ const xmaxIdx = lowerTokens.findIndex(t => ['xmax', 'right_x', 'right', 'br_x'].includes(t));
+
+ if (wIdx !== -1) {
+ wOrXmaxIdx = wIdx;
+ isWidthHeight = true;
+ } else if (xmaxIdx !== -1) {
+ wOrXmaxIdx = xmaxIdx;
+ isWidthHeight = false;
+ } else {
+ wOrXmaxIdx = 4;
+ }
+
+ const hIdx = lowerTokens.findIndex(t => ['h', 'height', 'box_h', 'bbox_h'].includes(t));
+ const ymaxIdx = lowerTokens.findIndex(t => ['ymax', 'bottom_y', 'bottom', 'br_y'].includes(t));
+
+ if (hIdx !== -1) {
+ hOrYmaxIdx = hIdx;
+ } else if (ymaxIdx !== -1) {
+ hOrYmaxIdx = ymaxIdx;
+ } else {
+ hOrYmaxIdx = 5;
+ }
+ }
+
+ const results = [];
+
+ for (let i = startLine; i < lines.length; i++) {
+ const row = lines[i].split(',').map(col => col.trim());
+ if (row.length <= Math.max(filenameIdx, classIdx, xIdx, yIdx, wOrXmaxIdx, hOrYmaxIdx)) {
+ continue;
+ }
+
+ const rawFilename = row[filenameIdx];
+ const rawClass = row[classIdx];
+
+ if (!rawFilename || !rawClass) {
+ continue;
+ }
+
+ const filename = path.basename(rawFilename.replace(/\\/g, '/'));
+ const className = rawClass.replace(/\s+/g, '_');
+
+ const xVal = parseFloat(row[xIdx]);
+ const yVal = parseFloat(row[yIdx]);
+ const val4 = parseFloat(row[wOrXmaxIdx]);
+ const val5 = parseFloat(row[hOrYmaxIdx]);
+
+ if (isNaN(xVal) || isNaN(yVal) || isNaN(val4) || isNaN(val5)) {
+ continue;
+ }
+
+ let x = Math.round(xVal);
+ let y = Math.round(yVal);
+ let w = 0;
+ let h = 0;
+
+ if (isWidthHeight) {
+ w = Math.round(val4);
+ h = Math.round(val5);
+ } else {
+ w = Math.round(val4 - xVal);
+ h = Math.round(val5 - yVal);
+ }
+
+ if (w <= 0 || h <= 0) {
+ continue;
+ }
+
+ results.push({
+ filename,
+ className,
+ x,
+ y,
+ w,
+ h
+ });
+ }
+
+ return results;
+}
+
+module.exports = parseKwCocoCsv;
diff --git a/utils/parseKwCocoJson.js b/utils/parseKwCocoJson.js
new file mode 100644
index 00000000..f81d0b06
--- /dev/null
+++ b/utils/parseKwCocoJson.js
@@ -0,0 +1,92 @@
+const path = require('path');
+
+/**
+ * Parses KW COCO / COCO-style JSON annotation content into the same
+ * {filename, className, x, y, w, h} shape produced by parseKwCocoCsv, so both
+ * formats can feed the same downstream label-import pipeline.
+ *
+ * Expects the standard COCO structure: {images: [...], annotations: [...], categories: [...]}
+ * with annotations referencing images/categories by id and bbox as [x, y, w, h].
+ *
+ * @param {string} jsonContent - Raw JSON string content
+ * @returns {Array<{filename: string, className: string, x: number, y: number, w: number, h: number}>} Parsed annotation objects
+ */
+function parseKwCocoJson(jsonContent) {
+ if (!jsonContent || typeof jsonContent !== 'string') {
+ return [];
+ }
+
+ let data;
+ try {
+ data = JSON.parse(jsonContent);
+ } catch (err) {
+ return [];
+ }
+
+ if (!data || typeof data !== 'object') {
+ return [];
+ }
+
+ const images = Array.isArray(data.images) ? data.images : [];
+ const annotations = Array.isArray(data.annotations) ? data.annotations : [];
+ const categories = Array.isArray(data.categories) ? data.categories : [];
+
+ if (images.length === 0 || annotations.length === 0) {
+ return [];
+ }
+
+ const imageIdToFilename = new Map();
+ for (const img of images) {
+ if (!img || img.id == null) continue;
+ const rawName = img.file_name || img.filename || img.name;
+ if (!rawName) continue;
+ imageIdToFilename.set(img.id, path.basename(String(rawName).replace(/\\/g, '/')));
+ }
+
+ const categoryIdToName = new Map();
+ for (const cat of categories) {
+ if (!cat || cat.id == null) continue;
+ const rawName = cat.name || cat.category_name;
+ if (!rawName) continue;
+ categoryIdToName.set(cat.id, String(rawName).replace(/\s+/g, '_'));
+ }
+
+ const results = [];
+
+ for (const ann of annotations) {
+ if (!ann) continue;
+
+ const filename = imageIdToFilename.get(ann.image_id);
+ if (!filename) continue;
+
+ const className = categoryIdToName.get(ann.category_id);
+ if (!className) continue;
+
+ const bbox = ann.bbox;
+ if (!Array.isArray(bbox) || bbox.length < 4) continue;
+
+ const xVal = parseFloat(bbox[0]);
+ const yVal = parseFloat(bbox[1]);
+ const wVal = parseFloat(bbox[2]);
+ const hVal = parseFloat(bbox[3]);
+
+ if (isNaN(xVal) || isNaN(yVal) || isNaN(wVal) || isNaN(hVal)) {
+ continue;
+ }
+
+ const x = Math.round(xVal);
+ const y = Math.round(yVal);
+ const w = Math.round(wVal);
+ const h = Math.round(hVal);
+
+ if (w <= 0 || h <= 0) {
+ continue;
+ }
+
+ results.push({ filename, className, x, y, w, h });
+ }
+
+ return results;
+}
+
+module.exports = parseKwCocoJson;
diff --git a/views/settings/projSettings.ejs b/views/settings/projSettings.ejs
index 8f9d1f3c..1f791f8f 100644
--- a/views/settings/projSettings.ejs
+++ b/views/settings/projSettings.ejs
@@ -93,6 +93,31 @@
+
+
+ Pre-Processing: Map KW COCO Annotations
+
+
+ Upload a `.csv` or `.json` annotation file (KW COCO format) to map bounding box labels directly to project images.
+
+
+
Delete Images Without Label