feat: add KW COCO CSV pre-processing annotation mapping

routes/api.js

@@ -59,6 +59,7 @@ const importDataset = require("./projects/importDataset");
 const importYolo = require("./projects/importYolo");
 const importKwCoco = require("./projects/importKwCoco");
 const importIfcb = require("./projects/importIfcb");
+const mapKwCocoCsv = require("./projects/mapKwCocoCsv");
 const mergeLocal = require("./projects/mergeLocal");
 const removeAccess = require("./projects/removeAccess");
 const transferAdmin = require("./projects/transferAdmin");
@@ -169,6 +170,8 @@ api.post("/import", importProject);
 api.post("/api/projects/import-dataset", importDataset);
 api.post("/api/projects/import-yolo", importYolo);
 api.post("/api/projects/import-kwcoco", importKwCoco);
+api.post("/api/projects/map-kwcoco-csv", mapKwCocoCsv);
+api.post("/mapKwCocoCsv", mapKwCocoCsv);
 api.post("/api/projects/import-ifcb", importIfcb);
 api.post("/mergeLocal", mergeLocal);
 api.post("/removeAccess", removeAccess);

routes/projects/mapKwCocoCsv.js

@@ -0,0 +1,127 @@
+const path = require('path');
+const fs = require('fs');
+const parseKwCocoCsv = require('../../utils/parseKwCocoCsv');
+const parseKwCocoJson = require('../../utils/parseKwCocoJson');
+const queries = require('../../queries/queries');
+const { Client } = require('../../queries/client');
+
+async function mapKwCocoCsv(req, res) {
+    try {
+        const projectName = req.body.PName || req.body.project_name || req.body.projectName;
+        const admin = req.body.Admin || req.cookies?.Username || 'admin';
+
+        if (!projectName) {
+            return res.status(400).json({ success: false, message: 'Project name is required.' });
+        }
+
+        if (!req.files || Object.keys(req.files).length === 0) {
+            return res.status(400).json({ success: false, message: 'No annotation file was uploaded.' });
+        }
+
+        const uploadedFile = req.files.kwcoco_csv || req.files.kwcoco_json || req.files.csv_file
+            || req.files.json_file || req.files.upload_csv || req.files.upload_json || Object.values(req.files)[0];
+        if (!uploadedFile) {
+            return res.status(400).json({ success: false, message: 'Invalid file upload payload.' });
+        }
+
+        const fileContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8');
+
+        const ext = path.extname(uploadedFile.name || '').toLowerCase();
+        const trimmedContent = fileContent.trim();
+        const isJson = ext === '.json' || (ext !== '.csv' && (trimmedContent.startsWith('{') || trimmedContent.startsWith('[')));
+
+        const parsedAnnotations = isJson ? parseKwCocoJson(fileContent) : parseKwCocoCsv(fileContent);
+        if (parsedAnnotations.length === 0) {
+            return res.status(400).json({ success: false, message: 'No valid KW COCO annotations found in file.' });
+        }
+
+        const mainPath = path.join(__dirname, '..', '..', 'public', 'projects');
+        const projectPath = path.join(mainPath, `${admin}-${projectName}`);
+
+        if (!fs.existsSync(projectPath)) {
+            return res.status(404).json({ success: false, message: `Project path not found: ${admin}-${projectName}` });
+        }
+
+        const dbPath = path.join(projectPath, `${projectName}.db`);
+        if (!global.projectDbClients[projectPath]) {
+            global.projectDbClients[projectPath] = new Client(dbPath);
+            const client = global.projectDbClients[projectPath];
+            if (typeof client.open === 'function') {
+                client.open();
+            }
+        }
+
+        // Migrate DB if needed
+        await queries.project.migrateProjectDb(projectPath);
+
+        // 1. Ensure all referenced classes exist in Classes table
+        const existingClassResult = await queries.project.getAllClasses(projectPath);
+        const existingClassRows = existingClassResult?.rows || [];
+        const existingClassSet = new Set(existingClassRows.map(c => c.CName));
+
+        const uniqueClasses = new Set(parsedAnnotations.map(a => a.className));
+        let classesAdded = 0;
+
+        for (const cname of uniqueClasses) {
+            if (!existingClassSet.has(cname)) {
+                await queries.project.createClass(projectPath, cname);
+                existingClassSet.add(cname);
+                classesAdded++;
+            }
+        }
+
+        // 2. Ensure referenced images exist in Images table
+        const existingImageResult = await queries.project.getAllImages(projectPath);
+        const existingImageRows = existingImageResult?.rows || [];
+        const existingImageSet = new Set(existingImageRows.map(i => i.IName));
+
+        const uniqueImages = new Set(parsedAnnotations.map(a => a.filename));
+        let imagesRegistered = 0;
+
+        for (const iname of uniqueImages) {
+            if (!existingImageSet.has(iname)) {
+                await queries.project.sql(projectPath, "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, 0, 0)", [iname]);
+                existingImageSet.add(iname);
+                imagesRegistered++;
+            }
+        }
+
+        // 3. Get current max LID in Labels table
+        const maxLidResult = await queries.project.getMaxLabelId(projectPath);
+        const maxLidRows = maxLidResult?.rows || [];
+        let nextLid = 1;
+        if (maxLidRows.length > 0 && maxLidRows[0].LID) {
+            nextLid = maxLidRows[0].LID + 1;
+        }
+
+        // 4. Insert labels
+        let labelsInserted = 0;
+        for (const ann of parsedAnnotations) {
+            await queries.project.createLabel(
+                projectPath,
+                nextLid++,
+                ann.className,
+                ann.x,
+                ann.y,
+                ann.w,
+                ann.h,
+                ann.filename
+            );
+            labelsInserted++;
+        }
+
+        return res.json({
+            success: true,
+            message: `Successfully mapped ${labelsInserted} KW COCO annotations.`,
+            labelsInserted,
+            classesAdded,
+            imagesRegistered
+        });
+
+    } catch (err) {
+        console.error('Error mapping KW COCO CSV annotations:', err);
+        return res.status(500).json({ success: false, message: err.message || 'Internal server error mapping KW COCO CSV.' });
+    }
+}
+
+module.exports = mapKwCocoCsv;

tests/integration/mapKwCocoCsv.test.js

@@ -0,0 +1,123 @@
+const request = require('supertest');
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const app = require('../../app');
+const queries = require('../../queries/queries');
+const { Client } = require('../../queries/client');
+
+describe('POST /api/projects/map-kwcoco-csv', () => {
+    let tmpDir;
+    let projectDir;
+    let originalProjectsPath;
+
+    beforeEach(() => {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'map-kwcoco-test-'));
+        projectDir = path.join(__dirname, '..', '..', 'public', 'projects', 'admin-testproj');
+        fs.mkdirSync(projectDir, { recursive: true });
+
+        const mockClient = {
+            open: jest.fn(),
+            all: jest.fn().mockImplementation((sql) => {
+                if (sql.includes('Classes')) return Promise.resolve({ success: true, rows: [] });
+                if (sql.includes('Images')) return Promise.resolve({ success: true, rows: [] });
+                if (sql.includes('Labels')) return Promise.resolve({ success: true, rows: [] });
+                return Promise.resolve({ success: true, rows: [] });
+            }),
+            get: jest.fn().mockResolvedValue({ success: true, row: null }),
+            run: jest.fn().mockResolvedValue({ success: true, changes: 1, lastID: 1 }),
+        };
+
+        global.projectDbClients = {
+            [projectDir]: mockClient
+        };
+    });
+
+    afterEach(() => {
+        fs.rmSync(projectDir, { recursive: true, force: true });
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+
+    test('returns 400 when project name is missing', async () => {
+        const res = await request(app)
+            .post('/api/projects/map-kwcoco-csv')
+            .set('Cookie', ['Username=admin']);
+
+        expect(res.statusCode).toBe(400);
+        expect(res.body.success).toBe(false);
+        expect(res.body.message).toMatch(/Project name is required/i);
+    });
+
+    test('returns 400 when no annotation file is uploaded', async () => {
+        const res = await request(app)
+            .post('/api/projects/map-kwcoco-csv')
+            .field('PName', 'testproj')
+            .field('Admin', 'admin');
+
+        expect(res.statusCode).toBe(400);
+        expect(res.body.success).toBe(false);
+        expect(res.body.message).toMatch(/No annotation file was uploaded/i);
+    });
+
+    test('returns 404 when project path does not exist', async () => {
+        const csvContent = 'filename,class,xmin,ymin,xmax,ymax\nimg1.jpg,dolphin,10,10,50,50';
+        const csvPath = path.join(tmpDir, 'test.csv');
+        fs.writeFileSync(csvPath, csvContent);
+
+        const res = await request(app)
+            .post('/api/projects/map-kwcoco-csv')
+            .field('PName', 'nonexistent_project')
+            .field('Admin', 'admin')
+            .attach('kwcoco_csv', csvPath);
+
+        expect(res.statusCode).toBe(404);
+        expect(res.body.success).toBe(false);
+    });
+
+    test('successfully maps KW COCO CSV annotations', async () => {
+        const csvContent = `filename,class,xmin,ymin,xmax,ymax
+img1.jpg,dolphin,10,20,100,150
+img2.jpg,shark,30,40,80,120`;
+        const csvPath = path.join(tmpDir, 'test.csv');
+        fs.writeFileSync(csvPath, csvContent);
+
+        const res = await request(app)
+            .post('/api/projects/map-kwcoco-csv')
+            .field('PName', 'testproj')
+            .field('Admin', 'admin')
+            .attach('kwcoco_csv', csvPath);
+
+        expect(res.statusCode).toBe(200);
+        expect(res.body.success).toBe(true);
+        expect(res.body.labelsInserted).toBe(2);
+    });
+
+    test('successfully maps KW COCO JSON annotations', async () => {
+        const jsonContent = JSON.stringify({
+            images: [
+                { id: 1, file_name: 'img1.jpg' },
+                { id: 2, file_name: 'img2.jpg' }
+            ],
+            annotations: [
+                { id: 1, image_id: 1, category_id: 1, bbox: [10, 20, 90, 130] },
+                { id: 2, image_id: 2, category_id: 2, bbox: [30, 40, 50, 80] }
+            ],
+            categories: [
+                { id: 1, name: 'dolphin' },
+                { id: 2, name: 'shark' }
+            ]
+        });
+        const jsonPath = path.join(tmpDir, 'test.json');
+        fs.writeFileSync(jsonPath, jsonContent);
+
+        const res = await request(app)
+            .post('/api/projects/map-kwcoco-csv')
+            .field('PName', 'testproj')
+            .field('Admin', 'admin')
+            .attach('kwcoco_json', jsonPath);
+
+        expect(res.statusCode).toBe(200);
+        expect(res.body.success).toBe(true);
+        expect(res.body.labelsInserted).toBe(2);
+    });
+});

tests/unit/parseKwCocoCsv.test.js

@@ -0,0 +1,52 @@
+const parseKwCocoCsv = require('../../utils/parseKwCocoCsv');
+
+describe('parseKwCocoCsv', () => {
+    test('returns empty array for invalid or empty input', () => {
+        expect(parseKwCocoCsv(null)).toEqual([]);
+        expect(parseKwCocoCsv('')).toEqual([]);
+        expect(parseKwCocoCsv('   ')).toEqual([]);
+    });
+
+    test('parses CSV with header containing filename, class, xmin, ymin, xmax, ymax', () => {
+        const csv = `filename,class,xmin,ymin,xmax,ymax
+image1.jpg,dolphin,10,20,100,150
+image2.png,blue whale,50,60,200,260`;
+
+        const result = parseKwCocoCsv(csv);
+        expect(result).toEqual([
+            { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 },
+            { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 }
+        ]);
+    });
+
+    test('parses CSV with header containing x, y, w, h', () => {
+        const csv = `file_name,category,x,y,w,h
+path/to/img3.jpg,sea turtle,15,25,80,95`;
+
+        const result = parseKwCocoCsv(csv);
+        expect(result).toEqual([
+            { filename: 'img3.jpg', className: 'sea_turtle', x: 15, y: 25, w: 80, h: 95 }
+        ]);
+    });
+
+    test('parses positional CSV without header', () => {
+        const csv = `img4.jpg,fish,5,10,45,60`;
+
+        const result = parseKwCocoCsv(csv);
+        expect(result).toEqual([
+            { filename: 'img4.jpg', className: 'fish', x: 5, y: 10, w: 40, h: 50 }
+        ]);
+    });
+
+    test('ignores invalid rows and negative width/height', () => {
+        const csv = `filename,class,xmin,ymin,xmax,ymax
+bad1.jpg,fish,100,100,50,50
+bad2.jpg,fish,invalid,10,20,30
+good.jpg,shark,10,10,30,30`;
+
+        const result = parseKwCocoCsv(csv);
+        expect(result).toEqual([
+            { filename: 'good.jpg', className: 'shark', x: 10, y: 10, w: 20, h: 20 }
+        ]);
+    });
+});

tests/unit/parseKwCocoJson.test.js

@@ -0,0 +1,64 @@
+const parseKwCocoJson = require('../../utils/parseKwCocoJson');
+
+describe('parseKwCocoJson', () => {
+    test('returns empty array for invalid or empty input', () => {
+        expect(parseKwCocoJson(null)).toEqual([]);
+        expect(parseKwCocoJson('')).toEqual([]);
+        expect(parseKwCocoJson('not json')).toEqual([]);
+        expect(parseKwCocoJson('{}')).toEqual([]);
+    });
+
+    test('parses standard COCO-style JSON with images/annotations/categories', () => {
+        const json = JSON.stringify({
+            images: [
+                { id: 1, file_name: 'image1.jpg' },
+                { id: 2, file_name: 'path/to/image2.png' }
+            ],
+            annotations: [
+                { id: 1, image_id: 1, category_id: 10, bbox: [10, 20, 90, 130] },
+                { id: 2, image_id: 2, category_id: 11, bbox: [50, 60, 150, 200] }
+            ],
+            categories: [
+                { id: 10, name: 'dolphin' },
+                { id: 11, name: 'blue whale' }
+            ]
+        });
+
+        const result = parseKwCocoJson(json);
+        expect(result).toEqual([
+            { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 },
+            { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 }
+        ]);
+    });
+
+    test('skips annotations referencing unknown images or categories', () => {
+        const json = JSON.stringify({
+            images: [{ id: 1, file_name: 'good.jpg' }],
+            annotations: [
+                { id: 1, image_id: 1, category_id: 5, bbox: [10, 10, 20, 20] },
+                { id: 2, image_id: 999, category_id: 5, bbox: [10, 10, 20, 20] },
+                { id: 3, image_id: 1, category_id: 999, bbox: [10, 10, 20, 20] }
+            ],
+            categories: [{ id: 5, name: 'fish' }]
+        });
+
+        const result = parseKwCocoJson(json);
+        expect(result).toEqual([
+            { filename: 'good.jpg', className: 'fish', x: 10, y: 10, w: 20, h: 20 }
+        ]);
+    });
+
+    test('ignores annotations with invalid or non-positive bbox dimensions', () => {
+        const json = JSON.stringify({
+            images: [{ id: 1, file_name: 'img.jpg' }],
+            annotations: [
+                { id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 0, 0] },
+                { id: 2, image_id: 1, category_id: 1, bbox: ['a', 'b', 'c', 'd'] },
+                { id: 3, image_id: 1, category_id: 1, bbox: [1, 2, 3] }
+            ],
+            categories: [{ id: 1, name: 'shark' }]
+        });
+
+        expect(parseKwCocoJson(json)).toEqual([]);
+    });
+});