From ceadbe6e1ee8356d609db3e56ee8dc371c7889ec Mon Sep 17 00:00:00 2001 From: FerretCode Date: Mon, 17 Aug 2026 16:15:27 -0700 Subject: [PATCH 1/3] feat: add KW COCO CSV pre-processing annotation mapping in project settings Co-authored-by: multica-agent --- routes/api.js | 3 + routes/projects/mapKwCocoCsv.js | 118 +++++++++++++++++++++ tests/integration/mapKwCocoCsv.test.js | 94 +++++++++++++++++ tests/unit/parseKwCocoCsv.test.js | 52 +++++++++ utils/parseKwCocoCsv.js | 141 +++++++++++++++++++++++++ views/settings/projSettings.ejs | 60 +++++++++++ 6 files changed, 468 insertions(+) create mode 100644 routes/projects/mapKwCocoCsv.js create mode 100644 tests/integration/mapKwCocoCsv.test.js create mode 100644 tests/unit/parseKwCocoCsv.test.js create mode 100644 utils/parseKwCocoCsv.js diff --git a/routes/api.js b/routes/api.js index 111e6915..72f9bd42 100755 --- a/routes/api.js +++ b/routes/api.js @@ -59,6 +59,7 @@ const importDataset = require("./projects/importDataset"); const importYolo = require("./projects/importYolo"); const importKwCoco = require("./projects/importKwCoco"); const importIfcb = require("./projects/importIfcb"); +const mapKwCocoCsv = require("./projects/mapKwCocoCsv"); const mergeLocal = require("./projects/mergeLocal"); const removeAccess = require("./projects/removeAccess"); const transferAdmin = require("./projects/transferAdmin"); @@ -169,6 +170,8 @@ api.post("/import", importProject); api.post("/api/projects/import-dataset", importDataset); api.post("/api/projects/import-yolo", importYolo); api.post("/api/projects/import-kwcoco", importKwCoco); +api.post("/api/projects/map-kwcoco-csv", mapKwCocoCsv); +api.post("/mapKwCocoCsv", mapKwCocoCsv); api.post("/api/projects/import-ifcb", importIfcb); api.post("/mergeLocal", mergeLocal); api.post("/removeAccess", removeAccess); diff --git a/routes/projects/mapKwCocoCsv.js b/routes/projects/mapKwCocoCsv.js new file mode 100644 index 00000000..3e28a447 --- /dev/null +++ b/routes/projects/mapKwCocoCsv.js @@ -0,0 +1,118 @@ +const path = require('path'); +const fs = require('fs'); +const parseKwCocoCsv = require('../../utils/parseKwCocoCsv'); +const queries = require('../../queries/queries'); +const { Client } = require('../../queries/client'); + +async function mapKwCocoCsv(req, res) { + try { + const projectName = req.body.PName || req.body.project_name || req.body.projectName; + const admin = req.body.Admin || req.cookies?.Username || 'admin'; + + if (!projectName) { + return res.status(400).json({ success: false, message: 'Project name is required.' }); + } + + if (!req.files || Object.keys(req.files).length === 0) { + return res.status(400).json({ success: false, message: 'No CSV file was uploaded.' }); + } + + const uploadedFile = req.files.kwcoco_csv || req.files.csv_file || req.files.upload_csv || Object.values(req.files)[0]; + if (!uploadedFile) { + return res.status(400).json({ success: false, message: 'Invalid file upload payload.' }); + } + + const csvContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8'); + + const parsedAnnotations = parseKwCocoCsv(csvContent); + if (parsedAnnotations.length === 0) { + return res.status(400).json({ success: false, message: 'No valid KW COCO CSV annotations found in file.' }); + } + + const mainPath = path.join(__dirname, '..', '..', 'public', 'projects'); + const projectPath = path.join(mainPath, `${admin}-${projectName}`); + + if (!fs.existsSync(projectPath)) { + return res.status(404).json({ success: false, message: `Project path not found: ${admin}-${projectName}` }); + } + + const dbPath = path.join(projectPath, `${projectName}.db`); + if (!global.projectDbClients[projectPath]) { + global.projectDbClients[projectPath] = new Client(dbPath); + const client = global.projectDbClients[projectPath]; + if (typeof client.open === 'function') { + client.open(); + } + } + + // Migrate DB if needed + await queries.project.migrateProjectDb(projectPath); + + // 1. Ensure all referenced classes exist in Classes table + const existingClassRows = await queries.project.getAllClasses(projectPath) || []; + const existingClassSet = new Set(existingClassRows.map(c => c.CName)); + + const uniqueClasses = new Set(parsedAnnotations.map(a => a.className)); + let classesAdded = 0; + + for (const cname of uniqueClasses) { + if (!existingClassSet.has(cname)) { + await queries.project.createClass(projectPath, cname); + existingClassSet.add(cname); + classesAdded++; + } + } + + // 2. Ensure referenced images exist in Images table + const existingImageRows = await queries.project.getAllImages(projectPath) || []; + const existingImageSet = new Set(existingImageRows.map(i => i.IName)); + + const uniqueImages = new Set(parsedAnnotations.map(a => a.filename)); + let imagesRegistered = 0; + + for (const iname of uniqueImages) { + if (!existingImageSet.has(iname)) { + await queries.project.sql(projectPath, "INSERT OR IGNORE INTO Images (IName, reviewImage, validateImage) VALUES (?, 0, 0)", [iname]); + existingImageSet.add(iname); + imagesRegistered++; + } + } + + // 3. Get current max LID in Labels table + const maxLidResult = await queries.project.getMaxLabelId(projectPath); + let nextLid = 1; + if (maxLidResult && maxLidResult.length > 0 && maxLidResult[0].LID) { + nextLid = maxLidResult[0].LID + 1; + } + + // 4. Insert labels + let labelsInserted = 0; + for (const ann of parsedAnnotations) { + await queries.project.createLabel( + projectPath, + nextLid++, + ann.className, + ann.x, + ann.y, + ann.w, + ann.h, + ann.filename + ); + labelsInserted++; + } + + return res.json({ + success: true, + message: `Successfully mapped ${labelsInserted} KW COCO annotations.`, + labelsInserted, + classesAdded, + imagesRegistered + }); + + } catch (err) { + console.error('Error mapping KW COCO CSV annotations:', err); + return res.status(500).json({ success: false, message: err.message || 'Internal server error mapping KW COCO CSV.' }); + } +} + +module.exports = mapKwCocoCsv; diff --git a/tests/integration/mapKwCocoCsv.test.js b/tests/integration/mapKwCocoCsv.test.js new file mode 100644 index 00000000..fdf16f2a --- /dev/null +++ b/tests/integration/mapKwCocoCsv.test.js @@ -0,0 +1,94 @@ +const request = require('supertest'); +const fs = require('fs'); +const path = require('path'); +const os = require('os'); +const app = require('../../app'); +const queries = require('../../queries/queries'); +const { Client } = require('../../queries/client'); + +describe('POST /api/projects/map-kwcoco-csv', () => { + let tmpDir; + let projectDir; + let originalProjectsPath; + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'map-kwcoco-test-')); + projectDir = path.join(__dirname, '..', '..', 'public', 'projects', 'admin-testproj'); + fs.mkdirSync(projectDir, { recursive: true }); + + const mockClient = { + open: jest.fn(), + all: jest.fn().mockImplementation((sql) => { + if (sql.includes('Classes')) return Promise.resolve([]); + if (sql.includes('Images')) return Promise.resolve([]); + if (sql.includes('Labels')) return Promise.resolve([]); + return Promise.resolve([]); + }), + get: jest.fn().mockResolvedValue(null), + run: jest.fn().mockResolvedValue({ changes: 1 }), + }; + + global.projectDbClients = { + [projectDir]: mockClient + }; + }); + + afterEach(() => { + fs.rmSync(projectDir, { recursive: true, force: true }); + fs.rmSync(tmpDir, { recursive: true, force: true }); + }); + + test('returns 400 when project name is missing', async () => { + const res = await request(app) + .post('/api/projects/map-kwcoco-csv') + .set('Cookie', ['Username=admin']); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + expect(res.body.message).toMatch(/Project name is required/i); + }); + + test('returns 400 when no CSV file is uploaded', async () => { + const res = await request(app) + .post('/api/projects/map-kwcoco-csv') + .field('PName', 'testproj') + .field('Admin', 'admin'); + + expect(res.statusCode).toBe(400); + expect(res.body.success).toBe(false); + expect(res.body.message).toMatch(/No CSV file was uploaded/i); + }); + + test('returns 404 when project path does not exist', async () => { + const csvContent = 'filename,class,xmin,ymin,xmax,ymax\nimg1.jpg,dolphin,10,10,50,50'; + const csvPath = path.join(tmpDir, 'test.csv'); + fs.writeFileSync(csvPath, csvContent); + + const res = await request(app) + .post('/api/projects/map-kwcoco-csv') + .field('PName', 'nonexistent_project') + .field('Admin', 'admin') + .attach('kwcoco_csv', csvPath); + + expect(res.statusCode).toBe(404); + expect(res.body.success).toBe(false); + }); + + test('successfully maps KW COCO CSV annotations', async () => { + const csvContent = `filename,class,xmin,ymin,xmax,ymax +img1.jpg,dolphin,10,20,100,150 +img2.jpg,shark,30,40,80,120`; + const csvPath = path.join(tmpDir, 'test.csv'); + fs.writeFileSync(csvPath, csvContent); + + const res = await request(app) + .post('/api/projects/map-kwcoco-csv') + .field('PName', 'testproj') + .field('Admin', 'admin') + .attach('kwcoco_csv', csvPath); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.labelsInserted).toBe(2); + }); +}); diff --git a/tests/unit/parseKwCocoCsv.test.js b/tests/unit/parseKwCocoCsv.test.js new file mode 100644 index 00000000..3f91e614 --- /dev/null +++ b/tests/unit/parseKwCocoCsv.test.js @@ -0,0 +1,52 @@ +const parseKwCocoCsv = require('../../utils/parseKwCocoCsv'); + +describe('parseKwCocoCsv', () => { + test('returns empty array for invalid or empty input', () => { + expect(parseKwCocoCsv(null)).toEqual([]); + expect(parseKwCocoCsv('')).toEqual([]); + expect(parseKwCocoCsv(' ')).toEqual([]); + }); + + test('parses CSV with header containing filename, class, xmin, ymin, xmax, ymax', () => { + const csv = `filename,class,xmin,ymin,xmax,ymax +image1.jpg,dolphin,10,20,100,150 +image2.png,blue whale,50,60,200,260`; + + const result = parseKwCocoCsv(csv); + expect(result).toEqual([ + { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 }, + { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 } + ]); + }); + + test('parses CSV with header containing x, y, w, h', () => { + const csv = `file_name,category,x,y,w,h +path/to/img3.jpg,sea turtle,15,25,80,95`; + + const result = parseKwCocoCsv(csv); + expect(result).toEqual([ + { filename: 'img3.jpg', className: 'sea_turtle', x: 15, y: 25, w: 80, h: 95 } + ]); + }); + + test('parses positional CSV without header', () => { + const csv = `img4.jpg,fish,5,10,45,60`; + + const result = parseKwCocoCsv(csv); + expect(result).toEqual([ + { filename: 'img4.jpg', className: 'fish', x: 5, y: 10, w: 40, h: 50 } + ]); + }); + + test('ignores invalid rows and negative width/height', () => { + const csv = `filename,class,xmin,ymin,xmax,ymax +bad1.jpg,fish,100,100,50,50 +bad2.jpg,fish,invalid,10,20,30 +good.jpg,shark,10,10,30,30`; + + const result = parseKwCocoCsv(csv); + expect(result).toEqual([ + { filename: 'good.jpg', className: 'shark', x: 10, y: 10, w: 20, h: 20 } + ]); + }); +}); diff --git a/utils/parseKwCocoCsv.js b/utils/parseKwCocoCsv.js new file mode 100644 index 00000000..b09bf5a3 --- /dev/null +++ b/utils/parseKwCocoCsv.js @@ -0,0 +1,141 @@ +const path = require('path'); + +/** + * Parses KW COCO CSV annotation strings or file contents into structured bounding box records. + * + * Supports header-based CSVs (filename/file_name, class/category/label, xmin/x/bbox_x, ymin/y/bbox_y, xmax/w/bbox_w, ymax/h/bbox_h) + * and positional/VIAME CSVs (filename, class, xmin/x, ymin/y, xmax/w, ymax/h). + * + * @param {string} csvContent - Raw CSV string content + * @returns {Array<{filename: string, className: string, x: number, y: number, w: number, h: number}>} Parsed annotation objects + */ +function parseKwCocoCsv(csvContent) { + if (!csvContent || typeof csvContent !== 'string') { + return []; + } + + const lines = csvContent + .split(/\r?\n/) + .map(line => line.trim()) + .filter(line => line.length > 0 && !line.startsWith('#')); + + if (lines.length === 0) { + return []; + } + + const firstLine = lines[0]; + const rawTokens = firstLine.split(',').map(t => t.trim()); + const lowerTokens = rawTokens.map(t => t.toLowerCase()); + + const hasHeader = lowerTokens.some(t => + ['filename', 'file_name', 'file', 'image', 'class', 'category', 'label', 'xmin', 'x', 'ymin', 'y', 'xmax', 'w', 'ymax', 'h', 'bbox_x'].includes(t) + ); + + let filenameIdx = 0; + let classIdx = 1; + let xIdx = 2; + let yIdx = 3; + let wOrXmaxIdx = 4; + let hOrYmaxIdx = 5; + let isWidthHeight = false; + + let startLine = 0; + + if (hasHeader) { + startLine = 1; + + filenameIdx = lowerTokens.findIndex(t => ['filename', 'file_name', 'file', 'image', 'iname', 'image_name'].includes(t)); + if (filenameIdx === -1) filenameIdx = 0; + + classIdx = lowerTokens.findIndex(t => ['class', 'category', 'label', 'cname', 'class_name'].includes(t)); + if (classIdx === -1) classIdx = 1; + + xIdx = lowerTokens.findIndex(t => ['xmin', 'x', 'left_x', 'left', 'bbox_x', 'tl_x'].includes(t)); + if (xIdx === -1) xIdx = 2; + + yIdx = lowerTokens.findIndex(t => ['ymin', 'y', 'top_y', 'top', 'bbox_y', 'tl_y'].includes(t)); + if (yIdx === -1) yIdx = 3; + + const wIdx = lowerTokens.findIndex(t => ['w', 'width', 'box_w', 'bbox_w'].includes(t)); + const xmaxIdx = lowerTokens.findIndex(t => ['xmax', 'right_x', 'right', 'br_x'].includes(t)); + + if (wIdx !== -1) { + wOrXmaxIdx = wIdx; + isWidthHeight = true; + } else if (xmaxIdx !== -1) { + wOrXmaxIdx = xmaxIdx; + isWidthHeight = false; + } else { + wOrXmaxIdx = 4; + } + + const hIdx = lowerTokens.findIndex(t => ['h', 'height', 'box_h', 'bbox_h'].includes(t)); + const ymaxIdx = lowerTokens.findIndex(t => ['ymax', 'bottom_y', 'bottom', 'br_y'].includes(t)); + + if (hIdx !== -1) { + hOrYmaxIdx = hIdx; + } else if (ymaxIdx !== -1) { + hOrYmaxIdx = ymaxIdx; + } else { + hOrYmaxIdx = 5; + } + } + + const results = []; + + for (let i = startLine; i < lines.length; i++) { + const row = lines[i].split(',').map(col => col.trim()); + if (row.length <= Math.max(filenameIdx, classIdx, xIdx, yIdx, wOrXmaxIdx, hOrYmaxIdx)) { + continue; + } + + const rawFilename = row[filenameIdx]; + const rawClass = row[classIdx]; + + if (!rawFilename || !rawClass) { + continue; + } + + const filename = path.basename(rawFilename.replace(/\\/g, '/')); + const className = rawClass.replace(/\s+/g, '_'); + + const xVal = parseFloat(row[xIdx]); + const yVal = parseFloat(row[yIdx]); + const val4 = parseFloat(row[wOrXmaxIdx]); + const val5 = parseFloat(row[hOrYmaxIdx]); + + if (isNaN(xVal) || isNaN(yVal) || isNaN(val4) || isNaN(val5)) { + continue; + } + + let x = Math.round(xVal); + let y = Math.round(yVal); + let w = 0; + let h = 0; + + if (isWidthHeight) { + w = Math.round(val4); + h = Math.round(val5); + } else { + w = Math.round(val4 - xVal); + h = Math.round(val5 - yVal); + } + + if (w <= 0 || h <= 0) { + continue; + } + + results.push({ + filename, + className, + x, + y, + w, + h + }); + } + + return results; +} + +module.exports = parseKwCocoCsv; diff --git a/views/settings/projSettings.ejs b/views/settings/projSettings.ejs index 8f9d1f3c..9fed6207 100644 --- a/views/settings/projSettings.ejs +++ b/views/settings/projSettings.ejs @@ -93,6 +93,31 @@
+
+

+ Pre-Processing: Map KW COCO CSV Annotations +

+

+ Upload a `.csv` annotation file (KW COCO format) to map bounding box labels directly to project images. +

+
+ + + +
+ + +
+
+ <% if(user == Admin) { %> + + <% } else { %> + Save + <% } %> +
+
+
+

Delete Images Without Label

@@ -237,4 +262,39 @@ request.send(formData); console.log(formData); }); + + if (document.forms['mapKwCocoCsvForm']) { + document.forms['mapKwCocoCsvForm'].addEventListener('submit', (event) => { + event.preventDefault(); + var formData = new FormData(); + const csvFileInput = document.getElementById('kwcoco_csv'); + if (!csvFileInput || !csvFileInput.files[0]) { + alert("Please select a .csv file to upload."); + return; + } + formData.append('kwcoco_csv', csvFileInput.files[0]); + formData.append("PName", "<%= PName %>"); + formData.append("Admin", "<%= Admin %>"); + formData.append("IDX", "<%= IDX %>"); + + var request = new XMLHttpRequest(); + request.open("post", event.target.action, true); + request.onreadystatechange = function receiveResponse() { + if (this.readyState == 4) { + if (this.status == 200) { + try { + var res = JSON.parse(this.response); + alert(res.message || "Annotations successfully mapped!"); + } catch (e) { + alert("Annotations mapped successfully."); + } + window.location.reload(); + } else { + alert("Failed to map annotations: " + this.response); + } + } + } + request.send(formData); + }); + } From 5a5cff03137d16aa74675ab2ea8ff946910cf8a0 Mon Sep 17 00:00:00 2001 From: FerretCode Date: Mon, 17 Aug 2026 16:38:26 -0700 Subject: [PATCH 2/3] feat: support KW COCO JSON alongside CSV in annotation pre-processing Adds a JSON parser (images/annotations/categories -> bbox records) and dispatches by file extension/content sniffing in the existing map-kwcoco-csv route so uploads of either .csv or .json are handled by the same endpoint. Co-authored-by: multica-agent --- routes/projects/mapKwCocoCsv.js | 16 +++-- tests/integration/mapKwCocoCsv.test.js | 33 ++++++++- tests/unit/parseKwCocoJson.test.js | 64 ++++++++++++++++++ utils/parseKwCocoJson.js | 92 ++++++++++++++++++++++++++ views/settings/projSettings.ejs | 8 +-- 5 files changed, 202 insertions(+), 11 deletions(-) create mode 100644 tests/unit/parseKwCocoJson.test.js create mode 100644 utils/parseKwCocoJson.js diff --git a/routes/projects/mapKwCocoCsv.js b/routes/projects/mapKwCocoCsv.js index 3e28a447..929d7996 100644 --- a/routes/projects/mapKwCocoCsv.js +++ b/routes/projects/mapKwCocoCsv.js @@ -1,6 +1,7 @@ const path = require('path'); const fs = require('fs'); const parseKwCocoCsv = require('../../utils/parseKwCocoCsv'); +const parseKwCocoJson = require('../../utils/parseKwCocoJson'); const queries = require('../../queries/queries'); const { Client } = require('../../queries/client'); @@ -14,19 +15,24 @@ async function mapKwCocoCsv(req, res) { } if (!req.files || Object.keys(req.files).length === 0) { - return res.status(400).json({ success: false, message: 'No CSV file was uploaded.' }); + return res.status(400).json({ success: false, message: 'No annotation file was uploaded.' }); } - const uploadedFile = req.files.kwcoco_csv || req.files.csv_file || req.files.upload_csv || Object.values(req.files)[0]; + const uploadedFile = req.files.kwcoco_csv || req.files.kwcoco_json || req.files.csv_file + || req.files.json_file || req.files.upload_csv || req.files.upload_json || Object.values(req.files)[0]; if (!uploadedFile) { return res.status(400).json({ success: false, message: 'Invalid file upload payload.' }); } - const csvContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8'); + const fileContent = uploadedFile.data ? uploadedFile.data.toString('utf8') : fs.readFileSync(uploadedFile.tempFilePath, 'utf8'); - const parsedAnnotations = parseKwCocoCsv(csvContent); + const ext = path.extname(uploadedFile.name || '').toLowerCase(); + const trimmedContent = fileContent.trim(); + const isJson = ext === '.json' || (ext !== '.csv' && (trimmedContent.startsWith('{') || trimmedContent.startsWith('['))); + + const parsedAnnotations = isJson ? parseKwCocoJson(fileContent) : parseKwCocoCsv(fileContent); if (parsedAnnotations.length === 0) { - return res.status(400).json({ success: false, message: 'No valid KW COCO CSV annotations found in file.' }); + return res.status(400).json({ success: false, message: 'No valid KW COCO annotations found in file.' }); } const mainPath = path.join(__dirname, '..', '..', 'public', 'projects'); diff --git a/tests/integration/mapKwCocoCsv.test.js b/tests/integration/mapKwCocoCsv.test.js index fdf16f2a..c592f842 100644 --- a/tests/integration/mapKwCocoCsv.test.js +++ b/tests/integration/mapKwCocoCsv.test.js @@ -48,7 +48,7 @@ describe('POST /api/projects/map-kwcoco-csv', () => { expect(res.body.message).toMatch(/Project name is required/i); }); - test('returns 400 when no CSV file is uploaded', async () => { + test('returns 400 when no annotation file is uploaded', async () => { const res = await request(app) .post('/api/projects/map-kwcoco-csv') .field('PName', 'testproj') @@ -56,7 +56,7 @@ describe('POST /api/projects/map-kwcoco-csv', () => { expect(res.statusCode).toBe(400); expect(res.body.success).toBe(false); - expect(res.body.message).toMatch(/No CSV file was uploaded/i); + expect(res.body.message).toMatch(/No annotation file was uploaded/i); }); test('returns 404 when project path does not exist', async () => { @@ -91,4 +91,33 @@ img2.jpg,shark,30,40,80,120`; expect(res.body.success).toBe(true); expect(res.body.labelsInserted).toBe(2); }); + + test('successfully maps KW COCO JSON annotations', async () => { + const jsonContent = JSON.stringify({ + images: [ + { id: 1, file_name: 'img1.jpg' }, + { id: 2, file_name: 'img2.jpg' } + ], + annotations: [ + { id: 1, image_id: 1, category_id: 1, bbox: [10, 20, 90, 130] }, + { id: 2, image_id: 2, category_id: 2, bbox: [30, 40, 50, 80] } + ], + categories: [ + { id: 1, name: 'dolphin' }, + { id: 2, name: 'shark' } + ] + }); + const jsonPath = path.join(tmpDir, 'test.json'); + fs.writeFileSync(jsonPath, jsonContent); + + const res = await request(app) + .post('/api/projects/map-kwcoco-csv') + .field('PName', 'testproj') + .field('Admin', 'admin') + .attach('kwcoco_json', jsonPath); + + expect(res.statusCode).toBe(200); + expect(res.body.success).toBe(true); + expect(res.body.labelsInserted).toBe(2); + }); }); diff --git a/tests/unit/parseKwCocoJson.test.js b/tests/unit/parseKwCocoJson.test.js new file mode 100644 index 00000000..cbb6b0de --- /dev/null +++ b/tests/unit/parseKwCocoJson.test.js @@ -0,0 +1,64 @@ +const parseKwCocoJson = require('../../utils/parseKwCocoJson'); + +describe('parseKwCocoJson', () => { + test('returns empty array for invalid or empty input', () => { + expect(parseKwCocoJson(null)).toEqual([]); + expect(parseKwCocoJson('')).toEqual([]); + expect(parseKwCocoJson('not json')).toEqual([]); + expect(parseKwCocoJson('{}')).toEqual([]); + }); + + test('parses standard COCO-style JSON with images/annotations/categories', () => { + const json = JSON.stringify({ + images: [ + { id: 1, file_name: 'image1.jpg' }, + { id: 2, file_name: 'path/to/image2.png' } + ], + annotations: [ + { id: 1, image_id: 1, category_id: 10, bbox: [10, 20, 90, 130] }, + { id: 2, image_id: 2, category_id: 11, bbox: [50, 60, 150, 200] } + ], + categories: [ + { id: 10, name: 'dolphin' }, + { id: 11, name: 'blue whale' } + ] + }); + + const result = parseKwCocoJson(json); + expect(result).toEqual([ + { filename: 'image1.jpg', className: 'dolphin', x: 10, y: 20, w: 90, h: 130 }, + { filename: 'image2.png', className: 'blue_whale', x: 50, y: 60, w: 150, h: 200 } + ]); + }); + + test('skips annotations referencing unknown images or categories', () => { + const json = JSON.stringify({ + images: [{ id: 1, file_name: 'good.jpg' }], + annotations: [ + { id: 1, image_id: 1, category_id: 5, bbox: [10, 10, 20, 20] }, + { id: 2, image_id: 999, category_id: 5, bbox: [10, 10, 20, 20] }, + { id: 3, image_id: 1, category_id: 999, bbox: [10, 10, 20, 20] } + ], + categories: [{ id: 5, name: 'fish' }] + }); + + const result = parseKwCocoJson(json); + expect(result).toEqual([ + { filename: 'good.jpg', className: 'fish', x: 10, y: 10, w: 20, h: 20 } + ]); + }); + + test('ignores annotations with invalid or non-positive bbox dimensions', () => { + const json = JSON.stringify({ + images: [{ id: 1, file_name: 'img.jpg' }], + annotations: [ + { id: 1, image_id: 1, category_id: 1, bbox: [0, 0, 0, 0] }, + { id: 2, image_id: 1, category_id: 1, bbox: ['a', 'b', 'c', 'd'] }, + { id: 3, image_id: 1, category_id: 1, bbox: [1, 2, 3] } + ], + categories: [{ id: 1, name: 'shark' }] + }); + + expect(parseKwCocoJson(json)).toEqual([]); + }); +}); diff --git a/utils/parseKwCocoJson.js b/utils/parseKwCocoJson.js new file mode 100644 index 00000000..f81d0b06 --- /dev/null +++ b/utils/parseKwCocoJson.js @@ -0,0 +1,92 @@ +const path = require('path'); + +/** + * Parses KW COCO / COCO-style JSON annotation content into the same + * {filename, className, x, y, w, h} shape produced by parseKwCocoCsv, so both + * formats can feed the same downstream label-import pipeline. + * + * Expects the standard COCO structure: {images: [...], annotations: [...], categories: [...]} + * with annotations referencing images/categories by id and bbox as [x, y, w, h]. + * + * @param {string} jsonContent - Raw JSON string content + * @returns {Array<{filename: string, className: string, x: number, y: number, w: number, h: number}>} Parsed annotation objects + */ +function parseKwCocoJson(jsonContent) { + if (!jsonContent || typeof jsonContent !== 'string') { + return []; + } + + let data; + try { + data = JSON.parse(jsonContent); + } catch (err) { + return []; + } + + if (!data || typeof data !== 'object') { + return []; + } + + const images = Array.isArray(data.images) ? data.images : []; + const annotations = Array.isArray(data.annotations) ? data.annotations : []; + const categories = Array.isArray(data.categories) ? data.categories : []; + + if (images.length === 0 || annotations.length === 0) { + return []; + } + + const imageIdToFilename = new Map(); + for (const img of images) { + if (!img || img.id == null) continue; + const rawName = img.file_name || img.filename || img.name; + if (!rawName) continue; + imageIdToFilename.set(img.id, path.basename(String(rawName).replace(/\\/g, '/'))); + } + + const categoryIdToName = new Map(); + for (const cat of categories) { + if (!cat || cat.id == null) continue; + const rawName = cat.name || cat.category_name; + if (!rawName) continue; + categoryIdToName.set(cat.id, String(rawName).replace(/\s+/g, '_')); + } + + const results = []; + + for (const ann of annotations) { + if (!ann) continue; + + const filename = imageIdToFilename.get(ann.image_id); + if (!filename) continue; + + const className = categoryIdToName.get(ann.category_id); + if (!className) continue; + + const bbox = ann.bbox; + if (!Array.isArray(bbox) || bbox.length < 4) continue; + + const xVal = parseFloat(bbox[0]); + const yVal = parseFloat(bbox[1]); + const wVal = parseFloat(bbox[2]); + const hVal = parseFloat(bbox[3]); + + if (isNaN(xVal) || isNaN(yVal) || isNaN(wVal) || isNaN(hVal)) { + continue; + } + + const x = Math.round(xVal); + const y = Math.round(yVal); + const w = Math.round(wVal); + const h = Math.round(hVal); + + if (w <= 0 || h <= 0) { + continue; + } + + results.push({ filename, className, x, y, w, h }); + } + + return results; +} + +module.exports = parseKwCocoJson; diff --git a/views/settings/projSettings.ejs b/views/settings/projSettings.ejs index 9fed6207..1f791f8f 100644 --- a/views/settings/projSettings.ejs +++ b/views/settings/projSettings.ejs @@ -95,18 +95,18 @@

- Pre-Processing: Map KW COCO CSV Annotations + Pre-Processing: Map KW COCO Annotations

- Upload a `.csv` annotation file (KW COCO format) to map bounding box labels directly to project images. + Upload a `.csv` or `.json` annotation file (KW COCO format) to map bounding box labels directly to project images.

- - + +
<% if(user == Admin) { %> From 2c6a523e9799e6dd8dd2540f62acef56af241f58 Mon Sep 17 00:00:00 2001 From: FerretCode Date: Mon, 17 Aug 2026 16:54:58 -0700 Subject: [PATCH 3/3] fix: use .rows when reading Client.all() results in mapKwCocoCsv getAllClasses/getAllImages/getMaxLabelId resolve to {success, rows} (matching queries/client.js's db.all()/db.get() contract used elsewhere, e.g. routes/inference/yoloInference.js), not a bare array, so existingClassRows.map threw at runtime. The integration test mock returned plain arrays instead of the real {rows: [...]} shape, which is why 14/14 passed despite the bug; the mock now matches the real client contract. Co-authored-by: multica-agent --- routes/projects/mapKwCocoCsv.js | 11 +++++++---- tests/integration/mapKwCocoCsv.test.js | 12 ++++++------ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/routes/projects/mapKwCocoCsv.js b/routes/projects/mapKwCocoCsv.js index 929d7996..6f09b827 100644 --- a/routes/projects/mapKwCocoCsv.js +++ b/routes/projects/mapKwCocoCsv.js @@ -55,7 +55,8 @@ async function mapKwCocoCsv(req, res) { await queries.project.migrateProjectDb(projectPath); // 1. Ensure all referenced classes exist in Classes table - const existingClassRows = await queries.project.getAllClasses(projectPath) || []; + const existingClassResult = await queries.project.getAllClasses(projectPath); + const existingClassRows = existingClassResult?.rows || []; const existingClassSet = new Set(existingClassRows.map(c => c.CName)); const uniqueClasses = new Set(parsedAnnotations.map(a => a.className)); @@ -70,7 +71,8 @@ async function mapKwCocoCsv(req, res) { } // 2. Ensure referenced images exist in Images table - const existingImageRows = await queries.project.getAllImages(projectPath) || []; + const existingImageResult = await queries.project.getAllImages(projectPath); + const existingImageRows = existingImageResult?.rows || []; const existingImageSet = new Set(existingImageRows.map(i => i.IName)); const uniqueImages = new Set(parsedAnnotations.map(a => a.filename)); @@ -86,9 +88,10 @@ async function mapKwCocoCsv(req, res) { // 3. Get current max LID in Labels table const maxLidResult = await queries.project.getMaxLabelId(projectPath); + const maxLidRows = maxLidResult?.rows || []; let nextLid = 1; - if (maxLidResult && maxLidResult.length > 0 && maxLidResult[0].LID) { - nextLid = maxLidResult[0].LID + 1; + if (maxLidRows.length > 0 && maxLidRows[0].LID) { + nextLid = maxLidRows[0].LID + 1; } // 4. Insert labels diff --git a/tests/integration/mapKwCocoCsv.test.js b/tests/integration/mapKwCocoCsv.test.js index c592f842..5005498f 100644 --- a/tests/integration/mapKwCocoCsv.test.js +++ b/tests/integration/mapKwCocoCsv.test.js @@ -19,13 +19,13 @@ describe('POST /api/projects/map-kwcoco-csv', () => { const mockClient = { open: jest.fn(), all: jest.fn().mockImplementation((sql) => { - if (sql.includes('Classes')) return Promise.resolve([]); - if (sql.includes('Images')) return Promise.resolve([]); - if (sql.includes('Labels')) return Promise.resolve([]); - return Promise.resolve([]); + if (sql.includes('Classes')) return Promise.resolve({ success: true, rows: [] }); + if (sql.includes('Images')) return Promise.resolve({ success: true, rows: [] }); + if (sql.includes('Labels')) return Promise.resolve({ success: true, rows: [] }); + return Promise.resolve({ success: true, rows: [] }); }), - get: jest.fn().mockResolvedValue(null), - run: jest.fn().mockResolvedValue({ changes: 1 }), + get: jest.fn().mockResolvedValue({ success: true, row: null }), + run: jest.fn().mockResolvedValue({ success: true, changes: 1, lastID: 1 }), }; global.projectDbClients = {