diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 5f768ce..bdd7021 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -17,6 +17,9 @@ "@tanstack/react-query": "^5.84.1", "@tanstack/react-query-devtools": "^5.84.1", "@tanstack/react-router": "1.19.1", + "@tensorflow-models/coco-ssd": "^2.2.2", + "@tensorflow-models/deeplab": "^0.2.2", + "@tensorflow/tfjs": "^3.21.0", "@types/leaflet": "^1.9.20", "@types/react-grid-layout": "^1.3.5", "@types/uuid": "^10.0.0", @@ -2552,6 +2555,169 @@ "url": "https://github.com/sponsors/tannerlinsley" } }, + "node_modules/@tensorflow-models/coco-ssd": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@tensorflow-models/coco-ssd/-/coco-ssd-2.2.2.tgz", + "integrity": "sha512-Jey2JscmKEValcFZH2ZLz14s8KPRmVtfJ0d0M3dPhvBp9dJiGNanVXr/pJAY5OS7emKj9uSciGhdkHWXY9Hovw==", + "license": "Apache-2.0", + "peerDependencies": { + "@tensorflow/tfjs-converter": "^3.3.0", + "@tensorflow/tfjs-core": "^3.3.0" + } + }, + "node_modules/@tensorflow-models/deeplab": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/@tensorflow-models/deeplab/-/deeplab-0.2.2.tgz", + "integrity": "sha512-tUB16D0fMbVuGR3JF7CXFseMxwnhQ+dtmAmZezT98ask7lVetxhJ5bcsKk/KGAQHcnSgJxZ2S4Cyt2xsrfE9NQ==", + "license": "Apache-2.0", + "peerDependencies": { + "@tensorflow/tfjs-converter": "^3.0.0", + "@tensorflow/tfjs-core": "^3.0.0" + } + }, + "node_modules/@tensorflow/tfjs": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs/-/tfjs-3.21.0.tgz", + "integrity": "sha512-khcARd3/872llL/oF4ouR40qlT71mylU66PGT8kHP/GJ5YKj44sv8lDRjU7lOVlJK7jsJFWEsNVHI3eMc/GWNQ==", + "license": "Apache-2.0", + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "3.21.0", + "@tensorflow/tfjs-backend-webgl": "3.21.0", + "@tensorflow/tfjs-converter": "3.21.0", + "@tensorflow/tfjs-core": "3.21.0", + "@tensorflow/tfjs-data": "3.21.0", + "@tensorflow/tfjs-layers": "3.21.0", + "argparse": "^1.0.10", + "chalk": "^4.1.0", + "core-js": "3", + "regenerator-runtime": "^0.13.5", + "yargs": "^16.0.3" + }, + "bin": { + "tfjs-custom-module": "dist/tools/custom_module/cli.js" + } + }, + "node_modules/@tensorflow/tfjs-backend-cpu": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-cpu/-/tfjs-backend-cpu-3.21.0.tgz", + "integrity": "sha512-88S21UAdzyK0CsLUrH17GPTD+26E85OP9CqmLZslaWjWUmBkeTQ5Zqyp6iK+gELnLxPx6q7JsNEeFuPv4254lQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/seedrandom": "^2.4.28", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs-backend-webgl": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-backend-webgl/-/tfjs-backend-webgl-3.21.0.tgz", + "integrity": "sha512-N4zitIAT9IX8B8oe489qM3f3VcESxGZIZvHmVP8varOQakTvTX859aaPo1s8hK1qCy4BjSGbweooZe4U8D4kTQ==", + "license": "Apache-2.0", + "dependencies": { + "@tensorflow/tfjs-backend-cpu": "3.21.0", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@types/webgl2": "0.0.6", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs-backend-webgl/node_modules/@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==", + "license": "MIT" + }, + "node_modules/@tensorflow/tfjs-converter": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-converter/-/tfjs-converter-3.21.0.tgz", + "integrity": "sha512-12Y4zVDq3yW+wSjSDpSv4HnpL2sDZrNiGSg8XNiDE4HQBdjdA+a+Q3sZF/8NV9y2yoBhL5L7V4mMLDdbZBd9/Q==", + "license": "Apache-2.0", + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs-core": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-3.21.0.tgz", + "integrity": "sha512-YSfsswOqWfd+M4bXIhT3hwtAb+IV8+ODwIxwdFR/7jTAPZP1wMVnSlpKnXHAN64HFOiP+Tm3HmKusEZ0+09A0w==", + "license": "Apache-2.0", + "dependencies": { + "@types/long": "^4.0.1", + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "^2.4.28", + "@types/webgl-ext": "0.0.30", + "@webgpu/types": "0.1.16", + "long": "4.0.0", + "node-fetch": "~2.6.1", + "seedrandom": "^3.0.5" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, + "node_modules/@tensorflow/tfjs-core/node_modules/@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==", + "license": "MIT" + }, + "node_modules/@tensorflow/tfjs-core/node_modules/@webgpu/types": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.16.tgz", + "integrity": "sha512-9E61voMP4+Rze02jlTXud++Htpjyyk8vw5Hyw9FGRrmhHQg2GqbuOfwf5Klrb8vTxc2XWI3EfO7RUHMpxTj26A==", + "license": "BSD-3-Clause" + }, + "node_modules/@tensorflow/tfjs-core/node_modules/long": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", + "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "license": "Apache-2.0" + }, + "node_modules/@tensorflow/tfjs-data": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-data/-/tfjs-data-3.21.0.tgz", + "integrity": "sha512-eFLfw2wIcFNxnP2Iv/SnVlihehzKMumk1b5Prcx1ixk/SbkCo5u0Lt7OVOWaEOKVqvB2sT+dJcTjAh6lrCC/QA==", + "license": "Apache-2.0", + "dependencies": { + "@types/node-fetch": "^2.1.2", + "node-fetch": "~2.6.1", + "string_decoder": "^1.3.0" + }, + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0", + "seedrandom": "^3.0.5" + } + }, + "node_modules/@tensorflow/tfjs-layers": { + "version": "3.21.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-layers/-/tfjs-layers-3.21.0.tgz", + "integrity": "sha512-CMVXsraakXgnXEnqD9QbtResA7nvV7Jz20pGmjFIodcQkClgmFFhdCG5N+zlVRHEz7VKG2OyfhltZ0dBq/OAhA==", + "license": "Apache-2.0 AND MIT", + "peerDependencies": { + "@tensorflow/tfjs-core": "3.21.0" + } + }, + "node_modules/@tensorflow/tfjs/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, "node_modules/@tweenjs/tween.js": { "version": "23.1.3", "license": "MIT" @@ -2588,6 +2754,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT" + }, "node_modules/@types/markdown-it": { "version": "14.1.2", "dev": true, @@ -2609,6 +2781,16 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.4" + } + }, "node_modules/@types/offscreencanvas": { "version": "2019.7.3", "license": "MIT" @@ -2659,6 +2841,12 @@ "@types/react": "*" } }, + "node_modules/@types/seedrandom": { + "version": "2.4.34", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.34.tgz", + "integrity": "sha512-ytDiArvrn/3Xk6/vtylys5tlY6eo7Ane0hvcx++TKo6RxQXuVfW0AF/oeWqAj9dN29SyhtawuXstgmPlwNcv/A==", + "license": "MIT" + }, "node_modules/@types/stats.js": { "version": "0.17.4", "license": "MIT" @@ -2684,6 +2872,18 @@ "version": "10.0.0", "license": "MIT" }, + "node_modules/@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==", + "license": "MIT" + }, + "node_modules/@types/webgl2": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.6.tgz", + "integrity": "sha512-50GQhDVTq/herLMiqSQkdtRu+d5q/cWHn4VvKJtrj4DJAjo1MNkWYa2MA41BaBO1q1HgsUjuQvEOk0QHvlnAaQ==", + "license": "MIT" + }, "node_modules/@types/webxr": { "version": "0.5.24", "license": "MIT" @@ -3494,9 +3694,17 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/ansi-styles": { "version": "4.3.0", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -3660,7 +3868,6 @@ }, "node_modules/chalk": { "version": "4.1.2", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -3715,6 +3922,17 @@ "date-fns": ">=2.0.0" } }, + "node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, "node_modules/clsx": { "version": "2.1.1", "license": "MIT", @@ -3724,7 +3942,6 @@ }, "node_modules/color-convert": { "version": "2.0.1", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -3735,7 +3952,6 @@ }, "node_modules/color-name": { "version": "1.1.4", - "dev": true, "license": "MIT" }, "node_modules/combined-stream": { @@ -3767,6 +3983,17 @@ "version": "1.9.0", "license": "MIT" }, + "node_modules/core-js": { + "version": "3.49.0", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.49.0.tgz", + "integrity": "sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/core-js" + } + }, "node_modules/cosmiconfig": { "version": "7.1.0", "license": "MIT", @@ -4235,6 +4462,12 @@ "node": ">= 0.4" } }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, "node_modules/entities": { "version": "4.5.0", "dev": true, @@ -4330,6 +4563,15 @@ "@esbuild/win32-x64": "0.25.12" } }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "license": "MIT", @@ -4539,6 +4781,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, "node_modules/get-intrinsic": { "version": "1.3.0", "license": "MIT", @@ -4619,7 +4870,6 @@ }, "node_modules/has-flag": { "version": "4.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4815,6 +5065,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/is-hexadecimal": { "version": "1.0.4", "license": "MIT", @@ -5127,6 +5386,26 @@ "react-dom": "^16.8 || ^17 || ^18 || ^19 || ^19.0.0-rc" } }, + "node_modules/node-fetch": { + "version": "2.6.13", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.13.tgz", + "integrity": "sha512-StxNAxh15zr77QvvkmveSQ8uCQ4+v5FkvNTj0OESmiHu+VRi/gXArXtkWMElOsOUNLtUEvI4yS+rdtOHZTwlQA==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/object-assign": { "version": "4.1.1", "license": "MIT", @@ -5716,6 +5995,21 @@ "node": ">=6" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "license": "MIT", @@ -5811,6 +6105,26 @@ "version": "1.3.3", "license": "BSD-3-Clause" }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safer-buffer": { "version": "2.1.2", "license": "MIT" @@ -5822,6 +6136,12 @@ "loose-envify": "^1.1.0" } }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", + "license": "MIT" + }, "node_modules/semver": { "version": "7.7.3", "dev": true, @@ -5873,6 +6193,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, "node_modules/stats-gl": { "version": "2.4.2", "license": "MIT", @@ -5893,6 +6219,41 @@ "version": "0.17.0", "license": "MIT" }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-json-comments": { "version": "3.1.1", "dev": true, @@ -5910,7 +6271,6 @@ }, "node_modules/supports-color": { "version": "7.2.0", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -6007,6 +6367,12 @@ "node": ">=14.14" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/troika-three-text": { "version": "0.52.4", "license": "MIT", @@ -6237,6 +6603,22 @@ "version": "1.1.1", "license": "MIT" }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "license": "ISC", @@ -6258,6 +6640,23 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "dev": true, @@ -6275,6 +6674,15 @@ "node": ">=0.4" } }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/yaml": { "version": "2.8.2", "license": "ISC", @@ -6288,6 +6696,33 @@ "url": "https://github.com/sponsors/eemeli" } }, + "node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "license": "MIT", + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, "node_modules/zod": { "version": "3.25.76", "dev": true, diff --git a/frontend/package.json b/frontend/package.json index e222b62..fba67f8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -23,6 +23,9 @@ "@tanstack/react-query": "^5.84.1", "@tanstack/react-query-devtools": "^5.84.1", "@tanstack/react-router": "1.19.1", + "@tensorflow-models/coco-ssd": "^2.2.2", + "@tensorflow-models/deeplab": "^0.2.2", + "@tensorflow/tfjs": "^3.21.0", "@types/leaflet": "^1.9.20", "@types/react-grid-layout": "^1.3.5", "@types/uuid": "^10.0.0", diff --git a/frontend/src/components/Dashboard/Widgets/ErrorWidget.tsx b/frontend/src/components/Dashboard/Widgets/ErrorWidget.tsx index 1a840c6..cce7c0e 100644 --- a/frontend/src/components/Dashboard/Widgets/ErrorWidget.tsx +++ b/frontend/src/components/Dashboard/Widgets/ErrorWidget.tsx @@ -31,14 +31,7 @@ export function ErrorWidget({ error }: ErrorWidgetProps) { - + Error @@ -50,4 +43,4 @@ export function ErrorWidget({ error }: ErrorWidgetProps) { ); -} \ No newline at end of file +} diff --git a/frontend/src/components/Dashboard/Widgets/MosaicWidgetContext.tsx b/frontend/src/components/Dashboard/Widgets/MosaicWidgetContext.tsx index ad3e3bf..4e775bf 100644 --- a/frontend/src/components/Dashboard/Widgets/MosaicWidgetContext.tsx +++ b/frontend/src/components/Dashboard/Widgets/MosaicWidgetContext.tsx @@ -12,4 +12,4 @@ export function useMosaicWidget(): WidgetConfig { return ctx; } -export { MosaicWidgetContext }; \ No newline at end of file +export { MosaicWidgetContext }; diff --git a/frontend/src/components/WidgetCustomize/StoreDataPanel.tsx b/frontend/src/components/WidgetCustomize/StoreDataPanel.tsx index 9d780a4..c02561b 100644 --- a/frontend/src/components/WidgetCustomize/StoreDataPanel.tsx +++ b/frontend/src/components/WidgetCustomize/StoreDataPanel.tsx @@ -155,4 +155,4 @@ export function StoreDataPanel({ )} ); -} \ No newline at end of file +} diff --git a/frontend/src/components/WidgetCustomize/WidgetConfigPanel.tsx b/frontend/src/components/WidgetCustomize/WidgetConfigPanel.tsx index 216aeb5..8876ecd 100644 --- a/frontend/src/components/WidgetCustomize/WidgetConfigPanel.tsx +++ b/frontend/src/components/WidgetCustomize/WidgetConfigPanel.tsx @@ -34,4 +34,4 @@ export function WidgetConfigPanel({ widgetConfig }: WidgetConfigPanelProps) { ); -} \ No newline at end of file +} diff --git a/frontend/src/widgets/JsonViewerWidget/JsonViewerSetting.tsx b/frontend/src/widgets/JsonViewerWidget/JsonViewerSetting.tsx index 10f6108..8898979 100644 --- a/frontend/src/widgets/JsonViewerWidget/JsonViewerSetting.tsx +++ b/frontend/src/widgets/JsonViewerWidget/JsonViewerSetting.tsx @@ -1,8 +1,8 @@ import { VStack, Text } from "@chakra-ui/react"; import { Controller, useForm } from "react-hook-form"; -import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; import { useMosaicWidget } from "@/components/Dashboard/Widgets/MosaicWidgetContext.tsx"; +import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; import { Checkbox } from "@/components/ui/checkbox.tsx"; import { Field } from "@/components/ui/field.tsx"; diff --git a/frontend/src/widgets/MediaViewerWidget/MediaViewerSetting.tsx b/frontend/src/widgets/MediaViewerWidget/MediaViewerSetting.tsx index 4a2b1cc..8f4631b 100644 --- a/frontend/src/widgets/MediaViewerWidget/MediaViewerSetting.tsx +++ b/frontend/src/widgets/MediaViewerWidget/MediaViewerSetting.tsx @@ -1,8 +1,8 @@ import { VStack, Text } from "@chakra-ui/react"; import { Controller, useForm } from "react-hook-form"; -import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; import { useMosaicWidget } from "@/components/Dashboard/Widgets/MosaicWidgetContext.tsx"; +import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; import { Checkbox } from "@/components/ui/checkbox.tsx"; import { Field } from "@/components/ui/field.tsx"; @@ -31,11 +31,7 @@ export function MediaViewerSetting() { Media Viewer Settings - + - + ); -} \ No newline at end of file +} diff --git a/frontend/src/widgets/MediaViewerWidget/MediaViewerWidget.tsx b/frontend/src/widgets/MediaViewerWidget/MediaViewerWidget.tsx index c4ff283..c20a8a0 100644 --- a/frontend/src/widgets/MediaViewerWidget/MediaViewerWidget.tsx +++ b/frontend/src/widgets/MediaViewerWidget/MediaViewerWidget.tsx @@ -135,7 +135,13 @@ export default function MediaViewerWidget({ widgetConfig }: WidgetProps) { {error ? ( - + ⚠️ @@ -193,4 +199,4 @@ export default function MediaViewerWidget({ widgetConfig }: WidgetProps) { ); -} \ No newline at end of file +} diff --git a/frontend/src/widgets/MediaViewerWidget/WidgetDescriptor.ts b/frontend/src/widgets/MediaViewerWidget/WidgetDescriptor.ts index 16890d9..97f4b24 100644 --- a/frontend/src/widgets/MediaViewerWidget/WidgetDescriptor.ts +++ b/frontend/src/widgets/MediaViewerWidget/WidgetDescriptor.ts @@ -18,4 +18,4 @@ export default class MediaViewerWidgetDescriptor extends WidgetDescriptor { public validateParams(_params: Record): string | null { return null; } -} \ No newline at end of file +} diff --git a/frontend/src/widgets/ObjectDetectionMediaWidget/ObjectDetectionMediaWidget.tsx b/frontend/src/widgets/ObjectDetectionMediaWidget/ObjectDetectionMediaWidget.tsx new file mode 100644 index 0000000..7374e86 --- /dev/null +++ b/frontend/src/widgets/ObjectDetectionMediaWidget/ObjectDetectionMediaWidget.tsx @@ -0,0 +1,422 @@ +import { Box, Flex, Text } from "@chakra-ui/react"; +import * as cocoSsd from "@tensorflow-models/coco-ssd"; +import * as tf from "@tensorflow/tfjs"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { MediaStreamStore } from "@/stores/MediaStreamStore/MediaStreamStore.ts"; +import type { WidgetProps } from "@/widgets/index.ts"; + +import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; +import { useMosaicStore } from "@/hooks/useMosaicStore.ts"; + +const DETECTION_INTERVAL_MS = 100; +const SCORE_THRESHOLD = 0.5; + +interface Detection { + class: string; + score: number; + bbox: [number, number, number, number]; +} + +export default function ObjectDetectionMediaWidget({ widgetConfig }: WidgetProps) { + const { getOrCreateStore, releaseStore } = useMosaicStore(); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const modelRef = useRef(null); + const animationFrameRef = useRef(null); + const timerRef = useRef(null); + const isDetectingRef = useRef(false); + const connector = widgetConfig.connectors[0]; + const connectorRobotId = connector?.robotId ?? ""; + const connectorId = connector?.connectorId ?? ""; + const [isModelLoading, setIsModelLoading] = useState(true); + const [isModelReady, setIsModelReady] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [isStreamReady, setIsStreamReady] = useState(false); + const [isDetecting, setIsDetecting] = useState(false); + const [detections, setDetections] = useState([]); + const [error, setError] = useState(null); + + const clearOverlay = useCallback(() => { + if (!canvasRef.current) return; + const ctx = canvasRef.current.getContext("2d"); + if (!ctx) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }, []); + + const stopDetectionLoop = useCallback(() => { + isDetectingRef.current = false; + setIsDetecting(false); + setDetections([]); + if (animationFrameRef.current !== null) { + cancelAnimationFrame(animationFrameRef.current); + animationFrameRef.current = null; + } + if (timerRef.current !== null) { + clearTimeout(timerRef.current); + timerRef.current = null; + } + clearOverlay(); + }, [clearOverlay]); + + const setupCanvas = useCallback(() => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas || video.videoWidth === 0 || video.videoHeight === 0) { + return; + } + + const videoWidth = video.videoWidth; + const videoHeight = video.videoHeight; + const elementWidth = video.clientWidth; + const elementHeight = video.clientHeight; + + if (elementWidth === 0 || elementHeight === 0) { + return; + } + + const videoAspectRatio = videoWidth / videoHeight; + const elementAspectRatio = elementWidth / elementHeight; + + let displayWidth = 0; + let displayHeight = 0; + let offsetX = 0; + let offsetY = 0; + + if (videoAspectRatio > elementAspectRatio) { + displayWidth = elementWidth; + displayHeight = elementWidth / videoAspectRatio; + offsetY = (elementHeight - displayHeight) / 2; + } else { + displayWidth = elementHeight * videoAspectRatio; + displayHeight = elementHeight; + offsetX = (elementWidth - displayWidth) / 2; + } + + canvas.width = videoWidth; + canvas.height = videoHeight; + canvas.style.width = `${displayWidth}px`; + canvas.style.height = `${displayHeight}px`; + canvas.style.left = `${offsetX}px`; + canvas.style.top = `${offsetY}px`; + }, []); + + const drawDetections = useCallback((items: Detection[]) => { + const canvas = canvasRef.current; + if (!canvas) return; + const ctx = canvas.getContext("2d"); + if (!ctx) return; + + ctx.clearRect(0, 0, canvas.width, canvas.height); + ctx.strokeStyle = "#00ff00"; + ctx.lineWidth = 3; + ctx.font = "20px Arial"; + + for (const detection of items) { + const [x, y, width, height] = detection.bbox; + ctx.strokeRect(x, y, width, height); + + const label = `${detection.class} ${(detection.score * 100).toFixed(1)}%`; + const textWidth = ctx.measureText(label).width; + const padding = 4; + const textHeight = 20; + const labelY = Math.max(y - (textHeight + padding * 2), 0); + + ctx.fillStyle = "rgba(0, 255, 0, 0.85)"; + ctx.fillRect(x, labelY, textWidth + padding * 2, textHeight + padding * 2); + + ctx.fillStyle = "#000"; + ctx.fillText(label, x + padding, labelY + textHeight); + } + }, []); + + const runDetection = useCallback(async () => { + if (!isDetectingRef.current) { + return; + } + + const video = videoRef.current; + const model = modelRef.current; + if (!video || !model || video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + timerRef.current = window.setTimeout(() => { + animationFrameRef.current = window.requestAnimationFrame(() => { + void runDetection(); + }); + }, DETECTION_INTERVAL_MS); + return; + } + + setupCanvas(); + + try { + const predictions = await model.detect(video); + if (!isDetectingRef.current) { + return; + } + + const filteredDetections: Detection[] = predictions + .filter((prediction) => prediction.score > SCORE_THRESHOLD) + .map((prediction) => ({ + class: prediction.class, + score: prediction.score, + bbox: prediction.bbox, + })); + + setDetections(filteredDetections); + drawDetections(filteredDetections); + } catch (detectionError) { + console.error("Object detection inference failed:", detectionError); + } + + if (!isDetectingRef.current) { + return; + } + + timerRef.current = window.setTimeout(() => { + animationFrameRef.current = window.requestAnimationFrame(() => { + void runDetection(); + }); + }, DETECTION_INTERVAL_MS); + }, [drawDetections, setupCanvas]); + + const configureVideo = (store: MediaStreamStore) => { + if (videoRef.current) { + store.setVideoElement(videoRef.current); + } + + const mediaStream = store.getMediaStream(); + if (mediaStream && videoRef.current && videoRef.current.srcObject === mediaStream) { + if (videoRef.current.paused) { + videoRef.current.play().catch((playError) => { + console.error("ObjectDetectionMediaWidget autoplay failed:", playError); + }); + } + } + + const videoElement = videoRef.current; + if (!videoElement) { + return; + } + + const syncVideoState = () => { + const hasMetadata = videoElement.readyState >= HTMLMediaElement.HAVE_METADATA; + setIsStreamReady(hasMetadata); + setIsPlaying(!videoElement.paused && !videoElement.ended); + if (hasMetadata) { + setError(null); + setupCanvas(); + } + }; + + const handlePlay = () => setIsPlaying(true); + const handlePause = () => setIsPlaying(false); + const handleLoadedMetadata = () => { + setError(null); + setIsStreamReady(true); + setupCanvas(); + }; + const handleLoadedData = () => { + setIsStreamReady(true); + setupCanvas(); + }; + const handleEmptied = () => { + setIsStreamReady(false); + setIsPlaying(false); + clearOverlay(); + }; + const handleEnded = () => setIsPlaying(false); + const handleStalled = () => setIsPlaying(false); + const handleError = (event: Event) => { + console.error("ObjectDetectionMediaWidget video load error:", event); + setError("Failed to load media stream."); + setIsStreamReady(false); + setIsPlaying(false); + }; + + videoElement.addEventListener("play", handlePlay); + videoElement.addEventListener("pause", handlePause); + videoElement.addEventListener("loadedmetadata", handleLoadedMetadata); + videoElement.addEventListener("loadeddata", handleLoadedData); + videoElement.addEventListener("emptied", handleEmptied); + videoElement.addEventListener("ended", handleEnded); + videoElement.addEventListener("stalled", handleStalled); + videoElement.addEventListener("error", handleError); + syncVideoState(); + + return () => { + videoElement.removeEventListener("play", handlePlay); + videoElement.removeEventListener("pause", handlePause); + videoElement.removeEventListener("loadedmetadata", handleLoadedMetadata); + videoElement.removeEventListener("loadeddata", handleLoadedData); + videoElement.removeEventListener("emptied", handleEmptied); + videoElement.removeEventListener("ended", handleEnded); + videoElement.removeEventListener("stalled", handleStalled); + videoElement.removeEventListener("error", handleError); + }; + }; + + useEffect(() => { + let cancelled = false; + + const loadModel = async () => { + setIsModelLoading(true); + setIsModelReady(false); + + try { + await tf.ready(); + const loadedModel = await cocoSsd.load(); + + if (cancelled) { + return; + } + + modelRef.current = loadedModel; + setIsModelReady(true); + } catch (modelError) { + console.error("COCO-SSD load failed:", modelError); + if (!cancelled) { + setError("Failed to load coco-ssd model."); + } + } finally { + if (!cancelled) { + setIsModelLoading(false); + } + } + }; + + void loadModel(); + + return () => { + cancelled = true; + stopDetectionLoop(); + modelRef.current = null; + }; + }, [stopDetectionLoop]); + + useEffect(() => { + if (!connector || !connectorRobotId || !connectorId) { + return; + } + + const store = getOrCreateStore(connector) as MediaStreamStore; + if (store === null) { + return; + } + + let cleanupVideoListeners = configureVideo(store); + const unsubscribeAfterConnected = store.onAfterConnected(() => { + cleanupVideoListeners?.(); + cleanupVideoListeners = configureVideo(store); + }); + + return () => { + cleanupVideoListeners?.(); + unsubscribeAfterConnected(); + stopDetectionLoop(); + setIsStreamReady(false); + setIsPlaying(false); + releaseStore(connector); + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }; + }, [connectorRobotId, connectorId, stopDetectionLoop]); + + useEffect(() => { + const video = videoRef.current; + if (!video) return; + const handleViewportChange = () => setupCanvas(); + const resizeObserver = new ResizeObserver(handleViewportChange); + resizeObserver.observe(video); + window.addEventListener("resize", handleViewportChange); + document.addEventListener("fullscreenchange", handleViewportChange); + return () => { + resizeObserver.disconnect(); + window.removeEventListener("resize", handleViewportChange); + document.removeEventListener("fullscreenchange", handleViewportChange); + }; + }, [setupCanvas]); + + useEffect(() => { + if (!isModelReady || !isStreamReady || !isPlaying || error) { + stopDetectionLoop(); + return; + } + + if (isDetectingRef.current) { + return; + } + + isDetectingRef.current = true; + setIsDetecting(true); + void runDetection(); + + return () => { + stopDetectionLoop(); + }; + }, [error, isModelReady, isPlaying, isStreamReady, runDetection, stopDetectionLoop]); + + return ( + + + + {error ? ( + + {error} + + ) : ( + + + )} + + + ); +} diff --git a/frontend/src/widgets/ObjectDetectionMediaWidget/WidgetDescriptor.ts b/frontend/src/widgets/ObjectDetectionMediaWidget/WidgetDescriptor.ts new file mode 100644 index 0000000..92fa871 --- /dev/null +++ b/frontend/src/widgets/ObjectDetectionMediaWidget/WidgetDescriptor.ts @@ -0,0 +1,21 @@ +import type { StoreType } from "@/mosaic/store/interface/mosaic-store.ts"; + +import { WidgetDescriptor } from "@/components/Dashboard/Widgets/WidgetDescriptor.ts"; + +export default class ObjectDetectionMediaWidgetDescriptor extends WidgetDescriptor { + public getName(): string { + return "ObjectDetectionMediaWidget"; + } + + public getRequiredStoreType(): StoreType { + return "media"; + } + + public validateParams(params: Record): string | null { + const model = params?.tfModel; + if (model !== undefined && model !== "coco-ssd") { + return "ObjectDetectionMediaWidget supports only coco-ssd."; + } + return null; + } +} diff --git a/frontend/src/widgets/SegmentationMediaWidget/SegmentationMediaWidget.tsx b/frontend/src/widgets/SegmentationMediaWidget/SegmentationMediaWidget.tsx new file mode 100644 index 0000000..fd4b153 --- /dev/null +++ b/frontend/src/widgets/SegmentationMediaWidget/SegmentationMediaWidget.tsx @@ -0,0 +1,437 @@ +import { Box, Flex, Text } from "@chakra-ui/react"; +import * as deepLab from "@tensorflow-models/deeplab"; +import * as tf from "@tensorflow/tfjs"; +import { useCallback, useEffect, useRef, useState } from "react"; + +import type { MediaStreamStore } from "@/stores/MediaStreamStore/MediaStreamStore.ts"; +import type { WidgetProps } from "@/widgets/index.ts"; + +import { MosaicWidget } from "@/components/Dashboard/Widgets/WidgetComponents.tsx"; +import { useMosaicStore } from "@/hooks/useMosaicStore.ts"; + +const SEGMENTATION_INTERVAL_MS = 100; +const DEEPLAB_MODEL_BASE = "pascal"; + +interface SegmentationOutput { + width: number; + height: number; + segmentationMap: Uint8Array | Uint8ClampedArray | number[]; +} + +export default function SegmentationMediaWidget({ widgetConfig }: WidgetProps) { + const { getOrCreateStore, releaseStore } = useMosaicStore(); + const videoRef = useRef(null); + const canvasRef = useRef(null); + const modelRef = useRef(null); + const tempCanvasRef = useRef(null); + const animationFrameRef = useRef(null); + const timerRef = useRef(null); + const isSegmentingRef = useRef(false); + const connector = widgetConfig.connectors[0]; + const connectorRobotId = connector?.robotId ?? ""; + const connectorId = connector?.connectorId ?? ""; + const [isModelLoading, setIsModelLoading] = useState(true); + const [isModelReady, setIsModelReady] = useState(false); + const [isPlaying, setIsPlaying] = useState(false); + const [isStreamReady, setIsStreamReady] = useState(false); + const [isSegmenting, setIsSegmenting] = useState(false); + const [error, setError] = useState(null); + + const clearOverlay = useCallback(() => { + if (!canvasRef.current) return; + const ctx = canvasRef.current.getContext("2d"); + if (!ctx) return; + ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); + }, []); + + const stopSegmentationLoop = useCallback(() => { + isSegmentingRef.current = false; + setIsSegmenting(false); + if (animationFrameRef.current !== null) { + cancelAnimationFrame(animationFrameRef.current); + animationFrameRef.current = null; + } + if (timerRef.current !== null) { + clearTimeout(timerRef.current); + timerRef.current = null; + } + clearOverlay(); + }, [clearOverlay]); + + const setupCanvas = useCallback(() => { + const video = videoRef.current; + const canvas = canvasRef.current; + if (!video || !canvas || video.videoWidth === 0 || video.videoHeight === 0) { + return; + } + + const videoWidth = video.videoWidth; + const videoHeight = video.videoHeight; + const elementWidth = video.clientWidth; + const elementHeight = video.clientHeight; + + if (elementWidth === 0 || elementHeight === 0) { + return; + } + + const videoAspectRatio = videoWidth / videoHeight; + const elementAspectRatio = elementWidth / elementHeight; + + let displayWidth = 0; + let displayHeight = 0; + let offsetX = 0; + let offsetY = 0; + + if (videoAspectRatio > elementAspectRatio) { + displayWidth = elementWidth; + displayHeight = elementWidth / videoAspectRatio; + offsetY = (elementHeight - displayHeight) / 2; + } else { + displayWidth = elementHeight * videoAspectRatio; + displayHeight = elementHeight; + offsetX = (elementWidth - displayWidth) / 2; + } + + canvas.width = videoWidth; + canvas.height = videoHeight; + canvas.style.width = `${displayWidth}px`; + canvas.style.height = `${displayHeight}px`; + canvas.style.left = `${offsetX}px`; + canvas.style.top = `${offsetY}px`; + }, []); + + const drawSegmentation = useCallback((output: SegmentationOutput) => { + const canvas = canvasRef.current; + if (!canvas) { + return; + } + + const canvasCtx = canvas.getContext("2d"); + if (!canvasCtx) { + return; + } + + if (!tempCanvasRef.current) { + tempCanvasRef.current = document.createElement("canvas"); + } + const tempCanvas = tempCanvasRef.current; + const tempCtx = tempCanvas.getContext("2d"); + if (!tempCtx) { + return; + } + + const { width, height, segmentationMap } = output; + tempCanvas.width = width; + tempCanvas.height = height; + const imageData = tempCtx.createImageData(width, height); + const data = imageData.data; + + for (let i = 0; i < segmentationMap.length; i += 4) { + const r = segmentationMap[i] ?? 0; + const g = segmentationMap[i + 1] ?? 0; + const b = segmentationMap[i + 2] ?? 0; + + if (r !== 0 || g !== 0 || b !== 0) { + data[i] = r; + data[i + 1] = g; + data[i + 2] = b; + data[i + 3] = 130; + } else { + data[i] = 0; + data[i + 1] = 0; + data[i + 2] = 0; + data[i + 3] = 0; + } + } + + tempCtx.putImageData(imageData, 0, 0); + canvasCtx.clearRect(0, 0, canvas.width, canvas.height); + canvasCtx.drawImage(tempCanvas, 0, 0, width, height, 0, 0, canvas.width, canvas.height); + }, []); + + const runSegmentation = useCallback(async () => { + if (!isSegmentingRef.current) { + return; + } + + const video = videoRef.current; + const model = modelRef.current; + if (!video || !model || video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) { + timerRef.current = window.setTimeout(() => { + animationFrameRef.current = window.requestAnimationFrame(() => { + void runSegmentation(); + }); + }, SEGMENTATION_INTERVAL_MS); + return; + } + + setupCanvas(); + try { + const output = (await model.segment(video)) as SegmentationOutput; + if (!isSegmentingRef.current) { + return; + } + drawSegmentation(output); + } catch (segmentError) { + console.error("Segmentation inference failed:", segmentError); + } + + if (!isSegmentingRef.current) { + return; + } + + timerRef.current = window.setTimeout(() => { + animationFrameRef.current = window.requestAnimationFrame(() => { + void runSegmentation(); + }); + }, SEGMENTATION_INTERVAL_MS); + }, [drawSegmentation, setupCanvas]); + + const configureVideo = (store: MediaStreamStore) => { + if (videoRef.current) { + store.setVideoElement(videoRef.current); + } + + const mediaStream = store.getMediaStream(); + if (mediaStream && videoRef.current && videoRef.current.srcObject === mediaStream) { + if (videoRef.current.paused) { + videoRef.current.play().catch((playError) => { + console.error("SegmentationMediaWidget autoplay failed:", playError); + }); + } + } + + const videoElement = videoRef.current; + if (!videoElement) { + return; + } + + const syncVideoState = () => { + const hasMetadata = videoElement.readyState >= HTMLMediaElement.HAVE_METADATA; + setIsStreamReady(hasMetadata); + setIsPlaying(!videoElement.paused && !videoElement.ended); + if (hasMetadata) { + setError(null); + setupCanvas(); + } + }; + + const handlePlay = () => setIsPlaying(true); + const handlePause = () => setIsPlaying(false); + const handleLoadedMetadata = () => { + setError(null); + setIsStreamReady(true); + setupCanvas(); + }; + const handleLoadedData = () => { + setIsStreamReady(true); + setupCanvas(); + }; + const handleEmptied = () => { + setIsStreamReady(false); + setIsPlaying(false); + clearOverlay(); + }; + const handleEnded = () => setIsPlaying(false); + const handleStalled = () => setIsPlaying(false); + const handleError = (event: Event) => { + console.error("SegmentationMediaWidget video load error:", event); + setError("Failed to load media stream."); + setIsStreamReady(false); + setIsPlaying(false); + }; + + videoElement.addEventListener("play", handlePlay); + videoElement.addEventListener("pause", handlePause); + videoElement.addEventListener("loadedmetadata", handleLoadedMetadata); + videoElement.addEventListener("loadeddata", handleLoadedData); + videoElement.addEventListener("emptied", handleEmptied); + videoElement.addEventListener("ended", handleEnded); + videoElement.addEventListener("stalled", handleStalled); + videoElement.addEventListener("error", handleError); + syncVideoState(); + + return () => { + videoElement.removeEventListener("play", handlePlay); + videoElement.removeEventListener("pause", handlePause); + videoElement.removeEventListener("loadedmetadata", handleLoadedMetadata); + videoElement.removeEventListener("loadeddata", handleLoadedData); + videoElement.removeEventListener("emptied", handleEmptied); + videoElement.removeEventListener("ended", handleEnded); + videoElement.removeEventListener("stalled", handleStalled); + videoElement.removeEventListener("error", handleError); + }; + }; + + useEffect(() => { + let cancelled = false; + + const loadModel = async () => { + setIsModelLoading(true); + setIsModelReady(false); + + try { + await tf.ready(); + const loadedModel = await deepLab.load({ + base: DEEPLAB_MODEL_BASE, + quantizationBytes: 2, + }); + + if (cancelled) { + const disposableModel = loadedModel as { dispose?: () => void }; + disposableModel.dispose?.(); + return; + } + + modelRef.current = loadedModel; + setIsModelReady(true); + } catch (modelError) { + console.error("DeepLab load failed:", modelError); + if (!cancelled) { + setError("Failed to load deeplab model."); + } + } finally { + if (!cancelled) { + setIsModelLoading(false); + } + } + }; + + void loadModel(); + + return () => { + cancelled = true; + stopSegmentationLoop(); + const disposableModel = modelRef.current as { dispose?: () => void } | null; + disposableModel?.dispose?.(); + modelRef.current = null; + tempCanvasRef.current = null; + }; + }, [stopSegmentationLoop]); + + useEffect(() => { + if (!connector || !connectorRobotId || !connectorId) { + return; + } + + const store = getOrCreateStore(connector) as MediaStreamStore; + if (store === null) { + return; + } + + let cleanupVideoListeners = configureVideo(store); + const unsubscribeAfterConnected = store.onAfterConnected(() => { + cleanupVideoListeners?.(); + cleanupVideoListeners = configureVideo(store); + }); + + return () => { + cleanupVideoListeners?.(); + unsubscribeAfterConnected(); + stopSegmentationLoop(); + setIsStreamReady(false); + setIsPlaying(false); + releaseStore(connector); + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }; + }, [connectorRobotId, connectorId, stopSegmentationLoop]); + + useEffect(() => { + const video = videoRef.current; + if (!video) return; + const handleViewportChange = () => setupCanvas(); + const resizeObserver = new ResizeObserver(handleViewportChange); + resizeObserver.observe(video); + window.addEventListener("resize", handleViewportChange); + document.addEventListener("fullscreenchange", handleViewportChange); + return () => { + resizeObserver.disconnect(); + window.removeEventListener("resize", handleViewportChange); + document.removeEventListener("fullscreenchange", handleViewportChange); + }; + }, [setupCanvas]); + + useEffect(() => { + if (!isModelReady || !isStreamReady || !isPlaying || error) { + stopSegmentationLoop(); + return; + } + + if (isSegmentingRef.current) { + return; + } + + isSegmentingRef.current = true; + setIsSegmenting(true); + void runSegmentation(); + + return () => { + stopSegmentationLoop(); + }; + }, [error, isModelReady, isPlaying, isStreamReady, runSegmentation, stopSegmentationLoop]); + + return ( + + + + {error ? ( + + {error} + + ) : ( + + + )} + + + ); +} diff --git a/frontend/src/widgets/SegmentationMediaWidget/WidgetDescriptor.ts b/frontend/src/widgets/SegmentationMediaWidget/WidgetDescriptor.ts new file mode 100644 index 0000000..de57abd --- /dev/null +++ b/frontend/src/widgets/SegmentationMediaWidget/WidgetDescriptor.ts @@ -0,0 +1,21 @@ +import type { StoreType } from "@/mosaic/store/interface/mosaic-store.ts"; + +import { WidgetDescriptor } from "@/components/Dashboard/Widgets/WidgetDescriptor.ts"; + +export default class SegmentationMediaWidgetDescriptor extends WidgetDescriptor { + public getName(): string { + return "SegmentationMediaWidget"; + } + + public getRequiredStoreType(): StoreType { + return "media"; + } + + public validateParams(params: Record): string | null { + const model = params?.tfModel; + if (model !== undefined && model !== "deeplab") { + return "SegmentationMediaWidget supports only deeplab."; + } + return null; + } +} diff --git a/frontend/src/widgets/WASDSenderWidget/WidgetDescriptor.ts b/frontend/src/widgets/WASDSenderWidget/WidgetDescriptor.ts index 7b9c42b..429d5ac 100644 --- a/frontend/src/widgets/WASDSenderWidget/WidgetDescriptor.ts +++ b/frontend/src/widgets/WASDSenderWidget/WidgetDescriptor.ts @@ -10,4 +10,4 @@ export default class WASDSenderWidgetDescriptor extends WidgetDescriptor { public getRequiredStoreType(): StoreType { return "sendable"; } -} \ No newline at end of file +}