From 1c14e42414b907fba9681f497b004b15fe85e4e0 Mon Sep 17 00:00:00 2001 From: Thiago Vinhas Date: Tue, 17 Feb 2026 19:47:34 -0600 Subject: [PATCH 1/8] [FEATURE] Added copy text to clipboard from kvm using OCR --- ui/localization/messages/en.json | 7 + ui/package-lock.json | 115 ++++++++ ui/package.json | 1 + ui/src/components/ActionBar.tsx | 29 +- ui/src/components/OcrOverlay.tsx | 474 ++++++++++++++++++++++++++++++ ui/src/components/WebRTCVideo.tsx | 25 +- ui/src/hooks/stores.ts | 10 + ui/vite.config.ts | 3 + 8 files changed, 654 insertions(+), 10 deletions(-) create mode 100644 ui/src/components/OcrOverlay.tsx diff --git a/ui/localization/messages/en.json b/ui/localization/messages/en.json index 6ad2c6b62..b147783d3 100644 --- a/ui/localization/messages/en.json +++ b/ui/localization/messages/en.json @@ -49,6 +49,7 @@ "access_update_tls_settings": "Update TLS Settings", "action_bar_connection_stats": "Connection Stats", "action_bar_extension": "Extension", + "action_bar_copy_text": "Copy text", "action_bar_fullscreen": "Fullscreen", "action_bar_settings": "Settings", "action_bar_virtual_keyboard": "Virtual Keyboard", @@ -709,6 +710,12 @@ "not_available": "N/A", "not_found": "Not found", "ntp_servers": "NTP Servers", + "ocr_copied": "Copied to clipboard", + "ocr_copy_hint": "Press {shortcut} to copy the selected text", + "ocr_drag_to_select": "Drag to select text region. Press Esc to cancel.", + "ocr_failed": "OCR failed. Please try again.", + "ocr_no_text_detected": "No text detected in selection", + "ocr_recognizing": "Recognizing text...", "oh_no": "Oh no!", "online": "Online", "other_session_detected": "Another Active Session Detected", diff --git a/ui/package-lock.json b/ui/package-lock.json index 0982f1a6c..1111441f6 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -36,6 +36,7 @@ "recharts": "^3.5.1", "semver": "^7.7.3", "tailwind-merge": "^3.4.0", + "tesseract.js": "^7.0.0", "tslog": "^4.10.2", "usehooks-ts": "^3.1.1", "validator": "^13.15.23", @@ -3127,6 +3128,12 @@ "baseline-browser-mapping": "dist/cli.js" } }, + "node_modules/bmp-js": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/bmp-js/-/bmp-js-0.1.0.tgz", + "integrity": "sha512-vHdS19CnY3hwiNdkaqk93DvjVLfbEcI8mys4UjuWrlX1haDmroo8o4xCzh4wD6DGV6HxRCyauwhHRqMTfERtjw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -4999,6 +5006,12 @@ "human-id": "dist/cli.js" } }, + "node_modules/idb-keyval": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/idb-keyval/-/idb-keyval-6.2.2.tgz", + "integrity": "sha512-yjD9nARJ/jb1g+CvD0tlhUHOrJ9Sy0P8T9MF3YaLlHnSRpwPfpTX0XIvpmw3gAJUmEu3FiICLBDPXVwyEvrleg==", + "license": "Apache-2.0" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -5406,6 +5419,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-url": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", + "integrity": "sha512-ITvGim8FhRiYe4IQ5uHSkj7pVaPDrCTkNd3yq3cV7iZAcJdHTUMPMEHcqSOy9xZ9qFenQCvi+2wjH9a1nXqHww==", + "license": "MIT" + }, "node_modules/is-weakmap": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.2.tgz", @@ -6025,6 +6044,26 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, "node_modules/node-releases": { "version": "2.0.27", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", @@ -6155,6 +6194,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/opencollective-postinstall": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.3.tgz", + "integrity": "sha512-8AV/sCtuzUeTo8gQK5qDZzARrulB3egtLzFgteqB2tcT4Mw7B8Kt7JcDHmltjz6FOAHsvTevk70gZEbhM4ZS9Q==", + "license": "MIT", + "bin": { + "opencollective-postinstall": "index.js" + } + }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -6755,6 +6803,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/regenerator-runtime": { + "version": "0.13.11", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz", + "integrity": "sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg==", + "license": "MIT" + }, "node_modules/regexp.prototype.flags": { "version": "1.5.4", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz", @@ -7315,6 +7369,30 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tesseract.js": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js/-/tesseract.js-7.0.0.tgz", + "integrity": "sha512-exPBkd+z+wM1BuMkx/Bjv43OeLBxhL5kKWsz/9JY+DXcXdiBjiAch0V49QR3oAJqCaL5qURE0vx9Eo+G5YE7mA==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "bmp-js": "^0.1.0", + "idb-keyval": "^6.2.0", + "is-url": "^1.2.4", + "node-fetch": "^2.6.9", + "opencollective-postinstall": "^2.0.3", + "regenerator-runtime": "^0.13.3", + "tesseract.js-core": "^7.0.0", + "wasm-feature-detect": "^1.8.0", + "zlibjs": "^0.3.1" + } + }, + "node_modules/tesseract.js-core": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/tesseract.js-core/-/tesseract.js-core-7.0.0.tgz", + "integrity": "sha512-WnNH518NzmbSq9zgTPeoF8c+xmilS8rFIl1YKbk/ptuuc7p6cLNELNuPAzcmsYw450ca6bLa8j3t0VAtq435Vw==", + "license": "Apache-2.0" + }, "node_modules/tiny-invariant": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", @@ -7337,6 +7415,12 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, "node_modules/ts-api-utils": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-2.1.0.tgz", @@ -7804,6 +7888,18 @@ } } }, + "node_modules/wasm-feature-detect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/wasm-feature-detect/-/wasm-feature-detect-1.8.0.tgz", + "integrity": "sha512-zksaLKM2fVlnB5jQQDqKXXwYHLQUVH9es+5TOOHwGOVJOCeRBCiPjwSg+3tN2AdTCzjgli4jijCH290kXb/zWQ==", + "license": "Apache-2.0" + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, "node_modules/webpack-virtual-modules": { "version": "0.6.2", "resolved": "https://registry.npmjs.org/webpack-virtual-modules/-/webpack-virtual-modules-0.6.2.tgz", @@ -7811,6 +7907,16 @@ "dev": true, "license": "MIT" }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -7946,6 +8052,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/zlibjs": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/zlibjs/-/zlibjs-0.3.1.tgz", + "integrity": "sha512-+J9RrgTKOmlxFSDHo0pI1xM6BLVUv+o0ZT9ANtCxGkjIVCCUdx9alUF8Gm+dGLKbkkkidWIHFDZHDMpfITt4+w==", + "license": "MIT", + "engines": { + "node": "*" + } + }, "node_modules/zod": { "version": "4.1.13", "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.13.tgz", diff --git a/ui/package.json b/ui/package.json index 756f45527..962a0dcc8 100644 --- a/ui/package.json +++ b/ui/package.json @@ -58,6 +58,7 @@ "semver": "^7.7.3", "recharts": "^3.5.1", "tailwind-merge": "^3.4.0", + "tesseract.js": "^7.0.0", "tslog": "^4.10.2", "usehooks-ts": "^3.1.1", "validator": "^13.15.23", diff --git a/ui/src/components/ActionBar.tsx b/ui/src/components/ActionBar.tsx index 364845193..beaaa2301 100644 --- a/ui/src/components/ActionBar.tsx +++ b/ui/src/components/ActionBar.tsx @@ -1,12 +1,18 @@ import { Fragment, useCallback, useRef } from "react"; import { MdOutlineContentPasteGo } from "react-icons/md"; -import { LuCable, LuHardDrive, LuMaximize, LuSettings, LuSignal } from "react-icons/lu"; +import { LuCable, LuHardDrive, LuMaximize, LuScanText, LuSettings, LuSignal } from "react-icons/lu"; import { FaKeyboard } from "react-icons/fa6"; import { Popover, PopoverButton, PopoverPanel } from "@headlessui/react"; import { CommandLineIcon } from "@heroicons/react/20/solid"; import { cx } from "@/cva.config"; -import { useHidStore, useMountMediaStore, useSettingsStore, useUiStore } from "@hooks/stores"; +import { + useHidStore, + useMountMediaStore, + useSettingsStore, + useUiStore, + useVideoStore, +} from "@hooks/stores"; import { useDeviceUiNavigation } from "@hooks/useAppNavigation"; import { Button } from "@components/Button"; import Container from "@components/Container"; @@ -23,9 +29,16 @@ export default function Actionbar({ }) { const { navigateTo } = useDeviceUiNavigation(); const { isVirtualKeyboardEnabled, setVirtualKeyboardEnabled } = useHidStore(); - const { setDisableVideoFocusTrap, terminalType, setTerminalType, toggleSidebarView } = - useUiStore(); + const { + setDisableVideoFocusTrap, + terminalType, + setTerminalType, + toggleSidebarView, + isOcrMode, + setOcrMode, + } = useUiStore(); const { remoteVirtualMediaState } = useMountMediaStore(); + const { width: videoWidth, height: videoHeight } = useVideoStore(); const { developerMode } = useSettingsStore(); // This is the only way to get a reliable state change for the popover @@ -63,6 +76,14 @@ export default function Actionbar({ onClick={() => setTerminalType(terminalType === "kvm" ? "none" : "kvm")} /> )} + + +
+