From 5440d7e17bc29b0f5c7287b0481cd969ebb7e4de Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Mon, 19 Oct 2026 16:27:51 -0400 Subject: [PATCH 01/13] chore(deps): bump minimum python version to 3.10 - needed to satisfy the trame >= 4 in dev. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c2ccc07..1be62f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "packaging", "pillow", ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.rst" license = {text = "Apache License 2.0"} keywords = ["Python", "Interactive", "Web", "Application", "Framework"] From 61a6b9fcf4c58dffb1de9c7ec856c009d33100b5 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Mon, 19 Oct 2026 16:21:50 -0400 Subject: [PATCH 02/13] feat(electron-shared-texture): Add new electron shared texture display area - this new display area shows the native GL texture in the `canvas` using operating system level handle to the GPU memory. --- .codespellrc | 1 + .github/workflows/test_and_release.yml | 9 + README.rst | 40 + electron/.eslintrc.cjs | 14 + electron/.gitignore | 1 + electron/README.md | 52 + electron/package-lock.json | 1743 +++++++++++++++++ electron/package.json | 34 + electron/src/main/dma/iosurface.js | 36 + electron/src/main/dma/pidfd.js | 34 + electron/src/main/dma/winhandle.js | 38 + electron/src/main/index.js | 229 +++ electron/src/main/protocol.js | 93 + electron/src/preload.js | 67 + electron/test/protocol.test.js | 78 + examples/06_electron/README.md | 21 + examples/06_electron/app.py | 249 +++ examples/06_electron/electron/.gitignore | 2 + examples/06_electron/electron/main.js | 28 + examples/06_electron/electron/package.json | 18 + js-components/src/style.css | 10 + pyproject.toml | 3 + src/trame_rca/electron/__init__.py | 96 + src/trame_rca/electron/comm.py | 66 + src/trame_rca/electron/passports/__init__.py | 29 + src/trame_rca/electron/passports/base.py | 20 + src/trame_rca/electron/passports/iosurface.py | 176 ++ src/trame_rca/electron/passports/nthandle.py | 493 +++++ .../electron/passports/wayland_dmabuf.py | 226 +++ src/trame_rca/schedulers/__init__.py | 5 + .../schedulers/shared_texture_scheduler.py | 269 +++ src/trame_rca/widgets/rca.py | 67 + tests/test_rca_electron.py | 165 ++ vue-components/src/components/DisplayArea.js | 3 + .../components/SharedTextureDisplayArea.js | 32 + vue-components/src/components/index.js | 2 + 36 files changed, 4449 insertions(+) create mode 100644 electron/.eslintrc.cjs create mode 100644 electron/.gitignore create mode 100644 electron/README.md create mode 100644 electron/package-lock.json create mode 100644 electron/package.json create mode 100644 electron/src/main/dma/iosurface.js create mode 100644 electron/src/main/dma/pidfd.js create mode 100644 electron/src/main/dma/winhandle.js create mode 100644 electron/src/main/index.js create mode 100644 electron/src/main/protocol.js create mode 100644 electron/src/preload.js create mode 100644 electron/test/protocol.test.js create mode 100644 examples/06_electron/README.md create mode 100644 examples/06_electron/app.py create mode 100644 examples/06_electron/electron/.gitignore create mode 100644 examples/06_electron/electron/main.js create mode 100644 examples/06_electron/electron/package.json create mode 100644 src/trame_rca/electron/__init__.py create mode 100644 src/trame_rca/electron/comm.py create mode 100644 src/trame_rca/electron/passports/__init__.py create mode 100644 src/trame_rca/electron/passports/base.py create mode 100644 src/trame_rca/electron/passports/iosurface.py create mode 100644 src/trame_rca/electron/passports/nthandle.py create mode 100644 src/trame_rca/electron/passports/wayland_dmabuf.py create mode 100644 src/trame_rca/schedulers/shared_texture_scheduler.py create mode 100644 tests/test_rca_electron.py create mode 100644 vue-components/src/components/SharedTextureDisplayArea.js diff --git a/.codespellrc b/.codespellrc index 189f46d..dcb49b3 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,2 +1,3 @@ [codespell] skip = **/package-lock.json,CHANGELOG.md +ignore-words-list = renderd diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index f315d9f..43e4b90 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -138,6 +138,15 @@ jobs: cd js-components npm run lint + - name: Test trame-rca-electron + run: | + cd electron + npm ci + npm test + npm run lint + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + test-npm-build: runs-on: ubuntu-latest diff --git a/README.rst b/README.rst index 76132bd..34e3f5d 100644 --- a/README.rst +++ b/README.rst @@ -128,3 +128,43 @@ up outside the trame-rca directory, checkout the `wheel` branch, and install it cd ../trame-rca python ./examples/05_video/vtk_cone_simple_video.py + +Shared textures with Electron +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In an Electron app, VTK can render into a GPU texture that Electron shows as is: without encoding +and without copying to the CPU. The Electron side is the ``@kitware/trame-rca-electron`` npm package in +``./electron``: it starts the trame server and loads its page. Supported on macOS (IOSurface), +Windows (D3D11 NT handles) and Linux under Wayland (dma-buf). + +- macOS: works out of the box (IOSurface). +- Windows: the GL driver needs ``WGL_NV_DX_interop2``, and VTK must render on the GPU + Chromium uses. Set ``D3D_ADAPTER=`` if that isn't DXGI adapter 0. +- Linux: Wayland only for now (dma-buf). Electron duplicates the server's file descriptors + with ``pidfd_getfd``, which needs ``kernel.yama.ptrace_scope`` at 1 or less: + ``sudo sysctl -w kernel.yama.ptrace_scope=1``. Set ``RENDER_NODE=/dev/dri/renderD129`` + if VTK picks another GPU than the compositor. + +.. code-block:: python + + from trame_rca.electron import init_vtk + from trame_rca.widgets import rca + + init_vtk() # before creating any VTK render window + + # ... create render_window, then in the UI: + view = rca.RemoteControlledArea(display="shared-texture") + view_handler = view.create_electron_handler(render_window) + +You can try our code example: + +.. code-block:: console + + pip install trame trame-vuetify + pip install "trame-rca[electron]" + + cd examples/06_electron/electron + npm i + npm start + +See ``electron/README.md`` for the Electron API and what packaging an app needs. diff --git a/electron/.eslintrc.cjs b/electron/.eslintrc.cjs new file mode 100644 index 0000000..e6ba8f2 --- /dev/null +++ b/electron/.eslintrc.cjs @@ -0,0 +1,14 @@ +/* eslint-env node */ +module.exports = { + root: true, + env: { + node: true, + browser: true, + es2022: true, + }, + extends: ['eslint:recommended'], + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'script', + }, +}; diff --git a/electron/.gitignore b/electron/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/electron/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/electron/README.md b/electron/README.md new file mode 100644 index 0000000..fa2d8c3 --- /dev/null +++ b/electron/README.md @@ -0,0 +1,52 @@ +# @kitware/trame-rca-electron + +Shows trame-rca `display="shared-texture"` views in Electron. The trame server renders VTK +into a GPU texture, and this package hands it to the page without encoding, or copying to the CPU. + +This package and the `trame-rca` python package speak a versioned protocol (see +`src/main/protocol.js` and `trame_rca/electron/comm.py`): use matching releases. + +## Use + +```js +const { app, BrowserWindow } = require('electron/main'); +const { launchTrame, webPreferences } = require('@kitware/trame-rca-electron'); + +app.whenReady().then(() => { + const win = new BrowserWindow({ webPreferences: webPreferences() }); + launchTrame(win, { command: 'python3', args: ['app.py'], onExit: () => app.quit() }); +}); +``` + +`launchTrame(win, options)` spawns `command` with `args` plus `--server --port 0 --emp-port `, +and loads the page in `win` once trame serves it. Options: + +- `command`, `args`, `cwd`, `env`: the trame server to run. Any executable works: a python + interpreter and a script, a frozen backend, etc. +- `onExit(code, signal)`: the server exited. +- `onStats({ fps, frames, info })`: once a second. +- `log`: defaults to `console`. + +It returns `{ process, close }`. `close()` kills the server; it is called when `win` closes. + +`webPreferences(extra)` sets the preload, and turns off the sandbox the preload needs for +`sharedTexture`. If the window needs its own preload, `require('@kitware/trame-rca-electron/preload')` +from it. + +On the python side, see `examples/06_electron` in trame-rca. + +## Packaging an app + +Packaging (electron-builder or Forge, a frozen python backend, signing) belongs to each app. +What this package needs from a packaged app: + +- `koffi` is a native module: unpack it from the asar archive (e.g. electron-builder's + `asarUnpack: ["**/node_modules/koffi/**"]`). +- Pass the packaged backend as `command`. It must be spawned by `launchTrame`, as a direct child + of the main process: on Linux, duplicating its dma-buf file descriptors needs ptrace access to + it (`kernel.yama.ptrace_scope` at 1 or less). +- The frozen backend must include `vtkmodules` and the `serve` directories of `trame_rca` and of + every other trame package it uses. +- macOS shares the texture as a global IOSurface, which does not work in an App Sandbox (so not on + the Mac App Store). +- Linux: X11 is not supported yet, Electron must run under Wayland. diff --git a/electron/package-lock.json b/electron/package-lock.json new file mode 100644 index 0000000..f7bff89 --- /dev/null +++ b/electron/package-lock.json @@ -0,0 +1,1743 @@ +{ + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "koffi": "^3.3.3" + }, + "devDependencies": { + "eslint": "^8.57.1" + }, + "peerDependencies": { + "electron": ">=44.7.0" + } + }, + "node_modules/@electron-internal/extract-zip": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@electron-internal/extract-zip/-/extract-zip-1.0.5.tgz", + "integrity": "sha512-+bqFCP98pLI0Tt0XQo1TmlXtwjWchISndDOxCkEcIuUgXWpBnLyRI+2DU+mesvnMMX6L1XDqYNA0lXNDHd/yiA==", + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@electron/get": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-5.1.0.tgz", + "integrity": "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA==", + "license": "MIT", + "peer": true, + "dependencies": { + "debug": "^4.1.1", + "env-paths": "^3.0.0", + "graceful-fs": "^4.2.11", + "progress": "^2.0.3", + "semver": "^7.6.3", + "sumchecker": "^3.0.1" + }, + "engines": { + "node": ">=22.12.0" + }, + "optionalDependencies": { + "undici": "^7.24.4" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.3.3.tgz", + "integrity": "sha512-c8iDq/mPXkgQmrmZ6SpqWIVF3ZnJEpjVQswzKzbrCSdr68pmSY/cNj1t4qvv32T8nNjW71/hB+E8Rq1+vVpmQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.3.3.tgz", + "integrity": "sha512-yAFu4oX6rpBCqr4HiGo3FpywmRN11LOSde4+GJZvfkFymqV3wZtx6TgVe8rZ4ogJZz9B8o1tDeKsZ6pqeoH40g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.3.3.tgz", + "integrity": "sha512-aFYI3YMdQWOAmHlVA+GfBlJZTpvfbUv/NoeBBkqblWN7BxuY02bkWMyoksUEl+mnx7ScoMonnjzXKJkNQXmR3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.3.3.tgz", + "integrity": "sha512-6Q8/DDhVADPMm3lFnZFJRETJtf485jZKzn1KogXeA9Gfztv/XMFiCOK4aEouFc256h9uWzC69mGimtcVcwK+Ww==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.3.3.tgz", + "integrity": "sha512-IynEDMMU9pp7EVVqQwXVU9x0JH7Opj8KB0wbcKgYNEhRM9eGzJ4ai7iwG8y8tdD/Y8niSVrVTj5WLNCCvUWXIw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.3.3.tgz", + "integrity": "sha512-5b4W9jX+wmfWn9vG7DvKIrNhhRpPDAx7YQV5i9epUR0Ogxz4s4a3APz/beLCvgHt4WhaSRrazxGYQz1oFOlfow==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.3.3.tgz", + "integrity": "sha512-+NmMrp3ji1NN4PTaLaXDCKMaJ5Fl+0+nrClmtsu/7wi+fjCrCSGMrJZUmTIJPm1nd8Z5fMRjWesiVvQiy/ETDg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.3.3.tgz", + "integrity": "sha512-K+NFO4/SiF4ANCS1ykem36wbEjfBJKsaakLKZ+6LbaLqyLjHHGky4fEIgipac/+XJtD4ZXjhexPIVjwzkO+rMQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.3.3.tgz", + "integrity": "sha512-y7UHjwOKbj2IjIpWtHmI8LEKXchidt47k605DCv5+YQMIDTbzhz/i3zTI1kRp6EYBLTBwRkR1X1PGt3uUi+rKQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.3.3.tgz", + "integrity": "sha512-dbcpe7IrWr18AAbIKa/RvJ8pM1lEpSELWYW6wsEBIYw/RTLfXqWJ6d9Ya0NWJgHTW+N7783KVQaV5c1Sr3ONDQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.3.3.tgz", + "integrity": "sha512-kbJFmWSoDHGZgxgDhInuvDi8MzrHafBfQ7iWN5P+nHv297beQXrHecFfCNBApnG9d2wrhEoY+EH/TdITuvaSwQ==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ppc64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ppc64/-/koffi-linux-ppc64-3.3.3.tgz", + "integrity": "sha512-YwGxjudYHsiDZ6qH4WmEawHXmTaZzjiZ4xdjcbzsVHqNvgZ3Nz7WG03PybecJ+BVH2vqBOntGZ2VsG/sL0WY1g==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.3.3.tgz", + "integrity": "sha512-k/txpUsOO/L4TexahQHxqdFRmBEGE+g++np9M5eNcsHWNXmazeWBrrfB9sjl6cZAP69KZFO0toP+ZMfDUDHmnQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.3.3.tgz", + "integrity": "sha512-AI4kcNNWeGmzQzsNb0S+ZL/ht4HMtDcQl5nYFkkuVUO2g1/2KdvNekkwNWhpp805mDbLSDoRZtlioHyH/v8poQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-arm64/-/koffi-openbsd-arm64-3.3.3.tgz", + "integrity": "sha512-NEbQMrixbAWSmWCh4XyHWuhgEqgO4tegWyHoJej7Q3PmyIlgxKHTcRZsjAJ9C3sfgBDNIqPWTjMy3KO2mqEp1g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.3.3.tgz", + "integrity": "sha512-zGFX0KwZGmrV7AZY0+oozBpMuQZT7yQEY3NCDSb0rEqKyOrYLBNiXiXA5T8NSY3ipaW0b71L5Cu0KLKabdLY0A==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.3.3.tgz", + "integrity": "sha512-c1HMWkZPAZKQesKj2HqA6Zk1YEcv7Doc6JU9YJepXG8KpK4XZmI0L5vESSfySmidmasm05QDyLAcwDnNSvtfRA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.3.3.tgz", + "integrity": "sha512-eL7gW4D22OBb3fi04s5ElzzNk+gkGEahaTNMB29UDLDeORe+0qIkhccBPovXW3/IRpdt8W7o8X2myUDXlTSYDQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.3.3.tgz", + "integrity": "sha512-3sw9vvQqro4yKsOWfKebzfTcdPKRwMrodGXWbfwkTsL61EQKdwsMNKA/0dDe0/ubAAZ0yPgstX8/gs94Svzfgw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.3.3.tgz", + "integrity": "sha512-tLTF17wl4CCwOlo1Tt0NbduwUduDvM6c8JSyearcLu6hpNHQo44RF3rKN2i7UkxwCqXyiudZK0lS421CC/TPZA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/electron": { + "version": "44.9.0", + "resolved": "https://registry.npmjs.org/electron/-/electron-44.9.0.tgz", + "integrity": "sha512-boP8zA5OIoMGYe3uWP4IqB1VJNoEzdHYgcBaVYnd1DBv5TogwXkbM4yJGSfNmYCzalVQ5J3TAIbaKOY+ILTnyg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@electron-internal/extract-zip": "^1.0.1", + "@electron/get": "^5.0.0", + "@types/node": "^24.9.0" + }, + "bin": { + "electron": "cli.js", + "install-electron": "install.js" + }, + "engines": { + "node": ">= 22.12.0" + } + }, + "node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "peer": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC", + "peer": true + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/koffi": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.3.3.tgz", + "integrity": "sha512-XWNI4eyxJJzR2MakPZzR3QeZ06JOklLbxglin0dHhx/qIQCby8lqHs9Acq4Q7CpCKnOrBSj+6b9J5y69wfr2ZA==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.3.3", + "@koromix/koffi-android-x64": "3.3.3", + "@koromix/koffi-darwin-arm64": "3.3.3", + "@koromix/koffi-darwin-x64": "3.3.3", + "@koromix/koffi-freebsd-arm64": "3.3.3", + "@koromix/koffi-freebsd-ia32": "3.3.3", + "@koromix/koffi-freebsd-x64": "3.3.3", + "@koromix/koffi-linux-arm": "3.3.3", + "@koromix/koffi-linux-arm64": "3.3.3", + "@koromix/koffi-linux-ia32": "3.3.3", + "@koromix/koffi-linux-loong64": "3.3.3", + "@koromix/koffi-linux-ppc64": "3.3.3", + "@koromix/koffi-linux-riscv64": "3.3.3", + "@koromix/koffi-linux-x64": "3.3.3", + "@koromix/koffi-openbsd-arm64": "3.3.3", + "@koromix/koffi-openbsd-ia32": "3.3.3", + "@koromix/koffi-openbsd-x64": "3.3.3", + "@koromix/koffi-win32-arm64": "3.3.3", + "@koromix/koffi-win32-ia32": "3.3.3", + "@koromix/koffi-win32-x64": "3.3.3" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "peer": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sumchecker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", + "integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "debug": "^4.1.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici": { + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT", + "peer": true + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/electron/package.json b/electron/package.json new file mode 100644 index 0000000..df08b7b --- /dev/null +++ b/electron/package.json @@ -0,0 +1,34 @@ +{ + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "description": "Show trame-rca shared-texture views in Electron, without encoding frames", + "license": "Apache-2.0", + "author": "Kitware Inc.", + "repository": { + "type": "git", + "url": "https://github.com/Kitware/trame-rca.git", + "directory": "electron" + }, + "type": "commonjs", + "main": "src/main/index.js", + "exports": { + ".": "./src/main/index.js", + "./preload": "./src/preload.js" + }, + "files": [ + "src/**/*.js" + ], + "scripts": { + "lint": "eslint src test", + "test": "node --test" + }, + "peerDependencies": { + "electron": ">=44.7.0" + }, + "dependencies": { + "koffi": "^3.3.3" + }, + "devDependencies": { + "eslint": "^8.57.1" + } +} diff --git a/electron/src/main/dma/iosurface.js b/electron/src/main/dma/iosurface.js new file mode 100644 index 0000000..667777c --- /dev/null +++ b/electron/src/main/dma/iosurface.js @@ -0,0 +1,36 @@ +// Open another process's IOSurface by its global ID with IOSurfaceLookup. +// Only works for surfaces created with kIOSurfaceIsGlobal (deprecated, unsandboxed only). +const koffi = require('koffi'); + +const iosurface = koffi.load('/System/Library/Frameworks/IOSurface.framework/IOSurface'); +const cf = koffi.load('/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation'); + +const IOSurfaceRef = koffi.pointer('IOSurfaceRef', koffi.opaque()); +const IOSurfaceLookup = iosurface.func('IOSurfaceLookup', IOSurfaceRef, ['uint32_t']); +const IOSurfaceGetWidth = iosurface.func('IOSurfaceGetWidth', 'size_t', [IOSurfaceRef]); +const IOSurfaceGetHeight = iosurface.func('IOSurfaceGetHeight', 'size_t', [IOSurfaceRef]); +const IOSurfaceGetPixelFormat = iosurface.func('IOSurfaceGetPixelFormat', 'uint32_t', [IOSurfaceRef]); +const CFRelease = cf.func('CFRelease', 'void', [IOSurfaceRef]); + +const fourccStr = (code) => String.fromCharCode((code >>> 24) & 0xff, (code >>> 16) & 0xff, (code >>> 8) & 0xff, code & 0xff); + +// Returns a retained IOSurfaceRef; release it with releaseSurface(). +const lookupSurface = (id) => { + const ref = IOSurfaceLookup(id); + if (!ref) throw new Error(`IOSurfaceLookup(${id}) failed (not global, or already freed)`); + return ref; +}; + +const describeSurface = (ref) => + `${IOSurfaceGetWidth(ref)}x${IOSurfaceGetHeight(ref)} ${fourccStr(IOSurfaceGetPixelFormat(ref))}`; + +// sharedTexture wants the IOSurfaceRef pointer value in a Buffer. +const surfaceHandle = (ref) => { + const buf = Buffer.alloc(8); + buf.writeBigUInt64LE(BigInt(koffi.address(ref))); + return buf; +}; + +const releaseSurface = (ref) => CFRelease(ref); + +module.exports = { lookupSurface, describeSurface, surfaceHandle, releaseSurface }; diff --git a/electron/src/main/dma/pidfd.js b/electron/src/main/dma/pidfd.js new file mode 100644 index 0000000..9eb736e --- /dev/null +++ b/electron/src/main/dma/pidfd.js @@ -0,0 +1,34 @@ +// Duplicate a file descriptor out of another process with pidfd_getfd(2). +// Requires ptrace access to the target (e.g. it is our child with ptrace_scope=1). +const koffi = require('koffi'); + +const libc = koffi.load('libc.so.6'); +const syscall = libc.func('long syscall(long nr, ...)'); +const close = libc.func('int close(int fd)'); +const strerror = libc.func('const char *strerror(int errnum)'); + +// x86_64 / asm-generic syscall numbers +const SYS_pidfd_open = 434; +const SYS_pidfd_getfd = 438; + +const check = (ret, what) => { + const r = Number(ret); + if (r < 0) { + const errno = koffi.errno(); + throw new Error(`${what} failed: ${strerror(errno)} (errno ${errno})`); + } + return r; +}; + +const dupFdFromProcess = (pid, targetFd) => { + const pidfd = check(syscall(SYS_pidfd_open, 'int', pid, 'uint', 0), `pidfd_open(${pid})`); + try { + return check(syscall(SYS_pidfd_getfd, 'int', pidfd, 'int', targetFd, 'uint', 0), `pidfd_getfd(${pid}, ${targetFd})`); + } finally { + close(pidfd); + } +}; + +const closeFd = (fd) => close(fd); + +module.exports = { dupFdFromProcess, closeFd }; diff --git a/electron/src/main/dma/winhandle.js b/electron/src/main/dma/winhandle.js new file mode 100644 index 0000000..44093a1 --- /dev/null +++ b/electron/src/main/dma/winhandle.js @@ -0,0 +1,38 @@ +// Duplicate an NT handle out of another process with DuplicateHandle. +// Requires PROCESS_DUP_HANDLE on the target (e.g. it is our child, running as the same user). +const koffi = require('koffi'); + +const kernel32 = koffi.load('kernel32.dll'); +const OpenProcess = kernel32.func('uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)'); +const GetCurrentProcess = kernel32.func('uintptr_t __stdcall GetCurrentProcess()'); +const DuplicateHandle = kernel32.func('int __stdcall DuplicateHandle(uintptr_t srcProcess, uintptr_t src, uintptr_t dstProcess, _Out_ uintptr_t *dst, uint32_t access, int inherit, uint32_t options)'); +const CloseHandle = kernel32.func('int __stdcall CloseHandle(uintptr_t handle)'); +const GetLastError = kernel32.func('uint32_t __stdcall GetLastError()'); + +const PROCESS_DUP_HANDLE = 0x0040; +const DUPLICATE_SAME_ACCESS = 0x2; + +const dupHandleFromProcess = (pid, remoteHandle) => { + const proc = OpenProcess(PROCESS_DUP_HANDLE, 0, pid); + if (!proc) throw new Error(`OpenProcess(${pid}) failed: error ${GetLastError()}`); + try { + const out = [0]; + if (!DuplicateHandle(proc, remoteHandle, GetCurrentProcess(), out, 0, 0, DUPLICATE_SAME_ACCESS)) { + throw new Error(`DuplicateHandle(${pid}, 0x${remoteHandle.toString(16)}) failed: error ${GetLastError()}`); + } + return out[0]; + } finally { + CloseHandle(proc); + } +}; + +// sharedTexture wants the HANDLE value in a Buffer. +const handleBuffer = (handle) => { + const buf = Buffer.alloc(8); + buf.writeBigUInt64LE(BigInt(handle)); + return buf; +}; + +const closeHandle = (handle) => CloseHandle(handle); + +module.exports = { dupHandleFromProcess, handleBuffer, closeHandle }; diff --git a/electron/src/main/index.js b/electron/src/main/index.js new file mode 100644 index 0000000..21fa5b8 --- /dev/null +++ b/electron/src/main/index.js @@ -0,0 +1,229 @@ +const { app, ipcMain, sharedTexture } = require('electron/main'); +const { spawn } = require('node:child_process'); +const net = require('node:net'); +const path = require('node:path'); + +const protocol = require('./protocol'); + +// Loaded on demand: pidfd.js needs Linux's libc, iosurface.js needs macOS frameworks, +// winhandle.js needs kernel32. +const pidfd = () => require('./dma/pidfd'); +const iosurface = () => require('./dma/iosurface'); +const winhandle = () => require('./dma/winhandle'); + +// The preload a BrowserWindow needs to show shared-texture views. +const preloadPath = path.join(__dirname, '..', 'preload.js'); + +// The webPreferences a BrowserWindow needs to show shared-texture views. +const webPreferences = (extra = {}) => ({ + preload: preloadPath, + // The preload needs require('electron').sharedTexture. + sandbox: false, + ...extra, +}); + +// Holds this process's reference to the producer's current texture. +class TextureHandle { + constructor(msg, log) { + this.localFds = []; + this.surfaceRef = null; + this.localHandle = null; + this.info = protocol.textureInfo(msg, this._open(msg, log)); + } + + _open(msg, log) { + const kind = protocol.handleKind(msg); + if (kind === 'ioSurface') { + // Open the producer's global IOSurface in this process. + const { lookupSurface, describeSurface, surfaceHandle } = iosurface(); + this.surfaceRef = lookupSurface(msg.ioSurfaceId); + log.info( + `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) IOSurface id=${msg.ioSurfaceId} ${describeSurface(this.surfaceRef)}`, + ); + return { ioSurface: surfaceHandle(this.surfaceRef) }; + } + if (kind === 'ntHandle') { + // Duplicate the producer's NT handle into this process. + const { dupHandleFromProcess, handleBuffer } = winhandle(); + this.localHandle = dupHandleFromProcess(msg.pid, msg.ntHandle); + log.info( + `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) handle 0x${msg.ntHandle.toString(16)} -> 0x${this.localHandle.toString(16)}`, + ); + return { ntHandle: handleBuffer(this.localHandle) }; + } + // Duplicate the producer's dma-buf fds into this process. + this.localFds = msg.planes.map((p) => pidfd().dupFdFromProcess(msg.pid, p.fd)); + log.info( + `[trame-rca] texture ${msg.width}x${msg.height} ${msg.fourcc} (${msg.pixelFormat}) modifier=${msg.modifier} fds ${msg.planes.map((p) => p.fd)} -> ${this.localFds}`, + ); + return protocol.nativePixmap(msg, this.localFds); + } + + release() { + this.localFds.forEach((fd) => pidfd().closeFd(fd)); + this.localFds = []; + if (this.surfaceRef) iosurface().releaseSurface(this.surfaceRef); + this.surfaceRef = null; + if (this.localHandle) winhandle().closeHandle(this.localHandle); + this.localHandle = null; + } +} + +// Spawn the trame server and load its page in `win` once it is ready. The page's +// shared-texture views then show the texture the server renders into. +// +// `command` is the executable to run (a python interpreter, a frozen backend, ...) and +// `args` its arguments, e.g. ['app.py']. +// trame-rca-electron appends the arguments the server needs: --server --port 0 --emp-port . +// +// Returns { process, close }. close() kills the server and releases the texture; it is +// called on its own when `win` closes. +const launchTrame = ( + win, + { command, args = [], cwd, env, onExit, onStats, log = console }, +) => { + if (!command) throw new Error('launchTrame: `command` is required'); + let socket = null; + let texture = null; + let closed = false; + const result = { process: null, close: null }; + + const send = (msg) => { + if (socket && !socket.destroyed) socket.write(protocol.encode(msg)); + }; + + const releaseTexture = () => { + texture?.release(); + texture = null; + }; + + const forwardFrame = async (info, msg) => { + // Once per frame: a frame that failed to transfer may still be released late by the renderer. + let done = false; + const frameDone = () => { + if (done) return; + done = true; + send({ type: 'frame_done', seq: msg.seq }); + }; + const frameDropped = () => { + if (done) return; + done = true; + send({ type: 'frame_done', seq: msg.seq, dropped: true }); + }; + let imported = null; + try { + // Electron retains the texture's handle, so it can be re-imported every frame. + imported = sharedTexture.importSharedTexture({ + textureInfo: { ...info, timestamp: msg.timestamp_us }, + allReferencesReleased: frameDone, + }); + await sharedTexture.sendSharedTexture({ + frame: win.webContents.mainFrame, + importedSharedTexture: imported, + }); + } catch (err) { + // sendSharedTexture uses a 1000ms timeout. If the ERP is still busy loading the page, + // or if some other error occurred, the page would never have shown this frame, so ask + // for it again. + log.warn(`[trame-rca] frame ${msg.seq} dropped: ${err.message}`); + frameDropped(); + } finally { + imported?.release(); + } + }; + + const onMessage = (conn) => (msg) => { + if (msg.type === 'server_ready') { + try { + protocol.checkProtocol(msg); + } catch (err) { + log.error(`[trame-rca] ${err.message}`); + conn.destroy(); + return; + } + log.info(`[trame-rca] loading ${msg.url}`); + win.loadURL(msg.url); + } else if (msg.type === 'texture') { + try { + // A new texture after a resize. Frames still in flight hold their own imports. + releaseTexture(); + texture = new TextureHandle(msg, log); + } catch (err) { + log.error('[trame-rca] failed to open the texture:', err); + conn.destroy(); + } + } else if (msg.type === 'frame' && texture) { + forwardFrame(texture.info, msg); + } + }; + + const handleConnection = (conn) => { + if (socket) { + // One server per window: ignore anything else knocking. + conn.destroy(); + return; + } + socket = conn; + // Without this, Nagle holds frame_done back until the producer's delayed ACK. + conn.setNoDelay(true); + send( + protocol.helloMessage( + protocol.passportName({ + platform: process.platform, + ozonePlatform: app.commandLine.getSwitchValue('ozone-platform'), + ozonePlatformHint: app.commandLine.getSwitchValue('ozone-platform-hint'), + waylandDisplay: process.env.WAYLAND_DISPLAY, + }), + ), + ); + conn.on( + 'data', + protocol.createLineParser(onMessage(conn), (line, err) => + log.error( + `[trame-rca] bad line from trame (${line.length} chars): ${JSON.stringify(line)}`, + err, + ), + ), + ); + conn.on('close', () => log.info('[trame-rca] trame disconnected')); + }; + + const statsListener = (event, stats) => { + if (event.sender === win.webContents) onStats?.(stats); + }; + ipcMain.on('trame-rca:stats', statsListener); + + const server = net.createServer(handleConnection); + + result.close = () => { + if (closed) return; + closed = true; + ipcMain.off('trame-rca:stats', statsListener); + result.process?.kill(); + socket?.destroy(); + socket = null; + server.close(); + releaseTexture(); + }; + win.once('closed', result.close); + + server.listen(0, '127.0.0.1', () => { + if (closed) return; + const { port } = server.address(); + const fullArgs = [...args, '--server', '--port', '0', '--emp-port', String(port)]; + log.info(`[trame-rca] spawning ${command} ${fullArgs.join(' ')}`); + result.process = spawn(command, fullArgs, { + cwd, + env: env ?? process.env, + stdio: ['ignore', 'inherit', 'inherit'], + }); + result.process.on('exit', (code, signal) => { + log.info(`[trame-rca] trame exited code=${code} signal=${signal}`); + onExit?.(code, signal); + }); + }); + + return result; +}; + +module.exports = { launchTrame, preloadPath, webPreferences, protocol }; diff --git a/electron/src/main/protocol.js b/electron/src/main/protocol.js new file mode 100644 index 0000000..7115740 --- /dev/null +++ b/electron/src/main/protocol.js @@ -0,0 +1,93 @@ +// The texture channel between the Electron main process (EMP) and trame-rca, as +// newline-delimited JSON on 127.0.0.1. See trame_rca/electron/comm.py for the messages. +// Nothing in here touches Electron or the OS, so it can be tested with plain node. + +// Must match trame_rca.electron.comm.PROTOCOL_VERSION. +const PROTOCOL_VERSION = 1; + +// The texture passport trame-rca should share its texture with, for this platform. +// On Linux, Electron uses Wayland when it's available, unless --ozone-platform=x11. +const passportName = ({ platform, ozonePlatform, ozonePlatformHint, waylandDisplay }) => { + if (platform === 'darwin') return 'iosurface'; + if (platform === 'win32') return 'nthandle'; + // --ozone-platform wins over --ozone-platform-hint, whose default is auto. + const ozone = ozonePlatform || ozonePlatformHint || 'auto'; + const wayland = ozone === 'auto' ? !!waylandDisplay : ozone === 'wayland'; + return wayland ? 'wayland-dmabuf' : 'x11-dmabuf'; +}; + +const helloMessage = (passport) => ({ type: 'hello', protocol: PROTOCOL_VERSION, passport }); + +const checkProtocol = (msg) => { + if (msg.protocol !== PROTOCOL_VERSION) { + throw new Error( + `trame-rca speaks shared texture protocol ${msg.protocol}, trame-rca-electron speaks ` + + `${PROTOCOL_VERSION}: update trame-rca or trame-rca-electron so they match`, + ); + } +}; + +// Calls onMessage with every complete JSON line fed in, and onBadLine with lines that +// aren't JSON instead of throwing out of the socket's data handler. +const createLineParser = (onMessage, onBadLine) => { + let buf = ''; + return (chunk) => { + buf += chunk.toString(); + let nl; + while ((nl = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + let msg; + try { + msg = JSON.parse(line); + } catch (err) { + onBadLine?.(line, err); + continue; + } + onMessage(msg); + } + }; +}; + +const encode = (msg) => JSON.stringify(msg) + '\n'; + +// Which OS handle a texture message carries. +const handleKind = (msg) => { + if (msg.ioSurfaceId !== undefined) return 'ioSurface'; + if (msg.ntHandle !== undefined) return 'ntHandle'; + return 'nativePixmap'; +}; + +// The nativePixmap handle of a dma-buf texture, once its fds are duplicated into this process. +const nativePixmap = (msg, localFds) => ({ + nativePixmap: { + planes: msg.planes.map((p, i) => ({ + fd: localFds[i], + stride: p.stride, + offset: p.offset, + size: p.size, + })), + modifier: msg.modifier, + supportsZeroCopyWebGpuImport: false, + }, +}); + +// The textureInfo of sharedTexture.importSharedTexture, for a texture message and its handle. +const textureInfo = (msg, handle) => ({ + pixelFormat: msg.pixelFormat, + codedSize: { width: msg.width, height: msg.height }, + visibleRect: { x: 0, y: 0, width: msg.width, height: msg.height }, + handle, +}); + +module.exports = { + PROTOCOL_VERSION, + checkProtocol, + createLineParser, + encode, + handleKind, + helloMessage, + nativePixmap, + passportName, + textureInfo, +}; diff --git a/electron/src/preload.js b/electron/src/preload.js new file mode 100644 index 0000000..1f2f794 --- /dev/null +++ b/electron/src/preload.js @@ -0,0 +1,67 @@ +// Draws the frames the main process sends into the page's shared-texture-display-area. +// Use it as a BrowserWindow's preload (see webPreferences()), or require it from yours. +const { contextBridge, ipcRenderer, sharedTexture } = require('electron'); + +const STATS_INTERVAL_MS = 1000; + +// The canvas of the page's shared-texture-display-area, attached when the component mounts. +let canvas = null; +let ctx = null; +let onStats = null; + +let frames = 0; +let windowStart = performance.now(); +let windowFrames = 0; +let info = ''; + +sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture }) => { + if (!canvas) { + importedSharedTexture.release(); + return; + } + const vf = importedSharedTexture.getVideoFrame(); + // The canvas's pixels follow the producer's texture, which follows the canvas's size. + if (canvas.width !== vf.displayWidth || canvas.height !== vf.displayHeight) { + canvas.width = vf.displayWidth; + canvas.height = vf.displayHeight; + } + // The producers render with GL, whose first row is the bottom, into textures whose first + // row is the top: draw the frame upside down to undo that. + ctx.setTransform(1, 0, 0, -1, 0, canvas.height); + ctx.drawImage(vf, 0, 0, canvas.width, canvas.height); + ctx.setTransform(1, 0, 0, 1, 0, 0); + info = `${vf.codedWidth}x${vf.codedHeight} ${vf.format}`; + vf.close(); + importedSharedTexture.release(); + frames++; + windowFrames++; +}); + +// Frames are rendered on demand, so report stats over a timed interval. +setInterval(() => { + const now = performance.now(); + const fps = (windowFrames * 1000) / (now - windowStart); + const hadFrames = windowFrames > 0; + windowStart = now; + windowFrames = 0; + onStats?.({ fps: Number(fps.toFixed(1)), frames, info }); + if (hadFrames && ctx) { + ipcRenderer.send('trame-rca:stats', { frames, fps: fps.toFixed(1), info }); + } +}, STATS_INTERVAL_MS); + +// The page's world can't touch sharedTexture or VideoFrames from here, but it can hand us its +// canvas, and a callback for the frame rate. +contextBridge.exposeInMainWorld('trameRcaSharedTexture', { + attach: (el, statsCallback) => { + canvas = el; + ctx = el.getContext('2d'); + onStats = statsCallback ?? null; + }, + detach: (el) => { + if (canvas !== el) return; + canvas = null; + ctx = null; + onStats = null; + }, +}); diff --git a/electron/test/protocol.test.js b/electron/test/protocol.test.js new file mode 100644 index 0000000..49c8290 --- /dev/null +++ b/electron/test/protocol.test.js @@ -0,0 +1,78 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); + +const protocol = require('../src/main/protocol'); + +test('passportName picks the platform passport', () => { + assert.equal(protocol.passportName({ platform: 'darwin' }), 'iosurface'); + assert.equal(protocol.passportName({ platform: 'win32' }), 'nthandle'); +}); + +test('passportName follows ozone switches and WAYLAND_DISPLAY on linux', () => { + const linux = (o) => protocol.passportName({ platform: 'linux', ...o }); + assert.equal(linux({ waylandDisplay: 'wayland-0' }), 'wayland-dmabuf'); + assert.equal(linux({}), 'x11-dmabuf'); + assert.equal(linux({ ozonePlatform: 'x11', waylandDisplay: 'wayland-0' }), 'x11-dmabuf'); + assert.equal(linux({ ozonePlatformHint: 'wayland' }), 'wayland-dmabuf'); + // --ozone-platform should win over the --ozone-platform-hint. + assert.equal(linux({ ozonePlatform: 'x11', ozonePlatformHint: 'wayland' }), 'x11-dmabuf'); +}); + +test('hello carries the protocol version', () => { + assert.deepEqual(protocol.helloMessage('iosurface'), { + type: 'hello', + protocol: protocol.PROTOCOL_VERSION, + passport: 'iosurface', + }); +}); + +test('checkProtocol rejects other versions', () => { + protocol.checkProtocol({ protocol: protocol.PROTOCOL_VERSION }); + assert.throws(() => protocol.checkProtocol({ protocol: protocol.PROTOCOL_VERSION + 1 }), /protocol/); + assert.throws(() => protocol.checkProtocol({}), /protocol/); +}); + +test('line parser joins chunks and reports bad lines', () => { + const messages = []; + const bad = []; + const feed = protocol.createLineParser( + (m) => messages.push(m), + (line) => bad.push(line), + ); + feed('{"type":"fr'); + feed('ame","seq":1}\n{oops}\n{"type":"frame","seq":2}\n{"type"'); + assert.deepEqual(messages, [ + { type: 'frame', seq: 1 }, + { type: 'frame', seq: 2 }, + ]); + assert.deepEqual(bad, ['{oops}']); + feed(':"frame","seq":3}\n'); + assert.equal(messages.length, 3); +}); + +test('handleKind and textureInfo', () => { + assert.equal(protocol.handleKind({ ioSurfaceId: 3 }), 'ioSurface'); + assert.equal(protocol.handleKind({ ntHandle: 4 }), 'ntHandle'); + assert.equal(protocol.handleKind({ planes: [] }), 'nativePixmap'); + const msg = { pixelFormat: 'bgra', width: 4, height: 2 }; + assert.deepEqual(protocol.textureInfo(msg, { h: 1 }), { + pixelFormat: 'bgra', + codedSize: { width: 4, height: 2 }, + visibleRect: { x: 0, y: 0, width: 4, height: 2 }, + handle: { h: 1 }, + }); +}); + +test('nativePixmap swaps in the local fds', () => { + const msg = { + modifier: '0', + planes: [{ fd: 30, stride: 16, offset: 0, size: 64 }], + }; + assert.deepEqual(protocol.nativePixmap(msg, [7]), { + nativePixmap: { + planes: [{ fd: 7, stride: 16, offset: 0, size: 64 }], + modifier: '0', + supportsZeroCopyWebGpuImport: false, + }, + }); +}); diff --git a/examples/06_electron/README.md b/examples/06_electron/README.md new file mode 100644 index 0000000..eb27d2b --- /dev/null +++ b/examples/06_electron/README.md @@ -0,0 +1,21 @@ +# Shared texture with Electron + +`app.py` renders its VTK view into a GPU texture that Electron shows as is: no encoding, +no copy to the CPU. The Electron app in `./electron` starts `app.py` and loads its page. + +## Run + +```sh +# From the repository root, in a virtual environment +pip install -e ".[electron]" trame trame-vuetify +cd electron +npm i +cd .. + +cd examples/06_electron/electron +npm i +npm start +``` + +`npm start` runs `app.py` with the `python3` of the active environment, or `$PYTHON`. + diff --git a/examples/06_electron/app.py b/examples/06_electron/app.py new file mode 100644 index 0000000..f35e591 --- /dev/null +++ b/examples/06_electron/app.py @@ -0,0 +1,249 @@ +"""A trame app whose VTK view is shared with Electron as a texture, not streamed. + +Started by the Electron app in ./electron, see the README. +""" + +import json +import platform +from importlib.metadata import version + +from trame.app import TrameApp +from trame.decorators import change +from trame.ui.vuetify3 import SinglePageWithDrawerLayout +from trame.widgets import html, vuetify3 as v3 +from trame_rca.electron import init_vtk +from trame_rca.widgets import rca + +# Before creating any VTK render window: VTK must render on the GPU Electron uses. +init_vtk() + +from vtkmodules.vtkCommonDataModel import vtkPiecewiseFunction # noqa: E402 +from vtkmodules.vtkFiltersSources import vtkPartitionedDataSetCollectionSource # noqa: E402 +from vtkmodules.vtkImagingCore import vtkRTAnalyticSource # noqa: E402 +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera # noqa: E402 +from vtkmodules.vtkRenderingCore import ( # noqa: E402 + vtkActor, + vtkColorTransferFunction, + vtkCompositePolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkVolume, + vtkVolumeProperty, +) +from vtkmodules.vtkRenderingUI import vtkGenericRenderWindowInteractor # noqa: E402 +from vtkmodules.vtkRenderingVolumeOpenGL2 import vtkSmartVolumeMapper # noqa: E402 + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: E402, F401 + + +def create_vtk_pipeline(): + wavelet = vtkRTAnalyticSource() + volume_mapper = vtkSmartVolumeMapper() + volume_mapper.SetInputConnection(wavelet.output_port) + color_tf = vtkColorTransferFunction() + color_tf.AddRGBPoint(0.0, 0.0, 0.0, 1.0) + color_tf.AddRGBPoint(255.0 / 2.0, 0.0, 1.0, 1.0) + color_tf.AddRGBPoint(255.0, 1.0, 0.0, 1.0) + opacity_tf = vtkPiecewiseFunction() + opacity_tf.AddPoint(0.0, 0.0) + opacity_tf.AddPoint(255.0, 0.1) + volume_property = vtkVolumeProperty() + volume_property.SetColor(color_tf) + volume_property.SetScalarOpacity(opacity_tf) + volume_property.ShadeOn() + volume_property.SetInterpolationTypeToLinear() + volume_actor = vtkVolume(mapper=volume_mapper, property=volume_property) + + shapes = vtkPartitionedDataSetCollectionSource() + cpdm = vtkCompositePolyDataMapper() + cpdm.SetInputConnection(shapes.output_port) + shapes_actor = vtkActor(mapper=cpdm) + + renderer = vtkRenderer() + renderer.AddVolume(volume_actor) + renderer.AddActor(shapes_actor) + # The texture's alpha reaches Electron, so keep the background opaque. + renderer.SetBackgroundAlpha(1.0) + renderer.ResetCamera() + + render_window = vtkRenderWindow() + render_window.AddRenderer(renderer) + # trame-rca replays the canvas's input on the interactor. + vtkGenericRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, renderer, opacity_tf, shapes + + +PYTHON_PACKAGES = [ + "vtk", + "trame", + "trame-client", + "trame-server", + "trame-vuetify", + "trame-rca", + "wslink", +] +ELECTRON_VERSIONS = [ + ("Electron", "electron"), + ("Chromium", "chrome"), + ("Node", "node"), + ("V8", "v8"), +] + + +class ElectronApp(TrameApp): + def __init__(self, server=None): + super().__init__(server) + self.server.cli.add_argument( + "--electron-versions", + default="{}", + help="The Electron main process's process.versions, as JSON", + ) + args, _ = self.server.cli.parse_known_args() + + electron_versions = json.loads(args.electron_versions) + self.state.electron_versions = [ + {"name": name, "version": electron_versions[key]} + for name, key in ELECTRON_VERSIONS + if key in electron_versions + ] + self.state.python_versions = [ + {"name": "Python", "version": platform.python_version()} + ] + [{"name": name, "version": version(name)} for name in PYTHON_PACKAGES] + self.state.spinning = False + # Frame stats from the canvas, once a second: only the footer needs them, not the server. + self.state.client_only("stats") + self.state.stats = {"fps": 0, "frames": 0, "info": ""} + + ( + self.render_window, + self.renderer, + self.volume_opacity_tf, + self.shapes_source, + ) = create_vtk_pipeline() + self._build_ui() + + def reset_camera(self): + self.renderer.ResetCamera() + self.view_handler.update() + + def before_render(self): + # Spin by a degree per frame: each frame asks for the next one. + if self.state.spinning: + # The interactor only hands its rates to the render window when an interaction + # starts or ends, so set the window's rate directly. A click mid-spin resets it. + self.render_window.desired_update_rate = ( + self.render_window.interactor.desired_update_rate + ) + self.renderer.GetActiveCamera().Azimuth(1.0) + self.renderer.ResetCameraClippingRange() + self.view_handler.update() + + @change("spinning") + def on_spinning_change(self, spinning, **_): + if not spinning: + # Back to a full quality still frame. + self.render_window.desired_update_rate = ( + self.render_window.interactor.still_update_rate + ) + self.view_handler.update() + + @change("number_of_shapes") + def on_number_of_shapes_change(self, number_of_shapes, **_): + self.shapes_source.number_of_shapes = min(12, max(1, int(number_of_shapes))) + self.view_handler.update() + + @change("opacity") + def on_opacity_change(self, opacity, **_): + # Node 1 is the top of the scalar range: (x, y, midpoint, sharpness). + self.volume_opacity_tf.SetNodeValue(1, [255.0, float(opacity), 0.5, 0.0]) + self.view_handler.update() + + def _build_ui(self): + with SinglePageWithDrawerLayout(self.server) as self.ui: + self.ui.title.set_text("Shared texture trame example") + + with self.ui.toolbar: + v3.VSpacer() + v3.VSlider( + v_model=("opacity", 0.1), + label="Opacity", + min=0.0, + max=1.0, + step=0.01, + thumb_label=True, + hide_details=True, + density="compact", + style="max-width: 250px", + classes="mx-2", + ) + v3.VSlider( + v_model=("number_of_shapes", 1), + label="Shapes", + min=1, + max=12, + step=1, + thumb_label=True, + hide_details=True, + density="compact", + style="max-width: 250px", + classes="mx-2", + ) + v3.VBtn(icon="mdi-crop-free", click=self.reset_camera) + v3.VBtn( + icon="mdi-rotate-3d-variant", + click="spinning = !spinning", + color=("spinning ? 'primary' : undefined",), + ) + + with self.ui.drawer: + with v3.VList(density="compact"): + v3.VListSubheader("Electron", v_if="electron_versions.length") + v3.VListItem( + v_for="item in electron_versions", + key=("item.name",), + title=("item.name",), + subtitle=("item.version",), + ) + v3.VListSubheader("Python") + v3.VListItem( + v_for="item in python_versions", + key=("item.name",), + title=("item.name",), + subtitle=("item.version",), + ) + + with self.ui.footer: + self.ui.footer.clear() + v3.VProgressCircular( + indeterminate=("!!trame__busy",), + color="#04a94d", + size=16, + width=3, + classes="ml-n3 mr-1", + ) + html.Span("{{ stats.fps }} fps", classes="text-caption") + v3.VSpacer() + html.Span( + "{{ stats.info }}", classes="text-caption text-grey-lighten-1" + ) + + with self.ui.content: + with v3.VContainer( + fluid=True, classes="pa-0 fill-height position-relative" + ): + view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + stats="stats = $event", + ) + self.view_handler = view.create_electron_handler( + self.render_window, before_render=self.before_render + ) + + +if __name__ == "__main__": + ElectronApp().server.start() diff --git a/examples/06_electron/electron/.gitignore b/examples/06_electron/electron/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/examples/06_electron/electron/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/examples/06_electron/electron/main.js b/examples/06_electron/electron/main.js new file mode 100644 index 0000000..338fb89 --- /dev/null +++ b/examples/06_electron/electron/main.js @@ -0,0 +1,28 @@ +const { app, BrowserWindow } = require('electron/main'); +const path = require('node:path'); +const { launchTrame, webPreferences } = require('@kitware/trame-rca-electron'); + +// Run ../app.py with the python of the active environment, unless PYTHON says otherwise. +const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); + +app.whenReady().then(() => { + const win = new BrowserWindow({ + width: 1920, + height: 1080, + useContentSize: true, + webPreferences: webPreferences(), + }); + const { electron, chrome, node, v8 } = process.versions; + launchTrame(win, { + command: python, + args: [ + path.join(__dirname, '..', 'app.py'), + '--electron-versions', + JSON.stringify({ electron, chrome, node, v8 }), + ], + onStats: (s) => console.log(`[renderer] frames=${s.frames} fps=${s.fps} ${s.info}`), + onExit: () => app.quit(), + }); +}); + +app.on('window-all-closed', () => app.quit()); diff --git a/examples/06_electron/electron/package.json b/examples/06_electron/electron/package.json new file mode 100644 index 0000000..6660099 --- /dev/null +++ b/examples/06_electron/electron/package.json @@ -0,0 +1,18 @@ +{ + "name": "trame-rca-electron-example", + "private": true, + "version": "0.0.0", + "description": "Electron shell for the shared texture trame-rca example", + "license": "Apache-2.0", + "type": "commonjs", + "main": "main.js", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "@kitware/trame-rca-electron": "file:../../../electron" + }, + "devDependencies": { + "electron": "^44.7.0" + } +} diff --git a/js-components/src/style.css b/js-components/src/style.css index c2ba355..69f0940 100644 --- a/js-components/src/style.css +++ b/js-components/src/style.css @@ -42,6 +42,16 @@ position: relative; } +.shared-texture-display-area { + display: block; + margin: 0; + outline: none; + padding: 0; + position: relative; + width: 100%; + height: 100%; +} + .image-region { margin: 0; padding: 0; diff --git a/pyproject.toml b/pyproject.toml index 1be62f4..eb32b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,9 @@ vtkstreaming = [ "vtk", "vtk-streaming>=0.4.0", ] +electron = [ + "vtk", # Shared textures with trame-rca-electron +] diff --git a/src/trame_rca/electron/__init__.py b/src/trame_rca/electron/__init__.py new file mode 100644 index 0000000..54f965c --- /dev/null +++ b/src/trame_rca/electron/__init__.py @@ -0,0 +1,96 @@ +"""Render VTK into a texture shared with an Electron app, instead of streaming images. + +The Electron side lives in the trame-rca-electron npm package: it spawns the trame +server with ``--emp-port`` and shows the shared texture in the page. + +Call :func:`init_vtk` before creating any VTK render window, then use +``RemoteControlledArea(display="shared-texture").create_electron_handler(...)``. +""" + +import argparse +import logging +import os +import sys + +from .comm import PROTOCOL_VERSION + +logger = logging.getLogger(__name__) + +__all__ = [ + "EMP_PORT_ARG", + "PROTOCOL_VERSION", + "emp_port", + "init_vtk", + "server_url", +] + +# The CLI argument that trame-rca-electron passes the port of its `texture` channel. +EMP_PORT_ARG = "--emp-port" + +_initialized = False + + +def emp_port(server): + """The port of the Electron main process's `texture` channel, from the CLI. + + Raises RuntimeError when the server wasn't started by trame-rca-electron. + """ + try: + server.cli.add_argument( + EMP_PORT_ARG, + type=int, + help="Port of the Electron main process's texture channel", + ) + except argparse.ArgumentError: + pass # Another view added it already. + args, _ = server.cli.parse_known_args() + if args.emp_port is None: + raise RuntimeError( + f"{EMP_PORT_ARG} is missing: shared-texture views only work when " + "the trame server is started by trame-rca-electron" + ) + return args.emp_port + + +def server_url(server): + """The URL of the page trame serves, once the server is ready.""" + return f"http://{server.server_options.host}:{server.port}/" + + +def init_vtk(): + """Make VTK render on the GPU Electron composites with. + + The shared texture can only be imported on the GPU Chromium uses, and VTK reads + some of these settings when its render window is constructed: call this before + creating any. It can be called more than once. + + Linux: ``RENDER_NODE`` overrides the render node (default: the GPU driving the + display). + Windows: ``D3D_ADAPTER`` overrides the DXGI adapter index (default 0). + MacOS: Calling this method is unnecessary. + """ + global _initialized + if _initialized: + return + _initialized = True + + if sys.platform == "linux": + from .passports import wayland_dmabuf + + if os.environ.get("RENDER_NODE"): + render_node, reason = os.environ["RENDER_NODE"], "env" + else: + render_node, reason = wayland_dmabuf.default_render_node() + logger.info("using render node %s (%s)", render_node, reason) + # vtkEGLRenderWindow::SetDeviceIndex doesn't reach the EGL display setup; the + # constructor only reads the device from this variable. + os.environ["VTK_EGL_DEVICE_INDEX"] = str( + wayland_dmabuf.device_index(render_node) + ) + os.environ["VTK_DEFAULT_OPENGL_WINDOW"] = "vtkEGLRenderWindow" + elif sys.platform == "win32": + from .passports import nthandle + + # The shared texture must live on the GPU Chromium uses, which is DXGI adapter 0 + # unless told otherwise. VTK's WGL context has to be on that GPU too. + nthandle.init_device(int(os.environ.get("D3D_ADAPTER", "0"))) diff --git a/src/trame_rca/electron/comm.py b/src/trame_rca/electron/comm.py new file mode 100644 index 0000000..25faef1 --- /dev/null +++ b/src/trame_rca/electron/comm.py @@ -0,0 +1,66 @@ +""" +Talks newline-delimited JSON to the Electron main process (EMP) on 127.0.0.1:. + +-> means python to EMP +<- means EMP to Python + + <- {"type":"hello", "protocol": N, "passport": name} at start, the texture passport to use + -> {"type":"server_ready", "protocol": N, "url": url} once trame serves its page + -> {"type":"texture", ...} at start and after every resize + -> {"type":"frame", "seq": n, "timestamp_us": t} after every render (finished on the GPU) + <- {"type":"frame_done", "seq": n} when Electron released the texture + <- {"type":"frame_done", "seq": n, "dropped": true} when the frame didn't reach the ERP + +Everything else (size, input, app state) goes from the renderer process to trame over its websocket. + +PROTOCOL_VERSION must match the one of the trame-rca-electron npm package. +""" + +import asyncio +import json +import socket + +PROTOCOL_VERSION = 1 + + +class ProtocolMismatchError(RuntimeError): + pass + + +def check_protocol(msg): + """Raise ProtocolMismatchError unless the EMP's hello speaks PROTOCOL_VERSION.""" + version = msg.get("protocol") + if version != PROTOCOL_VERSION: + raise ProtocolMismatchError( + f"Electron speaks shared texture protocol {version}, trame-rca speaks " + f"{PROTOCOL_VERSION}: update trame-rca or trame-rca-electron so they match" + ) + + +class LineConnection: + def __init__(self, reader, writer): + self.reader = reader + self.writer = writer + + @classmethod + async def open(cls, port, host="127.0.0.1"): + reader, writer = await asyncio.open_connection(host, port) + # Small request/response messages: don't let Nagle wait for delayed ACKs. + writer.get_extra_info("socket").setsockopt( + socket.IPPROTO_TCP, socket.TCP_NODELAY, 1 + ) + return cls(reader, writer) + + async def send(self, msg): + self.writer.write((json.dumps(msg) + "\n").encode()) + await self.writer.drain() + + async def recv(self): + """Return the next message. Raises EOFError on close.""" + line = await self.reader.readline() + if not line: + raise EOFError + return json.loads(line) + + def close(self): + self.writer.close() diff --git a/src/trame_rca/electron/passports/__init__.py b/src/trame_rca/electron/passports/__init__.py new file mode 100644 index 0000000..a31eefa --- /dev/null +++ b/src/trame_rca/electron/passports/__init__.py @@ -0,0 +1,29 @@ +"""Platform specific ways to share a VTK texture with Electron.""" + +from .base import TexturePassport + +__all__ = [ + "TexturePassport", + "texture_passport_class", +] + + +def texture_passport_class(name): + """The name of the passport class that the EMP wants to use.""" + if not isinstance(name, str): + raise TypeError("Passport name must be a string") + if name == "iosurface": + from .iosurface import IOSurfaceTexturePassport + + return IOSurfaceTexturePassport + if name == "nthandle": + from .nthandle import NTHandleTexturePassport + + return NTHandleTexturePassport + if name == "wayland-dmabuf": + from .wayland_dmabuf import WaylandDMABUFTexturePassport + + return WaylandDMABUFTexturePassport + if name == "x11-dmabuf": + raise NotImplementedError("x11-dmabuf passport is not supported yet") + raise ValueError(f"Passport name '{name}' is not recognized") diff --git a/src/trame_rca/electron/passports/base.py b/src/trame_rca/electron/passports/base.py new file mode 100644 index 0000000..054849c --- /dev/null +++ b/src/trame_rca/electron/passports/base.py @@ -0,0 +1,20 @@ +class TexturePassport: + """A cross-process abstraction for letting textures travel from Python to Electron. + The passport has everything needed to access VTK render window's texture from another process. + """ + + def __init__(self, render_window, width, height): + self.render_window = render_window + self.width = width + self.height = height + + def to_json(self): + """The ``texture`` message sent to the Electron main process.""" + return {} + + def release(self): + """Free the texture and any OS handle to it.""" + + async def flush_graphics_commands(self): + """Wait until the GPU finished rendering into the texture.""" + self.render_window.WaitForCompletion() diff --git a/src/trame_rca/electron/passports/iosurface.py b/src/trame_rca/electron/passports/iosurface.py new file mode 100644 index 0000000..5451147 --- /dev/null +++ b/src/trame_rca/electron/passports/iosurface.py @@ -0,0 +1,176 @@ +import ctypes +import logging +import os + +from .base import TexturePassport +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +vp = ctypes.c_void_p + +# --- CoreFoundation / IOSurface ---------------------------------------------- +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cf.CFNumberCreate.restype = vp +cf.CFNumberCreate.argtypes = [vp, ctypes.c_long, vp] +cf.CFDictionaryCreateMutable.restype = vp +cf.CFDictionaryCreateMutable.argtypes = [vp, ctypes.c_long, vp, vp] +cf.CFDictionarySetValue.argtypes = [vp, vp, vp] +cf.CFRelease.argtypes = [vp] +kCFNumberSInt32Type = 3 +kCFBooleanTrue = vp.in_dll(cf, "kCFBooleanTrue") + +iosurface = ctypes.CDLL("/System/Library/Frameworks/IOSurface.framework/IOSurface") +iosurface.IOSurfaceCreate.restype = vp +iosurface.IOSurfaceCreate.argtypes = [vp] +iosurface.IOSurfaceGetID.restype = ctypes.c_uint32 +iosurface.IOSurfaceGetID.argtypes = [vp] +iosurface.IOSurfaceGetBytesPerRow.restype = ctypes.c_size_t +iosurface.IOSurfaceGetBytesPerRow.argtypes = [vp] + +# --- CGL / GL ------------------------------------------------------------------ +cgl = ctypes.CDLL("/System/Library/Frameworks/OpenGL.framework/OpenGL") +cgl.CGLGetCurrentContext.restype = vp +cgl.CGLErrorString.restype = ctypes.c_char_p +cgl.CGLErrorString.argtypes = [ctypes.c_int] +cgl.CGLTexImageIOSurface2D.argtypes = [ + vp, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_uint32, + ctypes.c_uint32, + vp, + ctypes.c_uint32, +] +cgl.glGenTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +cgl.glDeleteTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +cgl.glBindTexture.argtypes = [ctypes.c_uint32, ctypes.c_uint32] +cgl.glGetString.restype = ctypes.c_char_p +cgl.glGetString.argtypes = [ctypes.c_uint32] + +GL_VERSION = 0x1F02 +GL_RENDERER = 0x1F01 +GL_TEXTURE_RECTANGLE = 0x84F5 +GL_RGBA = 0x1908 +GL_BGRA = 0x80E1 +GL_UNSIGNED_INT_8_8_8_8_REV = 0x8367 +GL_FRAMEBUFFER = 0x8D40 + +# IOSurface pixel format 'BGRA' -> electron pixelFormat +PIXEL_FORMAT_BGRA = 0x42475241 + + +def check(err, what): + if err != 0: + raise RuntimeError(f"{what} failed: {cgl.CGLErrorString(err).decode()} ({err})") + + +def _iosurface_key(name): + return vp.in_dll(iosurface, name) + + +def _create_surface(width, height): + """A BGRA IOSurface that other processes can open with IOSurfaceLookup(id).""" + props = cf.CFDictionaryCreateMutable( + None, + 0, + ctypes.addressof(ctypes.c_char.in_dll(cf, "kCFTypeDictionaryKeyCallBacks")), + ctypes.addressof(ctypes.c_char.in_dll(cf, "kCFTypeDictionaryValueCallBacks")), + ) + for key, value in ( + ("kIOSurfaceWidth", width), + ("kIOSurfaceHeight", height), + ("kIOSurfaceBytesPerElement", 4), + ("kIOSurfacePixelFormat", PIXEL_FORMAT_BGRA), + ): + v = ctypes.c_int32(value) + number = cf.CFNumberCreate(None, kCFNumberSInt32Type, ctypes.byref(v)) + cf.CFDictionarySetValue(props, _iosurface_key(key), number) + cf.CFRelease(number) + # Deprecated, but the only way to share an IOSurface without passing a Mach port. + cf.CFDictionarySetValue(props, _iosurface_key("kIOSurfaceIsGlobal"), kCFBooleanTrue) + surface = iosurface.IOSurfaceCreate(props) + cf.CFRelease(props) + if not surface: + raise RuntimeError(f"IOSurfaceCreate failed for {width}x{height}") + return surface + + +def _attach_surface(render_window, surface, width, height): + """Make the IOSurface the color attachment 0 of the window's DisplayFramebuffer.""" + render_window.MakeCurrent() + ctx = cgl.CGLGetCurrentContext() + + # CGL only binds IOSurfaces to rectangle textures. + handle = ctypes.c_uint32() + cgl.glGenTextures(1, ctypes.byref(handle)) + cgl.glBindTexture(GL_TEXTURE_RECTANGLE, handle.value) + check( + cgl.CGLTexImageIOSurface2D( + ctx, + GL_TEXTURE_RECTANGLE, + GL_RGBA, + width, + height, + GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, + surface, + 0, + ), + "CGLTexImageIOSurface2D", + ) + cgl.glBindTexture(GL_TEXTURE_RECTANGLE, 0) + + tex = vtkTextureObject() + tex.SetContext(render_window) + tex.AssignToExistingTexture(handle.value, GL_TEXTURE_RECTANGLE) + # Only records the size: an assigned texture has no dimensions, so Resize() never + # calls glTexImage2D (which would detach the IOSurface). The DisplayFramebuffer + # resizes its attachments on every Start(), and this keeps that a no-op too. + tex.Resize(width, height) + + display_fbo = render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError( + "DisplayFramebuffer incomplete with the IOSurface attachment" + ) + return tex, handle + + +class IOSurfaceTexturePassport(TexturePassport): + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + self.surface = _create_surface(width, height) + self.vtk_texture, self.handle = _attach_surface( + render_window, self.surface, width, height + ) + self.surface_id = iosurface.IOSurfaceGetID(self.surface) + logger.info( + f"IOSurface id={self.surface_id} {width}x{height} BGRA " + f"stride={iosurface.IOSurfaceGetBytesPerRow(self.surface)}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "pixelFormat": "bgra", + "ioSurfaceId": self.surface_id, + } + + def release(self): + self.render_window.MakeCurrent() + cgl.glDeleteTextures(1, ctypes.byref(self.handle)) + cf.CFRelease(self.surface) + + async def flush_graphics_commands(self): + # no need to call superclass method since we already call glFinish. + cgl.glFinish() diff --git a/src/trame_rca/electron/passports/nthandle.py b/src/trame_rca/electron/passports/nthandle.py new file mode 100644 index 0000000..c4da960 --- /dev/null +++ b/src/trame_rca/electron/passports/nthandle.py @@ -0,0 +1,493 @@ +"""Render into a D3D11 texture (WGL_NV_DX_interop2) and share it with Electron by NT handle (Windows). + +The vtkOpenGLRenderWindow resolves its RenderFramebuffer into the DisplayFramebuffer in Frame(). +We swap the DisplayFramebuffer's color attachment for a GL texture that aliases a D3D11 texture +through WGL_NV_DX_interop2. Each frame is then copied into a texture shared by NT handle. +""" + +import asyncio +import ctypes +import logging +import os +from ctypes import wintypes + +from .base import TexturePassport +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +vp = ctypes.c_void_p +HRESULT = ctypes.c_long + + +def check_hr(hr, what): + if hr < 0: + raise RuntimeError(f"{what} failed: HRESULT 0x{hr & 0xFFFFFFFF:08x}") + + +class GUID(ctypes.Structure): + _fields_ = [ + ("Data1", ctypes.c_uint32), + ("Data2", ctypes.c_uint16), + ("Data3", ctypes.c_uint16), + ("Data4", ctypes.c_ubyte * 8), + ] + + def __init__(self, s): + a, b, c, d, e = s.split("-") + super().__init__( + int(a, 16), + int(b, 16), + int(c, 16), + (ctypes.c_ubyte * 8)(*bytes.fromhex(d + e)), + ) + + +IID_IDXGIFactory1 = GUID("770aae78-f26f-4dba-a829-253c83d1b387") +IID_IDXGIResource1 = GUID("30961379-4609-4a41-998e-54fe567ee0c1") +IID_ID3D11Device5 = GUID("8ffde202-a0e7-45df-9e01-e837801b5ea0") +IID_ID3D11DeviceContext4 = GUID("917600da-f58c-4c33-98d8-3e15b390fa24") +IID_ID3D11Fence = GUID("affde9d1-1df7-4bb7-8a34-0f46251dab80") + + +def com_call(obj, index, restype, *args): + """Call method `index` of a COM object's vtable (indices from the Windows SDK headers).""" + vtbl = ctypes.cast(obj, ctypes.POINTER(ctypes.POINTER(vp))).contents + fn = ctypes.WINFUNCTYPE( + restype, + vp, + *(type(a) if isinstance(a, ctypes._SimpleCData) else vp for a in args), + )(vtbl[index]) + return fn(obj, *args) + + +def com_release(obj): + com_call(obj, 2, ctypes.c_ulong) # IUnknown::Release + + +# --- D3D11 / DXGI ---------------------------------------------------------------- +dxgi = ctypes.WinDLL("dxgi") +d3d11 = ctypes.WinDLL("d3d11") +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +kernel32.CloseHandle.argtypes = [wintypes.HANDLE] +kernel32.CreateEventW.restype = wintypes.HANDLE +kernel32.CreateEventW.argtypes = [vp, wintypes.BOOL, wintypes.BOOL, wintypes.LPCWSTR] +kernel32.WaitForSingleObject.restype = wintypes.DWORD +kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + + +class LUID(ctypes.Structure): + _fields_ = [("LowPart", wintypes.DWORD), ("HighPart", wintypes.LONG)] + + +class DXGI_ADAPTER_DESC1(ctypes.Structure): + _fields_ = [ + ("Description", ctypes.c_wchar * 128), + ("VendorId", ctypes.c_uint), + ("DeviceId", ctypes.c_uint), + ("SubSysId", ctypes.c_uint), + ("Revision", ctypes.c_uint), + ("DedicatedVideoMemory", ctypes.c_size_t), + ("DedicatedSystemMemory", ctypes.c_size_t), + ("SharedSystemMemory", ctypes.c_size_t), + ("AdapterLuid", LUID), + ("Flags", ctypes.c_uint), + ] + + +class D3D11_TEXTURE2D_DESC(ctypes.Structure): + _fields_ = [ + ("Width", ctypes.c_uint), + ("Height", ctypes.c_uint), + ("MipLevels", ctypes.c_uint), + ("ArraySize", ctypes.c_uint), + ("Format", ctypes.c_uint), + ("SampleCount", ctypes.c_uint), + ("SampleQuality", ctypes.c_uint), + ("Usage", ctypes.c_uint), + ("BindFlags", ctypes.c_uint), + ("CPUAccessFlags", ctypes.c_uint), + ("MiscFlags", ctypes.c_uint), + ] + + +DXGI_FORMAT_B8G8R8A8_UNORM = 87 +D3D11_BIND_SHADER_RESOURCE = 0x8 +D3D11_BIND_RENDER_TARGET = 0x20 +D3D11_RESOURCE_MISC_SHARED = 0x2 +D3D11_RESOURCE_MISC_SHARED_NTHANDLE = 0x800 +D3D11_CREATE_DEVICE_BGRA_SUPPORT = 0x20 +D3D11_SDK_VERSION = 7 +D3D_DRIVER_TYPE_UNKNOWN = 0 +D3D11_FENCE_FLAG_NONE = 0 +WAIT_OBJECT_0 = 0 +DXGI_SHARED_RESOURCE_READ = 0x80000000 +DXGI_SHARED_RESOURCE_WRITE = 1 + +# vtable indices +IDXGIFactory1_EnumAdapters1 = 12 +IDXGIAdapter1_GetDesc1 = 10 +ID3D11Device_CreateTexture2D = 5 +ID3D11Device5_CreateFence = 68 +ID3D11DeviceContext_CopyResource = 47 +ID3D11DeviceContext_Flush = 111 +ID3D11DeviceContext4_Signal = 147 +ID3D11Fence_SetEventOnCompletion = 9 +IDXGIResource1_CreateSharedHandle = 13 + + +def _create_texture(device, width, height, misc_flags=0): + desc = D3D11_TEXTURE2D_DESC( + Width=width, + Height=height, + MipLevels=1, + ArraySize=1, + Format=DXGI_FORMAT_B8G8R8A8_UNORM, + SampleCount=1, + BindFlags=D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE, + MiscFlags=misc_flags, + ) + texture = vp() + check_hr( + com_call( + device, + ID3D11Device_CreateTexture2D, + HRESULT, + ctypes.byref(desc), + None, + ctypes.byref(texture), + ), + "CreateTexture2D", + ) + return texture + + +def _create_shared_texture(device, width, height): + """A BGRA texture with an NT handle that Chromium can open.""" + texture = _create_texture( + device, + width, + height, + D3D11_RESOURCE_MISC_SHARED | D3D11_RESOURCE_MISC_SHARED_NTHANDLE, + ) + resource = vp() + check_hr( + com_call( + texture, + 0, + HRESULT, + ctypes.byref(IID_IDXGIResource1), + ctypes.byref(resource), + ), + "QueryInterface(IDXGIResource1)", + ) + handle = wintypes.HANDLE() + check_hr( + com_call( + resource, + IDXGIResource1_CreateSharedHandle, + HRESULT, + None, + wintypes.DWORD(DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE), + None, + ctypes.byref(handle), + ), + "CreateSharedHandle", + ) + com_release(resource) + return texture, handle.value + + +# --- WGL / GL -------------------------------------------------------------------- +# VTK owns the WGL context; this only registers the D3D11 texture with it. +# Everything framebuffer-related goes through VTK so its vtkOpenGLState cache stays in sync. +opengl32 = ctypes.WinDLL("opengl32") +opengl32.wglGetProcAddress.restype = vp +opengl32.wglGetProcAddress.argtypes = [ctypes.c_char_p] +opengl32.glGenTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +opengl32.glDeleteTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] + +GL_TEXTURE_2D = 0x0DE1 +GL_FRAMEBUFFER = 0x8D40 +WGL_ACCESS_WRITE_DISCARD_NV = 0x0002 + + +def _wgl_func(name, restype, *argtypes): + addr = opengl32.wglGetProcAddress(name.encode()) + if not addr: + raise RuntimeError( + f"{name} missing: the GL driver doesn't support WGL_NV_DX_interop2" + ) + return ctypes.WINFUNCTYPE(restype, *argtypes)(addr) + + +class DXInterop: + """WGL_NV_DX_interop2 entry points; needs a current WGL context to resolve.""" + + def __init__(self): + H = wintypes.HANDLE + self.open_device = _wgl_func("wglDXOpenDeviceNV", H, vp) + self.close_device = _wgl_func("wglDXCloseDeviceNV", wintypes.BOOL, H) + self.register = _wgl_func( + "wglDXRegisterObjectNV", + H, + H, + vp, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_uint32, + ) + self.unregister = _wgl_func("wglDXUnregisterObjectNV", wintypes.BOOL, H, H) + self.lock = _wgl_func( + "wglDXLockObjectsNV", wintypes.BOOL, H, ctypes.c_int32, ctypes.POINTER(H) + ) + self.unlock = _wgl_func( + "wglDXUnlockObjectsNV", wintypes.BOOL, H, ctypes.c_int32, ctypes.POINTER(H) + ) + + +class D3DDevice: + """The process's D3D11 device, shared by every passport, and its interop with VTK's GL context.""" + + def __init__(self, adapter_index): + factory = vp() + check_hr( + dxgi.CreateDXGIFactory1( + ctypes.byref(IID_IDXGIFactory1), ctypes.byref(factory) + ), + "CreateDXGIFactory1", + ) + adapter = vp() + check_hr( + com_call( + factory, + IDXGIFactory1_EnumAdapters1, + HRESULT, + ctypes.c_uint(adapter_index), + ctypes.byref(adapter), + ), + f"EnumAdapters1({adapter_index})", + ) + desc = DXGI_ADAPTER_DESC1() + check_hr( + com_call(adapter, IDXGIAdapter1_GetDesc1, HRESULT, ctypes.byref(desc)), + "GetDesc1", + ) + luid = (desc.AdapterLuid.HighPart << 32) | desc.AdapterLuid.LowPart + logger.info( + f"D3D11 adapter {adapter_index}: {desc.Description} (luid 0x{luid:x})" + ) + + self.device, self.context = vp(), vp() + check_hr( + d3d11.D3D11CreateDevice( + adapter, + D3D_DRIVER_TYPE_UNKNOWN, + None, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + None, + 0, + D3D11_SDK_VERSION, + ctypes.byref(self.device), + None, + ctypes.byref(self.context), + ), + "D3D11CreateDevice", + ) + com_release(adapter) + com_release(factory) + + # A fence signals an event once the GPU ran the commands before it, so a thread can + # wait for the copy without spinning or blocking the event loop. + device5, self.context4 = vp(), vp() + check_hr( + com_call( + self.device, + 0, + HRESULT, + ctypes.byref(IID_ID3D11Device5), + ctypes.byref(device5), + ), + "QueryInterface(ID3D11Device5)", + ) + check_hr( + com_call( + self.context, + 0, + HRESULT, + ctypes.byref(IID_ID3D11DeviceContext4), + ctypes.byref(self.context4), + ), + "QueryInterface(ID3D11DeviceContext4)", + ) + self.fence = vp() + check_hr( + com_call( + device5, + ID3D11Device5_CreateFence, + HRESULT, + ctypes.c_uint64(0), + ctypes.c_uint(D3D11_FENCE_FLAG_NONE), + ctypes.byref(IID_ID3D11Fence), + ctypes.byref(self.fence), + ), + "CreateFence", + ) + com_release(device5) + self.fence_value = 0 + # Auto-reset: SetEventOnCompletion arms it again for every frame. + self.fence_event = kernel32.CreateEventW(None, False, False, None) + if not self.fence_event: + raise RuntimeError(f"CreateEventW failed: {ctypes.get_last_error()}") + # Opened with the first passport: resolving the interop needs VTK's context. + self.interop = None + self.gl_device = None + + def open_interop(self, render_window): + if self.interop: + return + render_window.MakeCurrent() + self.interop = DXInterop() + self.gl_device = self.interop.open_device(self.device) + if not self.gl_device: + raise RuntimeError( + f"wglDXOpenDeviceNV failed: {ctypes.GetLastError()} " + "(GL and D3D11 on different GPUs? try D3D_ADAPTER)" + ) + + def copy(self, dst, src): + com_call(self.context, ID3D11DeviceContext_CopyResource, None, dst, src) + + async def wait_idle(self, timeout_ms=1000): + """Wait until the GPU ran every command submitted so far, without blocking the event loop.""" + self.fence_value += 1 + check_hr( + com_call( + self.context4, + ID3D11DeviceContext4_Signal, + HRESULT, + self.fence, + ctypes.c_uint64(self.fence_value), + ), + "Signal", + ) + check_hr( + com_call( + self.fence, + ID3D11Fence_SetEventOnCompletion, + HRESULT, + ctypes.c_uint64(self.fence_value), + wintypes.HANDLE(self.fence_event), + ), + "SetEventOnCompletion", + ) + # The signal only reaches the GPU once the context's commands are submitted. + com_call(self.context, ID3D11DeviceContext_Flush, None) + # Only the event is touched off the event loop's thread: the context isn't thread safe. + result = await asyncio.to_thread( + kernel32.WaitForSingleObject, self.fence_event, timeout_ms + ) + if result != WAIT_OBJECT_0: + raise RuntimeError( + f"the D3D11 copy didn't finish within {timeout_ms}ms (wait returned {result})" + ) + + +_d3d = None + + +def init_device(adapter_index=0): + """Create the D3D11 device on the DXGI adapter Chromium uses (0 unless told otherwise).""" + global _d3d + if _d3d is None: + _d3d = D3DDevice(adapter_index) + return _d3d + + +def _attach_texture(render_window, d3d, d3d_texture, width, height): + """Make the D3D11 texture the color attachment 0 of the window's DisplayFramebuffer. Leaves it locked.""" + render_window.MakeCurrent() + handle = ctypes.c_uint32() + opengl32.glGenTextures(1, ctypes.byref(handle)) + obj = d3d.interop.register( + d3d.gl_device, + d3d_texture, + handle.value, + GL_TEXTURE_2D, + WGL_ACCESS_WRITE_DISCARD_NV, + ) + if not obj: + raise RuntimeError(f"wglDXRegisterObjectNV failed: {ctypes.GetLastError()}") + objs = (wintypes.HANDLE * 1)(obj) + + tex = vtkTextureObject() + tex.SetContext(render_window) + tex.AssignToExistingTexture(handle.value, GL_TEXTURE_2D) + # Only records the size: an assigned texture has no dimensions, so Resize() never + # calls glTexImage2D (which would detach the D3D11 storage). The DisplayFramebuffer + # resizes its attachments on every Start(), and this keeps that a no-op too. + tex.Resize(width, height) + + display_fbo = render_window.GetDisplayFramebuffer() + d3d.interop.lock(d3d.gl_device, 1, objs) + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the D3D11 texture") + return tex, handle, objs + + +class NTHandleTexturePassport(TexturePassport): + """VTK's D3D11 render target and the texture shared with Electron, at one size. + + GL owns the render target (it stays locked) except while a frame is copied out in + flush_graphics_commands(), so VTK can render into it at any time. + """ + + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + self.d3d = init_device() + self.d3d.open_interop(render_window) + # NVIDIA's interop can't register textures created with SHARED_NTHANDLE (ERROR_OPEN_FAILED), + # so VTK renders into a private texture that gets copied into the shared one on the GPU. + self.d3d_render_texture = _create_texture(self.d3d.device, width, height) + self.d3d_shared_texture, self.nt_handle = _create_shared_texture( + self.d3d.device, width, height + ) + self.vtk_texture, self.d3d_texture_gl_handle, self.objs = _attach_texture( + render_window, self.d3d, self.d3d_render_texture, width, height + ) + logger.info( + f"D3D11 texture {width}x{height} BGRA NT handle=0x{self.nt_handle:x}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "pixelFormat": "bgra", + "ntHandle": self.nt_handle, + } + + def release(self): + self.render_window.MakeCurrent() + self.d3d.interop.unlock(self.d3d.gl_device, 1, self.objs) + self.d3d.interop.unregister(self.d3d.gl_device, self.objs[0]) + opengl32.glDeleteTextures(1, ctypes.byref(self.d3d_texture_gl_handle)) + kernel32.CloseHandle(self.nt_handle) + com_release(self.d3d_shared_texture) + com_release(self.d3d_render_texture) + + async def flush_graphics_commands(self): + # Unlocking hands the texture back to D3D11, ordered after VTK's GL commands. + self.d3d.interop.unlock(self.d3d.gl_device, 1, self.objs) + try: + self.d3d.copy(dst=self.d3d_shared_texture, src=self.d3d_render_texture) + # The shared texture has no keyed mutex, so wait until the GPU is done before telling Electron. + await self.d3d.wait_idle() + finally: + self.d3d.interop.lock(self.d3d.gl_device, 1, self.objs) diff --git a/src/trame_rca/electron/passports/wayland_dmabuf.py b/src/trame_rca/electron/passports/wayland_dmabuf.py new file mode 100644 index 0000000..fb8b190 --- /dev/null +++ b/src/trame_rca/electron/passports/wayland_dmabuf.py @@ -0,0 +1,226 @@ +import ctypes +import logging +import glob +import os + +from .base import TexturePassport +from vtkmodules.vtkCommonCore import VTK_UNSIGNED_CHAR +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +# --- EGL ----------------------------------------------------------------------- +EGL_NONE = 0x3038 +EGL_EXTENSIONS = 0x3055 +EGL_DRM_RENDER_NODE_FILE_EXT = 0x3377 +EGL_GL_TEXTURE_2D_KHR = 0x30B1 +EGL_GL_TEXTURE_LEVEL_KHR = 0x30BC +GL_FRAMEBUFFER = 0x8D40 + +EGLint = ctypes.c_int32 +EGLBoolean = ctypes.c_uint32 +EGLenum = ctypes.c_uint32 +EGLuint64KHR = ctypes.c_uint64 +vp = ctypes.c_void_p + +egl = ctypes.CDLL("libEGL.so.1") +egl.eglGetProcAddress.restype = vp +egl.eglGetProcAddress.argtypes = [ctypes.c_char_p] +egl.eglGetError.restype = EGLint +egl.eglQueryString.argtypes = [vp, EGLint] +egl.eglQueryString.restype = ctypes.c_char_p +egl.eglGetCurrentDisplay.restype = vp +egl.eglGetCurrentContext.restype = vp + + +def egl_ext(name, restype, *argtypes): + addr = egl.eglGetProcAddress(name.encode()) + if not addr: + raise RuntimeError(f"missing EGL function {name}") + return ctypes.CFUNCTYPE(restype, *argtypes)(addr) + + +eglQueryDevicesEXT = egl_ext( + "eglQueryDevicesEXT", EGLBoolean, EGLint, ctypes.POINTER(vp), ctypes.POINTER(EGLint) +) +eglQueryDeviceStringEXT = egl_ext( + "eglQueryDeviceStringEXT", ctypes.c_char_p, vp, EGLint +) +eglCreateImageKHR = egl_ext( + "eglCreateImageKHR", vp, vp, vp, EGLenum, vp, ctypes.POINTER(EGLint) +) +eglDestroyImageKHR = egl_ext("eglDestroyImageKHR", EGLBoolean, vp, vp) +eglExportDMABUFImageQueryMESA = egl_ext( + "eglExportDMABUFImageQueryMESA", + EGLBoolean, + vp, + vp, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(EGLuint64KHR), +) +eglExportDMABUFImageMESA = egl_ext( + "eglExportDMABUFImageMESA", + EGLBoolean, + vp, + vp, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(EGLint), + ctypes.POINTER(EGLint), +) + + +def attribs(*values): + return (EGLint * (len(values) + 1))(*values, EGL_NONE) + + +def check(ok, what): + if not ok: + raise RuntimeError(f"{what} failed: EGL error 0x{egl.eglGetError():04x}") + return ok + + +def fourcc_str(code): + return "".join(chr((code >> (8 * i)) & 0xFF) for i in range(4)) + + +# DRM fourcc -> electron pixelFormat +PIXEL_FORMATS = {"AB24": "rgba", "AR24": "bgra"} + + +def default_render_node(): + """Return (render node, reason). Prefers the GPU driving the display (boot_vga=1). + + Chromium can only import the dma-buf if it was allocated on the same GPU + that it's compositor runs on. In a multi-GPU box that isn't necessarily /dev/dri/renderD128. + """ + for boot_vga in sorted(glob.glob("/sys/class/drm/card*/device/boot_vga")): + try: + with open(boot_vga) as f: + if f.read().strip() != "1": + continue + except OSError: + continue + nodes = sorted( + glob.glob(os.path.join(os.path.dirname(boot_vga), "drm", "renderD*")) + ) + if nodes: + return f"/dev/dri/{os.path.basename(nodes[0])}", "boot_vga" + return "/dev/dri/renderD128", "fallback" + + +def device_index(render_node): + """The index of render_node among the EGL devices, just the way vtkEGLRenderWindow counts them.""" + devices = (vp * 16)() + count = EGLint() + check(eglQueryDevicesEXT(16, devices, ctypes.byref(count)), "eglQueryDevicesEXT") + for i in range(count.value): + node = eglQueryDeviceStringEXT(devices[i], EGL_DRM_RENDER_NODE_FILE_EXT) + node = node.decode() if node else None + logger.info(f"EGL device {i}: render node {node}") + if node == render_node: + return i + raise RuntimeError(f"no EGL device with render node {render_node}") + + +def attach_texture(render_window, width, height): + """Make a fresh RGBA8 texture the color attachment 0 of the window's DisplayFramebuffer.""" + tex = vtkTextureObject() + tex.SetContext(render_window) + if not tex.Allocate2D(width, height, 4, VTK_UNSIGNED_CHAR): + raise RuntimeError("allocating the RGBA8 texture failed") + + display_fbo = render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the exported texture") + return tex + + +def export_dmabuf(render_window, tex, height): + render_window.MakeCurrent() + dpy, ctx = egl.eglGetCurrentDisplay(), egl.eglGetCurrentContext() + exts = egl.eglQueryString(dpy, EGL_EXTENSIONS).decode().split() + for ext in ("EGL_MESA_image_dma_buf_export", "EGL_KHR_gl_texture_2D_image"): + if ext not in exts: + raise RuntimeError(f"display lacks {ext}") + + image = check( + eglCreateImageKHR( + dpy, + ctx, + EGL_GL_TEXTURE_2D_KHR, + vp(tex.GetHandle()), + attribs(EGL_GL_TEXTURE_LEVEL_KHR, 0), + ), + "eglCreateImageKHR", + ) + fourcc, num_planes = ctypes.c_int(), ctypes.c_int() + modifiers = (EGLuint64KHR * 4)() + check( + eglExportDMABUFImageQueryMESA( + dpy, image, ctypes.byref(fourcc), ctypes.byref(num_planes), modifiers + ), + "eglExportDMABUFImageQueryMESA", + ) + fourcc_strings = fourcc_str(fourcc.value) + if fourcc_strings not in PIXEL_FORMATS: + raise RuntimeError(f"unsupported fourcc {fourcc_strings}") + fds = (ctypes.c_int * 4)(-1, -1, -1, -1) + strides, offsets = (EGLint * 4)(), (EGLint * 4)() + check( + eglExportDMABUFImageMESA(dpy, image, fds, strides, offsets), + "eglExportDMABUFImageMESA", + ) + eglDestroyImageKHR(dpy, image) + render_window.WaitForCompletion() + + planes = [] + for i in range(num_planes.value): + # The dma-buf's real size; tiled layouts can be larger than stride*height. + try: + size = os.lseek(fds[i], 0, os.SEEK_END) - offsets[i] + except OSError: + size = strides[i] * height + planes.append( + {"fd": fds[i], "stride": strides[i], "offset": offsets[i], "size": size} + ) + + return fourcc_strings, modifiers[0], planes + + +class WaylandDMABUFTexturePassport(TexturePassport): + """The texture VTK resolves into, exported as a dma-buf, at one size.""" + + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + render_window.MakeCurrent() + self.vtk_texture = attach_texture(render_window, width, height) + self.fourcc, self.modifier, self.planes = export_dmabuf( + render_window, self.vtk_texture, height + ) + logger.info( + f"exported {width}x{height} fourcc={self.fourcc} modifier=0x{self.modifier:016x} " + f"planes={self.planes}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "fourcc": self.fourcc, + "pixelFormat": PIXEL_FORMATS[self.fourcc], + "modifier": str(self.modifier), + "planes": self.planes, + } + + def release(self): + for p in self.planes: + os.close(p["fd"]) + self.render_window.MakeCurrent() + self.vtk_texture.ReleaseGraphicsResources(self.render_window) diff --git a/src/trame_rca/schedulers/__init__.py b/src/trame_rca/schedulers/__init__.py index 3accf67..0f0f0a4 100644 --- a/src/trame_rca/schedulers/__init__.py +++ b/src/trame_rca/schedulers/__init__.py @@ -8,10 +8,15 @@ from .video_scheduler import RcaVideoRenderScheduler except (ModuleNotFoundError, ImportError) as e: logger.info(e.msg) +try: + from .shared_texture_scheduler import RcaSharedTextureRenderScheduler +except (ModuleNotFoundError, ImportError) as e: + logger.info(e.msg) __all__ = [ "RcaImageRenderScheduler", "RcaRenderSchedulerProtocol", + "RcaSharedTextureRenderScheduler", "RcaVideoRenderScheduler", ] diff --git a/src/trame_rca/schedulers/shared_texture_scheduler.py b/src/trame_rca/schedulers/shared_texture_scheduler.py new file mode 100644 index 0000000..481c571 --- /dev/null +++ b/src/trame_rca/schedulers/shared_texture_scheduler.py @@ -0,0 +1,269 @@ +"""A render scheduler that renders into a texture shared with the Electron main process (EMP). + +OS handles do not go through trame's websocket because that is wired to the Electron renderer +process (ERP), which does not have access to the OS API needed to import a texture. + +Instead the scheduler tells the EMP about the shared texture and each finished frame over +trame_rca.electron.comm, and the EMP hands them to the renderer process. Size and input still +come from the RemoteControlledArea through the RcaViewAdapter. + +Renders run on trame's event loop. +""" + +from __future__ import annotations +from asyncio.futures import Future + +import asyncio +import logging +import time +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from trame.app import asynchronous + +from trame_rca.electron.comm import ( + PROTOCOL_VERSION, + LineConnection, + check_protocol, +) +from trame_rca.electron.passports import texture_passport_class, TexturePassport +from trame_rca.rca import ( + RemoteControlledAreaProtocol, + VtkRemoteControlledArea, + window_wrapper, +) + +logger = logging.getLogger(__name__) + +FRAME_DONE_TIMEOUT_S = 1.0 + + +def _report_failure(task): + # Otherwise asyncio only reports it once the task is garbage collected. + if not task.cancelled() and task.exception(): + logger.error("render task failed", exc_info=task.exception()) + + +class RcaSharedTextureRenderScheduler: + """Implements RcaRenderSchedulerProtocol by sharing the render window's texture with Electron.""" + + def __init__( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + emp_port: int, + *, + target_fps: float = 60.0, + on_disconnect: Callable[[], None] | None = None, + before_render: Callable[[], None] | None = None, + ): + self._rca = window_wrapper(window) + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + self._emp_port = emp_port + self._target_fps = target_fps + self._on_disconnect = on_disconnect + # Called before every frame, e.g. to animate the scene. It may schedule_render() again. + self._before_render = before_render + self._push_callback = None + + self._conn = None + self._passport = None + self._passport_name_future = None + self._seq = 0 + self._frame_done = None + # Renders asked for before start() wait in here. + self._request_event = asyncio.Event() + self._tasks = [] + self._is_closing = False + + # Render into the FBOs only: Frame() then stops after resolving into the DisplayFramebuffer. + self._rca.render_window.SetUseOffScreenBuffers(True) + self._rca.render_window.SetDoubleBuffer(False) + self._rca.render_window.SetShowWindow(False) + self._rca.render_window.GetInteractor().Initialize() + + # Wait for the client to report it's canvas size before rendering to avoid + # sending a not-so-useful 300x300 default sized frame. + self._sized = False + process_resize_event = self._rca.process_resize_event + + def on_resize(width, height): + process_resize_event(width, height) + self._sized = True + self.schedule_render() # to ensure self._request_event is set + + self._rca.process_resize_event = on_resize + + @property + def rca(self): + return self._rca + + @property + def target_fps(self): + return self._target_fps + + @target_fps.setter + def target_fps(self, value): + self._target_fps = value + + def set_push_callback(self, callback): + # Frames go to the EMP, never through the push callback. + self._push_callback = callback + + def reset(self): + """Nothing to re-initialize.""" + + def schedule_render(self): + self._request_event.set() + + def start(self, url: str | None = None, **_): + """Connect to the EMP and start rendering. Call once trame's event loop runs. + + ``url`` is the page the EMP should load: the one trame serves. + """ + self._start_task(self._run(url)) + + def _start_task(self, coro): + task = asynchronous.create_task(coro) + task.add_done_callback(_report_failure) + self._tasks.append(task) + + async def close(self): + if self._is_closing: + return + self._is_closing = True + for task in self._tasks: + task.cancel() + if self._conn: + self._conn.close() + if self._passport: + self._passport.release() + self._passport = None + + async def _run(self, url): + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + self._passport_name_future = asyncio.get_running_loop().create_future() + self._conn = await LineConnection.open(self._emp_port) + logger.info("connected to the Electron main process on port %d", self._emp_port) + self._start_task(self._read()) + try: + passport_name = await self._passport_name_future + except Exception: + await self._disconnect() + raise + await self._conn.send( + {"type": "server_ready", "protocol": PROTOCOL_VERSION, "url": url} + ) + next_frame = 0 + period = 1.0 / self._target_fps + + while not self._is_closing: + await self._request_event.wait() + self._request_event.clear() + # Only render when the ERP has booted up ( is sized) + if not self._sized: + continue + + await self._update_texture(passport_name) + assert isinstance(self._passport, TexturePassport), "Passport unavailable!" + if self._before_render: + self._before_render() + self._rca.render_window.Render() + self._rca.render_window.MakeCurrent() + await self._passport.flush_graphics_commands() + + # Backpressure: don't touch the texture until Electron is done reading it. + # Input keeps setting the request event meanwhile, so it renders once afterwards. + seq = self._seq + self._seq += 1 + self._frame_done = asyncio.get_running_loop().create_future() + await self._conn.send( + {"type": "frame", "seq": seq, "timestamp_us": int(time.time() * 1e6)} + ) + try: + await asyncio.wait_for(self._frame_done, FRAME_DONE_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning( + "frame %d: no frame_done within %ss, continuing", + seq, + FRAME_DONE_TIMEOUT_S, + ) + self._frame_done = None + # Pace against fixed frame clock held in `next_frame` + next_frame = max(next_frame + period, time.monotonic()) + await asyncio.sleep(max(0.0, next_frame - time.monotonic())) + + async def _update_texture(self, passport_name): + """(Re)allocate the shared texture when the render window's size changed.""" + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + assert isinstance(self._conn, LineConnection), ( + "Connection to EMP is uninitialized!" + ) + width, height = self._rca.render_window.GetSize() + if self._passport and (self._passport.width, self._passport.height) == ( + width, + height, + ): + return + if self._passport is None: + # The first render creates the context and the framebuffers the passport patches. + self._rca.render_window.Render() + self._rca.render_window.MakeCurrent() + passport_class = texture_passport_class(passport_name) + old, self._passport = ( + self._passport, + passport_class(self._rca.render_window, width, height), + ) + if old: + # Frames still in flight hold their own imports in Electron. + old.release() + await self._conn.send(self._passport.to_json()) + + async def _read(self): + assert isinstance(self._conn, LineConnection), ( + "Connection to EMP is uninitialized!" + ) + assert isinstance(self._passport_name_future, Future), ( + "Passport name future was not created!" + ) + try: + while True: + msg = await self._conn.recv() + if msg["type"] == "hello": + try: + check_protocol(msg) + except Exception as e: + self._passport_name_future.set_exception(e) + return + self._passport_name_future.set_result(msg["passport"]) + elif msg["type"] == "frame_done": + if ( + msg["seq"] == self._seq - 1 + and self._frame_done + and not self._frame_done.done() + ): + self._frame_done.set_result(None) + if msg.get("dropped"): + # The page never processed it: e.g. it was still loading or busy.. + logger.info("frame %d dropped, rendering again", msg["seq"]) + self.schedule_render() + except (EOFError, ConnectionError): + logger.info("the Electron main process disconnected") + if not self._passport_name_future.done(): + self._passport_name_future.set_exception( + ConnectionError("disconnected before the hello message") + ) + return + await self._disconnect() + + async def _disconnect(self): + await self.close() + if self._on_disconnect: + self._on_disconnect() diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index c671aaa..5df44d6 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -26,6 +26,7 @@ "MediaSourceDisplayArea", "VideoDecoderDisplayArea", "RawImageDisplayArea", + "SharedTextureDisplayArea", "ImageStream", "ImageRegion", ] @@ -136,6 +137,59 @@ def create_view_handler( self.add_view_handler(view_handler) return view_handler + def create_electron_handler( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + target_fps: float = 60.0, + before_render: Callable[[], None] | None = None, + on_disconnect: Callable[[], None] | None = None, + ): + """ + Render ``render_window`` into a texture shared with the Electron app that + started this server (see @Kitware/trame-rca-electron), for ``display="shared-texture"``. + + Call ``trame_rca.electron.init_vtk()`` before creating ``render_window``. + ``before_render`` is called before every frame and may schedule another render, + e.g. to animate. ``on_disconnect`` is called when Electron goes away and + defaults to stopping the server. + """ + from trame.app import asynchronous + + from trame_rca.electron import emp_port, server_url + from trame_rca.schedulers import RcaSharedTextureRenderScheduler + + if self.display != "shared-texture": + raise ValueError( + "create_electron_handler needs a RemoteControlledArea with " + f'display="shared-texture", not "{self.display}"' + ) + + if on_disconnect is None: + + def on_disconnect(): + asynchronous.create_task(self.server.stop()) + + rca_protocol = window_wrapper(window) + scheduler = RcaSharedTextureRenderScheduler( + rca_protocol, + emp_port(self.server), + target_fps=target_fps, + on_disconnect=on_disconnect, + before_render=before_render, + ) + + def start(**_): + scheduler.start(url=server_url(self.server)) + + if self.server.running: + start() + else: + self.ctrl.on_server_ready.add(start) + + view_handler = RcaViewAdapter(scheduler.rca, self.name, scheduler=scheduler) + self.add_view_handler(view_handler) + return view_handler + def create_vtk_handler( self, render_window: vtkRenderWindow, @@ -252,6 +306,19 @@ def __init__(self, **kwargs): ] +class SharedTextureDisplayArea(HtmlElement): + """The canvas trame-rca-electron draws the shared texture into, once a frame.""" + + def __init__(self, **kwargs): + super().__init__( + "shared-texture-display-area", + **kwargs, + ) + self._event_names += [ + "stats", + ] + + class ImageStream(HtmlElement): ID = 0 NAMES = WeakKeyDictionary() diff --git a/tests/test_rca_electron.py b/tests/test_rca_electron.py new file mode 100644 index 0000000..1f15f76 --- /dev/null +++ b/tests/test_rca_electron.py @@ -0,0 +1,165 @@ +import asyncio +import json + +import pytest + +from trame_rca.electron import PROTOCOL_VERSION +from trame_rca.electron.comm import ProtocolMismatchError, check_protocol +from trame_rca.electron.passports import TexturePassport, texture_passport_class +from trame_rca.schedulers import RcaSharedTextureRenderScheduler + +TIMEOUT_S = 10 + + +class FakePassport(TexturePassport): + released = [] + + def to_json(self): + return {"type": "texture", "width": self.width, "height": self.height} + + def release(self): + FakePassport.released.append((self.width, self.height)) + + +class FakeEmp: + """The Electron main process's end of the texture channel.""" + + def __init__(self, hello): + self.hello = hello + self.connected = asyncio.Event() + self.reader = self.writer = None + + async def start(self): + self.server = await asyncio.start_server(self._on_connect, "127.0.0.1", 0) + return self.server.sockets[0].getsockname()[1] + + async def _on_connect(self, reader, writer): + self.reader, self.writer = reader, writer + await self.send(self.hello) + self.connected.set() + + async def send(self, msg): + self.writer.write((json.dumps(msg) + "\n").encode()) + await self.writer.drain() + + async def recv(self): + await asyncio.wait_for(self.connected.wait(), TIMEOUT_S) + line = await asyncio.wait_for(self.reader.readline(), TIMEOUT_S) + return json.loads(line) + + def close(self): + if self.writer: + self.writer.close() + self.server.close() + + +@pytest.fixture +def fake_passport(monkeypatch): + FakePassport.released = [] + monkeypatch.setattr( + "trame_rca.schedulers.shared_texture_scheduler.texture_passport_class", + lambda name: FakePassport, + ) + + +def test_texture_passport_class(): + with pytest.raises(TypeError): + texture_passport_class(1) + with pytest.raises(NotImplementedError): + texture_passport_class("x11-dmabuf") + with pytest.raises(ValueError): + texture_passport_class("nope") + + +def test_check_protocol(): + check_protocol({"protocol": PROTOCOL_VERSION}) + with pytest.raises(ProtocolMismatchError): + check_protocol({"protocol": PROTOCOL_VERSION + 1}) + with pytest.raises(ProtocolMismatchError): + check_protocol({}) + + +@pytest.mark.asyncio +async def test_scheduler_shares_frames(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + disconnected = asyncio.Event() + scheduler = RcaSharedTextureRenderScheduler( + a_render_window, port, on_disconnect=disconnected.set + ) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert await emp.recv() == { + "type": "server_ready", + "protocol": PROTOCOL_VERSION, + "url": "http://127.0.0.1:1234/", + } + + # Nothing to render into until the page reported its size. + scheduler.schedule_render() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + + # As the RcaViewAdapter does when the canvas reports its size. + scheduler.rca.process_resize_event(300, 300) + scheduler.schedule_render() + assert await emp.recv() == {"type": "texture", "width": 300, "height": 300} + frame = await emp.recv() + assert frame["type"] == "frame" and frame["seq"] == 0 + await emp.send({"type": "frame_done", "seq": 0}) + + # A resize makes a new texture and releases the old one. + scheduler.rca.process_resize_event(200, 100) + scheduler.schedule_render() + assert await emp.recv() == {"type": "texture", "width": 200, "height": 100} + assert FakePassport.released == [(300, 300)] + assert (await emp.recv())["seq"] == 1 + + # Electron going away closes the scheduler. + emp.close() + await asyncio.wait_for(disconnected.wait(), TIMEOUT_S) + assert FakePassport.released == [(300, 300), (200, 100)] + finally: + await scheduler.close() + emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_rejects_other_protocol(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION + 1, "passport": "x"}) + port = await emp.start() + disconnected = asyncio.Event() + scheduler = RcaSharedTextureRenderScheduler( + a_render_window, port, on_disconnect=disconnected.set + ) + try: + scheduler.start(url="http://127.0.0.1:1234/") + await asyncio.wait_for(disconnected.wait(), TIMEOUT_S) + # Never told Electron the page is ready. + await asyncio.wait_for(emp.connected.wait(), TIMEOUT_S) + assert await emp.reader.readline() == b"" + finally: + await scheduler.close() + emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_renders_dropped_frames_again(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + scheduler = RcaSharedTextureRenderScheduler(a_render_window, port) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert (await emp.recv())["type"] == "server_ready" + scheduler.rca.process_resize_event(300, 300) + scheduler.schedule_render() + assert (await emp.recv())["type"] == "texture" + assert (await emp.recv())["seq"] == 0 + + # e.g. sendSharedTexture timed out while the page was loading. + await emp.send({"type": "frame_done", "seq": 0, "dropped": True}) + frame = await emp.recv() + assert frame["type"] == "frame" and frame["seq"] == 1 + finally: + await scheduler.close() + emp.close() diff --git a/vue-components/src/components/DisplayArea.js b/vue-components/src/components/DisplayArea.js index ad8783d..7e0d0ee 100644 --- a/vue-components/src/components/DisplayArea.js +++ b/vue-components/src/components/DisplayArea.js @@ -2,6 +2,7 @@ import ImageDisplayArea from './ImageDisplayArea.js'; import MediaSourceDisplayArea from './MediaSourceDisplayArea.js'; import VideoDecoderDisplayArea from './VideoDecoderDisplayArea.js'; import RawImageDisplayArea from './RawImageDisplayArea.js'; +import SharedTextureDisplayArea from './SharedTextureDisplayArea.js'; export default { components: { @@ -9,6 +10,7 @@ export default { MediaSourceDisplayArea, VideoDecoderDisplayArea, RawImageDisplayArea, + SharedTextureDisplayArea, }, props: { name: { @@ -38,6 +40,7 @@ export default { + `, }; diff --git a/vue-components/src/components/SharedTextureDisplayArea.js b/vue-components/src/components/SharedTextureDisplayArea.js new file mode 100644 index 0000000..fe1aa82 --- /dev/null +++ b/vue-components/src/components/SharedTextureDisplayArea.js @@ -0,0 +1,32 @@ +// Shows the frames the trame-rca-electron preload draws from the texture VTK shares with +// Electron. Only works in a page loaded by trame-rca-electron: the preload defines the bridge. +export default { + emits: ['stats'], + methods: { + cleanup() { + window.trameRcaSharedTexture?.detach(this.$el); + }, + }, + mounted() { + if (!window.trameRcaSharedTexture) { + console.warn( + 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' + ); + return; + } + // { fps, frames, info } once a second. + window.trameRcaSharedTexture.attach(this.$el, (stats) => + this.$emit('stats', stats) + ); + }, + // support both vue2 and vue3 unmount callbacks + beforeDestroy() { + this.cleanup(); + }, + beforeUnmount() { + this.cleanup(); + }, + template: ` + + `, +}; diff --git a/vue-components/src/components/index.js b/vue-components/src/components/index.js index f6445b8..a8d27f7 100644 --- a/vue-components/src/components/index.js +++ b/vue-components/src/components/index.js @@ -5,6 +5,7 @@ import RemoteControlledArea from './RemoteControlledArea.js'; import MediaSourceDisplayArea from './MediaSourceDisplayArea.js'; import VideoDecoderDisplayArea from './VideoDecoderDisplayArea.js'; import RawImageDisplayArea from './RawImageDisplayArea.js'; +import SharedTextureDisplayArea from './SharedTextureDisplayArea.js'; import ImageStream from './ImageStream.js'; import ImageRegion from './ImageRegion.js'; @@ -16,6 +17,7 @@ export default { MediaSourceDisplayArea, VideoDecoderDisplayArea, RawImageDisplayArea, + SharedTextureDisplayArea, ImageStream, ImageRegion, }; From 91cda4e4d41a794bc78ecd557d8cc1b7814eac61 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Mon, 19 Oct 2026 18:33:04 -0400 Subject: [PATCH 03/13] fix(electron): add profiler for render and flush --- src/trame_rca/schedulers/shared_texture_scheduler.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/trame_rca/schedulers/shared_texture_scheduler.py b/src/trame_rca/schedulers/shared_texture_scheduler.py index 481c571..dd1d00d 100644 --- a/src/trame_rca/schedulers/shared_texture_scheduler.py +++ b/src/trame_rca/schedulers/shared_texture_scheduler.py @@ -22,6 +22,7 @@ from vtkmodules.vtkRenderingCore import vtkRenderWindow from trame.app import asynchronous +from trame_common.utils import profiler from trame_rca.electron.comm import ( PROTOCOL_VERSION, @@ -58,6 +59,8 @@ def __init__( on_disconnect: Callable[[], None] | None = None, before_render: Callable[[], None] | None = None, ): + self._timer_render = profiler.Timer("rca.shared_texture.render") + self._timer_flush = profiler.Timer("rca.shared_texture.flush") self._rca = window_wrapper(window) assert isinstance(self._rca, VtkRemoteControlledArea), ( "SharedTexture scheduler only works with VTK render windows!" @@ -173,9 +176,11 @@ async def _run(self, url): assert isinstance(self._passport, TexturePassport), "Passport unavailable!" if self._before_render: self._before_render() - self._rca.render_window.Render() - self._rca.render_window.MakeCurrent() - await self._passport.flush_graphics_commands() + with self._timer_render: + self._rca.render_window.Render() + with self._timer_flush: + self._rca.render_window.MakeCurrent() + await self._passport.flush_graphics_commands() # Backpressure: don't touch the texture until Electron is done reading it. # Input keeps setting the request event meanwhile, so it renders once afterwards. From 0b39070b4e2be98536a551a6600bfd4887bb8305 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Mon, 19 Oct 2026 18:40:59 -0400 Subject: [PATCH 04/13] fix(example): enable profiler in electron example --- examples/06_electron/app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/examples/06_electron/app.py b/examples/06_electron/app.py index f35e591..0d59e63 100644 --- a/examples/06_electron/app.py +++ b/examples/06_electron/app.py @@ -36,6 +36,12 @@ # Required for vtk factory import vtkmodules.vtkRenderingOpenGL2 # noqa: E402, F401 +# Uncomment to see per-frame profile +# from trame_common.utils import profiler +# profiler.enable() +# profiler.LOGGER.filter.include("rca.shared_texture") +# profiler.LOGGER.filter.exclude("rca.vtk") + def create_vtk_pipeline(): wavelet = vtkRTAnalyticSource() From c37f34849c0bff7f2581cdfe008ba221c2d0d6c1 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Mon, 19 Oct 2026 18:19:47 -0400 Subject: [PATCH 05/13] fix(windows): set timer resolution to 1ms --- src/trame_rca/electron/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/trame_rca/electron/__init__.py b/src/trame_rca/electron/__init__.py index 54f965c..f20eb39 100644 --- a/src/trame_rca/electron/__init__.py +++ b/src/trame_rca/electron/__init__.py @@ -94,3 +94,8 @@ def init_vtk(): # The shared texture must live on the GPU Chromium uses, which is DXGI adapter 0 # unless told otherwise. VTK's WGL context has to be on that GPU too. nthandle.init_device(int(os.environ.get("D3D_ADAPTER", "0"))) + # Ask for a 1ms resolution in the Python process. Otherwise, asyncio.sleep + # rounds short waits to ~15.6ms + import ctypes + + ctypes.WinDLL("winmm").timeBeginPeriod(1) From 42c5512a052ebd0b73a65ae04a074eee0d85901b Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 20 Oct 2026 08:11:54 -0400 Subject: [PATCH 06/13] fix(js): convert from CJS to module JS --- electron/.eslintrc.cjs | 2 +- electron/README.md | 11 +++-- electron/package.json | 9 ++-- electron/src/main/dma/index.js | 9 ++++ electron/src/main/dma/iosurface.js | 4 +- electron/src/main/dma/pidfd.js | 4 +- electron/src/main/dma/winhandle.js | 4 +- electron/src/main/index.js | 57 ++++++++++++---------- electron/src/main/protocol.js | 2 +- electron/src/{preload.js => preload.mjs} | 4 +- electron/test/protocol.test.js | 6 +-- examples/06_electron/electron/main.js | 8 +-- examples/06_electron/electron/package.json | 2 +- 13 files changed, 70 insertions(+), 52 deletions(-) create mode 100644 electron/src/main/dma/index.js rename electron/src/{preload.js => preload.mjs} (95%) diff --git a/electron/.eslintrc.cjs b/electron/.eslintrc.cjs index e6ba8f2..4338b47 100644 --- a/electron/.eslintrc.cjs +++ b/electron/.eslintrc.cjs @@ -9,6 +9,6 @@ module.exports = { extends: ['eslint:recommended'], parserOptions: { ecmaVersion: 'latest', - sourceType: 'script', + sourceType: 'module', }, }; diff --git a/electron/README.md b/electron/README.md index fa2d8c3..3264e01 100644 --- a/electron/README.md +++ b/electron/README.md @@ -9,8 +9,8 @@ This package and the `trame-rca` python package speak a versioned protocol (see ## Use ```js -const { app, BrowserWindow } = require('electron/main'); -const { launchTrame, webPreferences } = require('@kitware/trame-rca-electron'); +import { app, BrowserWindow } from 'electron/main'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; app.whenReady().then(() => { const win = new BrowserWindow({ webPreferences: webPreferences() }); @@ -29,9 +29,10 @@ and loads the page in `win` once trame serves it. Options: It returns `{ process, close }`. `close()` kills the server; it is called when `win` closes. -`webPreferences(extra)` sets the preload, and turns off the sandbox the preload needs for -`sharedTexture`. If the window needs its own preload, `require('@kitware/trame-rca-electron/preload')` -from it. +`webPreferences(extra)` sets the preload, and turns off the sandbox: the preload needs +`sharedTexture`, and Electron doesn't run ES module preloads in a sandbox. If the window needs its +own preload, `import '@kitware/trame-rca-electron/preload'` from it: it must then be an ES module +preload too, with the `.mjs` extension. On the python side, see `examples/06_electron` in trame-rca. diff --git a/electron/package.json b/electron/package.json index df08b7b..52e1185 100644 --- a/electron/package.json +++ b/electron/package.json @@ -9,17 +9,18 @@ "url": "https://github.com/Kitware/trame-rca.git", "directory": "electron" }, - "type": "commonjs", + "type": "module", "main": "src/main/index.js", "exports": { ".": "./src/main/index.js", - "./preload": "./src/preload.js" + "./preload": "./src/preload.mjs" }, "files": [ - "src/**/*.js" + "src/**/*.js", + "src/**/*.mjs" ], "scripts": { - "lint": "eslint src test", + "lint": "eslint src test --ext .js,.mjs", "test": "node --test" }, "peerDependencies": { diff --git a/electron/src/main/dma/index.js b/electron/src/main/dma/index.js new file mode 100644 index 0000000..509a864 --- /dev/null +++ b/electron/src/main/dma/index.js @@ -0,0 +1,9 @@ +// Each module loads its platform's libraries when imported: pidfd.js needs Linux's libc, +// iosurface.js needs macOS frameworks, winhandle.js needs kernel32. So import only this +// platform's, and asynchronously: a top-level await would stop CommonJS apps from +// require()-ing this package. +export const loadDma = (platform) => { + if (platform === 'darwin') return import('./iosurface.js'); + if (platform === 'win32') return import('./winhandle.js'); + return import('./pidfd.js'); +}; diff --git a/electron/src/main/dma/iosurface.js b/electron/src/main/dma/iosurface.js index 667777c..dde2fd0 100644 --- a/electron/src/main/dma/iosurface.js +++ b/electron/src/main/dma/iosurface.js @@ -1,6 +1,6 @@ // Open another process's IOSurface by its global ID with IOSurfaceLookup. // Only works for surfaces created with kIOSurfaceIsGlobal (deprecated, unsandboxed only). -const koffi = require('koffi'); +import koffi from 'koffi'; const iosurface = koffi.load('/System/Library/Frameworks/IOSurface.framework/IOSurface'); const cf = koffi.load('/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation'); @@ -33,4 +33,4 @@ const surfaceHandle = (ref) => { const releaseSurface = (ref) => CFRelease(ref); -module.exports = { lookupSurface, describeSurface, surfaceHandle, releaseSurface }; +export { lookupSurface, describeSurface, surfaceHandle, releaseSurface }; diff --git a/electron/src/main/dma/pidfd.js b/electron/src/main/dma/pidfd.js index 9eb736e..51eddae 100644 --- a/electron/src/main/dma/pidfd.js +++ b/electron/src/main/dma/pidfd.js @@ -1,6 +1,6 @@ // Duplicate a file descriptor out of another process with pidfd_getfd(2). // Requires ptrace access to the target (e.g. it is our child with ptrace_scope=1). -const koffi = require('koffi'); +import koffi from 'koffi'; const libc = koffi.load('libc.so.6'); const syscall = libc.func('long syscall(long nr, ...)'); @@ -31,4 +31,4 @@ const dupFdFromProcess = (pid, targetFd) => { const closeFd = (fd) => close(fd); -module.exports = { dupFdFromProcess, closeFd }; +export { dupFdFromProcess, closeFd }; diff --git a/electron/src/main/dma/winhandle.js b/electron/src/main/dma/winhandle.js index 44093a1..fac1039 100644 --- a/electron/src/main/dma/winhandle.js +++ b/electron/src/main/dma/winhandle.js @@ -1,6 +1,6 @@ // Duplicate an NT handle out of another process with DuplicateHandle. // Requires PROCESS_DUP_HANDLE on the target (e.g. it is our child, running as the same user). -const koffi = require('koffi'); +import koffi from 'koffi'; const kernel32 = koffi.load('kernel32.dll'); const OpenProcess = kernel32.func('uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)'); @@ -35,4 +35,4 @@ const handleBuffer = (handle) => { const closeHandle = (handle) => CloseHandle(handle); -module.exports = { dupHandleFromProcess, handleBuffer, closeHandle }; +export { dupHandleFromProcess, handleBuffer, closeHandle }; diff --git a/electron/src/main/index.js b/electron/src/main/index.js index 21fa5b8..a164d46 100644 --- a/electron/src/main/index.js +++ b/electron/src/main/index.js @@ -1,30 +1,28 @@ -const { app, ipcMain, sharedTexture } = require('electron/main'); -const { spawn } = require('node:child_process'); -const net = require('node:net'); -const path = require('node:path'); +import { app, ipcMain, sharedTexture } from 'electron/main'; +import { spawn } from 'node:child_process'; +import net from 'node:net'; +import { fileURLToPath } from 'node:url'; -const protocol = require('./protocol'); +import { loadDma } from './dma/index.js'; +import * as protocol from './protocol.js'; -// Loaded on demand: pidfd.js needs Linux's libc, iosurface.js needs macOS frameworks, -// winhandle.js needs kernel32. -const pidfd = () => require('./dma/pidfd'); -const iosurface = () => require('./dma/iosurface'); -const winhandle = () => require('./dma/winhandle'); - -// The preload a BrowserWindow needs to show shared-texture views. -const preloadPath = path.join(__dirname, '..', 'preload.js'); +// The preload a BrowserWindow needs to show shared-texture views. Electron only runs +// ES module preloads with the .mjs extension. +const preloadPath = fileURLToPath(new URL('../preload.mjs', import.meta.url)); // The webPreferences a BrowserWindow needs to show shared-texture views. const webPreferences = (extra = {}) => ({ preload: preloadPath, - // The preload needs require('electron').sharedTexture. + // The preload needs electron's sharedTexture, and ES module preloads can't be sandboxed. sandbox: false, ...extra, }); -// Holds this process's reference to the producer's current texture. +// Holds this process's reference to the producer's current texture. `dma` is this +// platform's module from loadDma(). class TextureHandle { - constructor(msg, log) { + constructor(msg, dma, log) { + this.dma = dma; this.localFds = []; this.surfaceRef = null; this.localHandle = null; @@ -35,7 +33,7 @@ class TextureHandle { const kind = protocol.handleKind(msg); if (kind === 'ioSurface') { // Open the producer's global IOSurface in this process. - const { lookupSurface, describeSurface, surfaceHandle } = iosurface(); + const { lookupSurface, describeSurface, surfaceHandle } = this.dma; this.surfaceRef = lookupSurface(msg.ioSurfaceId); log.info( `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) IOSurface id=${msg.ioSurfaceId} ${describeSurface(this.surfaceRef)}`, @@ -44,7 +42,7 @@ class TextureHandle { } if (kind === 'ntHandle') { // Duplicate the producer's NT handle into this process. - const { dupHandleFromProcess, handleBuffer } = winhandle(); + const { dupHandleFromProcess, handleBuffer } = this.dma; this.localHandle = dupHandleFromProcess(msg.pid, msg.ntHandle); log.info( `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) handle 0x${msg.ntHandle.toString(16)} -> 0x${this.localHandle.toString(16)}`, @@ -52,7 +50,7 @@ class TextureHandle { return { ntHandle: handleBuffer(this.localHandle) }; } // Duplicate the producer's dma-buf fds into this process. - this.localFds = msg.planes.map((p) => pidfd().dupFdFromProcess(msg.pid, p.fd)); + this.localFds = msg.planes.map((p) => this.dma.dupFdFromProcess(msg.pid, p.fd)); log.info( `[trame-rca] texture ${msg.width}x${msg.height} ${msg.fourcc} (${msg.pixelFormat}) modifier=${msg.modifier} fds ${msg.planes.map((p) => p.fd)} -> ${this.localFds}`, ); @@ -60,11 +58,11 @@ class TextureHandle { } release() { - this.localFds.forEach((fd) => pidfd().closeFd(fd)); + this.localFds.forEach((fd) => this.dma.closeFd(fd)); this.localFds = []; - if (this.surfaceRef) iosurface().releaseSurface(this.surfaceRef); + if (this.surfaceRef) this.dma.releaseSurface(this.surfaceRef); this.surfaceRef = null; - if (this.localHandle) winhandle().closeHandle(this.localHandle); + if (this.localHandle) this.dma.closeHandle(this.localHandle); this.localHandle = null; } } @@ -85,6 +83,8 @@ const launchTrame = ( if (!command) throw new Error('launchTrame: `command` is required'); let socket = null; let texture = null; + // This platform's dma module, loaded before the server is spawned. + let dma = null; let closed = false; const result = { process: null, close: null }; @@ -147,7 +147,7 @@ const launchTrame = ( try { // A new texture after a resize. Frames still in flight hold their own imports. releaseTexture(); - texture = new TextureHandle(msg, log); + texture = new TextureHandle(msg, dma, log); } catch (err) { log.error('[trame-rca] failed to open the texture:', err); conn.destroy(); @@ -207,7 +207,14 @@ const launchTrame = ( }; win.once('closed', result.close); - server.listen(0, '127.0.0.1', () => { + server.listen(0, '127.0.0.1', async () => { + try { + dma = await loadDma(process.platform); + } catch (err) { + log.error('[trame-rca] failed to load the OS texture API:', err); + result.close(); + return; + } if (closed) return; const { port } = server.address(); const fullArgs = [...args, '--server', '--port', '0', '--emp-port', String(port)]; @@ -226,4 +233,4 @@ const launchTrame = ( return result; }; -module.exports = { launchTrame, preloadPath, webPreferences, protocol }; +export { launchTrame, preloadPath, webPreferences, protocol }; diff --git a/electron/src/main/protocol.js b/electron/src/main/protocol.js index 7115740..ddc0c62 100644 --- a/electron/src/main/protocol.js +++ b/electron/src/main/protocol.js @@ -80,7 +80,7 @@ const textureInfo = (msg, handle) => ({ handle, }); -module.exports = { +export { PROTOCOL_VERSION, checkProtocol, createLineParser, diff --git a/electron/src/preload.js b/electron/src/preload.mjs similarity index 95% rename from electron/src/preload.js rename to electron/src/preload.mjs index 1f2f794..93cbaef 100644 --- a/electron/src/preload.js +++ b/electron/src/preload.mjs @@ -1,6 +1,6 @@ // Draws the frames the main process sends into the page's shared-texture-display-area. -// Use it as a BrowserWindow's preload (see webPreferences()), or require it from yours. -const { contextBridge, ipcRenderer, sharedTexture } = require('electron'); +// Use it as a BrowserWindow's preload (see webPreferences()), or import it from yours. +import { contextBridge, ipcRenderer, sharedTexture } from 'electron'; const STATS_INTERVAL_MS = 1000; diff --git a/electron/test/protocol.test.js b/electron/test/protocol.test.js index 49c8290..2bf94c9 100644 --- a/electron/test/protocol.test.js +++ b/electron/test/protocol.test.js @@ -1,7 +1,7 @@ -const test = require('node:test'); -const assert = require('node:assert/strict'); +import test from 'node:test'; +import assert from 'node:assert/strict'; -const protocol = require('../src/main/protocol'); +import * as protocol from '../src/main/protocol.js'; test('passportName picks the platform passport', () => { assert.equal(protocol.passportName({ platform: 'darwin' }), 'iosurface'); diff --git a/examples/06_electron/electron/main.js b/examples/06_electron/electron/main.js index 338fb89..5df7045 100644 --- a/examples/06_electron/electron/main.js +++ b/examples/06_electron/electron/main.js @@ -1,6 +1,6 @@ -const { app, BrowserWindow } = require('electron/main'); -const path = require('node:path'); -const { launchTrame, webPreferences } = require('@kitware/trame-rca-electron'); +import { app, BrowserWindow } from 'electron/main'; +import { fileURLToPath } from 'node:url'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; // Run ../app.py with the python of the active environment, unless PYTHON says otherwise. const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); @@ -16,7 +16,7 @@ app.whenReady().then(() => { launchTrame(win, { command: python, args: [ - path.join(__dirname, '..', 'app.py'), + fileURLToPath(new URL('../app.py', import.meta.url)), '--electron-versions', JSON.stringify({ electron, chrome, node, v8 }), ], diff --git a/examples/06_electron/electron/package.json b/examples/06_electron/electron/package.json index 6660099..82e58f4 100644 --- a/examples/06_electron/electron/package.json +++ b/examples/06_electron/electron/package.json @@ -4,7 +4,7 @@ "version": "0.0.0", "description": "Electron shell for the shared texture trame-rca example", "license": "Apache-2.0", - "type": "commonjs", + "type": "module", "main": "main.js", "scripts": { "start": "electron ." From ba4b11294d76a1715a35b73fa14bfefbc250d4a6 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 20 Oct 2026 08:16:34 -0400 Subject: [PATCH 07/13] fix(rca): move init function trame_rca.widgets --- README.rst | 4 +-- examples/06_electron/app.py | 5 ++- src/trame_rca/electron/__init__.py | 55 ++---------------------------- src/trame_rca/widgets/__init__.py | 5 +++ src/trame_rca/widgets/rca.py | 55 +++++++++++++++++++++++++++++- 5 files changed, 65 insertions(+), 59 deletions(-) diff --git a/README.rst b/README.rst index 34e3f5d..d5d530e 100644 --- a/README.rst +++ b/README.rst @@ -147,10 +147,10 @@ Windows (D3D11 NT handles) and Linux under Wayland (dma-buf). .. code-block:: python - from trame_rca.electron import init_vtk + from trame_rca.widgets import init_vtk_for_shared_textures from trame_rca.widgets import rca - init_vtk() # before creating any VTK render window + init_vtk_for_shared_textures() # before creating any VTK render window # ... create render_window, then in the UI: view = rca.RemoteControlledArea(display="shared-texture") diff --git a/examples/06_electron/app.py b/examples/06_electron/app.py index 0d59e63..3335d52 100644 --- a/examples/06_electron/app.py +++ b/examples/06_electron/app.py @@ -11,11 +11,10 @@ from trame.decorators import change from trame.ui.vuetify3 import SinglePageWithDrawerLayout from trame.widgets import html, vuetify3 as v3 -from trame_rca.electron import init_vtk -from trame_rca.widgets import rca +from trame_rca.widgets import init_vtk_for_shared_textures, rca # Before creating any VTK render window: VTK must render on the GPU Electron uses. -init_vtk() +init_vtk_for_shared_textures() from vtkmodules.vtkCommonDataModel import vtkPiecewiseFunction # noqa: E402 from vtkmodules.vtkFiltersSources import vtkPartitionedDataSetCollectionSource # noqa: E402 diff --git a/src/trame_rca/electron/__init__.py b/src/trame_rca/electron/__init__.py index f20eb39..e427587 100644 --- a/src/trame_rca/electron/__init__.py +++ b/src/trame_rca/electron/__init__.py @@ -3,32 +3,25 @@ The Electron side lives in the trame-rca-electron npm package: it spawns the trame server with ``--emp-port`` and shows the shared texture in the page. -Call :func:`init_vtk` before creating any VTK render window, then use +Call ``trame_rca.widgets.init_vtk_for_shared_textures()`` before creating any VTK render +window, then use ``RemoteControlledArea(display="shared-texture").create_electron_handler(...)``. """ import argparse -import logging -import os -import sys from .comm import PROTOCOL_VERSION -logger = logging.getLogger(__name__) - __all__ = [ "EMP_PORT_ARG", "PROTOCOL_VERSION", "emp_port", - "init_vtk", "server_url", ] # The CLI argument that trame-rca-electron passes the port of its `texture` channel. EMP_PORT_ARG = "--emp-port" -_initialized = False - def emp_port(server): """The port of the Electron main process's `texture` channel, from the CLI. @@ -55,47 +48,3 @@ def emp_port(server): def server_url(server): """The URL of the page trame serves, once the server is ready.""" return f"http://{server.server_options.host}:{server.port}/" - - -def init_vtk(): - """Make VTK render on the GPU Electron composites with. - - The shared texture can only be imported on the GPU Chromium uses, and VTK reads - some of these settings when its render window is constructed: call this before - creating any. It can be called more than once. - - Linux: ``RENDER_NODE`` overrides the render node (default: the GPU driving the - display). - Windows: ``D3D_ADAPTER`` overrides the DXGI adapter index (default 0). - MacOS: Calling this method is unnecessary. - """ - global _initialized - if _initialized: - return - _initialized = True - - if sys.platform == "linux": - from .passports import wayland_dmabuf - - if os.environ.get("RENDER_NODE"): - render_node, reason = os.environ["RENDER_NODE"], "env" - else: - render_node, reason = wayland_dmabuf.default_render_node() - logger.info("using render node %s (%s)", render_node, reason) - # vtkEGLRenderWindow::SetDeviceIndex doesn't reach the EGL display setup; the - # constructor only reads the device from this variable. - os.environ["VTK_EGL_DEVICE_INDEX"] = str( - wayland_dmabuf.device_index(render_node) - ) - os.environ["VTK_DEFAULT_OPENGL_WINDOW"] = "vtkEGLRenderWindow" - elif sys.platform == "win32": - from .passports import nthandle - - # The shared texture must live on the GPU Chromium uses, which is DXGI adapter 0 - # unless told otherwise. VTK's WGL context has to be on that GPU too. - nthandle.init_device(int(os.environ.get("D3D_ADAPTER", "0"))) - # Ask for a 1ms resolution in the Python process. Otherwise, asyncio.sleep - # rounds short waits to ~15.6ms - import ctypes - - ctypes.WinDLL("winmm").timeBeginPeriod(1) diff --git a/src/trame_rca/widgets/__init__.py b/src/trame_rca/widgets/__init__.py index e69de29..be9d04f 100644 --- a/src/trame_rca/widgets/__init__.py +++ b/src/trame_rca/widgets/__init__.py @@ -0,0 +1,5 @@ +from .rca import init_vtk_for_shared_textures + +__all__ = [ + "init_vtk_for_shared_textures", +] diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index 5df44d6..1117fcf 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -2,6 +2,9 @@ from __future__ import annotations +import logging +import os +import sys import warnings from typing import TYPE_CHECKING, Callable from weakref import WeakKeyDictionary, WeakValueDictionary @@ -18,7 +21,10 @@ from .. import module +logger = logging.getLogger(__name__) + __all__ = [ + "init_vtk_for_shared_textures", "RemoteControlledArea", "DisplayArea", "StatisticsDisplay", @@ -39,6 +45,53 @@ def __init__(self, _elem_name, children=None, **kwargs): self.server.enable_module(module) +_shared_textures_initialized = False + + +def init_vtk_for_shared_textures(): + """Make VTK render on the GPU Electron composites with. + + The shared texture can only be imported on the GPU Chromium uses, and VTK reads + some of these settings when its render window is constructed: call this before + creating any. It can be called more than once. + + Linux: ``RENDER_NODE`` overrides the render node (default: the GPU driving the + display). + Windows: ``D3D_ADAPTER`` overrides the DXGI adapter index (default 0). + MacOS: Calling this method is unnecessary. + """ + global _shared_textures_initialized + if _shared_textures_initialized: + return + _shared_textures_initialized = True + + if sys.platform == "linux": + from trame_rca.electron.passports import wayland_dmabuf + + if os.environ.get("RENDER_NODE"): + render_node, reason = os.environ["RENDER_NODE"], "env" + else: + render_node, reason = wayland_dmabuf.default_render_node() + logger.info("using render node %s (%s)", render_node, reason) + # vtkEGLRenderWindow::SetDeviceIndex doesn't reach the EGL display setup; the + # constructor only reads the device from this variable. + os.environ["VTK_EGL_DEVICE_INDEX"] = str( + wayland_dmabuf.device_index(render_node) + ) + os.environ["VTK_DEFAULT_OPENGL_WINDOW"] = "vtkEGLRenderWindow" + elif sys.platform == "win32": + from trame_rca.electron.passports import nthandle + + # The shared texture must live on the GPU Chromium uses, which is DXGI adapter 0 + # unless told otherwise. VTK's WGL context has to be on that GPU too. + nthandle.init_device(int(os.environ.get("D3D_ADAPTER", "0"))) + # Ask for a 1ms resolution in the Python process. Otherwise, asyncio.sleep + # rounds short waits to ~15.6ms + import ctypes + + ctypes.WinDLL("winmm").timeBeginPeriod(1) + + # Expose your vue component(s) class RemoteControlledArea(HtmlElement): _next_id = 0 @@ -148,7 +201,7 @@ def create_electron_handler( Render ``render_window`` into a texture shared with the Electron app that started this server (see @Kitware/trame-rca-electron), for ``display="shared-texture"``. - Call ``trame_rca.electron.init_vtk()`` before creating ``render_window``. + Call ``init_vtk_for_shared_textures()`` before creating ``render_window``. ``before_render`` is called before every frame and may schedule another render, e.g. to animate. ``on_disconnect`` is called when Electron goes away and defaults to stopping the server. From da1e8514d961af0c94f14ebb1a4d0ccd9a188155 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 27 Oct 2026 15:30:13 -0400 Subject: [PATCH 08/13] fix(rca): do not render in render window getter --- src/trame_rca/rca/vtk_rca.py | 5 ++--- src/trame_rca/schedulers/video_scheduler.py | 4 ++++ tests/test_benchmark.py | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/trame_rca/rca/vtk_rca.py b/src/trame_rca/rca/vtk_rca.py index 06360e3..ff4b007 100644 --- a/src/trame_rca/rca/vtk_rca.py +++ b/src/trame_rca/rca/vtk_rca.py @@ -32,7 +32,7 @@ def __init__(self, vtk_render_window: vtkRenderWindow): @property def img_cols_rows(self): - self._render() + self.render() with self._timer_capture: self._window_to_image.Modified() self._window_to_image.Update() @@ -50,10 +50,9 @@ def img_cols_rows(self): @property def render_window(self): - self._render() return self._vtk_render_window - def _render(self): + def render(self): with self._timer_render: self._vtk_render_window.Render() diff --git a/src/trame_rca/schedulers/video_scheduler.py b/src/trame_rca/schedulers/video_scheduler.py index a2bf15d..ee16037 100644 --- a/src/trame_rca/schedulers/video_scheduler.py +++ b/src/trame_rca/schedulers/video_scheduler.py @@ -45,6 +45,8 @@ def __init__( self._rca: VtkRemoteControlledArea = window_wrapper(window) self._push_callback = push_callback self.on_codec_changed = on_codec_changed + # The encoder needs the render window's GL context, which the first render creates. + self._rca.render() self._rca_encoder = RcaVideoEncoder( self._rca.render_window, push_callback=self._push, defer=negotiate ) @@ -115,6 +117,7 @@ async def _render(self): self._request_event.clear() started = monotonic() + self._rca.render() self._rca_encoder.encode(self._rca.render_window) # Sleep only for what is left of the _target_period_s. await sleep(max(0.0, self._target_period_s - (monotonic() - started))) @@ -124,4 +127,5 @@ def _push(self, content: bytes, meta: dict, _m_time: int): self._push_callback(content, meta) def reset(self): + self._rca.render() self._rca_encoder.reset(self._rca.render_window) diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 9f67be4..f1a710e 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -54,7 +54,7 @@ def full_hd_scene(): interactor.SetRenderWindow(render_window) rca = VtkRemoteControlledArea(render_window) - rca_render = rca._render + rca_render = rca.render def render(): rca_render() @@ -68,10 +68,10 @@ def recording_render(): rendered.append(True) render() - rca._render = recording_render + rca.render = recording_render rca.img_cols_rows - assert rendered, "img_cols_rows no longer renders through _render" - rca._render = lambda: None + assert rendered, "img_cols_rows no longer renders through render()" + rca.render = lambda: None yield rca, render, renderer.GetActiveCamera() render_window.Finalize() From 912c36183d2eae1937eb3a1600b270fe5d0229f6 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 27 Oct 2026 18:36:45 -0400 Subject: [PATCH 09/13] fix(shared-texture): use ring of textures to reduce latency --- README.rst | 9 ++ electron/src/main/index.js | 28 ++-- src/trame_rca/electron/comm.py | 9 +- src/trame_rca/electron/passports/base.py | 22 +++ .../schedulers/shared_texture_scheduler.py | 141 ++++++++++++------ src/trame_rca/view_adapter.py | 9 ++ src/trame_rca/widgets/rca.py | 6 +- tests/test_rca_electron.py | 73 ++++++++- 8 files changed, 233 insertions(+), 64 deletions(-) diff --git a/README.rst b/README.rst index d5d530e..d9496ae 100644 --- a/README.rst +++ b/README.rst @@ -156,6 +156,15 @@ Windows (D3D11 NT handles) and Linux under Wayland (dma-buf). view = rca.RemoteControlledArea(display="shared-texture") view_handler = view.create_electron_handler(render_window) +VTK renders into a ring of shared textures, by default this is 2. This allows VTK +to draw the next frame, while Electron still shows the previous one. If you're on a +mid-tier to high-end GPU, you normally do not need to use more than 1 texture because +VTK can render faster. However, on low-end hardware, it's more efficient to render the +N+k frames while electron is presenting the N'th frame. This hides the cost of each render. +The ``create_electron_handler(..., texture_count=n)`` sets the size of this ring texture container, +and you can change it at runtime with ``view_handler.texture_count = n``. A value of 1 +waits for Electron after every frame. + You can try our code example: .. code-block:: console diff --git a/electron/src/main/index.js b/electron/src/main/index.js index a164d46..e24598f 100644 --- a/electron/src/main/index.js +++ b/electron/src/main/index.js @@ -18,7 +18,7 @@ const webPreferences = (extra = {}) => ({ ...extra, }); -// Holds this process's reference to the producer's current texture. `dma` is this +// Holds this process's reference to one of the producer's textures. `dma` is this // platform's module from loadDma(). class TextureHandle { constructor(msg, dma, log) { @@ -74,7 +74,7 @@ class TextureHandle { // `args` its arguments, e.g. ['app.py']. // trame-rca-electron appends the arguments the server needs: --server --port 0 --emp-port . // -// Returns { process, close }. close() kills the server and releases the texture; it is +// Returns { process, close }. close() kills the server and releases the textures; it is // called on its own when `win` closes. const launchTrame = ( win, @@ -82,7 +82,8 @@ const launchTrame = ( ) => { if (!command) throw new Error('launchTrame: `command` is required'); let socket = null; - let texture = null; + // The producer's ring of textures, by slot. + let textures = []; // This platform's dma module, loaded before the server is spawned. let dma = null; let closed = false; @@ -92,9 +93,9 @@ const launchTrame = ( if (socket && !socket.destroyed) socket.write(protocol.encode(msg)); }; - const releaseTexture = () => { - texture?.release(); - texture = null; + const releaseTextures = (fromSlot = 0) => { + textures.slice(fromSlot).forEach((t) => t?.release()); + textures = textures.slice(0, fromSlot); }; const forwardFrame = async (info, msg) => { @@ -145,15 +146,18 @@ const launchTrame = ( win.loadURL(msg.url); } else if (msg.type === 'texture') { try { - // A new texture after a resize. Frames still in flight hold their own imports. - releaseTexture(); - texture = new TextureHandle(msg, dma, log); + const { slot, slots } = msg; + textures[slot]?.release(); + textures[slot] = new TextureHandle(msg, dma, log); + // release textures beyond `slots` to handle shrink + releaseTextures(slots); } catch (err) { log.error('[trame-rca] failed to open the texture:', err); conn.destroy(); } - } else if (msg.type === 'frame' && texture) { - forwardFrame(texture.info, msg); + } else if (msg.type === 'frame') { + const texture = textures[msg.slot]; + if (texture) forwardFrame(texture.info, msg); } }; @@ -203,7 +207,7 @@ const launchTrame = ( socket?.destroy(); socket = null; server.close(); - releaseTexture(); + releaseTextures(); }; win.once('closed', result.close); diff --git a/src/trame_rca/electron/comm.py b/src/trame_rca/electron/comm.py index 25faef1..6c63c64 100644 --- a/src/trame_rca/electron/comm.py +++ b/src/trame_rca/electron/comm.py @@ -6,11 +6,16 @@ <- {"type":"hello", "protocol": N, "passport": name} at start, the texture passport to use -> {"type":"server_ready", "protocol": N, "url": url} once trame serves its page - -> {"type":"texture", ...} at start and after every resize - -> {"type":"frame", "seq": n, "timestamp_us": t} after every render (finished on the GPU) + -> {"type":"texture", "slot": i, "slots": k, ...} at start, after every resize and + change of texture count, one per slot + -> {"type":"frame", "seq": n, "slot": i, "timestamp_us": t} + after every render into slot i + (finished on the GPU) <- {"type":"frame_done", "seq": n} when Electron released the texture <- {"type":"frame_done", "seq": n, "dropped": true} when the frame didn't reach the ERP +The k textures form a ring: several frames can be in flight, one per slot. Slots past k are gone. + Everything else (size, input, app state) goes from the renderer process to trame over its websocket. PROTOCOL_VERSION must match the one of the trame-rca-electron npm package. diff --git a/src/trame_rca/electron/passports/base.py b/src/trame_rca/electron/passports/base.py index 054849c..e8a77b4 100644 --- a/src/trame_rca/electron/passports/base.py +++ b/src/trame_rca/electron/passports/base.py @@ -1,8 +1,14 @@ +GL_FRAMEBUFFER = 0x8D40 + + class TexturePassport: """A cross-process abstraction for letting textures travel from Python to Electron. The passport has everything needed to access VTK render window's texture from another process. """ + # The vtkTextureObject VTK resolves its frames into, set by the subclasses. + vtk_texture = None + def __init__(self, render_window, width, height): self.render_window = render_window self.width = width @@ -12,6 +18,22 @@ def to_json(self): """The ``texture`` message sent to the Electron main process.""" return {} + def attach(self): + """Make this texture the one the render window resolves its next frames into. + + A passport attaches its texture when created: only needed when several take turns. + """ + if self.vtk_texture is None: + return + self.render_window.MakeCurrent() + display_fbo = self.render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, self.vtk_texture) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the shared texture") + def release(self): """Free the texture and any OS handle to it.""" diff --git a/src/trame_rca/schedulers/shared_texture_scheduler.py b/src/trame_rca/schedulers/shared_texture_scheduler.py index dd1d00d..73db6bb 100644 --- a/src/trame_rca/schedulers/shared_texture_scheduler.py +++ b/src/trame_rca/schedulers/shared_texture_scheduler.py @@ -3,10 +3,14 @@ OS handles do not go through trame's websocket because that is wired to the Electron renderer process (ERP), which does not have access to the OS API needed to import a texture. -Instead the scheduler tells the EMP about the shared texture and each finished frame over +Instead the scheduler tells the EMP about the shared textures and each finished frame over trame_rca.electron.comm, and the EMP hands them to the renderer process. Size and input still come from the RemoteControlledArea through the RcaViewAdapter. +The textures form a ring: VTK renders the next frame into the next texture while Electron +still shows the previous ones, and only waits when the texture it's about to reuse is still +held by Electron. + Renders run on trame's event loop. """ @@ -41,6 +45,12 @@ FRAME_DONE_TIMEOUT_S = 1.0 +def _check_texture_count(value): + if int(value) != value or value < 1: + raise ValueError(f"texture_count must be a positive integer, not {value!r}") + return int(value) + + def _report_failure(task): # Otherwise asyncio only reports it once the task is garbage collected. if not task.cancelled() and task.exception(): @@ -58,6 +68,7 @@ def __init__( target_fps: float = 60.0, on_disconnect: Callable[[], None] | None = None, before_render: Callable[[], None] | None = None, + texture_count: int = 2, ): self._timer_render = profiler.Timer("rca.shared_texture.render") self._timer_flush = profiler.Timer("rca.shared_texture.flush") @@ -73,10 +84,16 @@ def __init__( self._push_callback = None self._conn = None - self._passport = None + self._texture_count = _check_texture_count(texture_count) + # The ring of shared textures + self._passports = [] + # the frame_done of the last frame per shared texture. + self._slot_frames = [] + self._next_slot = 0 + # `seq`:`frame_done` future map valid until Electron fulfils with the "type":"frame_done" message. + self._frames_in_flight = {} self._passport_name_future = None self._seq = 0 - self._frame_done = None # Renders asked for before start() wait in here. self._request_event = asyncio.Event() self._tasks = [] @@ -112,6 +129,19 @@ def target_fps(self): def target_fps(self, value): self._target_fps = value + @property + def texture_count(self): + """How many shared textures VTK renders into in turn. 1 waits for Electron after every frame.""" + return self._texture_count + + @texture_count.setter + def texture_count(self, value): + value = _check_texture_count(value) + if value != self._texture_count: + self._texture_count = value + # The ring is rebuilt before the next frame. + self.schedule_render() + def set_push_callback(self, callback): # Frames go to the EMP, never through the push callback. self._push_callback = callback @@ -142,9 +172,9 @@ async def close(self): task.cancel() if self._conn: self._conn.close() - if self._passport: - self._passport.release() - self._passport = None + for passport in self._passports: + passport.release() + self._passports = [] async def _run(self, url): assert isinstance(self._rca, VtkRemoteControlledArea), ( @@ -172,39 +202,63 @@ async def _run(self, url): if not self._sized: continue - await self._update_texture(passport_name) - assert isinstance(self._passport, TexturePassport), "Passport unavailable!" + # Do not touch a texture in the ring until Electron is done reading it. + # Wait until a slot frees up in the ring. + await self._wait_for_slot(self._next_slot) + await self._update_textures(passport_name) + slot = self._next_slot + passport = self._passports[slot] + if (passport.width, passport.height) != tuple( + self._rca.render_window.GetSize() + ): + # Resized while waiting: rendering now would reallocate the shared texture. + self.schedule_render() + continue + self._next_slot = (slot + 1) % len(self._passports) + + passport.attach() if self._before_render: self._before_render() with self._timer_render: self._rca.render_window.Render() with self._timer_flush: self._rca.render_window.MakeCurrent() - await self._passport.flush_graphics_commands() + await passport.flush_graphics_commands() - # Backpressure: don't touch the texture until Electron is done reading it. - # Input keeps setting the request event meanwhile, so it renders once afterwards. seq = self._seq self._seq += 1 - self._frame_done = asyncio.get_running_loop().create_future() + frame_done = asyncio.get_running_loop().create_future() + self._slot_frames[slot] = frame_done + self._frames_in_flight[seq] = frame_done await self._conn.send( - {"type": "frame", "seq": seq, "timestamp_us": int(time.time() * 1e6)} + { + "type": "frame", + "seq": seq, + "slot": slot, + "timestamp_us": int(time.time() * 1e6), + } ) - try: - await asyncio.wait_for(self._frame_done, FRAME_DONE_TIMEOUT_S) - except asyncio.TimeoutError: - logger.warning( - "frame %d: no frame_done within %ss, continuing", - seq, - FRAME_DONE_TIMEOUT_S, - ) - self._frame_done = None # Pace against fixed frame clock held in `next_frame` next_frame = max(next_frame + period, time.monotonic()) await asyncio.sleep(max(0.0, next_frame - time.monotonic())) - async def _update_texture(self, passport_name): - """(Re)allocate the shared texture when the render window's size changed.""" + async def _wait_for_slot(self, slot): + """Wait until Electron released the last frame rendered into the texture at ``slot``.""" + frame_done = self._slot_frames[slot] if slot < len(self._slot_frames) else None + if frame_done is None or frame_done.done(): + return + try: + # shield: a frame_done arriving late must still find its future. + await asyncio.wait_for(asyncio.shield(frame_done), FRAME_DONE_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning( + "texture %d: no frame_done within %ss, continuing", + slot, + FRAME_DONE_TIMEOUT_S, + ) + + async def _update_textures(self, passport_name): + """(Re)allocate the shared textures when the render window's size or their count changed.""" assert isinstance(self._rca, VtkRemoteControlledArea), ( "SharedTexture scheduler only works with VTK render windows!" ) @@ -212,24 +266,30 @@ async def _update_texture(self, passport_name): "Connection to EMP is uninitialized!" ) width, height = self._rca.render_window.GetSize() - if self._passport and (self._passport.width, self._passport.height) == ( - width, - height, + count = self._texture_count + if len(self._passports) == count and all( + (p.width, p.height) == (width, height) for p in self._passports ): return - if self._passport is None: - # The first render creates the context and the framebuffers the passport patches. + if not self._passports: + # The first render will create the context and the framebuffers. self._rca.render_window.Render() self._rca.render_window.MakeCurrent() passport_class = texture_passport_class(passport_name) - old, self._passport = ( - self._passport, - passport_class(self._rca.render_window, width, height), + old, self._passports = ( + self._passports, + [ + passport_class(self._rca.render_window, width, height) + for _ in range(count) + ], ) - if old: - # Frames still in flight hold their own imports in Electron. - old.release() - await self._conn.send(self._passport.to_json()) + for passport in old: + passport.release() + self._slot_frames = [None] * count + self._next_slot = 0 + for slot, passport in enumerate(self._passports): + assert isinstance(passport, TexturePassport) + await self._conn.send({**passport.to_json(), "slot": slot, "slots": count}) async def _read(self): assert isinstance(self._conn, LineConnection), ( @@ -249,12 +309,9 @@ async def _read(self): return self._passport_name_future.set_result(msg["passport"]) elif msg["type"] == "frame_done": - if ( - msg["seq"] == self._seq - 1 - and self._frame_done - and not self._frame_done.done() - ): - self._frame_done.set_result(None) + frame_done = self._frames_in_flight.pop(msg["seq"], None) + if frame_done and not frame_done.done(): + frame_done.set_result(None) if msg.get("dropped"): # The page never processed it: e.g. it was still loading or busy.. logger.info("frame %d dropped, rendering again", msg["seq"]) diff --git a/src/trame_rca/view_adapter.py b/src/trame_rca/view_adapter.py index 0d960b8..a53893d 100644 --- a/src/trame_rca/view_adapter.py +++ b/src/trame_rca/view_adapter.py @@ -76,6 +76,15 @@ def target_fps(self) -> float: def target_fps(self, value) -> None: self._scheduler.target_fps = value + @property + def texture_count(self) -> int: + """How many shared textures the scheduler renders into in turn (shared-texture only)""" + return self._scheduler.texture_count + + @texture_count.setter + def texture_count(self, value) -> None: + self._scheduler.texture_count = value + @property def max_pixel_count(self): """Use 0 to disable capping of pixel count""" diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index 1117fcf..a2c08f0 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -196,6 +196,7 @@ def create_electron_handler( target_fps: float = 60.0, before_render: Callable[[], None] | None = None, on_disconnect: Callable[[], None] | None = None, + texture_count: int = 2, ): """ Render ``render_window`` into a texture shared with the Electron app that @@ -204,7 +205,9 @@ def create_electron_handler( Call ``init_vtk_for_shared_textures()`` before creating ``render_window``. ``before_render`` is called before every frame and may schedule another render, e.g. to animate. ``on_disconnect`` is called when Electron goes away and - defaults to stopping the server. + defaults to stopping the server. ``texture_count`` is how many shared textures + VTK renders into in turn, so it can render the next frame while Electron shows the + previous ones. The returned handler's ``texture_count`` changes it at runtime. """ from trame.app import asynchronous @@ -229,6 +232,7 @@ def on_disconnect(): target_fps=target_fps, on_disconnect=on_disconnect, before_render=before_render, + texture_count=texture_count, ) def start(**_): diff --git a/tests/test_rca_electron.py b/tests/test_rca_electron.py index 1f15f76..3e99b15 100644 --- a/tests/test_rca_electron.py +++ b/tests/test_rca_electron.py @@ -103,22 +103,32 @@ async def test_scheduler_shares_frames(a_render_window, fake_passport): # As the RcaViewAdapter does when the canvas reports its size. scheduler.rca.process_resize_event(300, 300) scheduler.schedule_render() - assert await emp.recv() == {"type": "texture", "width": 300, "height": 300} + # One texture per slot of the ring, two by default. + for slot in range(2): + assert await emp.recv() == { + "type": "texture", + "width": 300, + "height": 300, + "slot": slot, + "slots": 2, + } frame = await emp.recv() - assert frame["type"] == "frame" and frame["seq"] == 0 + assert frame["type"] == "frame" and frame["seq"] == 0 and frame["slot"] == 0 await emp.send({"type": "frame_done", "seq": 0}) - # A resize makes a new texture and releases the old one. + # A resize makes new textures and releases the old ones. scheduler.rca.process_resize_event(200, 100) scheduler.schedule_render() - assert await emp.recv() == {"type": "texture", "width": 200, "height": 100} - assert FakePassport.released == [(300, 300)] - assert (await emp.recv())["seq"] == 1 + for slot in range(2): + assert (await emp.recv())["width"] == 200 + assert FakePassport.released == [(300, 300)] * 2 + frame = await emp.recv() + assert frame["seq"] == 1 and frame["slot"] == 0 # Electron going away closes the scheduler. emp.close() await asyncio.wait_for(disconnected.wait(), TIMEOUT_S) - assert FakePassport.released == [(300, 300), (200, 100)] + assert FakePassport.released == [(300, 300)] * 2 + [(200, 100)] * 2 finally: await scheduler.close() emp.close() @@ -154,6 +164,7 @@ async def test_scheduler_renders_dropped_frames_again(a_render_window, fake_pass scheduler.rca.process_resize_event(300, 300) scheduler.schedule_render() assert (await emp.recv())["type"] == "texture" + assert (await emp.recv())["type"] == "texture" assert (await emp.recv())["seq"] == 0 # e.g. sendSharedTexture timed out while the page was loading. @@ -163,3 +174,51 @@ async def test_scheduler_renders_dropped_frames_again(a_render_window, fake_pass finally: await scheduler.close() emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_texture_ring(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + scheduler = RcaSharedTextureRenderScheduler(a_render_window, port, texture_count=3) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert (await emp.recv())["type"] == "server_ready" + scheduler.rca.process_resize_event(300, 300) + for _ in range(3): + assert (await emp.recv())["slots"] == 3 + + # Up to one frame in flight per texture, without waiting for frame_done. + for seq in range(3): + scheduler.schedule_render() + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (seq, seq) + + # Back to slot 0: waits until Electron released its frame. + scheduler.schedule_render() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + await emp.send({"type": "frame_done", "seq": 0}) + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (3, 0) + for seq in range(1, 4): + await emp.send({"type": "frame_done", "seq": seq}) + + # Changing the count at runtime rebuilds the ring before the next frame. + scheduler.texture_count = 1 + assert await emp.recv() == { + "type": "texture", + "width": 300, + "height": 300, + "slot": 0, + "slots": 1, + } + assert len(FakePassport.released) == 3 + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (4, 0) + + with pytest.raises(ValueError): + scheduler.texture_count = 0 + finally: + await scheduler.close() + emp.close() From ec3cd50364b7893e9803b611203a55d14cdc41d9 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 27 Oct 2026 18:37:03 -0400 Subject: [PATCH 10/13] docs(example): add native desktop app for comparison --- examples/06_electron/app_native.py | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 examples/06_electron/app_native.py diff --git a/examples/06_electron/app_native.py b/examples/06_electron/app_native.py new file mode 100644 index 0000000..ea5e12a --- /dev/null +++ b/examples/06_electron/app_native.py @@ -0,0 +1,114 @@ +"""The same VTK scene as app.py, rendered in a native desktop window without trame. + +Starts spinning the camera and prints the fps to the console once a second, +as a baseline to compare the shared texture path against. +""" + +import argparse +import time + +from vtkmodules.vtkCommonDataModel import vtkPiecewiseFunction +from vtkmodules.vtkFiltersSources import vtkPartitionedDataSetCollectionSource +from vtkmodules.vtkImagingCore import vtkRTAnalyticSource +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera +from vtkmodules.vtkRenderingCore import ( + vtkActor, + vtkColorTransferFunction, + vtkCompositePolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, + vtkVolume, + vtkVolumeProperty, +) +from vtkmodules.vtkRenderingVolumeOpenGL2 import vtkSmartVolumeMapper + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: F401 + + +def create_vtk_pipeline(): + wavelet = vtkRTAnalyticSource() + volume_mapper = vtkSmartVolumeMapper() + volume_mapper.SetInputConnection(wavelet.output_port) + color_tf = vtkColorTransferFunction() + color_tf.AddRGBPoint(0.0, 0.0, 0.0, 1.0) + color_tf.AddRGBPoint(255.0 / 2.0, 0.0, 1.0, 1.0) + color_tf.AddRGBPoint(255.0, 1.0, 0.0, 1.0) + opacity_tf = vtkPiecewiseFunction() + opacity_tf.AddPoint(0.0, 0.0) + opacity_tf.AddPoint(255.0, 0.1) + volume_property = vtkVolumeProperty() + volume_property.SetColor(color_tf) + volume_property.SetScalarOpacity(opacity_tf) + volume_property.ShadeOn() + volume_property.SetInterpolationTypeToLinear() + volume_actor = vtkVolume(mapper=volume_mapper, property=volume_property) + + shapes = vtkPartitionedDataSetCollectionSource() + cpdm = vtkCompositePolyDataMapper() + cpdm.SetInputConnection(shapes.output_port) + shapes_actor = vtkActor(mapper=cpdm) + + renderer = vtkRenderer() + renderer.AddVolume(volume_actor) + renderer.AddActor(shapes_actor) + renderer.ResetCamera() + + render_window = vtkRenderWindow(size=(1024, 768)) + render_window.SetWindowName("Shared texture native example") + render_window.AddRenderer(renderer) + interactor = vtkRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, renderer, interactor + + +class FpsCounter: + def __init__(self): + self.frames = 0 + self.start = time.perf_counter() + + def tick(self): + self.frames += 1 + elapsed = time.perf_counter() - self.start + if elapsed >= 1.0: + print(f"{self.frames / elapsed:.1f} fps", flush=True) + self.frames = 0 + self.start = time.perf_counter() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--no-vsync", action="store_true", help="Don't cap the fps at the display rate" + ) + args = parser.parse_args() + + render_window, renderer, interactor = create_vtk_pipeline() + interactor.Initialize() + if args.no_vsync: + # Needs a created window, so after Initialize(). + render_window.SetSwapControl(0) + # Match app.py while spinning: render at the interactor's interactive rate. + render_window.desired_update_rate = interactor.desired_update_rate + + camera = renderer.GetActiveCamera() + fps = FpsCounter() + render_window.AddObserver("EndEvent", lambda *_: fps.tick()) + + def spin(*_): + # Spin by a degree per frame, like app.py. + camera.Azimuth(1.0) + renderer.ResetCameraClippingRange() + render_window.Render() + + interactor.AddObserver("TimerEvent", spin) + interactor.CreateRepeatingTimer(1) + render_window.Render() + interactor.Start() + + +if __name__ == "__main__": + main() From c159b293a42a6a3092ad25624a7ad1a090cd2300 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Tue, 27 Oct 2026 18:37:24 -0400 Subject: [PATCH 11/13] docs(example): add point cloud streaming example --- examples/07_electron_point_cloud/README.md | 36 + examples/07_electron_point_cloud/app.py | 644 ++++++++++++++++++ .../07_electron_point_cloud/app_native.py | 252 +++++++ .../electron/.gitignore | 2 + .../07_electron_point_cloud/electron/main.js | 26 + .../electron/package.json | 18 + 6 files changed, 978 insertions(+) create mode 100644 examples/07_electron_point_cloud/README.md create mode 100644 examples/07_electron_point_cloud/app.py create mode 100644 examples/07_electron_point_cloud/app_native.py create mode 100644 examples/07_electron_point_cloud/electron/.gitignore create mode 100644 examples/07_electron_point_cloud/electron/main.js create mode 100644 examples/07_electron_point_cloud/electron/package.json diff --git a/examples/07_electron_point_cloud/README.md b/examples/07_electron_point_cloud/README.md new file mode 100644 index 0000000..8eb3cd1 --- /dev/null +++ b/examples/07_electron_point_cloud/README.md @@ -0,0 +1,36 @@ +# Dynamic point cloud with Electron + +`app.py` moves every point of a point cloud and gives it a new color each frame, then +renders its VTK view into a GPU texture that Electron shows as is. There is a a panel that +shows the updates/s, the time spent per stage and, after a few +point counts, how many points fit in 30 or 60 updates/s. + +`app_native.py` renders the same scene in a native desktop window without trame, as a +baseline: `python app_native.py [--points N] [--frames K] [--no-vsync]`. + +## Run + +```sh +# From the repository root, in a virtual environment +pip install -e ".[electron]" trame +cd electron +npm i +cd .. + +cd examples/07_electron_point_cloud/electron +npm i +npm start +# Or pass arguments to app.py +npm start -- --points 500000 --step 250000 --target-fps 1000 +``` + +`npm start` runs `app.py` with the `python3` of the active environment, or `$PYTHON`. +`--target-fps` (default 60) caps the update rate. `--textures` (default 2) is how many +shared textures VTK renders into in turn, also switchable at runtime in the panel: with 1, +each update waits for Electron to show the previous frame, with 2 VTK renders the next +frame meanwhile. + +The frames are computed ahead each time the point count changes, `--frames` of them +(default 8), and played back and forth: an update only swaps in the next frame's arrays, +so the app work is ~0 and the timings show the pipeline alone. That costs +`--frames` × 15 bytes per point of memory. `--frames 0` computes each frame live instead. diff --git a/examples/07_electron_point_cloud/app.py b/examples/07_electron_point_cloud/app.py new file mode 100644 index 0000000..222557d --- /dev/null +++ b/examples/07_electron_point_cloud/app.py @@ -0,0 +1,644 @@ +"""A dynamic point cloud whose VTK view is shared with Electron as a texture, not streamed. + +Every frame the server moves each point randomly, assigns new random colors and renders +into the texture Electron shows. The frames are computed ahead, on resize, so that the +app work is ~0: --frames 0 computes each one live instead. The next frame starts once Electron is done with the +previous one, so the measured updates/s is the maximum rate at which new data goes +through the full pipeline: + + numpy -> vtk arrays -> render (upload to the GPU + draw) -> shared texture + -> Electron composites the frame -> "frame_done" back to Python + +Started by the Electron app in ./electron, see the README. app_native.py renders the +same scene in a desktop window, as a baseline. +""" + +import asyncio +import time + +import numpy as np + +from trame.app import TrameApp +from trame.decorators import change +from trame.ui.html import DivLayout +from trame.widgets import client, html +from trame_common.utils import profiler +from trame_rca.widgets import init_vtk_for_shared_textures, rca + +# Before creating any VTK render window: VTK must render on the GPU Electron uses. +init_vtk_for_shared_textures() + +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray # noqa: E402 +from vtkmodules.vtkCommonCore import vtkPoints # noqa: E402 +from vtkmodules.vtkCommonDataModel import vtkCellArray, vtkPolyData # noqa: E402 +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera # noqa: E402 +from vtkmodules.vtkRenderingCore import ( # noqa: E402 + vtkActor, + vtkPolyDataMapper, + vtkRenderer, + vtkRenderWindow, +) +from vtkmodules.vtkRenderingUI import vtkGenericRenderWindowInteractor # noqa: E402 + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: E402, F401 + +WARMUP_FRAMES = 5 # frames skipped after a resize (topology + first upload) +ROLLING_FRAMES = 30 # frames averaged in the live panel +UI_REFRESH_S = 0.25 +BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb +DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod + +STAGES = [ + ("generate", "app work: swap in the next precomputed frame, or compute it"), + ("render", "Render + WaitForCompletion: upload to the GPU + draw, until done"), + ( + "electron", + "until the next texture is free: import, IPC, canvas draw, frame_done, pacing", + ), +] + +PANEL_STYLE = """ +position:absolute; top:1rem; left:1rem; z-index:10; width:38rem; +max-height:calc(100vh - 2rem); overflow:auto; +background:rgba(20,22,28,0.85); color:#e8e8e8; border-radius:8px; +padding:0.75rem 1rem; font:12px/1.4 ui-monospace,Menlo,monospace; +""" + +CSS = """ +body { margin: 0; } +.bench table { width:100%; border-collapse:collapse; margin:0.25rem 0 0.75rem; } +.bench td, .bench th { padding:1px 4px; text-align:right; white-space:nowrap; } +.bench td:first-child, .bench th:first-child { text-align:left; } +.bench td.stage { white-space:normal; } +.bench th { color:#9aa; font-weight:normal; border-bottom:1px solid #444; } +.bench .bar { height:8px; background:#4c9be8; border-radius:2px; } +.bench .big { font-size:20px; font-weight:bold; } +.bench button { font:inherit; padding:4px 10px; margin-right:6px; cursor:pointer; } +.bench .muted { color:#9aa; } +""" + + +# ----------------------------------------------------------------------------- +# Data +# ----------------------------------------------------------------------------- + + +class PointCloud: + """vtkPolyData whose point and color arrays change every frame + + With ``n_frames`` > 1 the frames are computed once, on resize, and each update only + points the polydata at the next frame's arrays: the app work drops to ~0 while every + frame still uploads new data. They play back and forth so the loop has no jump. + With ``n_frames`` = 0 each update computes a new frame, in place. + """ + + def __init__(self, n_points, n_frames=8, seed=0): + self.rng = np.random.default_rng(seed) + self.polydata = vtkPolyData() + self.n_points = 0 + self.n_frames = n_frames + self.positions = np.zeros((0, 3), dtype=np.float32) + self.frames = [] + self._t = 0 + self.resize(n_points) + + def resize(self, n_points): + n_old = self.n_points + positions = np.empty((n_points, 3), dtype=np.float32) + positions[: min(n_old, n_points)] = self.positions[:n_points] + if n_points > n_old: + positions[n_old:] = self.rng.standard_normal( + (n_points - n_old, 3), dtype=np.float32 + ) + colors = np.empty((n_points, 3), dtype=np.uint8) + self._noise = np.empty_like(positions) + self.n_points = n_points + + # Keep python references alive: the VTK arrays share numpy memory. + self.frames = [] + for _ in range(max(1, self.n_frames)): + self._walk(positions, colors) + self.frames.append(self._to_vtk(positions, colors)) + positions, colors = positions.copy(), colors.copy() + self.positions = self.frames[-1][0] + self.colors = self.frames[-1][1] + self._t = 0 + + # A single polyvertex cell with every point. The mapper rebuilds its index buffer + # whenever the points change, and that is about twice as fast as from one vertex + # cell per point. + verts = vtkCellArray() + verts.SetData( + numpy_to_vtkIdTypeArray(np.array([0, n_points], dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(np.arange(n_points, dtype=np.int64), deep=True), + ) + + self.polydata.SetPoints(vtkPoints()) + self.polydata.SetVerts(verts) + self._show(self.frames[-1]) + self.polydata.Modified() + + def animate(self): + if self.n_frames == 0: + self._walk(self.positions, self.colors) + self._show(self.frames[0]) + return + # Ping-pong: 0, 1, ..., K-1, K-2, ..., 1, 0, 1, ... + k = len(self.frames) + period = max(1, 2 * k - 2) + i = self._t % period + self._t += 1 + self._show(self.frames[i if i < k else period - i]) + + def _walk(self, positions, colors, step=0.02, pull=0.995): + # Random walk with a weak pull toward the origin so the cloud stays put. + self.rng.standard_normal(out=self._noise, dtype=np.float32) + self._noise *= step + positions *= pull + positions += self._noise + colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) + + @staticmethod + def _to_vtk(positions, colors): + vtk_positions = numpy_to_vtk(positions, deep=False) + vtk_colors = numpy_to_vtk(colors, deep=False) + vtk_colors.SetName("colors") + return positions, colors, vtk_positions, vtk_colors + + def _show(self, frame): + _, _, vtk_positions, vtk_colors = frame + # Modified() also covers a single frame, or the live mode, reusing the arrays. + vtk_positions.Modified() + vtk_colors.Modified() + points = self.polydata.GetPoints() + points.SetData(vtk_positions) + points.Modified() + self.polydata.point_data.SetScalars(vtk_colors) + + @property + def payload_bytes(self): + return self.n_points * BYTES_PER_POINT + + +def create_vtk_pipeline(polydata): + mapper = vtkPolyDataMapper() + mapper.SetInputData(polydata) + mapper.SetColorModeToDirectScalars() + mapper.SetScalarModeToUsePointData() + # The cloud stays around the origin, so float32 coordinates need no shift and scale: + # skipping it saves a pass over the points on every upload. + mapper.SetVBOShiftScaleMethod(DISABLE_SHIFT_SCALE) + + actor = vtkActor(mapper=mapper) + actor.property.point_size = 2 + + renderer = vtkRenderer(background=(0.1, 0.1, 0.12)) + renderer.AddActor(actor) + # The texture's alpha reaches Electron, so keep the background opaque. + renderer.SetBackgroundAlpha(1.0) + renderer.ResetCamera() + + render_window = vtkRenderWindow() + render_window.AddRenderer(renderer) + # MSAA costs more than drawing the points themselves, ~6 ms per frame at 1080p. + render_window.SetMultiSamples(0) + # trame-rca replays the canvas's input on the interactor. + vtkGenericRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window + + +# ----------------------------------------------------------------------------- +# Frame bookkeeping +# ----------------------------------------------------------------------------- + + +class Frame: + __slots__ = ["n_points", "t_start", "stages", "total_ms"] + + def __init__(self, n_points): + self.n_points = n_points + self.t_start = time.perf_counter() + self.stages = {} + self.total_ms = 0.0 + + def finalize(self, t_end): + """The frame ends when the next one starts: whatever is left is Electron's""" + self.total_ms = (t_end - self.t_start) * 1000 + self.stages["electron"] = max( + 0.0, + self.total_ms - self.stages["generate"] - self.stages["render"], + ) + + +class LevelStats: + """Aggregated frames recorded at one point count and number of shared textures""" + + def __init__(self, n_points, textures): + self.n_points = n_points + self.textures = textures + self.seen = 0 + self.count = 0 + self.total_ms = 0.0 + self.work_ms = 0.0 + + def add(self, frame): + self.seen += 1 + if self.seen <= WARMUP_FRAMES: + return + self.count += 1 + self.total_ms += frame.total_ms + self.work_ms += frame.stages["generate"] + + @property + def mean_ms(self): + return self.total_ms / self.count if self.count else 0.0 + + def row(self): + mean = self.mean_ms + mb = self.n_points * BYTES_PER_POINT / 1e6 + overhead = mean - self.work_ms / self.count + return { + "points": f"{self.n_points:,}", + "textures": self.textures, + "frames": self.count, + "ms": f"{mean:.1f}", + "ups": f"{1000 / mean:.1f}" if mean else "-", + "mbs": f"{mb / (overhead / 1000):.0f}" if overhead > 0 else "-", + } + + +def fit_capacity(levels): + """Least squares fit total_ms = a + b * n_points over the measured levels""" + pts = [(lv.n_points, lv.mean_ms) for lv in levels if lv.count >= 10] + if len(pts) < 2: + return None + n = np.array([p[0] for p in pts], dtype=float) + t = np.array([p[1] for p in pts], dtype=float) + b, a = np.polyfit(n, t, 1) + if b <= 0: + return None + + def capacity(ups): + return max(0, int((1000.0 / ups - a) / b)) + + return { + "fixed_ms": f"{a:.2f}", + "ns_per_point": f"{b * 1e6:.1f}", + "at30": f"{capacity(30):,}", + "at60": f"{capacity(60):,}", + } + + +# ----------------------------------------------------------------------------- +# Application +# ----------------------------------------------------------------------------- + + +class PointCloudBenchmark(TrameApp): + def __init__(self, server=None): + super().__init__(server) + + cli = self.server.cli + cli.add_argument("--points", type=int, default=100_000, help="Initial points") + cli.add_argument( + "--step", type=int, default=100_000, help="Points per +N click" + ) + cli.add_argument( + "--frames", + type=int, + default=8, + help="Frames computed ahead on resize, 0 to compute each one live", + ) + cli.add_argument( + "--target-fps", type=float, default=60.0, help="Cap on the frame rate" + ) + cli.add_argument( + "--textures", + type=int, + default=2, + help="Shared textures VTK renders into in turn, also changeable in the panel", + ) + args, _ = cli.parse_known_args() + + self.step_size = args.step + self.target_fps = args.target_fps + self.texture_count = args.textures + self.cloud = PointCloud(args.points, args.frames) + self.pending_points = None + self.running = True + self.frame = None + self.recent = [] + self.levels = [] + + self._timer_generate = profiler.Timer("rca.bench.generate") + self._render_start = 0.0 + + self.render_window = create_vtk_pipeline(self.cloud.polydata) + self.render_window.AddObserver("StartEvent", self._on_render_start) + self.render_window.AddObserver("EndEvent", self._on_render_end) + + self._build_ui() + self._new_level() + self.server.controller.on_server_ready.add(self._on_server_ready) + + # Frame loop ----------------------------------------------------------------- + + def _on_server_ready(self, **_): + asyncio.get_running_loop().create_task(self._publish_stats()) + + def before_render(self): + # Called by the scheduler before every frame: each frame asks for the next one. + t_now = time.perf_counter() + if self.frame is not None: + self.frame.finalize(t_now) + self.recent.append(self.frame) + del self.recent[:-ROLLING_FRAMES] + self.levels[-1].add(self.frame) + self.frame = None + + if not self.running: + return + + if self.pending_points is not None: + if self.pending_points != self.cloud.n_points: + self.cloud.resize(self.pending_points) + self._new_level() + self.pending_points = None + + frame = Frame(self.cloud.n_points) + with self._timer_generate: + self.cloud.animate() + frame.stages["generate"] = self._timer_generate.dt + self.frame = frame + self.view_handler.update() + + def _on_render_start(self, *_): + self._render_start = time.perf_counter() + + def _on_render_end(self, *_): + if self.frame is None: + return + # GL only queues the work: wait for the GPU here so that it counts as rendering. + # The scheduler's flush waits for the same thing right after, for free. + self.render_window.WaitForCompletion() + self.frame.stages["render"] = (time.perf_counter() - self._render_start) * 1000 + + # Controls ------------------------------------------------------------------- + + def add_points(self): + self._change_points(+self.step_size) + + def remove_points(self): + self._change_points(-self.step_size) + + def _change_points(self, delta): + base = self.pending_points or self.cloud.n_points + self.pending_points = max(1, base + delta) + self.state.n_points_pending = self.pending_points + + @change("textures") + def on_textures_change(self, textures, **_): + textures = int(textures) + if textures == self.texture_count: + return + # The scheduler rebuilds its ring of textures before the next frame. + self.texture_count = textures + self.view_handler.texture_count = textures + self._new_level() + + def toggle_running(self): + self.running = not self.running + self.state.running = self.running + if self.running: + self.view_handler.update() + + def _new_level(self): + if self.levels and self.levels[-1].count: + row = self.levels[-1].row() + print( + f"[bench] points={row['points']:>12} textures={row['textures']} " + f"updates={row['frames']:>5} " + f"update={row['ms']:>8} ms updates/s={row['ups']:>7} overhead MB/s={row['mbs']:>6}", + flush=True, + ) + if self.levels and not self.levels[-1].count: + self.levels.pop() + + # Coming back to a measured point count keeps adding to its row so the + # fit sees each count once. Warmup is skipped again after the resize. + key = (self.cloud.n_points, self.texture_count) + level = next( + (lv for lv in self.levels if (lv.n_points, lv.textures) == key), None + ) + if level is None: + level = LevelStats(*key) + else: + self.levels.remove(level) + level.seen = 0 + self.levels.append(level) + self.recent.clear() + + # Stats ------------------------------------------------------------------------ + + async def _publish_stats(self): + t_print = time.perf_counter() + while True: + await asyncio.sleep(UI_REFRESH_S) + if not self.recent: + continue + stats = self._stats() + with self.state: + self.state.update(stats) + # Also once a second on the console, to compare with app_native.py. + if time.perf_counter() - t_print >= 1.0: + t_print = time.perf_counter() + stages = " ".join( + f"{s['name']}={s['ms']:>7} ms" for s in stats["stages"] + ) + print( + f"points={stats['n_points']:>12} updates/s={stats['ups']:>7} " + f"update={stats['frame_ms']:>7} ms {stages}", + flush=True, + ) + + def _stats(self): + frames = self.recent + k = len(frames) + total = sum(f.total_ms for f in frames) / k + stage_ms = { + name: sum(f.stages.get(name, 0) for f in frames) / k for name, _ in STAGES + } + overhead = total - stage_ms["generate"] + payload_mb = self.cloud.payload_bytes / 1e6 + + return { + "n_points": f"{self.cloud.n_points:,}", + "n_points_pending": self.pending_points, + "payload_mb": f"{payload_mb:.1f}", + "ups": f"{1000 / total:.1f}", + "frame_ms": f"{total:.1f}", + "overhead_ms": f"{overhead:.1f}", + "overhead_mbs": f"{payload_mb / (overhead / 1000):.0f}" + if overhead > 0 + else "-", + "stages": [ + { + "name": name, + "help": help, + "ms": f"{stage_ms[name]:.2f}", + "pct": f"{100 * stage_ms[name] / total:.0f}", + } + for name, help in STAGES + ], + "levels": [ + lv.row() + for lv in sorted(self.levels, key=lambda lv: (lv.textures, lv.n_points)) + if lv.count + ], + "fit": fit_capacity( + [lv for lv in self.levels if lv.textures == self.texture_count] + ), + } + + # UI ----------------------------------------------------------------------------- + + def _build_ui(self): + self.state.update( + { + "running": True, + "n_points": f"{self.cloud.n_points:,}", + "n_points_pending": None, + "payload_mb": "-", + "ups": "-", + "frame_ms": "-", + "overhead_ms": "-", + "overhead_mbs": "-", + "stages": [], + "levels": [], + "textures": self.texture_count, + "fit": None, + } + ) + # Frame stats from the canvas, once a second: only the panel needs them, not the server. + self.state.client_only("stats") + self.state.stats = {"fps": 0, "frames": 0, "info": ""} + + with DivLayout(self.server) as self.ui: + self.ui.root.style = "height:100vh;position:relative;" + client.Style(CSS) + + with html.Div(style="position:absolute;inset:0;"): + view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + stats="stats = $event", + ) + self.view_handler = view.create_electron_handler( + self.render_window, + target_fps=self.target_fps, + texture_count=self.texture_count, + before_render=self.before_render, + ) + + with html.Div(classes="bench", style=PANEL_STYLE): + with html.Div(style="margin-bottom:0.5rem;"): + html.Button( + "{{ running ? 'Pause' : 'Resume' }}", + click=self.toggle_running, + ) + html.Button(f"−{self.step_size:,}", click=self.remove_points) + html.Button(f"+{self.step_size:,}", click=self.add_points) + html.Span("textures:", classes="muted", style="margin:0 6px;") + for n in range(1, 5): + html.Button( + str(n), + click=f"textures = {n}", + style=( + "{ fontWeight: textures === %d ? 'bold' : 'normal' }" + % n, + ), + ) + html.Span( + "→ {{ n_points_pending.toLocaleString() }} on next update", + v_if="n_points_pending", + classes="muted", + ) + + with html.Div(style="display:flex;gap:1.5rem;margin-bottom:0.5rem;"): + with html.Div(): + html.Div("{{ stats.fps }} fps", classes="big") + html.Div("client render rate", classes="muted") + with html.Div(): + html.Div("{{ ups }} updates/s", classes="big") + html.Div("{{ frame_ms }} ms / update", classes="muted") + with html.Div(): + html.Div("{{ n_points }} pts", classes="big") + html.Div("{{ payload_mb }} MB / update", classes="muted") + + html.Div( + "Overhead (update − app work): {{ overhead_ms }} ms " + "→ {{ overhead_mbs }} MB/s" + f", capped at {self.target_fps:g} updates/s" + ) + + with html.Table(): + with html.Tr(): + html.Th("stage") + html.Th("ms") + html.Th("%") + html.Th("", style="width:35%;") + with html.Tr(v_for="s in stages", key="s.name"): + with html.Td(classes="stage"): + html.Span("{{ s.name }} ") + html.Span("({{ s.help }})", classes="muted") + html.Td("{{ s.ms }}") + html.Td("{{ s.pct }}") + with html.Td(): + html.Div(classes="bar", style=("{ width: s.pct + '%' }",)) + + html.Div( + f"Per point count (first {WARMUP_FRAMES} updates skipped)", + classes="muted", + ) + with html.Table(): + with html.Tr(): + html.Th("points") + html.Th("textures") + html.Th("updates") + html.Th("ms") + html.Th("updates/s") + html.Th("MB/s") + with html.Tr( + v_for="l in levels", key="l.points + '/' + l.textures" + ): + html.Td("{{ l.points }}") + html.Td("{{ l.textures }}") + html.Td("{{ l.frames }}") + html.Td("{{ l.ms }}") + html.Td("{{ l.ups }}") + html.Td("{{ l.mbs }}") + + with html.Div(v_if="fit"): + html.Div( + "Fit with {{ textures }} textures: {{ fit.fixed_ms }} ms fixed " + "+ {{ fit.ns_per_point }} ns/point" + ) + html.Div( + "Realtime capacity: {{ fit.at30 }} pts @ 30 updates/s, " + "{{ fit.at60 }} pts @ 60 updates/s", + classes="big", + style="font-size:14px;", + ) + html.Div( + "Add points at least twice to fit the per-point cost", + v_else=True, + classes="muted", + ) + + +if __name__ == "__main__": + PointCloudBenchmark().server.start() diff --git a/examples/07_electron_point_cloud/app_native.py b/examples/07_electron_point_cloud/app_native.py new file mode 100644 index 0000000..7e204f2 --- /dev/null +++ b/examples/07_electron_point_cloud/app_native.py @@ -0,0 +1,252 @@ +"""A dynamic point cloud rendered in a native desktop window without trame. + +Every frame moves each point randomly and assigns new random colors, then renders. +The frames are computed ahead, on resize: --frames 0 computes each one live instead. +Prints the updates/s and the time spent per stage to the console once a second, +as a baseline to compare the shared texture path against. + +Keys: + / - add or remove --step points, space pauses or resumes the animation. +""" + +import argparse +import time + +import numpy as np +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray +from vtkmodules.vtkCommonCore import vtkPoints +from vtkmodules.vtkCommonDataModel import vtkCellArray, vtkPolyData +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera +from vtkmodules.vtkRenderingCore import ( + vtkActor, + vtkPolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, +) + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: F401 + +BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb +DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod + + +class PointCloud: + """vtkPolyData whose point and color arrays change every frame + + With ``n_frames`` > 1 the frames are computed once, on resize, and each update only + points the polydata at the next frame's arrays: the app work drops to ~0 while every + frame still uploads new data. They play back and forth so the loop has no jump. + With ``n_frames`` = 0 each update computes a new frame, in place. + """ + + def __init__(self, n_points, n_frames=8, seed=0): + self.rng = np.random.default_rng(seed) + self.polydata = vtkPolyData() + self.n_points = 0 + self.n_frames = n_frames + self.positions = np.zeros((0, 3), dtype=np.float32) + self.frames = [] + self._t = 0 + self.resize(n_points) + + def resize(self, n_points): + n_old = self.n_points + positions = np.empty((n_points, 3), dtype=np.float32) + positions[: min(n_old, n_points)] = self.positions[:n_points] + if n_points > n_old: + positions[n_old:] = self.rng.standard_normal( + (n_points - n_old, 3), dtype=np.float32 + ) + colors = np.empty((n_points, 3), dtype=np.uint8) + self._noise = np.empty_like(positions) + self.n_points = n_points + + # Keep python references alive: the VTK arrays share numpy memory. + self.frames = [] + for _ in range(max(1, self.n_frames)): + self._walk(positions, colors) + self.frames.append(self._to_vtk(positions, colors)) + positions, colors = positions.copy(), colors.copy() + self.positions = self.frames[-1][0] + self.colors = self.frames[-1][1] + self._t = 0 + + # A single polyvertex cell with every point. The mapper rebuilds its index buffer + # whenever the points change, and that is about twice as fast as from one vertex + # cell per point. + verts = vtkCellArray() + verts.SetData( + numpy_to_vtkIdTypeArray(np.array([0, n_points], dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(np.arange(n_points, dtype=np.int64), deep=True), + ) + + self.polydata.SetPoints(vtkPoints()) + self.polydata.SetVerts(verts) + self._show(self.frames[-1]) + self.polydata.Modified() + + def animate(self): + if self.n_frames == 0: + self._walk(self.positions, self.colors) + self._show(self.frames[0]) + return + # Ping-pong: 0, 1, ..., K-1, K-2, ..., 1, 0, 1, ... + k = len(self.frames) + period = max(1, 2 * k - 2) + i = self._t % period + self._t += 1 + self._show(self.frames[i if i < k else period - i]) + + def _walk(self, positions, colors, step=0.02, pull=0.995): + # Random walk with a weak pull toward the origin so the cloud stays put. + self.rng.standard_normal(out=self._noise, dtype=np.float32) + self._noise *= step + positions *= pull + positions += self._noise + colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) + + @staticmethod + def _to_vtk(positions, colors): + vtk_positions = numpy_to_vtk(positions, deep=False) + vtk_colors = numpy_to_vtk(colors, deep=False) + vtk_colors.SetName("colors") + return positions, colors, vtk_positions, vtk_colors + + def _show(self, frame): + _, _, vtk_positions, vtk_colors = frame + # Modified() also covers a single frame, or the live mode, reusing the arrays. + vtk_positions.Modified() + vtk_colors.Modified() + points = self.polydata.GetPoints() + points.SetData(vtk_positions) + points.Modified() + self.polydata.point_data.SetScalars(vtk_colors) + + @property + def payload_bytes(self): + return self.n_points * BYTES_PER_POINT + + +def create_vtk_pipeline(polydata): + mapper = vtkPolyDataMapper() + mapper.SetInputData(polydata) + mapper.SetColorModeToDirectScalars() + mapper.SetScalarModeToUsePointData() + # The cloud stays around the origin, so float32 coordinates need no shift and scale: + # skipping it saves a pass over the points on every upload. + mapper.SetVBOShiftScaleMethod(DISABLE_SHIFT_SCALE) + + actor = vtkActor(mapper=mapper) + actor.property.point_size = 2 + + renderer = vtkRenderer(background=(0.1, 0.1, 0.12)) + renderer.AddActor(actor) + renderer.ResetCamera() + + render_window = vtkRenderWindow(size=(1024, 768)) + # MSAA costs more than drawing the points themselves, ~6 ms per frame at 1080p. + render_window.SetMultiSamples(0) + render_window.SetWindowName("Dynamic point cloud native example") + render_window.AddRenderer(renderer) + interactor = vtkRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, interactor + + +class Stats: + """Accumulates per-frame stage timings and prints their mean once a second""" + + def __init__(self): + self.reset() + + def reset(self): + self.frames = 0 + self.generate_ms = 0.0 + self.render_ms = 0.0 + self.start = time.perf_counter() + + def add(self, generate_ms, render_ms, cloud): + self.frames += 1 + self.generate_ms += generate_ms + self.render_ms += render_ms + elapsed = time.perf_counter() - self.start + if elapsed < 1.0: + return + n = self.frames + frame_ms = 1000 * elapsed / n + render_ms = self.render_ms / n + payload_mb = cloud.payload_bytes / 1e6 + print( + f"points={cloud.n_points:>12,} updates/s={n / elapsed:>7.1f} " + f"update={frame_ms:>7.1f} ms generate={self.generate_ms / n:>7.2f} ms " + f"render={render_ms:>7.2f} ms ({payload_mb:.1f} MB/update" + + (f", {payload_mb / (render_ms / 1000):.0f} MB/s" if render_ms else "") + + ")", + flush=True, + ) + self.reset() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--points", type=int, default=100_000, help="Initial points") + parser.add_argument( + "--step", type=int, default=100_000, help="Points per + or - key press" + ) + parser.add_argument( + "--frames", + type=int, + default=8, + help="Frames computed ahead on resize, 0 to compute each one live", + ) + parser.add_argument( + "--no-vsync", action="store_true", help="Don't cap the fps at the display rate" + ) + args = parser.parse_args() + + cloud = PointCloud(args.points, args.frames) + render_window, interactor = create_vtk_pipeline(cloud.polydata) + interactor.Initialize() + if args.no_vsync: + # Needs a created window, so after Initialize(). + render_window.SetSwapControl(0) + + stats = Stats() + running = True + + def step(*_): + if not running: + return + t0 = time.perf_counter() + cloud.animate() + t1 = time.perf_counter() + render_window.Render() + t2 = time.perf_counter() + stats.add(1000 * (t1 - t0), 1000 * (t2 - t1), cloud) + + def on_key(obj, _): + nonlocal running + key = obj.GetKeySym() + if key in ("plus", "equal", "KP_Add"): + cloud.resize(cloud.n_points + args.step) + elif key in ("minus", "KP_Subtract"): + cloud.resize(max(1, cloud.n_points - args.step)) + elif key == "space": + running = not running + else: + return + stats.reset() + render_window.Render() + + interactor.AddObserver("KeyPressEvent", on_key) + interactor.AddObserver("TimerEvent", step) + interactor.CreateRepeatingTimer(1) + render_window.Render() + interactor.Start() + + +if __name__ == "__main__": + main() diff --git a/examples/07_electron_point_cloud/electron/.gitignore b/examples/07_electron_point_cloud/electron/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/examples/07_electron_point_cloud/electron/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/examples/07_electron_point_cloud/electron/main.js b/examples/07_electron_point_cloud/electron/main.js new file mode 100644 index 0000000..4ccb2d8 --- /dev/null +++ b/examples/07_electron_point_cloud/electron/main.js @@ -0,0 +1,26 @@ +import { app, BrowserWindow } from 'electron/main'; +import { fileURLToPath } from 'node:url'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; + +// Run ../app.py with the python of the active environment, unless PYTHON says otherwise. +const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); + +// Forward the arguments after `npm start --` to app.py, e.g. --points 500000. +const appArgs = process.argv.slice(app.isPackaged ? 1 : 2); + +app.whenReady().then(() => { + const win = new BrowserWindow({ + width: 1920, + height: 1080, + useContentSize: true, + webPreferences: webPreferences(), + }); + launchTrame(win, { + command: python, + args: [fileURLToPath(new URL('../app.py', import.meta.url)), ...appArgs], + onStats: (s) => console.log(`[renderer] frames=${s.frames} fps=${s.fps} ${s.info}`), + onExit: () => app.quit(), + }); +}); + +app.on('window-all-closed', () => app.quit()); diff --git a/examples/07_electron_point_cloud/electron/package.json b/examples/07_electron_point_cloud/electron/package.json new file mode 100644 index 0000000..cc16118 --- /dev/null +++ b/examples/07_electron_point_cloud/electron/package.json @@ -0,0 +1,18 @@ +{ + "name": "trame-rca-electron-point-cloud-example", + "private": true, + "version": "0.0.0", + "description": "Electron shell for the shared texture dynamic point cloud trame-rca example", + "license": "Apache-2.0", + "type": "module", + "main": "main.js", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "@kitware/trame-rca-electron": "file:../../../electron" + }, + "devDependencies": { + "electron": "^44.7.0" + } +} From 2f756ce905562d6cf19672b9bfffd1a5a009ef21 Mon Sep 17 00:00:00 2001 From: Jaswant Panchumarti Date: Wed, 28 Oct 2026 11:55:08 -0400 Subject: [PATCH 12/13] feat(shared-texture): support multiple views --- electron/README.md | 2 +- electron/src/main/index.js | 55 ++++--- electron/src/preload.mjs | 69 ++++----- examples/06_electron/electron/main.js | 2 +- examples/07_electron_point_cloud/README.md | 9 +- examples/07_electron_point_cloud/app.py | 120 +++++++++++++-- .../07_electron_point_cloud/app_native.py | 21 ++- .../07_electron_point_cloud/electron/main.js | 2 +- src/trame_rca/electron/comm.py | 140 +++++++++++++++++- .../schedulers/shared_texture_scheduler.py | 109 ++++++-------- src/trame_rca/widgets/rca.py | 8 +- tests/test_rca_electron.py | 106 ++++++++++++- vue-components/src/components/DisplayArea.js | 2 +- .../components/SharedTextureDisplayArea.js | 41 +++-- 14 files changed, 520 insertions(+), 166 deletions(-) diff --git a/electron/README.md b/electron/README.md index 3264e01..8d406d1 100644 --- a/electron/README.md +++ b/electron/README.md @@ -24,7 +24,7 @@ and loads the page in `win` once trame serves it. Options: - `command`, `args`, `cwd`, `env`: the trame server to run. Any executable works: a python interpreter and a script, a frozen backend, etc. - `onExit(code, signal)`: the server exited. -- `onStats({ fps, frames, info })`: once a second. +- `onStats({ view, fps, frames, info })`: once a second, per shared-texture view that drew frames. - `log`: defaults to `console`. It returns `{ process, close }`. `close()` kills the server; it is called when `win` closes. diff --git a/electron/src/main/index.js b/electron/src/main/index.js index e24598f..46c97ff 100644 --- a/electron/src/main/index.js +++ b/electron/src/main/index.js @@ -36,7 +36,7 @@ class TextureHandle { const { lookupSurface, describeSurface, surfaceHandle } = this.dma; this.surfaceRef = lookupSurface(msg.ioSurfaceId); log.info( - `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) IOSurface id=${msg.ioSurfaceId} ${describeSurface(this.surfaceRef)}`, + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} (${msg.pixelFormat}) IOSurface id=${msg.ioSurfaceId} ${describeSurface(this.surfaceRef)}`, ); return { ioSurface: surfaceHandle(this.surfaceRef) }; } @@ -45,14 +45,14 @@ class TextureHandle { const { dupHandleFromProcess, handleBuffer } = this.dma; this.localHandle = dupHandleFromProcess(msg.pid, msg.ntHandle); log.info( - `[trame-rca] texture ${msg.width}x${msg.height} (${msg.pixelFormat}) handle 0x${msg.ntHandle.toString(16)} -> 0x${this.localHandle.toString(16)}`, + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} (${msg.pixelFormat}) handle 0x${msg.ntHandle.toString(16)} -> 0x${this.localHandle.toString(16)}`, ); return { ntHandle: handleBuffer(this.localHandle) }; } // Duplicate the producer's dma-buf fds into this process. this.localFds = msg.planes.map((p) => this.dma.dupFdFromProcess(msg.pid, p.fd)); log.info( - `[trame-rca] texture ${msg.width}x${msg.height} ${msg.fourcc} (${msg.pixelFormat}) modifier=${msg.modifier} fds ${msg.planes.map((p) => p.fd)} -> ${this.localFds}`, + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} ${msg.fourcc} (${msg.pixelFormat}) modifier=${msg.modifier} fds ${msg.planes.map((p) => p.fd)} -> ${this.localFds}`, ); return protocol.nativePixmap(msg, this.localFds); } @@ -68,7 +68,7 @@ class TextureHandle { } // Spawn the trame server and load its page in `win` once it is ready. The page's -// shared-texture views then show the texture the server renders into. +// shared-texture views then show the textures the server renders into, one ring per view. // // `command` is the executable to run (a python interpreter, a frozen backend, ...) and // `args` its arguments, e.g. ['app.py']. @@ -82,8 +82,8 @@ const launchTrame = ( ) => { if (!command) throw new Error('launchTrame: `command` is required'); let socket = null; - // The producer's ring of textures, by slot. - let textures = []; + // The producer's rings of textures, by view name, then by slot. + const textures = new Map(); // This platform's dma module, loaded before the server is spawned. let dma = null; let closed = false; @@ -93,9 +93,17 @@ const launchTrame = ( if (socket && !socket.destroyed) socket.write(protocol.encode(msg)); }; - const releaseTextures = (fromSlot = 0) => { - textures.slice(fromSlot).forEach((t) => t?.release()); - textures = textures.slice(0, fromSlot); + // Release the textures of `view` from `fromSlot` on. + const releaseTextures = (view, fromSlot = 0) => { + const ring = textures.get(view) ?? []; + ring.slice(fromSlot).forEach((t) => t?.release()); + ring.length = Math.min(ring.length, fromSlot); + if (!ring.length) textures.delete(view); + }; + + const releaseAllTextures = () => { + textures.forEach((ring) => ring.forEach((t) => t?.release())); + textures.clear(); }; const forwardFrame = async (info, msg) => { @@ -104,12 +112,12 @@ const launchTrame = ( const frameDone = () => { if (done) return; done = true; - send({ type: 'frame_done', seq: msg.seq }); + send({ type: 'frame_done', view: msg.view, seq: msg.seq }); }; const frameDropped = () => { if (done) return; done = true; - send({ type: 'frame_done', seq: msg.seq, dropped: true }); + send({ type: 'frame_done', view: msg.view, seq: msg.seq, dropped: true }); }; let imported = null; try { @@ -118,15 +126,16 @@ const launchTrame = ( textureInfo: { ...info, timestamp: msg.timestamp_us }, allReferencesReleased: frameDone, }); - await sharedTexture.sendSharedTexture({ - frame: win.webContents.mainFrame, - importedSharedTexture: imported, - }); + // The preload's receiver gets the view's name as its second argument. + await sharedTexture.sendSharedTexture( + { frame: win.webContents.mainFrame, importedSharedTexture: imported }, + msg.view, + ); } catch (err) { // sendSharedTexture uses a 1000ms timeout. If the ERP is still busy loading the page, // or if some other error occurred, the page would never have shown this frame, so ask // for it again. - log.warn(`[trame-rca] frame ${msg.seq} dropped: ${err.message}`); + log.warn(`[trame-rca] view ${msg.view}: frame ${msg.seq} dropped: ${err.message}`); frameDropped(); } finally { imported?.release(); @@ -146,17 +155,19 @@ const launchTrame = ( win.loadURL(msg.url); } else if (msg.type === 'texture') { try { - const { slot, slots } = msg; - textures[slot]?.release(); - textures[slot] = new TextureHandle(msg, dma, log); + const { view, slot, slots } = msg; + if (!textures.has(view)) textures.set(view, []); + const ring = textures.get(view); + ring[slot]?.release(); + ring[slot] = new TextureHandle(msg, dma, log); // release textures beyond `slots` to handle shrink - releaseTextures(slots); + releaseTextures(view, slots); } catch (err) { log.error('[trame-rca] failed to open the texture:', err); conn.destroy(); } } else if (msg.type === 'frame') { - const texture = textures[msg.slot]; + const texture = textures.get(msg.view)?.[msg.slot]; if (texture) forwardFrame(texture.info, msg); } }; @@ -207,7 +218,7 @@ const launchTrame = ( socket?.destroy(); socket = null; server.close(); - releaseTextures(); + releaseAllTextures(); }; win.once('closed', result.close); diff --git a/electron/src/preload.mjs b/electron/src/preload.mjs index 93cbaef..51a154b 100644 --- a/electron/src/preload.mjs +++ b/electron/src/preload.mjs @@ -1,24 +1,21 @@ -// Draws the frames the main process sends into the page's shared-texture-display-area. +// Draws the frames the main process sends into the page's shared-texture-display-areas. // Use it as a BrowserWindow's preload (see webPreferences()), or import it from yours. import { contextBridge, ipcRenderer, sharedTexture } from 'electron'; const STATS_INTERVAL_MS = 1000; -// The canvas of the page's shared-texture-display-area, attached when the component mounts. -let canvas = null; -let ctx = null; -let onStats = null; +// The canvas of each shared-texture-display-area, by view name, attached when the component +// mounts. +const views = new Map(); -let frames = 0; -let windowStart = performance.now(); -let windowFrames = 0; -let info = ''; - -sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture }) => { - if (!canvas) { +// The main process sends the view's name along with every frame. +sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture }, view) => { + const target = views.get(view); + if (!target) { importedSharedTexture.release(); return; } + const { canvas, ctx } = target; const vf = importedSharedTexture.getVideoFrame(); // The canvas's pixels follow the producer's texture, which follows the canvas's size. if (canvas.width !== vf.displayWidth || canvas.height !== vf.displayHeight) { @@ -30,38 +27,44 @@ sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture }) => { ctx.setTransform(1, 0, 0, -1, 0, canvas.height); ctx.drawImage(vf, 0, 0, canvas.width, canvas.height); ctx.setTransform(1, 0, 0, 1, 0, 0); - info = `${vf.codedWidth}x${vf.codedHeight} ${vf.format}`; + target.info = `${vf.codedWidth}x${vf.codedHeight} ${vf.format}`; vf.close(); importedSharedTexture.release(); - frames++; - windowFrames++; + target.frames++; + target.windowFrames++; }); // Frames are rendered on demand, so report stats over a timed interval. setInterval(() => { const now = performance.now(); - const fps = (windowFrames * 1000) / (now - windowStart); - const hadFrames = windowFrames > 0; - windowStart = now; - windowFrames = 0; - onStats?.({ fps: Number(fps.toFixed(1)), frames, info }); - if (hadFrames && ctx) { - ipcRenderer.send('trame-rca:stats', { frames, fps: fps.toFixed(1), info }); - } + views.forEach((target, view) => { + const fps = (target.windowFrames * 1000) / (now - target.windowStart); + const hadFrames = target.windowFrames > 0; + target.windowStart = now; + target.windowFrames = 0; + const { frames, info } = target; + target.onStats?.({ fps: Number(fps.toFixed(1)), frames, info }); + if (hadFrames) { + ipcRenderer.send('trame-rca:stats', { view, frames, fps: fps.toFixed(1), info }); + } + }); }, STATS_INTERVAL_MS); // The page's world can't touch sharedTexture or VideoFrames from here, but it can hand us its -// canvas, and a callback for the frame rate. +// canvases, and a callback for their frame rate. contextBridge.exposeInMainWorld('trameRcaSharedTexture', { - attach: (el, statsCallback) => { - canvas = el; - ctx = el.getContext('2d'); - onStats = statsCallback ?? null; + attach: (view, el, statsCallback) => { + views.set(view, { + canvas: el, + ctx: el.getContext('2d'), + onStats: statsCallback ?? null, + frames: 0, + windowStart: performance.now(), + windowFrames: 0, + info: '', + }); }, - detach: (el) => { - if (canvas !== el) return; - canvas = null; - ctx = null; - onStats = null; + detach: (view, el) => { + if (views.get(view)?.canvas === el) views.delete(view); }, }); diff --git a/examples/06_electron/electron/main.js b/examples/06_electron/electron/main.js index 5df7045..2eca7bf 100644 --- a/examples/06_electron/electron/main.js +++ b/examples/06_electron/electron/main.js @@ -20,7 +20,7 @@ app.whenReady().then(() => { '--electron-versions', JSON.stringify({ electron, chrome, node, v8 }), ], - onStats: (s) => console.log(`[renderer] frames=${s.frames} fps=${s.fps} ${s.info}`), + onStats: (s) => console.log(`[renderer] ${s.view} frames=${s.frames} fps=${s.fps} ${s.info}`), onExit: () => app.quit(), }); }); diff --git a/examples/07_electron_point_cloud/README.md b/examples/07_electron_point_cloud/README.md index 8eb3cd1..142b81f 100644 --- a/examples/07_electron_point_cloud/README.md +++ b/examples/07_electron_point_cloud/README.md @@ -6,7 +6,7 @@ shows the updates/s, the time spent per stage and, after a few point counts, how many points fit in 30 or 60 updates/s. `app_native.py` renders the same scene in a native desktop window without trame, as a -baseline: `python app_native.py [--points N] [--frames K] [--no-vsync]`. +baseline: `python app_native.py [--points N] [--frames K] [--radii X Y Z] [--no-vsync]`. ## Run @@ -24,11 +24,16 @@ npm start npm start -- --points 500000 --step 250000 --target-fps 1000 ``` +`npm start -- --second-view` adds a second shared-texture view of the same cloud next to the +first, and spins the cameras of both views in opposite directions. The panel's timings stay +those of the first view, which now shares the GPU and the event loop with the second. + `npm start` runs `app.py` with the `python3` of the active environment, or `$PYTHON`. `--target-fps` (default 60) caps the update rate. `--textures` (default 2) is how many shared textures VTK renders into in turn, also switchable at runtime in the panel: with 1, each update waits for Electron to show the previous frame, with 2 VTK renders the next -frame meanwhile. +frame meanwhile. `--radii X Y Z` (default 2 1 0.5) is how far the ellipsoidal cloud spreads +along each axis. The frames are computed ahead each time the point count changes, `--frames` of them (default 8), and played back and forth: an update only swaps in the next frame's arrays, diff --git a/examples/07_electron_point_cloud/app.py b/examples/07_electron_point_cloud/app.py index 222557d..eae3085 100644 --- a/examples/07_electron_point_cloud/app.py +++ b/examples/07_electron_point_cloud/app.py @@ -9,6 +9,9 @@ numpy -> vtk arrays -> render (upload to the GPU + draw) -> shared texture -> Electron composites the frame -> "frame_done" back to Python +--second-view adds a second shared-texture view of the same cloud next to the first, and +spins the camera of both views, in opposite directions. + Started by the Electron app in ./electron, see the README. app_native.py renders the same scene in a desktop window, as a baseline. """ @@ -48,6 +51,8 @@ UI_REFRESH_S = 0.25 BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod +RADII = (2.0, 1.0, 0.5) # standard deviation of the cloud along x, y and z +SPIN_DEG_PER_S = 30.0 # camera spin with --second-view STAGES = [ ("generate", "app work: swap in the next precomputed frame, or compute it"), @@ -91,10 +96,12 @@ class PointCloud: points the polydata at the next frame's arrays: the app work drops to ~0 while every frame still uploads new data. They play back and forth so the loop has no jump. With ``n_frames`` = 0 each update computes a new frame, in place. + The points spread out along x, y and z by ``radii``: an ellipsoidal cloud. """ - def __init__(self, n_points, n_frames=8, seed=0): + def __init__(self, n_points, n_frames=8, seed=0, radii=RADII): self.rng = np.random.default_rng(seed) + self.radii = np.asarray(radii, dtype=np.float32) self.polydata = vtkPolyData() self.n_points = 0 self.n_frames = n_frames @@ -111,6 +118,7 @@ def resize(self, n_points): positions[n_old:] = self.rng.standard_normal( (n_points - n_old, 3), dtype=np.float32 ) + positions[n_old:] *= self.radii colors = np.empty((n_points, 3), dtype=np.uint8) self._noise = np.empty_like(positions) self.n_points = n_points @@ -152,9 +160,10 @@ def animate(self): self._show(self.frames[i if i < k else period - i]) def _walk(self, positions, colors, step=0.02, pull=0.995): - # Random walk with a weak pull toward the origin so the cloud stays put. + # Random walk with a weak pull toward the origin so the cloud stays put. The steps + # scale with the radii too, so the walk keeps the cloud's shape. self.rng.standard_normal(out=self._noise, dtype=np.float32) - self._noise *= step + self._noise *= step * self.radii positions *= pull positions += self._noise colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) @@ -211,6 +220,29 @@ def create_vtk_pipeline(polydata): return render_window +class CameraSpinner: + """Turns the camera of a render window around its view up at a steady rate. + + The angle follows the elapsed time, so views rendering at different rates spin alike. + """ + + def __init__(self, render_window, deg_per_s): + renderer = render_window.GetRenderers().GetFirstRenderer() + self.camera = renderer.GetActiveCamera() + self.deg_per_s = deg_per_s + self._t_last = None + + def step(self): + t = time.perf_counter() + if self._t_last is not None: + self.camera.Azimuth(self.deg_per_s * (t - self._t_last)) + self._t_last = t + + def pause(self): + # Resume from where it stopped, without catching up on the pause. + self._t_last = None + + # ----------------------------------------------------------------------------- # Frame bookkeeping # ----------------------------------------------------------------------------- @@ -322,12 +354,25 @@ def __init__(self, server=None): default=2, help="Shared textures VTK renders into in turn, also changeable in the panel", ) + cli.add_argument( + "--radii", + type=float, + nargs=3, + default=RADII, + metavar=("X", "Y", "Z"), + help="Spread of the ellipsoidal cloud along x, y and z", + ) + cli.add_argument( + "--second-view", + action="store_true", + help="Add a second view of the cloud and spin both cameras", + ) args, _ = cli.parse_known_args() self.step_size = args.step self.target_fps = args.target_fps self.texture_count = args.textures - self.cloud = PointCloud(args.points, args.frames) + self.cloud = PointCloud(args.points, args.frames, radii=args.radii) self.pending_points = None self.running = True self.frame = None @@ -341,6 +386,17 @@ def __init__(self, server=None): self.render_window.AddObserver("StartEvent", self._on_render_start) self.render_window.AddObserver("EndEvent", self._on_render_end) + # The second view shares the cloud: the first view's frames move its points. + self.second_render_window = None + self.spinners = [] + if args.second_view: + self.second_render_window = create_vtk_pipeline(self.cloud.polydata) + self.spinners = [ + CameraSpinner(self.render_window, SPIN_DEG_PER_S), + CameraSpinner(self.second_render_window, -SPIN_DEG_PER_S), + ] + self.second_view_handler = None + self._build_ui() self._new_level() self.server.controller.on_server_ready.add(self._on_server_ready) @@ -369,6 +425,9 @@ def before_render(self): self._new_level() self.pending_points = None + if self.spinners: + self.spinners[0].step() + frame = Frame(self.cloud.n_points) with self._timer_generate: self.cloud.animate() @@ -376,6 +435,13 @@ def before_render(self): self.frame = frame self.view_handler.update() + def before_render_second(self): + # The second view only spins: it picks up the cloud's latest points on its own. + if not self.running: + return + self.spinners[1].step() + self.second_view_handler.update() + def _on_render_start(self, *_): self._render_start = time.perf_counter() @@ -408,6 +474,8 @@ def on_textures_change(self, textures, **_): # The scheduler rebuilds its ring of textures before the next frame. self.texture_count = textures self.view_handler.texture_count = textures + if self.second_view_handler: + self.second_view_handler.texture_count = textures self._new_level() def toggle_running(self): @@ -415,6 +483,11 @@ def toggle_running(self): self.state.running = self.running if self.running: self.view_handler.update() + if self.second_view_handler: + self.second_view_handler.update() + else: + for spinner in self.spinners: + spinner.pause() def _new_level(self): if self.levels and self.levels[-1].count: @@ -531,18 +604,33 @@ def _build_ui(self): self.ui.root.style = "height:100vh;position:relative;" client.Style(CSS) - with html.Div(style="position:absolute;inset:0;"): - view = rca.RemoteControlledArea( - display="shared-texture", - event_throttle_ms=10, - stats="stats = $event", - ) - self.view_handler = view.create_electron_handler( - self.render_window, - target_fps=self.target_fps, - texture_count=self.texture_count, - before_render=self.before_render, - ) + with html.Div(style="position:absolute;inset:0;display:flex;"): + with html.Div(style="flex:1;min-width:0;position:relative;"): + view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + stats="stats = $event", + ) + self.view_handler = view.create_electron_handler( + self.render_window, + target_fps=self.target_fps, + texture_count=self.texture_count, + before_render=self.before_render, + ) + if self.second_render_window: + with html.Div( + style="flex:1;min-width:0;position:relative;border-left:1px solid #333;" + ): + second_view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + ) + self.second_view_handler = second_view.create_electron_handler( + self.second_render_window, + target_fps=self.target_fps, + texture_count=self.texture_count, + before_render=self.before_render_second, + ) with html.Div(classes="bench", style=PANEL_STYLE): with html.Div(style="margin-bottom:0.5rem;"): diff --git a/examples/07_electron_point_cloud/app_native.py b/examples/07_electron_point_cloud/app_native.py index 7e204f2..c8b102b 100644 --- a/examples/07_electron_point_cloud/app_native.py +++ b/examples/07_electron_point_cloud/app_native.py @@ -29,6 +29,7 @@ BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod +RADII = (2.0, 1.0, 0.5) # standard deviation of the cloud along x, y and z class PointCloud: @@ -38,10 +39,12 @@ class PointCloud: points the polydata at the next frame's arrays: the app work drops to ~0 while every frame still uploads new data. They play back and forth so the loop has no jump. With ``n_frames`` = 0 each update computes a new frame, in place. + The points spread out along x, y and z by ``radii``: an ellipsoidal cloud. """ - def __init__(self, n_points, n_frames=8, seed=0): + def __init__(self, n_points, n_frames=8, seed=0, radii=RADII): self.rng = np.random.default_rng(seed) + self.radii = np.asarray(radii, dtype=np.float32) self.polydata = vtkPolyData() self.n_points = 0 self.n_frames = n_frames @@ -58,6 +61,7 @@ def resize(self, n_points): positions[n_old:] = self.rng.standard_normal( (n_points - n_old, 3), dtype=np.float32 ) + positions[n_old:] *= self.radii colors = np.empty((n_points, 3), dtype=np.uint8) self._noise = np.empty_like(positions) self.n_points = n_points @@ -99,9 +103,10 @@ def animate(self): self._show(self.frames[i if i < k else period - i]) def _walk(self, positions, colors, step=0.02, pull=0.995): - # Random walk with a weak pull toward the origin so the cloud stays put. + # Random walk with a weak pull toward the origin so the cloud stays put. The steps + # scale with the radii too, so the walk keeps the cloud's shape. self.rng.standard_normal(out=self._noise, dtype=np.float32) - self._noise *= step + self._noise *= step * self.radii positions *= pull positions += self._noise colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) @@ -202,12 +207,20 @@ def main(): default=8, help="Frames computed ahead on resize, 0 to compute each one live", ) + parser.add_argument( + "--radii", + type=float, + nargs=3, + default=RADII, + metavar=("X", "Y", "Z"), + help="Spread of the ellipsoidal cloud along x, y and z", + ) parser.add_argument( "--no-vsync", action="store_true", help="Don't cap the fps at the display rate" ) args = parser.parse_args() - cloud = PointCloud(args.points, args.frames) + cloud = PointCloud(args.points, args.frames, radii=args.radii) render_window, interactor = create_vtk_pipeline(cloud.polydata) interactor.Initialize() if args.no_vsync: diff --git a/examples/07_electron_point_cloud/electron/main.js b/examples/07_electron_point_cloud/electron/main.js index 4ccb2d8..05ae456 100644 --- a/examples/07_electron_point_cloud/electron/main.js +++ b/examples/07_electron_point_cloud/electron/main.js @@ -18,7 +18,7 @@ app.whenReady().then(() => { launchTrame(win, { command: python, args: [fileURLToPath(new URL('../app.py', import.meta.url)), ...appArgs], - onStats: (s) => console.log(`[renderer] frames=${s.frames} fps=${s.fps} ${s.info}`), + onStats: (s) => console.log(`[renderer] ${s.view} frames=${s.frames} fps=${s.fps} ${s.info}`), onExit: () => app.quit(), }); }); diff --git a/src/trame_rca/electron/comm.py b/src/trame_rca/electron/comm.py index 6c63c64..fe80cdf 100644 --- a/src/trame_rca/electron/comm.py +++ b/src/trame_rca/electron/comm.py @@ -5,16 +5,21 @@ <- means EMP to Python <- {"type":"hello", "protocol": N, "passport": name} at start, the texture passport to use - -> {"type":"server_ready", "protocol": N, "url": url} once trame serves its page - -> {"type":"texture", "slot": i, "slots": k, ...} at start, after every resize and + -> {"type":"server_ready", "protocol": N, "url": url} once, when trame serves its page + -> {"type":"texture", "view": v, "slot": i, "slots": k, ...} + at start, after every resize and change of texture count, one per slot - -> {"type":"frame", "seq": n, "slot": i, "timestamp_us": t} + -> {"type":"frame", "view": v, "seq": n, "slot": i, "timestamp_us": t} after every render into slot i (finished on the GPU) - <- {"type":"frame_done", "seq": n} when Electron released the texture - <- {"type":"frame_done", "seq": n, "dropped": true} when the frame didn't reach the ERP + <- {"type":"frame_done", "view": v, "seq": n} when Electron released the texture + <- {"type":"frame_done", "view": v, "seq": n, "dropped": true} + when the frame didn't reach the ERP -The k textures form a ring: several frames can be in flight, one per slot. Slots past k are gone. +Every shared-texture view of the server talks over the same connection: `view` is the name of +its RemoteControlledArea, and `seq` and `slot` count per view. +The k textures of a view form a ring: several frames can be in flight, one per slot. Slots past +k are gone. Everything else (size, input, app state) goes from the renderer process to trame over its websocket. @@ -23,7 +28,11 @@ import asyncio import json +import logging import socket +from typing import Callable + +logger = logging.getLogger(__name__) PROTOCOL_VERSION = 1 @@ -46,6 +55,8 @@ class LineConnection: def __init__(self, reader, writer): self.reader = reader self.writer = writer + # Several views send on the same connection. + self._send_lock = asyncio.Lock() @classmethod async def open(cls, port, host="127.0.0.1"): @@ -57,8 +68,9 @@ async def open(cls, port, host="127.0.0.1"): return cls(reader, writer) async def send(self, msg): - self.writer.write((json.dumps(msg) + "\n").encode()) - await self.writer.drain() + async with self._send_lock: + self.writer.write((json.dumps(msg) + "\n").encode()) + await self.writer.drain() async def recv(self): """Return the next message. Raises EOFError on close.""" @@ -69,3 +81,115 @@ async def recv(self): def close(self): self.writer.close() + + +class EmpClient: + """The connection to the EMP, shared by all the shared-texture views of the server. + + The EMP takes a single connection per window, so views register here by name and get + the frame_done messages of their own frames. + """ + + # The open clients, by EMP port. + _clients: dict[int, "EmpClient"] = {} + + @classmethod + def for_port(cls, port: int) -> "EmpClient": + """The client connected (or connecting) to the EMP on ``port``.""" + client = cls._clients.get(port) + if client is None: + client = cls._clients[port] = cls(port) + return client + + def __init__(self, port: int): + self.port = port + self._conn: LineConnection | None = None + self._connect_task: asyncio.Task | None = None + self._read_task: asyncio.Task | None = None + # view name: (on_frame_done(msg), on_disconnect()) + self._views: dict[str, tuple[Callable[[dict], None], Callable[[], None]]] = {} + self._server_ready_sent = False + self._closed = False + + def add_view( + self, + view: str, + on_frame_done: Callable[[dict], None], + on_disconnect: Callable[[], None], + ): + if view in self._views: + raise ValueError(f"a shared-texture view named {view!r} already exists") + self._views[view] = (on_frame_done, on_disconnect) + + def remove_view(self, view: str): + """Forget ``view``. The connection closes with the last view.""" + self._views.pop(view, None) + if not self._views: + self.close() + + async def connect(self) -> str: + """Connect to the EMP once, and return the texture passport name of its hello.""" + if self._connect_task is None: + self._connect_task = asyncio.ensure_future(self._connect()) + # shield: one view giving up must not cancel the connection of the others. + return await asyncio.shield(self._connect_task) + + async def _connect(self): + try: + self._conn = await LineConnection.open(self.port) + logger.info("connected to the Electron main process on port %d", self.port) + try: + msg = await self._conn.recv() + except EOFError: + raise ConnectionError("disconnected before the hello message") from None + if msg.get("type") != "hello": + raise ConnectionError(f"expected a hello message, got {msg!r}") + check_protocol(msg) + except Exception: + self.close() + raise + self._read_task = asyncio.ensure_future(self._read()) + return msg["passport"] + + async def server_ready(self, url: str | None): + """Tell the EMP which page to load. Only the first call sends it.""" + if self._server_ready_sent: + return + self._server_ready_sent = True + await self.send( + {"type": "server_ready", "protocol": PROTOCOL_VERSION, "url": url} + ) + + async def send(self, msg): + assert self._conn is not None, "Connection to EMP is uninitialized!" + await self._conn.send(msg) + + async def _read(self): + assert self._conn is not None, "Connection to EMP is uninitialized!" + try: + while True: + msg = await self._conn.recv() + if msg["type"] == "frame_done": + callbacks = self._views.get(msg.get("view")) + if callbacks: + callbacks[0](msg) + except (EOFError, ConnectionError): + logger.info("the Electron main process disconnected") + self.close() + + def close(self): + """Close the connection and tell every view it is gone.""" + if self._closed: + return + self._closed = True + if EmpClient._clients.get(self.port) is self: + del EmpClient._clients[self.port] + current = asyncio.current_task() + for task in (self._read_task, self._connect_task): + if task and task is not current: + task.cancel() + if self._conn: + self._conn.close() + views, self._views = self._views, {} + for _, on_disconnect in views.values(): + on_disconnect() diff --git a/src/trame_rca/schedulers/shared_texture_scheduler.py b/src/trame_rca/schedulers/shared_texture_scheduler.py index 73db6bb..c6b6d7a 100644 --- a/src/trame_rca/schedulers/shared_texture_scheduler.py +++ b/src/trame_rca/schedulers/shared_texture_scheduler.py @@ -4,7 +4,8 @@ process (ERP), which does not have access to the OS API needed to import a texture. Instead the scheduler tells the EMP about the shared textures and each finished frame over -trame_rca.electron.comm, and the EMP hands them to the renderer process. Size and input still +trame_rca.electron.comm, and the EMP hands them to the renderer process. All the schedulers of +a server share one connection to the EMP, and tag their messages with their view's name. Size and input still come from the RemoteControlledArea through the RcaViewAdapter. The textures form a ring: VTK renders the next frame into the next texture while Electron @@ -15,8 +16,6 @@ """ from __future__ import annotations -from asyncio.futures import Future - import asyncio import logging import time @@ -28,11 +27,7 @@ from trame.app import asynchronous from trame_common.utils import profiler -from trame_rca.electron.comm import ( - PROTOCOL_VERSION, - LineConnection, - check_protocol, -) +from trame_rca.electron.comm import EmpClient from trame_rca.electron.passports import texture_passport_class, TexturePassport from trame_rca.rca import ( RemoteControlledAreaProtocol, @@ -69,7 +64,9 @@ def __init__( on_disconnect: Callable[[], None] | None = None, before_render: Callable[[], None] | None = None, texture_count: int = 2, + view: str = "default", ): + """``view`` is the name of the RemoteControlledArea that shows the frames.""" self._timer_render = profiler.Timer("rca.shared_texture.render") self._timer_flush = profiler.Timer("rca.shared_texture.flush") self._rca = window_wrapper(window) @@ -77,13 +74,14 @@ def __init__( "SharedTexture scheduler only works with VTK render windows!" ) self._emp_port = emp_port + self._view = view self._target_fps = target_fps self._on_disconnect = on_disconnect # Called before every frame, e.g. to animate the scene. It may schedule_render() again. self._before_render = before_render self._push_callback = None - self._conn = None + self._client: EmpClient | None = None self._texture_count = _check_texture_count(texture_count) # The ring of shared textures self._passports = [] @@ -92,7 +90,6 @@ def __init__( self._next_slot = 0 # `seq`:`frame_done` future map valid until Electron fulfils with the "type":"frame_done" message. self._frames_in_flight = {} - self._passport_name_future = None self._seq = 0 # Renders asked for before start() wait in here. self._request_event = asyncio.Event() @@ -121,6 +118,10 @@ def on_resize(width, height): def rca(self): return self._rca + @property + def view(self): + return self._view + @property def target_fps(self): return self._target_fps @@ -155,8 +156,11 @@ def schedule_render(self): def start(self, url: str | None = None, **_): """Connect to the EMP and start rendering. Call once trame's event loop runs. - ``url`` is the page the EMP should load: the one trame serves. + ``url`` is the page the EMP should load: the one trame serves. Only the first + scheduler to connect tells the EMP about it. """ + self._client = EmpClient.for_port(self._emp_port) + self._client.add_view(self._view, self._on_frame_done, self._on_client_closed) self._start_task(self._run(url)) def _start_task(self, coro): @@ -169,9 +173,10 @@ async def close(self): return self._is_closing = True for task in self._tasks: - task.cancel() - if self._conn: - self._conn.close() + if task is not asyncio.current_task(): + task.cancel() + if self._client: + self._client.remove_view(self._view) for passport in self._passports: passport.release() self._passports = [] @@ -180,18 +185,13 @@ async def _run(self, url): assert isinstance(self._rca, VtkRemoteControlledArea), ( "SharedTexture scheduler only works with VTK render windows!" ) - self._passport_name_future = asyncio.get_running_loop().create_future() - self._conn = await LineConnection.open(self._emp_port) - logger.info("connected to the Electron main process on port %d", self._emp_port) - self._start_task(self._read()) + assert isinstance(self._client, EmpClient), "start() did not run!" try: - passport_name = await self._passport_name_future + passport_name = await self._client.connect() except Exception: await self._disconnect() raise - await self._conn.send( - {"type": "server_ready", "protocol": PROTOCOL_VERSION, "url": url} - ) + await self._client.server_ready(url) next_frame = 0 period = 1.0 / self._target_fps @@ -230,9 +230,10 @@ async def _run(self, url): frame_done = asyncio.get_running_loop().create_future() self._slot_frames[slot] = frame_done self._frames_in_flight[seq] = frame_done - await self._conn.send( + await self._client.send( { "type": "frame", + "view": self._view, "seq": seq, "slot": slot, "timestamp_us": int(time.time() * 1e6), @@ -262,9 +263,7 @@ async def _update_textures(self, passport_name): assert isinstance(self._rca, VtkRemoteControlledArea), ( "SharedTexture scheduler only works with VTK render windows!" ) - assert isinstance(self._conn, LineConnection), ( - "Connection to EMP is uninitialized!" - ) + assert isinstance(self._client, EmpClient), "start() did not run!" width, height = self._rca.render_window.GetSize() count = self._texture_count if len(self._passports) == count and all( @@ -289,43 +288,33 @@ async def _update_textures(self, passport_name): self._next_slot = 0 for slot, passport in enumerate(self._passports): assert isinstance(passport, TexturePassport) - await self._conn.send({**passport.to_json(), "slot": slot, "slots": count}) + await self._client.send( + { + **passport.to_json(), + "view": self._view, + "slot": slot, + "slots": count, + } + ) - async def _read(self): - assert isinstance(self._conn, LineConnection), ( - "Connection to EMP is uninitialized!" - ) - assert isinstance(self._passport_name_future, Future), ( - "Passport name future was not created!" - ) - try: - while True: - msg = await self._conn.recv() - if msg["type"] == "hello": - try: - check_protocol(msg) - except Exception as e: - self._passport_name_future.set_exception(e) - return - self._passport_name_future.set_result(msg["passport"]) - elif msg["type"] == "frame_done": - frame_done = self._frames_in_flight.pop(msg["seq"], None) - if frame_done and not frame_done.done(): - frame_done.set_result(None) - if msg.get("dropped"): - # The page never processed it: e.g. it was still loading or busy.. - logger.info("frame %d dropped, rendering again", msg["seq"]) - self.schedule_render() - except (EOFError, ConnectionError): - logger.info("the Electron main process disconnected") - if not self._passport_name_future.done(): - self._passport_name_future.set_exception( - ConnectionError("disconnected before the hello message") - ) - return - await self._disconnect() + def _on_frame_done(self, msg): + frame_done = self._frames_in_flight.pop(msg["seq"], None) + if frame_done and not frame_done.done(): + frame_done.set_result(None) + if msg.get("dropped"): + # The page never processed it: e.g. it was still loading or busy.. + logger.info( + "view %s: frame %d dropped, rendering again", self._view, msg["seq"] + ) + self.schedule_render() + + def _on_client_closed(self): + if not self._is_closing: + self._start_task(self._disconnect()) async def _disconnect(self): + if self._is_closing: + return await self.close() if self._on_disconnect: self._on_disconnect() diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index a2c08f0..cbe08b6 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -208,6 +208,8 @@ def create_electron_handler( defaults to stopping the server. ``texture_count`` is how many shared textures VTK renders into in turn, so it can render the next frame while Electron shows the previous ones. The returned handler's ``texture_count`` changes it at runtime. + + Each shared-texture view of a page needs its own ``name``. """ from trame.app import asynchronous @@ -233,6 +235,7 @@ def on_disconnect(): on_disconnect=on_disconnect, before_render=before_render, texture_count=texture_count, + view=self.name, ) def start(**_): @@ -364,13 +367,16 @@ def __init__(self, **kwargs): class SharedTextureDisplayArea(HtmlElement): - """The canvas trame-rca-electron draws the shared texture into, once a frame.""" + """The canvas trame-rca-electron draws the shared texture of view ``name`` into, once a frame.""" def __init__(self, **kwargs): super().__init__( "shared-texture-display-area", **kwargs, ) + self._attr_names += [ + "name", + ] self._event_names += [ "stats", ] diff --git a/tests/test_rca_electron.py b/tests/test_rca_electron.py index 3e99b15..7357943 100644 --- a/tests/test_rca_electron.py +++ b/tests/test_rca_electron.py @@ -109,12 +109,14 @@ async def test_scheduler_shares_frames(a_render_window, fake_passport): "type": "texture", "width": 300, "height": 300, + "view": "default", "slot": slot, "slots": 2, } frame = await emp.recv() assert frame["type"] == "frame" and frame["seq"] == 0 and frame["slot"] == 0 - await emp.send({"type": "frame_done", "seq": 0}) + assert frame["view"] == "default" + await emp.send({"type": "frame_done", "view": "default", "seq": 0}) # A resize makes new textures and releases the old ones. scheduler.rca.process_resize_event(200, 100) @@ -168,7 +170,9 @@ async def test_scheduler_renders_dropped_frames_again(a_render_window, fake_pass assert (await emp.recv())["seq"] == 0 # e.g. sendSharedTexture timed out while the page was loading. - await emp.send({"type": "frame_done", "seq": 0, "dropped": True}) + await emp.send( + {"type": "frame_done", "view": "default", "seq": 0, "dropped": True} + ) frame = await emp.recv() assert frame["type"] == "frame" and frame["seq"] == 1 finally: @@ -198,11 +202,11 @@ async def test_scheduler_texture_ring(a_render_window, fake_passport): scheduler.schedule_render() with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(emp.reader.readline(), 0.2) - await emp.send({"type": "frame_done", "seq": 0}) + await emp.send({"type": "frame_done", "view": "default", "seq": 0}) frame = await emp.recv() assert (frame["seq"], frame["slot"]) == (3, 0) for seq in range(1, 4): - await emp.send({"type": "frame_done", "seq": seq}) + await emp.send({"type": "frame_done", "view": "default", "seq": seq}) # Changing the count at runtime rebuilds the ring before the next frame. scheduler.texture_count = 1 @@ -210,6 +214,7 @@ async def test_scheduler_texture_ring(a_render_window, fake_passport): "type": "texture", "width": 300, "height": 300, + "view": "default", "slot": 0, "slots": 1, } @@ -222,3 +227,96 @@ async def test_scheduler_texture_ring(a_render_window, fake_passport): finally: await scheduler.close() emp.close() + + +@pytest.fixture +def two_render_windows(): + from vtkmodules.vtkRenderingCore import ( + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, + ) + + windows = [] + for _ in range(2): + window = vtkRenderWindow() + window.AddRenderer(vtkRenderer()) + window.ShowWindowOff() + vtkRenderWindowInteractor().SetRenderWindow(window) + windows.append(window) + yield windows + for window in windows: + window.Finalize() + + +@pytest.mark.asyncio +async def test_schedulers_share_the_connection(two_render_windows, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + connections = [] + on_connect = emp._on_connect + + async def count_connections(reader, writer): + connections.append(writer) + if len(connections) == 1: + await on_connect(reader, writer) + + emp._on_connect = count_connections + port = await emp.start() + disconnected = {"a": asyncio.Event(), "b": asyncio.Event()} + schedulers = { + view: RcaSharedTextureRenderScheduler( + window, port, view=view, on_disconnect=disconnected[view].set + ) + for view, window in zip(("a", "b"), two_render_windows) + } + try: + for scheduler in schedulers.values(): + scheduler.start(url="http://127.0.0.1:1234/") + # The page is loaded once, for both views. + assert (await emp.recv())["type"] == "server_ready" + + schedulers["a"].rca.process_resize_event(300, 300) + schedulers["b"].rca.process_resize_event(200, 100) + textures = {} + frames = {} + while len(frames) < 2: + msg = await emp.recv() + assert msg["type"] in ("texture", "frame") + if msg["type"] == "texture": + textures.setdefault(msg["view"], []).append(msg) + else: + frames[msg["view"]] = msg + assert len(connections) == 1 + assert [t["width"] for t in textures["a"]] == [300, 300] + assert [t["width"] for t in textures["b"]] == [200, 200] + # seq counts per view. + assert frames["a"]["seq"] == frames["b"]["seq"] == 0 + + # A frame_done only reaches its own view. + await emp.send({"type": "frame_done", "view": "b", "seq": 0, "dropped": True}) + frame = await emp.recv() + assert (frame["view"], frame["seq"]) == ("b", 1) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + + # Two views can't share a name. + with pytest.raises(ValueError): + RcaSharedTextureRenderScheduler( + two_render_windows[0], port, view="a" + ).start() + + # Closing one view keeps the connection of the other. + await schedulers["a"].close() + schedulers["b"].schedule_render() + frame = await emp.recv() + assert frame["view"] == "b" + assert not disconnected["b"].is_set() + + # Electron going away disconnects the remaining views. + emp.close() + await asyncio.wait_for(disconnected["b"].wait(), TIMEOUT_S) + assert not disconnected["a"].is_set() + finally: + for scheduler in schedulers.values(): + await scheduler.close() + emp.close() diff --git a/vue-components/src/components/DisplayArea.js b/vue-components/src/components/DisplayArea.js index 7e0d0ee..5692d86 100644 --- a/vue-components/src/components/DisplayArea.js +++ b/vue-components/src/components/DisplayArea.js @@ -40,7 +40,7 @@ export default { - + `, }; diff --git a/vue-components/src/components/SharedTextureDisplayArea.js b/vue-components/src/components/SharedTextureDisplayArea.js index fe1aa82..799d502 100644 --- a/vue-components/src/components/SharedTextureDisplayArea.js +++ b/vue-components/src/components/SharedTextureDisplayArea.js @@ -1,23 +1,40 @@ // Shows the frames the trame-rca-electron preload draws from the texture VTK shares with -// Electron. Only works in a page loaded by trame-rca-electron: the preload defines the bridge. +// Electron, for the view `name`. Only works in a page loaded by trame-rca-electron: the preload +// defines the bridge. export default { + props: { + name: { + type: String, + default: 'default', + }, + }, emits: ['stats'], methods: { + attach() { + if (!window.trameRcaSharedTexture) { + console.warn( + 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' + ); + return; + } + this.attachedName = this.name; + // { fps, frames, info } once a second. + window.trameRcaSharedTexture.attach(this.name, this.$el, (stats) => + this.$emit('stats', stats) + ); + }, cleanup() { - window.trameRcaSharedTexture?.detach(this.$el); + window.trameRcaSharedTexture?.detach(this.attachedName, this.$el); + }, + }, + watch: { + name() { + this.cleanup(); + this.attach(); }, }, mounted() { - if (!window.trameRcaSharedTexture) { - console.warn( - 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' - ); - return; - } - // { fps, frames, info } once a second. - window.trameRcaSharedTexture.attach(this.$el, (stats) => - this.$emit('stats', stats) - ); + this.attach(); }, // support both vue2 and vue3 unmount callbacks beforeDestroy() { From 0d4a1abb77381b060c9bcf5543a5c9fb4486be47 Mon Sep 17 00:00:00 2001 From: Sebastien Jourdain Date: Mon, 2 Nov 2026 18:58:02 -0700 Subject: [PATCH 13/13] fix: cleanup vue component --- .../components/SharedTextureDisplayArea.js | 65 ++++++++++--------- 1 file changed, 33 insertions(+), 32 deletions(-) diff --git a/vue-components/src/components/SharedTextureDisplayArea.js b/vue-components/src/components/SharedTextureDisplayArea.js index 799d502..bb91d78 100644 --- a/vue-components/src/components/SharedTextureDisplayArea.js +++ b/vue-components/src/components/SharedTextureDisplayArea.js @@ -1,3 +1,5 @@ +const { ref, onBeforeUnmount, watchEffect } = window.Vue; + // Shows the frames the trame-rca-electron preload draws from the texture VTK shares with // Electron, for the view `name`. Only works in a page loaded by trame-rca-electron: the preload // defines the bridge. @@ -9,41 +11,40 @@ export default { }, }, emits: ['stats'], - methods: { - attach() { - if (!window.trameRcaSharedTexture) { - console.warn( - 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' - ); - return; + setup(props, { emit }) { + const el = ref(null); + const attached = []; + + function detachAll() { + while (attached.length) { + const [n, c] = attached.pop(); + window.trameRcaSharedTexture.detach(n, c); } - this.attachedName = this.name; - // { fps, frames, info } once a second. - window.trameRcaSharedTexture.attach(this.name, this.$el, (stats) => - this.$emit('stats', stats) + } + onBeforeUnmount(detachAll); + + if (!window.trameRcaSharedTexture) { + console.warn( + 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' ); - }, - cleanup() { - window.trameRcaSharedTexture?.detach(this.attachedName, this.$el); - }, - }, - watch: { - name() { - this.cleanup(); - this.attach(); - }, - }, - mounted() { - this.attach(); - }, - // support both vue2 and vue3 unmount callbacks - beforeDestroy() { - this.cleanup(); - }, - beforeUnmount() { - this.cleanup(); + } + + function emitStats(stats) { + emit('stats', stats); + } + + watchEffect(() => { + const name = props.name; + const container = el.value; + detachAll(); + if (!container) return; + attached.push([name, container]); + window.trameRcaSharedTexture.attach(name, container, emitStats); + }); + + return { el }; }, template: ` - + `, };