diff --git a/.codespellrc b/.codespellrc index 189f46d..dcb49b3 100644 --- a/.codespellrc +++ b/.codespellrc @@ -1,2 +1,3 @@ [codespell] skip = **/package-lock.json,CHANGELOG.md +ignore-words-list = renderd diff --git a/.github/workflows/test_and_release.yml b/.github/workflows/test_and_release.yml index f315d9f..43e4b90 100644 --- a/.github/workflows/test_and_release.yml +++ b/.github/workflows/test_and_release.yml @@ -138,6 +138,15 @@ jobs: cd js-components npm run lint + - name: Test trame-rca-electron + run: | + cd electron + npm ci + npm test + npm run lint + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: 1 + test-npm-build: runs-on: ubuntu-latest diff --git a/README.rst b/README.rst index 76132bd..d9496ae 100644 --- a/README.rst +++ b/README.rst @@ -128,3 +128,52 @@ up outside the trame-rca directory, checkout the `wheel` branch, and install it cd ../trame-rca python ./examples/05_video/vtk_cone_simple_video.py + +Shared textures with Electron +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In an Electron app, VTK can render into a GPU texture that Electron shows as is: without encoding +and without copying to the CPU. The Electron side is the ``@kitware/trame-rca-electron`` npm package in +``./electron``: it starts the trame server and loads its page. Supported on macOS (IOSurface), +Windows (D3D11 NT handles) and Linux under Wayland (dma-buf). + +- macOS: works out of the box (IOSurface). +- Windows: the GL driver needs ``WGL_NV_DX_interop2``, and VTK must render on the GPU + Chromium uses. Set ``D3D_ADAPTER=`` if that isn't DXGI adapter 0. +- Linux: Wayland only for now (dma-buf). Electron duplicates the server's file descriptors + with ``pidfd_getfd``, which needs ``kernel.yama.ptrace_scope`` at 1 or less: + ``sudo sysctl -w kernel.yama.ptrace_scope=1``. Set ``RENDER_NODE=/dev/dri/renderD129`` + if VTK picks another GPU than the compositor. + +.. code-block:: python + + from trame_rca.widgets import init_vtk_for_shared_textures + from trame_rca.widgets import rca + + init_vtk_for_shared_textures() # before creating any VTK render window + + # ... create render_window, then in the UI: + view = rca.RemoteControlledArea(display="shared-texture") + view_handler = view.create_electron_handler(render_window) + +VTK renders into a ring of shared textures, by default this is 2. This allows VTK +to draw the next frame, while Electron still shows the previous one. If you're on a +mid-tier to high-end GPU, you normally do not need to use more than 1 texture because +VTK can render faster. However, on low-end hardware, it's more efficient to render the +N+k frames while electron is presenting the N'th frame. This hides the cost of each render. +The ``create_electron_handler(..., texture_count=n)`` sets the size of this ring texture container, +and you can change it at runtime with ``view_handler.texture_count = n``. A value of 1 +waits for Electron after every frame. + +You can try our code example: + +.. code-block:: console + + pip install trame trame-vuetify + pip install "trame-rca[electron]" + + cd examples/06_electron/electron + npm i + npm start + +See ``electron/README.md`` for the Electron API and what packaging an app needs. diff --git a/electron/.eslintrc.cjs b/electron/.eslintrc.cjs new file mode 100644 index 0000000..4338b47 --- /dev/null +++ b/electron/.eslintrc.cjs @@ -0,0 +1,14 @@ +/* eslint-env node */ +module.exports = { + root: true, + env: { + node: true, + browser: true, + es2022: true, + }, + extends: ['eslint:recommended'], + parserOptions: { + ecmaVersion: 'latest', + sourceType: 'module', + }, +}; diff --git a/electron/.gitignore b/electron/.gitignore new file mode 100644 index 0000000..c2658d7 --- /dev/null +++ b/electron/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/electron/README.md b/electron/README.md new file mode 100644 index 0000000..8d406d1 --- /dev/null +++ b/electron/README.md @@ -0,0 +1,53 @@ +# @kitware/trame-rca-electron + +Shows trame-rca `display="shared-texture"` views in Electron. The trame server renders VTK +into a GPU texture, and this package hands it to the page without encoding, or copying to the CPU. + +This package and the `trame-rca` python package speak a versioned protocol (see +`src/main/protocol.js` and `trame_rca/electron/comm.py`): use matching releases. + +## Use + +```js +import { app, BrowserWindow } from 'electron/main'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; + +app.whenReady().then(() => { + const win = new BrowserWindow({ webPreferences: webPreferences() }); + launchTrame(win, { command: 'python3', args: ['app.py'], onExit: () => app.quit() }); +}); +``` + +`launchTrame(win, options)` spawns `command` with `args` plus `--server --port 0 --emp-port `, +and loads the page in `win` once trame serves it. Options: + +- `command`, `args`, `cwd`, `env`: the trame server to run. Any executable works: a python + interpreter and a script, a frozen backend, etc. +- `onExit(code, signal)`: the server exited. +- `onStats({ view, fps, frames, info })`: once a second, per shared-texture view that drew frames. +- `log`: defaults to `console`. + +It returns `{ process, close }`. `close()` kills the server; it is called when `win` closes. + +`webPreferences(extra)` sets the preload, and turns off the sandbox: the preload needs +`sharedTexture`, and Electron doesn't run ES module preloads in a sandbox. If the window needs its +own preload, `import '@kitware/trame-rca-electron/preload'` from it: it must then be an ES module +preload too, with the `.mjs` extension. + +On the python side, see `examples/06_electron` in trame-rca. + +## Packaging an app + +Packaging (electron-builder or Forge, a frozen python backend, signing) belongs to each app. +What this package needs from a packaged app: + +- `koffi` is a native module: unpack it from the asar archive (e.g. electron-builder's + `asarUnpack: ["**/node_modules/koffi/**"]`). +- Pass the packaged backend as `command`. It must be spawned by `launchTrame`, as a direct child + of the main process: on Linux, duplicating its dma-buf file descriptors needs ptrace access to + it (`kernel.yama.ptrace_scope` at 1 or less). +- The frozen backend must include `vtkmodules` and the `serve` directories of `trame_rca` and of + every other trame package it uses. +- macOS shares the texture as a global IOSurface, which does not work in an App Sandbox (so not on + the Mac App Store). +- Linux: X11 is not supported yet, Electron must run under Wayland. diff --git a/electron/package-lock.json b/electron/package-lock.json new file mode 100644 index 0000000..f7bff89 --- /dev/null +++ b/electron/package-lock.json @@ -0,0 +1,1743 @@ +{ + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "koffi": "^3.3.3" + }, + "devDependencies": { + "eslint": "^8.57.1" + }, + "peerDependencies": { + "electron": ">=44.7.0" + } + }, + "node_modules/@electron-internal/extract-zip": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@electron-internal/extract-zip/-/extract-zip-1.0.5.tgz", + "integrity": "sha512-+bqFCP98pLI0Tt0XQo1TmlXtwjWchISndDOxCkEcIuUgXWpBnLyRI+2DU+mesvnMMX6L1XDqYNA0lXNDHd/yiA==", + "license": "BSD-2-Clause", + "peer": true, + "engines": { + "node": ">=22.12.0" + } + }, + "node_modules/@electron/get": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/@electron/get/-/get-5.1.0.tgz", + "integrity": "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA==", + "license": "MIT", + "peer": true, + "dependencies": { + "debug": "^4.1.1", + "env-paths": "^3.0.0", + "graceful-fs": "^4.2.11", + "progress": "^2.0.3", + "semver": "^7.6.3", + "sumchecker": "^3.0.1" + }, + "engines": { + "node": ">=22.12.0" + }, + "optionalDependencies": { + "undici": "^7.24.4" + } + }, + "node_modules/@eslint-community/eslint-utils": { + "version": "4.10.1", + "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.10.1.tgz", + "integrity": "sha512-cuadcxVFE8sDK6iWJbs8Sn0av2Nrh2QSGQhVlBW9AaAHqHwjWsZHT8LJ4hFGPh7ASBV2deFdM7H/DPjulmh8rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "eslint-visitor-keys": "^3.4.3" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + }, + "peerDependencies": { + "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" + } + }, + "node_modules/@eslint-community/regexpp": { + "version": "4.12.2", + "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.12.2.tgz", + "integrity": "sha512-EriSTlt5OC9/7SXkRSCAhfSxxoSUgBm33OH+IkwbdpgoqsSsUg7y3uh+IICI/Qg4BBWr3U2i39RpmycbxMq4ew==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.0.0 || ^14.0.0 || >=16.0.0" + } + }, + "node_modules/@eslint/eslintrc": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", + "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ajv": "^6.12.4", + "debug": "^4.3.2", + "espree": "^9.6.0", + "globals": "^13.19.0", + "ignore": "^5.2.0", + "import-fresh": "^3.2.1", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/@eslint/js": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.1.tgz", + "integrity": "sha512-d9zaMRSTIKDLhctzH12MtXvJKSSUhaHcjV+2Z+GK+EEY7XKpP5yR4x+N3TAcHTcu963nIr+TMcCb4DBCYX1z6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + } + }, + "node_modules/@humanwhocodes/config-array": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.13.0.tgz", + "integrity": "sha512-DZLEEqFWQFiyK6h5YIeynKx7JlvCYWL0cImfSRXZ9l4Sg2efkFGTuFf6vzXjK1cq6IYkU+Eg/JizXw+TD2vRNw==", + "deprecated": "Use @eslint/config-array instead", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@humanwhocodes/object-schema": "^2.0.3", + "debug": "^4.3.1", + "minimatch": "^3.0.5" + }, + "engines": { + "node": ">=10.10.0" + } + }, + "node_modules/@humanwhocodes/module-importer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", + "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.22" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/nzakas" + } + }, + "node_modules/@humanwhocodes/object-schema": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", + "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", + "deprecated": "Use @eslint/object-schema instead", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/@koromix/koffi-android-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-arm64/-/koffi-android-arm64-3.3.3.tgz", + "integrity": "sha512-c8iDq/mPXkgQmrmZ6SpqWIVF3ZnJEpjVQswzKzbrCSdr68pmSY/cNj1t4qvv32T8nNjW71/hB+E8Rq1+vVpmQg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-android-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-android-x64/-/koffi-android-x64-3.3.3.tgz", + "integrity": "sha512-yAFu4oX6rpBCqr4HiGo3FpywmRN11LOSde4+GJZvfkFymqV3wZtx6TgVe8rZ4ogJZz9B8o1tDeKsZ6pqeoH40g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-arm64/-/koffi-darwin-arm64-3.3.3.tgz", + "integrity": "sha512-aFYI3YMdQWOAmHlVA+GfBlJZTpvfbUv/NoeBBkqblWN7BxuY02bkWMyoksUEl+mnx7ScoMonnjzXKJkNQXmR3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-darwin-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-darwin-x64/-/koffi-darwin-x64-3.3.3.tgz", + "integrity": "sha512-6Q8/DDhVADPMm3lFnZFJRETJtf485jZKzn1KogXeA9Gfztv/XMFiCOK4aEouFc256h9uWzC69mGimtcVcwK+Ww==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-arm64/-/koffi-freebsd-arm64-3.3.3.tgz", + "integrity": "sha512-IynEDMMU9pp7EVVqQwXVU9x0JH7Opj8KB0wbcKgYNEhRM9eGzJ4ai7iwG8y8tdD/Y8niSVrVTj5WLNCCvUWXIw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-ia32/-/koffi-freebsd-ia32-3.3.3.tgz", + "integrity": "sha512-5b4W9jX+wmfWn9vG7DvKIrNhhRpPDAx7YQV5i9epUR0Ogxz4s4a3APz/beLCvgHt4WhaSRrazxGYQz1oFOlfow==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-freebsd-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-freebsd-x64/-/koffi-freebsd-x64-3.3.3.tgz", + "integrity": "sha512-+NmMrp3ji1NN4PTaLaXDCKMaJ5Fl+0+nrClmtsu/7wi+fjCrCSGMrJZUmTIJPm1nd8Z5fMRjWesiVvQiy/ETDg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm/-/koffi-linux-arm-3.3.3.tgz", + "integrity": "sha512-K+NFO4/SiF4ANCS1ykem36wbEjfBJKsaakLKZ+6LbaLqyLjHHGky4fEIgipac/+XJtD4ZXjhexPIVjwzkO+rMQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-arm64/-/koffi-linux-arm64-3.3.3.tgz", + "integrity": "sha512-y7UHjwOKbj2IjIpWtHmI8LEKXchidt47k605DCv5+YQMIDTbzhz/i3zTI1kRp6EYBLTBwRkR1X1PGt3uUi+rKQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ia32/-/koffi-linux-ia32-3.3.3.tgz", + "integrity": "sha512-dbcpe7IrWr18AAbIKa/RvJ8pM1lEpSELWYW6wsEBIYw/RTLfXqWJ6d9Ya0NWJgHTW+N7783KVQaV5c1Sr3ONDQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-loong64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-loong64/-/koffi-linux-loong64-3.3.3.tgz", + "integrity": "sha512-kbJFmWSoDHGZgxgDhInuvDi8MzrHafBfQ7iWN5P+nHv297beQXrHecFfCNBApnG9d2wrhEoY+EH/TdITuvaSwQ==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-ppc64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-ppc64/-/koffi-linux-ppc64-3.3.3.tgz", + "integrity": "sha512-YwGxjudYHsiDZ6qH4WmEawHXmTaZzjiZ4xdjcbzsVHqNvgZ3Nz7WG03PybecJ+BVH2vqBOntGZ2VsG/sL0WY1g==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-riscv64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-riscv64/-/koffi-linux-riscv64-3.3.3.tgz", + "integrity": "sha512-k/txpUsOO/L4TexahQHxqdFRmBEGE+g++np9M5eNcsHWNXmazeWBrrfB9sjl6cZAP69KZFO0toP+ZMfDUDHmnQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-linux-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-linux-x64/-/koffi-linux-x64-3.3.3.tgz", + "integrity": "sha512-AI4kcNNWeGmzQzsNb0S+ZL/ht4HMtDcQl5nYFkkuVUO2g1/2KdvNekkwNWhpp805mDbLSDoRZtlioHyH/v8poQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-arm64/-/koffi-openbsd-arm64-3.3.3.tgz", + "integrity": "sha512-NEbQMrixbAWSmWCh4XyHWuhgEqgO4tegWyHoJej7Q3PmyIlgxKHTcRZsjAJ9C3sfgBDNIqPWTjMy3KO2mqEp1g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-ia32/-/koffi-openbsd-ia32-3.3.3.tgz", + "integrity": "sha512-zGFX0KwZGmrV7AZY0+oozBpMuQZT7yQEY3NCDSb0rEqKyOrYLBNiXiXA5T8NSY3ipaW0b71L5Cu0KLKabdLY0A==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-openbsd-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-openbsd-x64/-/koffi-openbsd-x64-3.3.3.tgz", + "integrity": "sha512-c1HMWkZPAZKQesKj2HqA6Zk1YEcv7Doc6JU9YJepXG8KpK4XZmI0L5vESSfySmidmasm05QDyLAcwDnNSvtfRA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-arm64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-arm64/-/koffi-win32-arm64-3.3.3.tgz", + "integrity": "sha512-eL7gW4D22OBb3fi04s5ElzzNk+gkGEahaTNMB29UDLDeORe+0qIkhccBPovXW3/IRpdt8W7o8X2myUDXlTSYDQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-ia32": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-ia32/-/koffi-win32-ia32-3.3.3.tgz", + "integrity": "sha512-3sw9vvQqro4yKsOWfKebzfTcdPKRwMrodGXWbfwkTsL61EQKdwsMNKA/0dDe0/ubAAZ0yPgstX8/gs94Svzfgw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@koromix/koffi-win32-x64": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/@koromix/koffi-win32-x64/-/koffi-win32-x64-3.3.3.tgz", + "integrity": "sha512-tLTF17wl4CCwOlo1Tt0NbduwUduDvM6c8JSyearcLu6hpNHQo44RF3rKN2i7UkxwCqXyiudZK0lS421CC/TPZA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "funding": { + "url": "https://liberapay.com/Koromix" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/@ungap/structured-clone": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", + "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ajv": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.15.0.tgz", + "integrity": "sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/brace-expansion": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.21.tgz", + "integrity": "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deep-is": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/electron": { + "version": "44.9.0", + "resolved": "https://registry.npmjs.org/electron/-/electron-44.9.0.tgz", + "integrity": "sha512-boP8zA5OIoMGYe3uWP4IqB1VJNoEzdHYgcBaVYnd1DBv5TogwXkbM4yJGSfNmYCzalVQ5J3TAIbaKOY+ILTnyg==", + "license": "MIT", + "peer": true, + "dependencies": { + "@electron-internal/extract-zip": "^1.0.1", + "@electron/get": "^5.0.0", + "@types/node": "^24.9.0" + }, + "bin": { + "electron": "cli.js", + "install-electron": "install.js" + }, + "engines": { + "node": ">= 22.12.0" + } + }, + "node_modules/env-paths": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-3.0.0.tgz", + "integrity": "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==", + "license": "MIT", + "peer": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/eslint": { + "version": "8.57.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.1.tgz", + "integrity": "sha512-ypowyDxpVSYpkXr9WPv2PAZCtNip1Mv5KTW0SCurXv/9iOpcrH9PaqUElksqEB6pChqHGDRCFTyrZlGhnLNGiA==", + "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", + "dev": true, + "license": "MIT", + "dependencies": { + "@eslint-community/eslint-utils": "^4.2.0", + "@eslint-community/regexpp": "^4.6.1", + "@eslint/eslintrc": "^2.1.4", + "@eslint/js": "8.57.1", + "@humanwhocodes/config-array": "^0.13.0", + "@humanwhocodes/module-importer": "^1.0.1", + "@nodelib/fs.walk": "^1.2.8", + "@ungap/structured-clone": "^1.2.0", + "ajv": "^6.12.4", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.3.2", + "doctrine": "^3.0.0", + "escape-string-regexp": "^4.0.0", + "eslint-scope": "^7.2.2", + "eslint-visitor-keys": "^3.4.3", + "espree": "^9.6.1", + "esquery": "^1.4.2", + "esutils": "^2.0.2", + "fast-deep-equal": "^3.1.3", + "file-entry-cache": "^6.0.1", + "find-up": "^5.0.0", + "glob-parent": "^6.0.2", + "globals": "^13.19.0", + "graphemer": "^1.4.0", + "ignore": "^5.2.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "is-path-inside": "^3.0.3", + "js-yaml": "^4.1.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash.merge": "^4.6.2", + "minimatch": "^3.1.2", + "natural-compare": "^1.4.0", + "optionator": "^0.9.3", + "strip-ansi": "^6.0.1", + "text-table": "^0.2.0" + }, + "bin": { + "eslint": "bin/eslint.js" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-scope": { + "version": "7.2.2", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", + "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "esrecurse": "^4.3.0", + "estraverse": "^5.2.0" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/espree": { + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "acorn": "^8.9.0", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.4.1" + }, + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, + "node_modules/esquery": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.7.0.tgz", + "integrity": "sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "estraverse": "^5.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "estraverse": "^5.2.0" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "dev": true, + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/file-entry-cache": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", + "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "flat-cache": "^3.0.4" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/flat-cache": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", + "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "flatted": "^3.2.9", + "keyv": "^4.5.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": "^10.12.0 || >=12.0.0" + } + }, + "node_modules/flatted": { + "version": "3.4.4", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", + "integrity": "sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==", + "dev": true, + "license": "ISC" + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/globals": { + "version": "13.24.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", + "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC", + "peer": true + }, + "node_modules/graphemer": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", + "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", + "dev": true, + "license": "MIT" + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ignore": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", + "integrity": "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", + "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/js-yaml": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", + "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/puzrin" + }, + { + "type": "github", + "url": "https://github.com/sponsors/nodeca" + } + ], + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-buffer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", + "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true, + "license": "MIT" + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/keyv": { + "version": "4.5.4", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", + "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "json-buffer": "3.0.1" + } + }, + "node_modules/koffi": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/koffi/-/koffi-3.3.3.tgz", + "integrity": "sha512-XWNI4eyxJJzR2MakPZzR3QeZ06JOklLbxglin0dHhx/qIQCby8lqHs9Acq4Q7CpCKnOrBSj+6b9J5y69wfr2ZA==", + "hasInstallScript": true, + "license": "MIT", + "funding": { + "url": "https://liberapay.com/Koromix" + }, + "optionalDependencies": { + "@koromix/koffi-android-arm64": "3.3.3", + "@koromix/koffi-android-x64": "3.3.3", + "@koromix/koffi-darwin-arm64": "3.3.3", + "@koromix/koffi-darwin-x64": "3.3.3", + "@koromix/koffi-freebsd-arm64": "3.3.3", + "@koromix/koffi-freebsd-ia32": "3.3.3", + "@koromix/koffi-freebsd-x64": "3.3.3", + "@koromix/koffi-linux-arm": "3.3.3", + "@koromix/koffi-linux-arm64": "3.3.3", + "@koromix/koffi-linux-ia32": "3.3.3", + "@koromix/koffi-linux-loong64": "3.3.3", + "@koromix/koffi-linux-ppc64": "3.3.3", + "@koromix/koffi-linux-riscv64": "3.3.3", + "@koromix/koffi-linux-x64": "3.3.3", + "@koromix/koffi-openbsd-arm64": "3.3.3", + "@koromix/koffi-openbsd-ia32": "3.3.3", + "@koromix/koffi-openbsd-x64": "3.3.3", + "@koromix/koffi-win32-arm64": "3.3.3", + "@koromix/koffi-win32-ia32": "3.3.3", + "@koromix/koffi-win32-x64": "3.3.3" + } + }, + "node_modules/levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lodash.merge": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", + "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/minimatch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", + "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/optionator": { + "version": "0.9.4", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", + "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", + "dev": true, + "license": "MIT", + "dependencies": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.5" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "peer": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sumchecker": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/sumchecker/-/sumchecker-3.0.1.tgz", + "integrity": "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "debug": "^4.1.0" + }, + "engines": { + "node": ">= 8.0" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", + "dev": true, + "license": "MIT" + }, + "node_modules/type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "license": "MIT", + "dependencies": { + "prelude-ls": "^1.2.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/type-fest": { + "version": "0.20.2", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", + "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici": { + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", + "license": "MIT", + "optional": true, + "peer": true, + "engines": { + "node": ">=20.18.1" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT", + "peer": true + }, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dev": true, + "license": "BSD-2-Clause", + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/electron/package.json b/electron/package.json new file mode 100644 index 0000000..52e1185 --- /dev/null +++ b/electron/package.json @@ -0,0 +1,35 @@ +{ + "name": "@kitware/trame-rca-electron", + "version": "0.1.0", + "description": "Show trame-rca shared-texture views in Electron, without encoding frames", + "license": "Apache-2.0", + "author": "Kitware Inc.", + "repository": { + "type": "git", + "url": "https://github.com/Kitware/trame-rca.git", + "directory": "electron" + }, + "type": "module", + "main": "src/main/index.js", + "exports": { + ".": "./src/main/index.js", + "./preload": "./src/preload.mjs" + }, + "files": [ + "src/**/*.js", + "src/**/*.mjs" + ], + "scripts": { + "lint": "eslint src test --ext .js,.mjs", + "test": "node --test" + }, + "peerDependencies": { + "electron": ">=44.7.0" + }, + "dependencies": { + "koffi": "^3.3.3" + }, + "devDependencies": { + "eslint": "^8.57.1" + } +} diff --git a/electron/src/main/dma/index.js b/electron/src/main/dma/index.js new file mode 100644 index 0000000..509a864 --- /dev/null +++ b/electron/src/main/dma/index.js @@ -0,0 +1,9 @@ +// Each module loads its platform's libraries when imported: pidfd.js needs Linux's libc, +// iosurface.js needs macOS frameworks, winhandle.js needs kernel32. So import only this +// platform's, and asynchronously: a top-level await would stop CommonJS apps from +// require()-ing this package. +export const loadDma = (platform) => { + if (platform === 'darwin') return import('./iosurface.js'); + if (platform === 'win32') return import('./winhandle.js'); + return import('./pidfd.js'); +}; diff --git a/electron/src/main/dma/iosurface.js b/electron/src/main/dma/iosurface.js new file mode 100644 index 0000000..dde2fd0 --- /dev/null +++ b/electron/src/main/dma/iosurface.js @@ -0,0 +1,36 @@ +// Open another process's IOSurface by its global ID with IOSurfaceLookup. +// Only works for surfaces created with kIOSurfaceIsGlobal (deprecated, unsandboxed only). +import koffi from 'koffi'; + +const iosurface = koffi.load('/System/Library/Frameworks/IOSurface.framework/IOSurface'); +const cf = koffi.load('/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation'); + +const IOSurfaceRef = koffi.pointer('IOSurfaceRef', koffi.opaque()); +const IOSurfaceLookup = iosurface.func('IOSurfaceLookup', IOSurfaceRef, ['uint32_t']); +const IOSurfaceGetWidth = iosurface.func('IOSurfaceGetWidth', 'size_t', [IOSurfaceRef]); +const IOSurfaceGetHeight = iosurface.func('IOSurfaceGetHeight', 'size_t', [IOSurfaceRef]); +const IOSurfaceGetPixelFormat = iosurface.func('IOSurfaceGetPixelFormat', 'uint32_t', [IOSurfaceRef]); +const CFRelease = cf.func('CFRelease', 'void', [IOSurfaceRef]); + +const fourccStr = (code) => String.fromCharCode((code >>> 24) & 0xff, (code >>> 16) & 0xff, (code >>> 8) & 0xff, code & 0xff); + +// Returns a retained IOSurfaceRef; release it with releaseSurface(). +const lookupSurface = (id) => { + const ref = IOSurfaceLookup(id); + if (!ref) throw new Error(`IOSurfaceLookup(${id}) failed (not global, or already freed)`); + return ref; +}; + +const describeSurface = (ref) => + `${IOSurfaceGetWidth(ref)}x${IOSurfaceGetHeight(ref)} ${fourccStr(IOSurfaceGetPixelFormat(ref))}`; + +// sharedTexture wants the IOSurfaceRef pointer value in a Buffer. +const surfaceHandle = (ref) => { + const buf = Buffer.alloc(8); + buf.writeBigUInt64LE(BigInt(koffi.address(ref))); + return buf; +}; + +const releaseSurface = (ref) => CFRelease(ref); + +export { lookupSurface, describeSurface, surfaceHandle, releaseSurface }; diff --git a/electron/src/main/dma/pidfd.js b/electron/src/main/dma/pidfd.js new file mode 100644 index 0000000..51eddae --- /dev/null +++ b/electron/src/main/dma/pidfd.js @@ -0,0 +1,34 @@ +// Duplicate a file descriptor out of another process with pidfd_getfd(2). +// Requires ptrace access to the target (e.g. it is our child with ptrace_scope=1). +import koffi from 'koffi'; + +const libc = koffi.load('libc.so.6'); +const syscall = libc.func('long syscall(long nr, ...)'); +const close = libc.func('int close(int fd)'); +const strerror = libc.func('const char *strerror(int errnum)'); + +// x86_64 / asm-generic syscall numbers +const SYS_pidfd_open = 434; +const SYS_pidfd_getfd = 438; + +const check = (ret, what) => { + const r = Number(ret); + if (r < 0) { + const errno = koffi.errno(); + throw new Error(`${what} failed: ${strerror(errno)} (errno ${errno})`); + } + return r; +}; + +const dupFdFromProcess = (pid, targetFd) => { + const pidfd = check(syscall(SYS_pidfd_open, 'int', pid, 'uint', 0), `pidfd_open(${pid})`); + try { + return check(syscall(SYS_pidfd_getfd, 'int', pidfd, 'int', targetFd, 'uint', 0), `pidfd_getfd(${pid}, ${targetFd})`); + } finally { + close(pidfd); + } +}; + +const closeFd = (fd) => close(fd); + +export { dupFdFromProcess, closeFd }; diff --git a/electron/src/main/dma/winhandle.js b/electron/src/main/dma/winhandle.js new file mode 100644 index 0000000..fac1039 --- /dev/null +++ b/electron/src/main/dma/winhandle.js @@ -0,0 +1,38 @@ +// Duplicate an NT handle out of another process with DuplicateHandle. +// Requires PROCESS_DUP_HANDLE on the target (e.g. it is our child, running as the same user). +import koffi from 'koffi'; + +const kernel32 = koffi.load('kernel32.dll'); +const OpenProcess = kernel32.func('uintptr_t __stdcall OpenProcess(uint32_t access, int inherit, uint32_t pid)'); +const GetCurrentProcess = kernel32.func('uintptr_t __stdcall GetCurrentProcess()'); +const DuplicateHandle = kernel32.func('int __stdcall DuplicateHandle(uintptr_t srcProcess, uintptr_t src, uintptr_t dstProcess, _Out_ uintptr_t *dst, uint32_t access, int inherit, uint32_t options)'); +const CloseHandle = kernel32.func('int __stdcall CloseHandle(uintptr_t handle)'); +const GetLastError = kernel32.func('uint32_t __stdcall GetLastError()'); + +const PROCESS_DUP_HANDLE = 0x0040; +const DUPLICATE_SAME_ACCESS = 0x2; + +const dupHandleFromProcess = (pid, remoteHandle) => { + const proc = OpenProcess(PROCESS_DUP_HANDLE, 0, pid); + if (!proc) throw new Error(`OpenProcess(${pid}) failed: error ${GetLastError()}`); + try { + const out = [0]; + if (!DuplicateHandle(proc, remoteHandle, GetCurrentProcess(), out, 0, 0, DUPLICATE_SAME_ACCESS)) { + throw new Error(`DuplicateHandle(${pid}, 0x${remoteHandle.toString(16)}) failed: error ${GetLastError()}`); + } + return out[0]; + } finally { + CloseHandle(proc); + } +}; + +// sharedTexture wants the HANDLE value in a Buffer. +const handleBuffer = (handle) => { + const buf = Buffer.alloc(8); + buf.writeBigUInt64LE(BigInt(handle)); + return buf; +}; + +const closeHandle = (handle) => CloseHandle(handle); + +export { dupHandleFromProcess, handleBuffer, closeHandle }; diff --git a/electron/src/main/index.js b/electron/src/main/index.js new file mode 100644 index 0000000..46c97ff --- /dev/null +++ b/electron/src/main/index.js @@ -0,0 +1,251 @@ +import { app, ipcMain, sharedTexture } from 'electron/main'; +import { spawn } from 'node:child_process'; +import net from 'node:net'; +import { fileURLToPath } from 'node:url'; + +import { loadDma } from './dma/index.js'; +import * as protocol from './protocol.js'; + +// The preload a BrowserWindow needs to show shared-texture views. Electron only runs +// ES module preloads with the .mjs extension. +const preloadPath = fileURLToPath(new URL('../preload.mjs', import.meta.url)); + +// The webPreferences a BrowserWindow needs to show shared-texture views. +const webPreferences = (extra = {}) => ({ + preload: preloadPath, + // The preload needs electron's sharedTexture, and ES module preloads can't be sandboxed. + sandbox: false, + ...extra, +}); + +// Holds this process's reference to one of the producer's textures. `dma` is this +// platform's module from loadDma(). +class TextureHandle { + constructor(msg, dma, log) { + this.dma = dma; + this.localFds = []; + this.surfaceRef = null; + this.localHandle = null; + this.info = protocol.textureInfo(msg, this._open(msg, log)); + } + + _open(msg, log) { + const kind = protocol.handleKind(msg); + if (kind === 'ioSurface') { + // Open the producer's global IOSurface in this process. + const { lookupSurface, describeSurface, surfaceHandle } = this.dma; + this.surfaceRef = lookupSurface(msg.ioSurfaceId); + log.info( + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} (${msg.pixelFormat}) IOSurface id=${msg.ioSurfaceId} ${describeSurface(this.surfaceRef)}`, + ); + return { ioSurface: surfaceHandle(this.surfaceRef) }; + } + if (kind === 'ntHandle') { + // Duplicate the producer's NT handle into this process. + const { dupHandleFromProcess, handleBuffer } = this.dma; + this.localHandle = dupHandleFromProcess(msg.pid, msg.ntHandle); + log.info( + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} (${msg.pixelFormat}) handle 0x${msg.ntHandle.toString(16)} -> 0x${this.localHandle.toString(16)}`, + ); + return { ntHandle: handleBuffer(this.localHandle) }; + } + // Duplicate the producer's dma-buf fds into this process. + this.localFds = msg.planes.map((p) => this.dma.dupFdFromProcess(msg.pid, p.fd)); + log.info( + `[trame-rca] view ${msg.view}: texture ${msg.width}x${msg.height} ${msg.fourcc} (${msg.pixelFormat}) modifier=${msg.modifier} fds ${msg.planes.map((p) => p.fd)} -> ${this.localFds}`, + ); + return protocol.nativePixmap(msg, this.localFds); + } + + release() { + this.localFds.forEach((fd) => this.dma.closeFd(fd)); + this.localFds = []; + if (this.surfaceRef) this.dma.releaseSurface(this.surfaceRef); + this.surfaceRef = null; + if (this.localHandle) this.dma.closeHandle(this.localHandle); + this.localHandle = null; + } +} + +// Spawn the trame server and load its page in `win` once it is ready. The page's +// shared-texture views then show the textures the server renders into, one ring per view. +// +// `command` is the executable to run (a python interpreter, a frozen backend, ...) and +// `args` its arguments, e.g. ['app.py']. +// trame-rca-electron appends the arguments the server needs: --server --port 0 --emp-port . +// +// Returns { process, close }. close() kills the server and releases the textures; it is +// called on its own when `win` closes. +const launchTrame = ( + win, + { command, args = [], cwd, env, onExit, onStats, log = console }, +) => { + if (!command) throw new Error('launchTrame: `command` is required'); + let socket = null; + // The producer's rings of textures, by view name, then by slot. + const textures = new Map(); + // This platform's dma module, loaded before the server is spawned. + let dma = null; + let closed = false; + const result = { process: null, close: null }; + + const send = (msg) => { + if (socket && !socket.destroyed) socket.write(protocol.encode(msg)); + }; + + // Release the textures of `view` from `fromSlot` on. + const releaseTextures = (view, fromSlot = 0) => { + const ring = textures.get(view) ?? []; + ring.slice(fromSlot).forEach((t) => t?.release()); + ring.length = Math.min(ring.length, fromSlot); + if (!ring.length) textures.delete(view); + }; + + const releaseAllTextures = () => { + textures.forEach((ring) => ring.forEach((t) => t?.release())); + textures.clear(); + }; + + const forwardFrame = async (info, msg) => { + // Once per frame: a frame that failed to transfer may still be released late by the renderer. + let done = false; + const frameDone = () => { + if (done) return; + done = true; + send({ type: 'frame_done', view: msg.view, seq: msg.seq }); + }; + const frameDropped = () => { + if (done) return; + done = true; + send({ type: 'frame_done', view: msg.view, seq: msg.seq, dropped: true }); + }; + let imported = null; + try { + // Electron retains the texture's handle, so it can be re-imported every frame. + imported = sharedTexture.importSharedTexture({ + textureInfo: { ...info, timestamp: msg.timestamp_us }, + allReferencesReleased: frameDone, + }); + // The preload's receiver gets the view's name as its second argument. + await sharedTexture.sendSharedTexture( + { frame: win.webContents.mainFrame, importedSharedTexture: imported }, + msg.view, + ); + } catch (err) { + // sendSharedTexture uses a 1000ms timeout. If the ERP is still busy loading the page, + // or if some other error occurred, the page would never have shown this frame, so ask + // for it again. + log.warn(`[trame-rca] view ${msg.view}: frame ${msg.seq} dropped: ${err.message}`); + frameDropped(); + } finally { + imported?.release(); + } + }; + + const onMessage = (conn) => (msg) => { + if (msg.type === 'server_ready') { + try { + protocol.checkProtocol(msg); + } catch (err) { + log.error(`[trame-rca] ${err.message}`); + conn.destroy(); + return; + } + log.info(`[trame-rca] loading ${msg.url}`); + win.loadURL(msg.url); + } else if (msg.type === 'texture') { + try { + const { view, slot, slots } = msg; + if (!textures.has(view)) textures.set(view, []); + const ring = textures.get(view); + ring[slot]?.release(); + ring[slot] = new TextureHandle(msg, dma, log); + // release textures beyond `slots` to handle shrink + releaseTextures(view, slots); + } catch (err) { + log.error('[trame-rca] failed to open the texture:', err); + conn.destroy(); + } + } else if (msg.type === 'frame') { + const texture = textures.get(msg.view)?.[msg.slot]; + if (texture) forwardFrame(texture.info, msg); + } + }; + + const handleConnection = (conn) => { + if (socket) { + // One server per window: ignore anything else knocking. + conn.destroy(); + return; + } + socket = conn; + // Without this, Nagle holds frame_done back until the producer's delayed ACK. + conn.setNoDelay(true); + send( + protocol.helloMessage( + protocol.passportName({ + platform: process.platform, + ozonePlatform: app.commandLine.getSwitchValue('ozone-platform'), + ozonePlatformHint: app.commandLine.getSwitchValue('ozone-platform-hint'), + waylandDisplay: process.env.WAYLAND_DISPLAY, + }), + ), + ); + conn.on( + 'data', + protocol.createLineParser(onMessage(conn), (line, err) => + log.error( + `[trame-rca] bad line from trame (${line.length} chars): ${JSON.stringify(line)}`, + err, + ), + ), + ); + conn.on('close', () => log.info('[trame-rca] trame disconnected')); + }; + + const statsListener = (event, stats) => { + if (event.sender === win.webContents) onStats?.(stats); + }; + ipcMain.on('trame-rca:stats', statsListener); + + const server = net.createServer(handleConnection); + + result.close = () => { + if (closed) return; + closed = true; + ipcMain.off('trame-rca:stats', statsListener); + result.process?.kill(); + socket?.destroy(); + socket = null; + server.close(); + releaseAllTextures(); + }; + win.once('closed', result.close); + + server.listen(0, '127.0.0.1', async () => { + try { + dma = await loadDma(process.platform); + } catch (err) { + log.error('[trame-rca] failed to load the OS texture API:', err); + result.close(); + return; + } + if (closed) return; + const { port } = server.address(); + const fullArgs = [...args, '--server', '--port', '0', '--emp-port', String(port)]; + log.info(`[trame-rca] spawning ${command} ${fullArgs.join(' ')}`); + result.process = spawn(command, fullArgs, { + cwd, + env: env ?? process.env, + stdio: ['ignore', 'inherit', 'inherit'], + }); + result.process.on('exit', (code, signal) => { + log.info(`[trame-rca] trame exited code=${code} signal=${signal}`); + onExit?.(code, signal); + }); + }); + + return result; +}; + +export { launchTrame, preloadPath, webPreferences, protocol }; diff --git a/electron/src/main/protocol.js b/electron/src/main/protocol.js new file mode 100644 index 0000000..ddc0c62 --- /dev/null +++ b/electron/src/main/protocol.js @@ -0,0 +1,93 @@ +// The texture channel between the Electron main process (EMP) and trame-rca, as +// newline-delimited JSON on 127.0.0.1. See trame_rca/electron/comm.py for the messages. +// Nothing in here touches Electron or the OS, so it can be tested with plain node. + +// Must match trame_rca.electron.comm.PROTOCOL_VERSION. +const PROTOCOL_VERSION = 1; + +// The texture passport trame-rca should share its texture with, for this platform. +// On Linux, Electron uses Wayland when it's available, unless --ozone-platform=x11. +const passportName = ({ platform, ozonePlatform, ozonePlatformHint, waylandDisplay }) => { + if (platform === 'darwin') return 'iosurface'; + if (platform === 'win32') return 'nthandle'; + // --ozone-platform wins over --ozone-platform-hint, whose default is auto. + const ozone = ozonePlatform || ozonePlatformHint || 'auto'; + const wayland = ozone === 'auto' ? !!waylandDisplay : ozone === 'wayland'; + return wayland ? 'wayland-dmabuf' : 'x11-dmabuf'; +}; + +const helloMessage = (passport) => ({ type: 'hello', protocol: PROTOCOL_VERSION, passport }); + +const checkProtocol = (msg) => { + if (msg.protocol !== PROTOCOL_VERSION) { + throw new Error( + `trame-rca speaks shared texture protocol ${msg.protocol}, trame-rca-electron speaks ` + + `${PROTOCOL_VERSION}: update trame-rca or trame-rca-electron so they match`, + ); + } +}; + +// Calls onMessage with every complete JSON line fed in, and onBadLine with lines that +// aren't JSON instead of throwing out of the socket's data handler. +const createLineParser = (onMessage, onBadLine) => { + let buf = ''; + return (chunk) => { + buf += chunk.toString(); + let nl; + while ((nl = buf.indexOf('\n')) >= 0) { + const line = buf.slice(0, nl); + buf = buf.slice(nl + 1); + let msg; + try { + msg = JSON.parse(line); + } catch (err) { + onBadLine?.(line, err); + continue; + } + onMessage(msg); + } + }; +}; + +const encode = (msg) => JSON.stringify(msg) + '\n'; + +// Which OS handle a texture message carries. +const handleKind = (msg) => { + if (msg.ioSurfaceId !== undefined) return 'ioSurface'; + if (msg.ntHandle !== undefined) return 'ntHandle'; + return 'nativePixmap'; +}; + +// The nativePixmap handle of a dma-buf texture, once its fds are duplicated into this process. +const nativePixmap = (msg, localFds) => ({ + nativePixmap: { + planes: msg.planes.map((p, i) => ({ + fd: localFds[i], + stride: p.stride, + offset: p.offset, + size: p.size, + })), + modifier: msg.modifier, + supportsZeroCopyWebGpuImport: false, + }, +}); + +// The textureInfo of sharedTexture.importSharedTexture, for a texture message and its handle. +const textureInfo = (msg, handle) => ({ + pixelFormat: msg.pixelFormat, + codedSize: { width: msg.width, height: msg.height }, + visibleRect: { x: 0, y: 0, width: msg.width, height: msg.height }, + handle, +}); + +export { + PROTOCOL_VERSION, + checkProtocol, + createLineParser, + encode, + handleKind, + helloMessage, + nativePixmap, + passportName, + textureInfo, +}; diff --git a/electron/src/preload.mjs b/electron/src/preload.mjs new file mode 100644 index 0000000..51a154b --- /dev/null +++ b/electron/src/preload.mjs @@ -0,0 +1,70 @@ +// Draws the frames the main process sends into the page's shared-texture-display-areas. +// Use it as a BrowserWindow's preload (see webPreferences()), or import it from yours. +import { contextBridge, ipcRenderer, sharedTexture } from 'electron'; + +const STATS_INTERVAL_MS = 1000; + +// The canvas of each shared-texture-display-area, by view name, attached when the component +// mounts. +const views = new Map(); + +// The main process sends the view's name along with every frame. +sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture }, view) => { + const target = views.get(view); + if (!target) { + importedSharedTexture.release(); + return; + } + const { canvas, ctx } = target; + const vf = importedSharedTexture.getVideoFrame(); + // The canvas's pixels follow the producer's texture, which follows the canvas's size. + if (canvas.width !== vf.displayWidth || canvas.height !== vf.displayHeight) { + canvas.width = vf.displayWidth; + canvas.height = vf.displayHeight; + } + // The producers render with GL, whose first row is the bottom, into textures whose first + // row is the top: draw the frame upside down to undo that. + ctx.setTransform(1, 0, 0, -1, 0, canvas.height); + ctx.drawImage(vf, 0, 0, canvas.width, canvas.height); + ctx.setTransform(1, 0, 0, 1, 0, 0); + target.info = `${vf.codedWidth}x${vf.codedHeight} ${vf.format}`; + vf.close(); + importedSharedTexture.release(); + target.frames++; + target.windowFrames++; +}); + +// Frames are rendered on demand, so report stats over a timed interval. +setInterval(() => { + const now = performance.now(); + views.forEach((target, view) => { + const fps = (target.windowFrames * 1000) / (now - target.windowStart); + const hadFrames = target.windowFrames > 0; + target.windowStart = now; + target.windowFrames = 0; + const { frames, info } = target; + target.onStats?.({ fps: Number(fps.toFixed(1)), frames, info }); + if (hadFrames) { + ipcRenderer.send('trame-rca:stats', { view, frames, fps: fps.toFixed(1), info }); + } + }); +}, STATS_INTERVAL_MS); + +// The page's world can't touch sharedTexture or VideoFrames from here, but it can hand us its +// canvases, and a callback for their frame rate. +contextBridge.exposeInMainWorld('trameRcaSharedTexture', { + attach: (view, el, statsCallback) => { + views.set(view, { + canvas: el, + ctx: el.getContext('2d'), + onStats: statsCallback ?? null, + frames: 0, + windowStart: performance.now(), + windowFrames: 0, + info: '', + }); + }, + detach: (view, el) => { + if (views.get(view)?.canvas === el) views.delete(view); + }, +}); diff --git a/electron/test/protocol.test.js b/electron/test/protocol.test.js new file mode 100644 index 0000000..2bf94c9 --- /dev/null +++ b/electron/test/protocol.test.js @@ -0,0 +1,78 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import * as protocol from '../src/main/protocol.js'; + +test('passportName picks the platform passport', () => { + assert.equal(protocol.passportName({ platform: 'darwin' }), 'iosurface'); + assert.equal(protocol.passportName({ platform: 'win32' }), 'nthandle'); +}); + +test('passportName follows ozone switches and WAYLAND_DISPLAY on linux', () => { + const linux = (o) => protocol.passportName({ platform: 'linux', ...o }); + assert.equal(linux({ waylandDisplay: 'wayland-0' }), 'wayland-dmabuf'); + assert.equal(linux({}), 'x11-dmabuf'); + assert.equal(linux({ ozonePlatform: 'x11', waylandDisplay: 'wayland-0' }), 'x11-dmabuf'); + assert.equal(linux({ ozonePlatformHint: 'wayland' }), 'wayland-dmabuf'); + // --ozone-platform should win over the --ozone-platform-hint. + assert.equal(linux({ ozonePlatform: 'x11', ozonePlatformHint: 'wayland' }), 'x11-dmabuf'); +}); + +test('hello carries the protocol version', () => { + assert.deepEqual(protocol.helloMessage('iosurface'), { + type: 'hello', + protocol: protocol.PROTOCOL_VERSION, + passport: 'iosurface', + }); +}); + +test('checkProtocol rejects other versions', () => { + protocol.checkProtocol({ protocol: protocol.PROTOCOL_VERSION }); + assert.throws(() => protocol.checkProtocol({ protocol: protocol.PROTOCOL_VERSION + 1 }), /protocol/); + assert.throws(() => protocol.checkProtocol({}), /protocol/); +}); + +test('line parser joins chunks and reports bad lines', () => { + const messages = []; + const bad = []; + const feed = protocol.createLineParser( + (m) => messages.push(m), + (line) => bad.push(line), + ); + feed('{"type":"fr'); + feed('ame","seq":1}\n{oops}\n{"type":"frame","seq":2}\n{"type"'); + assert.deepEqual(messages, [ + { type: 'frame', seq: 1 }, + { type: 'frame', seq: 2 }, + ]); + assert.deepEqual(bad, ['{oops}']); + feed(':"frame","seq":3}\n'); + assert.equal(messages.length, 3); +}); + +test('handleKind and textureInfo', () => { + assert.equal(protocol.handleKind({ ioSurfaceId: 3 }), 'ioSurface'); + assert.equal(protocol.handleKind({ ntHandle: 4 }), 'ntHandle'); + assert.equal(protocol.handleKind({ planes: [] }), 'nativePixmap'); + const msg = { pixelFormat: 'bgra', width: 4, height: 2 }; + assert.deepEqual(protocol.textureInfo(msg, { h: 1 }), { + pixelFormat: 'bgra', + codedSize: { width: 4, height: 2 }, + visibleRect: { x: 0, y: 0, width: 4, height: 2 }, + handle: { h: 1 }, + }); +}); + +test('nativePixmap swaps in the local fds', () => { + const msg = { + modifier: '0', + planes: [{ fd: 30, stride: 16, offset: 0, size: 64 }], + }; + assert.deepEqual(protocol.nativePixmap(msg, [7]), { + nativePixmap: { + planes: [{ fd: 7, stride: 16, offset: 0, size: 64 }], + modifier: '0', + supportsZeroCopyWebGpuImport: false, + }, + }); +}); diff --git a/examples/06_electron/README.md b/examples/06_electron/README.md new file mode 100644 index 0000000..eb27d2b --- /dev/null +++ b/examples/06_electron/README.md @@ -0,0 +1,21 @@ +# Shared texture with Electron + +`app.py` renders its VTK view into a GPU texture that Electron shows as is: no encoding, +no copy to the CPU. The Electron app in `./electron` starts `app.py` and loads its page. + +## Run + +```sh +# From the repository root, in a virtual environment +pip install -e ".[electron]" trame trame-vuetify +cd electron +npm i +cd .. + +cd examples/06_electron/electron +npm i +npm start +``` + +`npm start` runs `app.py` with the `python3` of the active environment, or `$PYTHON`. + diff --git a/examples/06_electron/app.py b/examples/06_electron/app.py new file mode 100644 index 0000000..3335d52 --- /dev/null +++ b/examples/06_electron/app.py @@ -0,0 +1,254 @@ +"""A trame app whose VTK view is shared with Electron as a texture, not streamed. + +Started by the Electron app in ./electron, see the README. +""" + +import json +import platform +from importlib.metadata import version + +from trame.app import TrameApp +from trame.decorators import change +from trame.ui.vuetify3 import SinglePageWithDrawerLayout +from trame.widgets import html, vuetify3 as v3 +from trame_rca.widgets import init_vtk_for_shared_textures, rca + +# Before creating any VTK render window: VTK must render on the GPU Electron uses. +init_vtk_for_shared_textures() + +from vtkmodules.vtkCommonDataModel import vtkPiecewiseFunction # noqa: E402 +from vtkmodules.vtkFiltersSources import vtkPartitionedDataSetCollectionSource # noqa: E402 +from vtkmodules.vtkImagingCore import vtkRTAnalyticSource # noqa: E402 +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera # noqa: E402 +from vtkmodules.vtkRenderingCore import ( # noqa: E402 + vtkActor, + vtkColorTransferFunction, + vtkCompositePolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkVolume, + vtkVolumeProperty, +) +from vtkmodules.vtkRenderingUI import vtkGenericRenderWindowInteractor # noqa: E402 +from vtkmodules.vtkRenderingVolumeOpenGL2 import vtkSmartVolumeMapper # noqa: E402 + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: E402, F401 + +# Uncomment to see per-frame profile +# from trame_common.utils import profiler +# profiler.enable() +# profiler.LOGGER.filter.include("rca.shared_texture") +# profiler.LOGGER.filter.exclude("rca.vtk") + + +def create_vtk_pipeline(): + wavelet = vtkRTAnalyticSource() + volume_mapper = vtkSmartVolumeMapper() + volume_mapper.SetInputConnection(wavelet.output_port) + color_tf = vtkColorTransferFunction() + color_tf.AddRGBPoint(0.0, 0.0, 0.0, 1.0) + color_tf.AddRGBPoint(255.0 / 2.0, 0.0, 1.0, 1.0) + color_tf.AddRGBPoint(255.0, 1.0, 0.0, 1.0) + opacity_tf = vtkPiecewiseFunction() + opacity_tf.AddPoint(0.0, 0.0) + opacity_tf.AddPoint(255.0, 0.1) + volume_property = vtkVolumeProperty() + volume_property.SetColor(color_tf) + volume_property.SetScalarOpacity(opacity_tf) + volume_property.ShadeOn() + volume_property.SetInterpolationTypeToLinear() + volume_actor = vtkVolume(mapper=volume_mapper, property=volume_property) + + shapes = vtkPartitionedDataSetCollectionSource() + cpdm = vtkCompositePolyDataMapper() + cpdm.SetInputConnection(shapes.output_port) + shapes_actor = vtkActor(mapper=cpdm) + + renderer = vtkRenderer() + renderer.AddVolume(volume_actor) + renderer.AddActor(shapes_actor) + # The texture's alpha reaches Electron, so keep the background opaque. + renderer.SetBackgroundAlpha(1.0) + renderer.ResetCamera() + + render_window = vtkRenderWindow() + render_window.AddRenderer(renderer) + # trame-rca replays the canvas's input on the interactor. + vtkGenericRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, renderer, opacity_tf, shapes + + +PYTHON_PACKAGES = [ + "vtk", + "trame", + "trame-client", + "trame-server", + "trame-vuetify", + "trame-rca", + "wslink", +] +ELECTRON_VERSIONS = [ + ("Electron", "electron"), + ("Chromium", "chrome"), + ("Node", "node"), + ("V8", "v8"), +] + + +class ElectronApp(TrameApp): + def __init__(self, server=None): + super().__init__(server) + self.server.cli.add_argument( + "--electron-versions", + default="{}", + help="The Electron main process's process.versions, as JSON", + ) + args, _ = self.server.cli.parse_known_args() + + electron_versions = json.loads(args.electron_versions) + self.state.electron_versions = [ + {"name": name, "version": electron_versions[key]} + for name, key in ELECTRON_VERSIONS + if key in electron_versions + ] + self.state.python_versions = [ + {"name": "Python", "version": platform.python_version()} + ] + [{"name": name, "version": version(name)} for name in PYTHON_PACKAGES] + self.state.spinning = False + # Frame stats from the canvas, once a second: only the footer needs them, not the server. + self.state.client_only("stats") + self.state.stats = {"fps": 0, "frames": 0, "info": ""} + + ( + self.render_window, + self.renderer, + self.volume_opacity_tf, + self.shapes_source, + ) = create_vtk_pipeline() + self._build_ui() + + def reset_camera(self): + self.renderer.ResetCamera() + self.view_handler.update() + + def before_render(self): + # Spin by a degree per frame: each frame asks for the next one. + if self.state.spinning: + # The interactor only hands its rates to the render window when an interaction + # starts or ends, so set the window's rate directly. A click mid-spin resets it. + self.render_window.desired_update_rate = ( + self.render_window.interactor.desired_update_rate + ) + self.renderer.GetActiveCamera().Azimuth(1.0) + self.renderer.ResetCameraClippingRange() + self.view_handler.update() + + @change("spinning") + def on_spinning_change(self, spinning, **_): + if not spinning: + # Back to a full quality still frame. + self.render_window.desired_update_rate = ( + self.render_window.interactor.still_update_rate + ) + self.view_handler.update() + + @change("number_of_shapes") + def on_number_of_shapes_change(self, number_of_shapes, **_): + self.shapes_source.number_of_shapes = min(12, max(1, int(number_of_shapes))) + self.view_handler.update() + + @change("opacity") + def on_opacity_change(self, opacity, **_): + # Node 1 is the top of the scalar range: (x, y, midpoint, sharpness). + self.volume_opacity_tf.SetNodeValue(1, [255.0, float(opacity), 0.5, 0.0]) + self.view_handler.update() + + def _build_ui(self): + with SinglePageWithDrawerLayout(self.server) as self.ui: + self.ui.title.set_text("Shared texture trame example") + + with self.ui.toolbar: + v3.VSpacer() + v3.VSlider( + v_model=("opacity", 0.1), + label="Opacity", + min=0.0, + max=1.0, + step=0.01, + thumb_label=True, + hide_details=True, + density="compact", + style="max-width: 250px", + classes="mx-2", + ) + v3.VSlider( + v_model=("number_of_shapes", 1), + label="Shapes", + min=1, + max=12, + step=1, + thumb_label=True, + hide_details=True, + density="compact", + style="max-width: 250px", + classes="mx-2", + ) + v3.VBtn(icon="mdi-crop-free", click=self.reset_camera) + v3.VBtn( + icon="mdi-rotate-3d-variant", + click="spinning = !spinning", + color=("spinning ? 'primary' : undefined",), + ) + + with self.ui.drawer: + with v3.VList(density="compact"): + v3.VListSubheader("Electron", v_if="electron_versions.length") + v3.VListItem( + v_for="item in electron_versions", + key=("item.name",), + title=("item.name",), + subtitle=("item.version",), + ) + v3.VListSubheader("Python") + v3.VListItem( + v_for="item in python_versions", + key=("item.name",), + title=("item.name",), + subtitle=("item.version",), + ) + + with self.ui.footer: + self.ui.footer.clear() + v3.VProgressCircular( + indeterminate=("!!trame__busy",), + color="#04a94d", + size=16, + width=3, + classes="ml-n3 mr-1", + ) + html.Span("{{ stats.fps }} fps", classes="text-caption") + v3.VSpacer() + html.Span( + "{{ stats.info }}", classes="text-caption text-grey-lighten-1" + ) + + with self.ui.content: + with v3.VContainer( + fluid=True, classes="pa-0 fill-height position-relative" + ): + view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + stats="stats = $event", + ) + self.view_handler = view.create_electron_handler( + self.render_window, before_render=self.before_render + ) + + +if __name__ == "__main__": + ElectronApp().server.start() diff --git a/examples/06_electron/app_native.py b/examples/06_electron/app_native.py new file mode 100644 index 0000000..ea5e12a --- /dev/null +++ b/examples/06_electron/app_native.py @@ -0,0 +1,114 @@ +"""The same VTK scene as app.py, rendered in a native desktop window without trame. + +Starts spinning the camera and prints the fps to the console once a second, +as a baseline to compare the shared texture path against. +""" + +import argparse +import time + +from vtkmodules.vtkCommonDataModel import vtkPiecewiseFunction +from vtkmodules.vtkFiltersSources import vtkPartitionedDataSetCollectionSource +from vtkmodules.vtkImagingCore import vtkRTAnalyticSource +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera +from vtkmodules.vtkRenderingCore import ( + vtkActor, + vtkColorTransferFunction, + vtkCompositePolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, + vtkVolume, + vtkVolumeProperty, +) +from vtkmodules.vtkRenderingVolumeOpenGL2 import vtkSmartVolumeMapper + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: F401 + + +def create_vtk_pipeline(): + wavelet = vtkRTAnalyticSource() + volume_mapper = vtkSmartVolumeMapper() + volume_mapper.SetInputConnection(wavelet.output_port) + color_tf = vtkColorTransferFunction() + color_tf.AddRGBPoint(0.0, 0.0, 0.0, 1.0) + color_tf.AddRGBPoint(255.0 / 2.0, 0.0, 1.0, 1.0) + color_tf.AddRGBPoint(255.0, 1.0, 0.0, 1.0) + opacity_tf = vtkPiecewiseFunction() + opacity_tf.AddPoint(0.0, 0.0) + opacity_tf.AddPoint(255.0, 0.1) + volume_property = vtkVolumeProperty() + volume_property.SetColor(color_tf) + volume_property.SetScalarOpacity(opacity_tf) + volume_property.ShadeOn() + volume_property.SetInterpolationTypeToLinear() + volume_actor = vtkVolume(mapper=volume_mapper, property=volume_property) + + shapes = vtkPartitionedDataSetCollectionSource() + cpdm = vtkCompositePolyDataMapper() + cpdm.SetInputConnection(shapes.output_port) + shapes_actor = vtkActor(mapper=cpdm) + + renderer = vtkRenderer() + renderer.AddVolume(volume_actor) + renderer.AddActor(shapes_actor) + renderer.ResetCamera() + + render_window = vtkRenderWindow(size=(1024, 768)) + render_window.SetWindowName("Shared texture native example") + render_window.AddRenderer(renderer) + interactor = vtkRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, renderer, interactor + + +class FpsCounter: + def __init__(self): + self.frames = 0 + self.start = time.perf_counter() + + def tick(self): + self.frames += 1 + elapsed = time.perf_counter() - self.start + if elapsed >= 1.0: + print(f"{self.frames / elapsed:.1f} fps", flush=True) + self.frames = 0 + self.start = time.perf_counter() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--no-vsync", action="store_true", help="Don't cap the fps at the display rate" + ) + args = parser.parse_args() + + render_window, renderer, interactor = create_vtk_pipeline() + interactor.Initialize() + if args.no_vsync: + # Needs a created window, so after Initialize(). + render_window.SetSwapControl(0) + # Match app.py while spinning: render at the interactor's interactive rate. + render_window.desired_update_rate = interactor.desired_update_rate + + camera = renderer.GetActiveCamera() + fps = FpsCounter() + render_window.AddObserver("EndEvent", lambda *_: fps.tick()) + + def spin(*_): + # Spin by a degree per frame, like app.py. + camera.Azimuth(1.0) + renderer.ResetCameraClippingRange() + render_window.Render() + + interactor.AddObserver("TimerEvent", spin) + interactor.CreateRepeatingTimer(1) + render_window.Render() + interactor.Start() + + +if __name__ == "__main__": + main() diff --git a/examples/06_electron/electron/.gitignore b/examples/06_electron/electron/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/examples/06_electron/electron/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/examples/06_electron/electron/main.js b/examples/06_electron/electron/main.js new file mode 100644 index 0000000..2eca7bf --- /dev/null +++ b/examples/06_electron/electron/main.js @@ -0,0 +1,28 @@ +import { app, BrowserWindow } from 'electron/main'; +import { fileURLToPath } from 'node:url'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; + +// Run ../app.py with the python of the active environment, unless PYTHON says otherwise. +const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); + +app.whenReady().then(() => { + const win = new BrowserWindow({ + width: 1920, + height: 1080, + useContentSize: true, + webPreferences: webPreferences(), + }); + const { electron, chrome, node, v8 } = process.versions; + launchTrame(win, { + command: python, + args: [ + fileURLToPath(new URL('../app.py', import.meta.url)), + '--electron-versions', + JSON.stringify({ electron, chrome, node, v8 }), + ], + onStats: (s) => console.log(`[renderer] ${s.view} frames=${s.frames} fps=${s.fps} ${s.info}`), + onExit: () => app.quit(), + }); +}); + +app.on('window-all-closed', () => app.quit()); diff --git a/examples/06_electron/electron/package.json b/examples/06_electron/electron/package.json new file mode 100644 index 0000000..82e58f4 --- /dev/null +++ b/examples/06_electron/electron/package.json @@ -0,0 +1,18 @@ +{ + "name": "trame-rca-electron-example", + "private": true, + "version": "0.0.0", + "description": "Electron shell for the shared texture trame-rca example", + "license": "Apache-2.0", + "type": "module", + "main": "main.js", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "@kitware/trame-rca-electron": "file:../../../electron" + }, + "devDependencies": { + "electron": "^44.7.0" + } +} diff --git a/examples/07_electron_point_cloud/README.md b/examples/07_electron_point_cloud/README.md new file mode 100644 index 0000000..142b81f --- /dev/null +++ b/examples/07_electron_point_cloud/README.md @@ -0,0 +1,41 @@ +# Dynamic point cloud with Electron + +`app.py` moves every point of a point cloud and gives it a new color each frame, then +renders its VTK view into a GPU texture that Electron shows as is. There is a a panel that +shows the updates/s, the time spent per stage and, after a few +point counts, how many points fit in 30 or 60 updates/s. + +`app_native.py` renders the same scene in a native desktop window without trame, as a +baseline: `python app_native.py [--points N] [--frames K] [--radii X Y Z] [--no-vsync]`. + +## Run + +```sh +# From the repository root, in a virtual environment +pip install -e ".[electron]" trame +cd electron +npm i +cd .. + +cd examples/07_electron_point_cloud/electron +npm i +npm start +# Or pass arguments to app.py +npm start -- --points 500000 --step 250000 --target-fps 1000 +``` + +`npm start -- --second-view` adds a second shared-texture view of the same cloud next to the +first, and spins the cameras of both views in opposite directions. The panel's timings stay +those of the first view, which now shares the GPU and the event loop with the second. + +`npm start` runs `app.py` with the `python3` of the active environment, or `$PYTHON`. +`--target-fps` (default 60) caps the update rate. `--textures` (default 2) is how many +shared textures VTK renders into in turn, also switchable at runtime in the panel: with 1, +each update waits for Electron to show the previous frame, with 2 VTK renders the next +frame meanwhile. `--radii X Y Z` (default 2 1 0.5) is how far the ellipsoidal cloud spreads +along each axis. + +The frames are computed ahead each time the point count changes, `--frames` of them +(default 8), and played back and forth: an update only swaps in the next frame's arrays, +so the app work is ~0 and the timings show the pipeline alone. That costs +`--frames` × 15 bytes per point of memory. `--frames 0` computes each frame live instead. diff --git a/examples/07_electron_point_cloud/app.py b/examples/07_electron_point_cloud/app.py new file mode 100644 index 0000000..eae3085 --- /dev/null +++ b/examples/07_electron_point_cloud/app.py @@ -0,0 +1,732 @@ +"""A dynamic point cloud whose VTK view is shared with Electron as a texture, not streamed. + +Every frame the server moves each point randomly, assigns new random colors and renders +into the texture Electron shows. The frames are computed ahead, on resize, so that the +app work is ~0: --frames 0 computes each one live instead. The next frame starts once Electron is done with the +previous one, so the measured updates/s is the maximum rate at which new data goes +through the full pipeline: + + numpy -> vtk arrays -> render (upload to the GPU + draw) -> shared texture + -> Electron composites the frame -> "frame_done" back to Python + +--second-view adds a second shared-texture view of the same cloud next to the first, and +spins the camera of both views, in opposite directions. + +Started by the Electron app in ./electron, see the README. app_native.py renders the +same scene in a desktop window, as a baseline. +""" + +import asyncio +import time + +import numpy as np + +from trame.app import TrameApp +from trame.decorators import change +from trame.ui.html import DivLayout +from trame.widgets import client, html +from trame_common.utils import profiler +from trame_rca.widgets import init_vtk_for_shared_textures, rca + +# Before creating any VTK render window: VTK must render on the GPU Electron uses. +init_vtk_for_shared_textures() + +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray # noqa: E402 +from vtkmodules.vtkCommonCore import vtkPoints # noqa: E402 +from vtkmodules.vtkCommonDataModel import vtkCellArray, vtkPolyData # noqa: E402 +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera # noqa: E402 +from vtkmodules.vtkRenderingCore import ( # noqa: E402 + vtkActor, + vtkPolyDataMapper, + vtkRenderer, + vtkRenderWindow, +) +from vtkmodules.vtkRenderingUI import vtkGenericRenderWindowInteractor # noqa: E402 + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: E402, F401 + +WARMUP_FRAMES = 5 # frames skipped after a resize (topology + first upload) +ROLLING_FRAMES = 30 # frames averaged in the live panel +UI_REFRESH_S = 0.25 +BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb +DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod +RADII = (2.0, 1.0, 0.5) # standard deviation of the cloud along x, y and z +SPIN_DEG_PER_S = 30.0 # camera spin with --second-view + +STAGES = [ + ("generate", "app work: swap in the next precomputed frame, or compute it"), + ("render", "Render + WaitForCompletion: upload to the GPU + draw, until done"), + ( + "electron", + "until the next texture is free: import, IPC, canvas draw, frame_done, pacing", + ), +] + +PANEL_STYLE = """ +position:absolute; top:1rem; left:1rem; z-index:10; width:38rem; +max-height:calc(100vh - 2rem); overflow:auto; +background:rgba(20,22,28,0.85); color:#e8e8e8; border-radius:8px; +padding:0.75rem 1rem; font:12px/1.4 ui-monospace,Menlo,monospace; +""" + +CSS = """ +body { margin: 0; } +.bench table { width:100%; border-collapse:collapse; margin:0.25rem 0 0.75rem; } +.bench td, .bench th { padding:1px 4px; text-align:right; white-space:nowrap; } +.bench td:first-child, .bench th:first-child { text-align:left; } +.bench td.stage { white-space:normal; } +.bench th { color:#9aa; font-weight:normal; border-bottom:1px solid #444; } +.bench .bar { height:8px; background:#4c9be8; border-radius:2px; } +.bench .big { font-size:20px; font-weight:bold; } +.bench button { font:inherit; padding:4px 10px; margin-right:6px; cursor:pointer; } +.bench .muted { color:#9aa; } +""" + + +# ----------------------------------------------------------------------------- +# Data +# ----------------------------------------------------------------------------- + + +class PointCloud: + """vtkPolyData whose point and color arrays change every frame + + With ``n_frames`` > 1 the frames are computed once, on resize, and each update only + points the polydata at the next frame's arrays: the app work drops to ~0 while every + frame still uploads new data. They play back and forth so the loop has no jump. + With ``n_frames`` = 0 each update computes a new frame, in place. + The points spread out along x, y and z by ``radii``: an ellipsoidal cloud. + """ + + def __init__(self, n_points, n_frames=8, seed=0, radii=RADII): + self.rng = np.random.default_rng(seed) + self.radii = np.asarray(radii, dtype=np.float32) + self.polydata = vtkPolyData() + self.n_points = 0 + self.n_frames = n_frames + self.positions = np.zeros((0, 3), dtype=np.float32) + self.frames = [] + self._t = 0 + self.resize(n_points) + + def resize(self, n_points): + n_old = self.n_points + positions = np.empty((n_points, 3), dtype=np.float32) + positions[: min(n_old, n_points)] = self.positions[:n_points] + if n_points > n_old: + positions[n_old:] = self.rng.standard_normal( + (n_points - n_old, 3), dtype=np.float32 + ) + positions[n_old:] *= self.radii + colors = np.empty((n_points, 3), dtype=np.uint8) + self._noise = np.empty_like(positions) + self.n_points = n_points + + # Keep python references alive: the VTK arrays share numpy memory. + self.frames = [] + for _ in range(max(1, self.n_frames)): + self._walk(positions, colors) + self.frames.append(self._to_vtk(positions, colors)) + positions, colors = positions.copy(), colors.copy() + self.positions = self.frames[-1][0] + self.colors = self.frames[-1][1] + self._t = 0 + + # A single polyvertex cell with every point. The mapper rebuilds its index buffer + # whenever the points change, and that is about twice as fast as from one vertex + # cell per point. + verts = vtkCellArray() + verts.SetData( + numpy_to_vtkIdTypeArray(np.array([0, n_points], dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(np.arange(n_points, dtype=np.int64), deep=True), + ) + + self.polydata.SetPoints(vtkPoints()) + self.polydata.SetVerts(verts) + self._show(self.frames[-1]) + self.polydata.Modified() + + def animate(self): + if self.n_frames == 0: + self._walk(self.positions, self.colors) + self._show(self.frames[0]) + return + # Ping-pong: 0, 1, ..., K-1, K-2, ..., 1, 0, 1, ... + k = len(self.frames) + period = max(1, 2 * k - 2) + i = self._t % period + self._t += 1 + self._show(self.frames[i if i < k else period - i]) + + def _walk(self, positions, colors, step=0.02, pull=0.995): + # Random walk with a weak pull toward the origin so the cloud stays put. The steps + # scale with the radii too, so the walk keeps the cloud's shape. + self.rng.standard_normal(out=self._noise, dtype=np.float32) + self._noise *= step * self.radii + positions *= pull + positions += self._noise + colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) + + @staticmethod + def _to_vtk(positions, colors): + vtk_positions = numpy_to_vtk(positions, deep=False) + vtk_colors = numpy_to_vtk(colors, deep=False) + vtk_colors.SetName("colors") + return positions, colors, vtk_positions, vtk_colors + + def _show(self, frame): + _, _, vtk_positions, vtk_colors = frame + # Modified() also covers a single frame, or the live mode, reusing the arrays. + vtk_positions.Modified() + vtk_colors.Modified() + points = self.polydata.GetPoints() + points.SetData(vtk_positions) + points.Modified() + self.polydata.point_data.SetScalars(vtk_colors) + + @property + def payload_bytes(self): + return self.n_points * BYTES_PER_POINT + + +def create_vtk_pipeline(polydata): + mapper = vtkPolyDataMapper() + mapper.SetInputData(polydata) + mapper.SetColorModeToDirectScalars() + mapper.SetScalarModeToUsePointData() + # The cloud stays around the origin, so float32 coordinates need no shift and scale: + # skipping it saves a pass over the points on every upload. + mapper.SetVBOShiftScaleMethod(DISABLE_SHIFT_SCALE) + + actor = vtkActor(mapper=mapper) + actor.property.point_size = 2 + + renderer = vtkRenderer(background=(0.1, 0.1, 0.12)) + renderer.AddActor(actor) + # The texture's alpha reaches Electron, so keep the background opaque. + renderer.SetBackgroundAlpha(1.0) + renderer.ResetCamera() + + render_window = vtkRenderWindow() + render_window.AddRenderer(renderer) + # MSAA costs more than drawing the points themselves, ~6 ms per frame at 1080p. + render_window.SetMultiSamples(0) + # trame-rca replays the canvas's input on the interactor. + vtkGenericRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window + + +class CameraSpinner: + """Turns the camera of a render window around its view up at a steady rate. + + The angle follows the elapsed time, so views rendering at different rates spin alike. + """ + + def __init__(self, render_window, deg_per_s): + renderer = render_window.GetRenderers().GetFirstRenderer() + self.camera = renderer.GetActiveCamera() + self.deg_per_s = deg_per_s + self._t_last = None + + def step(self): + t = time.perf_counter() + if self._t_last is not None: + self.camera.Azimuth(self.deg_per_s * (t - self._t_last)) + self._t_last = t + + def pause(self): + # Resume from where it stopped, without catching up on the pause. + self._t_last = None + + +# ----------------------------------------------------------------------------- +# Frame bookkeeping +# ----------------------------------------------------------------------------- + + +class Frame: + __slots__ = ["n_points", "t_start", "stages", "total_ms"] + + def __init__(self, n_points): + self.n_points = n_points + self.t_start = time.perf_counter() + self.stages = {} + self.total_ms = 0.0 + + def finalize(self, t_end): + """The frame ends when the next one starts: whatever is left is Electron's""" + self.total_ms = (t_end - self.t_start) * 1000 + self.stages["electron"] = max( + 0.0, + self.total_ms - self.stages["generate"] - self.stages["render"], + ) + + +class LevelStats: + """Aggregated frames recorded at one point count and number of shared textures""" + + def __init__(self, n_points, textures): + self.n_points = n_points + self.textures = textures + self.seen = 0 + self.count = 0 + self.total_ms = 0.0 + self.work_ms = 0.0 + + def add(self, frame): + self.seen += 1 + if self.seen <= WARMUP_FRAMES: + return + self.count += 1 + self.total_ms += frame.total_ms + self.work_ms += frame.stages["generate"] + + @property + def mean_ms(self): + return self.total_ms / self.count if self.count else 0.0 + + def row(self): + mean = self.mean_ms + mb = self.n_points * BYTES_PER_POINT / 1e6 + overhead = mean - self.work_ms / self.count + return { + "points": f"{self.n_points:,}", + "textures": self.textures, + "frames": self.count, + "ms": f"{mean:.1f}", + "ups": f"{1000 / mean:.1f}" if mean else "-", + "mbs": f"{mb / (overhead / 1000):.0f}" if overhead > 0 else "-", + } + + +def fit_capacity(levels): + """Least squares fit total_ms = a + b * n_points over the measured levels""" + pts = [(lv.n_points, lv.mean_ms) for lv in levels if lv.count >= 10] + if len(pts) < 2: + return None + n = np.array([p[0] for p in pts], dtype=float) + t = np.array([p[1] for p in pts], dtype=float) + b, a = np.polyfit(n, t, 1) + if b <= 0: + return None + + def capacity(ups): + return max(0, int((1000.0 / ups - a) / b)) + + return { + "fixed_ms": f"{a:.2f}", + "ns_per_point": f"{b * 1e6:.1f}", + "at30": f"{capacity(30):,}", + "at60": f"{capacity(60):,}", + } + + +# ----------------------------------------------------------------------------- +# Application +# ----------------------------------------------------------------------------- + + +class PointCloudBenchmark(TrameApp): + def __init__(self, server=None): + super().__init__(server) + + cli = self.server.cli + cli.add_argument("--points", type=int, default=100_000, help="Initial points") + cli.add_argument( + "--step", type=int, default=100_000, help="Points per +N click" + ) + cli.add_argument( + "--frames", + type=int, + default=8, + help="Frames computed ahead on resize, 0 to compute each one live", + ) + cli.add_argument( + "--target-fps", type=float, default=60.0, help="Cap on the frame rate" + ) + cli.add_argument( + "--textures", + type=int, + default=2, + help="Shared textures VTK renders into in turn, also changeable in the panel", + ) + cli.add_argument( + "--radii", + type=float, + nargs=3, + default=RADII, + metavar=("X", "Y", "Z"), + help="Spread of the ellipsoidal cloud along x, y and z", + ) + cli.add_argument( + "--second-view", + action="store_true", + help="Add a second view of the cloud and spin both cameras", + ) + args, _ = cli.parse_known_args() + + self.step_size = args.step + self.target_fps = args.target_fps + self.texture_count = args.textures + self.cloud = PointCloud(args.points, args.frames, radii=args.radii) + self.pending_points = None + self.running = True + self.frame = None + self.recent = [] + self.levels = [] + + self._timer_generate = profiler.Timer("rca.bench.generate") + self._render_start = 0.0 + + self.render_window = create_vtk_pipeline(self.cloud.polydata) + self.render_window.AddObserver("StartEvent", self._on_render_start) + self.render_window.AddObserver("EndEvent", self._on_render_end) + + # The second view shares the cloud: the first view's frames move its points. + self.second_render_window = None + self.spinners = [] + if args.second_view: + self.second_render_window = create_vtk_pipeline(self.cloud.polydata) + self.spinners = [ + CameraSpinner(self.render_window, SPIN_DEG_PER_S), + CameraSpinner(self.second_render_window, -SPIN_DEG_PER_S), + ] + self.second_view_handler = None + + self._build_ui() + self._new_level() + self.server.controller.on_server_ready.add(self._on_server_ready) + + # Frame loop ----------------------------------------------------------------- + + def _on_server_ready(self, **_): + asyncio.get_running_loop().create_task(self._publish_stats()) + + def before_render(self): + # Called by the scheduler before every frame: each frame asks for the next one. + t_now = time.perf_counter() + if self.frame is not None: + self.frame.finalize(t_now) + self.recent.append(self.frame) + del self.recent[:-ROLLING_FRAMES] + self.levels[-1].add(self.frame) + self.frame = None + + if not self.running: + return + + if self.pending_points is not None: + if self.pending_points != self.cloud.n_points: + self.cloud.resize(self.pending_points) + self._new_level() + self.pending_points = None + + if self.spinners: + self.spinners[0].step() + + frame = Frame(self.cloud.n_points) + with self._timer_generate: + self.cloud.animate() + frame.stages["generate"] = self._timer_generate.dt + self.frame = frame + self.view_handler.update() + + def before_render_second(self): + # The second view only spins: it picks up the cloud's latest points on its own. + if not self.running: + return + self.spinners[1].step() + self.second_view_handler.update() + + def _on_render_start(self, *_): + self._render_start = time.perf_counter() + + def _on_render_end(self, *_): + if self.frame is None: + return + # GL only queues the work: wait for the GPU here so that it counts as rendering. + # The scheduler's flush waits for the same thing right after, for free. + self.render_window.WaitForCompletion() + self.frame.stages["render"] = (time.perf_counter() - self._render_start) * 1000 + + # Controls ------------------------------------------------------------------- + + def add_points(self): + self._change_points(+self.step_size) + + def remove_points(self): + self._change_points(-self.step_size) + + def _change_points(self, delta): + base = self.pending_points or self.cloud.n_points + self.pending_points = max(1, base + delta) + self.state.n_points_pending = self.pending_points + + @change("textures") + def on_textures_change(self, textures, **_): + textures = int(textures) + if textures == self.texture_count: + return + # The scheduler rebuilds its ring of textures before the next frame. + self.texture_count = textures + self.view_handler.texture_count = textures + if self.second_view_handler: + self.second_view_handler.texture_count = textures + self._new_level() + + def toggle_running(self): + self.running = not self.running + self.state.running = self.running + if self.running: + self.view_handler.update() + if self.second_view_handler: + self.second_view_handler.update() + else: + for spinner in self.spinners: + spinner.pause() + + def _new_level(self): + if self.levels and self.levels[-1].count: + row = self.levels[-1].row() + print( + f"[bench] points={row['points']:>12} textures={row['textures']} " + f"updates={row['frames']:>5} " + f"update={row['ms']:>8} ms updates/s={row['ups']:>7} overhead MB/s={row['mbs']:>6}", + flush=True, + ) + if self.levels and not self.levels[-1].count: + self.levels.pop() + + # Coming back to a measured point count keeps adding to its row so the + # fit sees each count once. Warmup is skipped again after the resize. + key = (self.cloud.n_points, self.texture_count) + level = next( + (lv for lv in self.levels if (lv.n_points, lv.textures) == key), None + ) + if level is None: + level = LevelStats(*key) + else: + self.levels.remove(level) + level.seen = 0 + self.levels.append(level) + self.recent.clear() + + # Stats ------------------------------------------------------------------------ + + async def _publish_stats(self): + t_print = time.perf_counter() + while True: + await asyncio.sleep(UI_REFRESH_S) + if not self.recent: + continue + stats = self._stats() + with self.state: + self.state.update(stats) + # Also once a second on the console, to compare with app_native.py. + if time.perf_counter() - t_print >= 1.0: + t_print = time.perf_counter() + stages = " ".join( + f"{s['name']}={s['ms']:>7} ms" for s in stats["stages"] + ) + print( + f"points={stats['n_points']:>12} updates/s={stats['ups']:>7} " + f"update={stats['frame_ms']:>7} ms {stages}", + flush=True, + ) + + def _stats(self): + frames = self.recent + k = len(frames) + total = sum(f.total_ms for f in frames) / k + stage_ms = { + name: sum(f.stages.get(name, 0) for f in frames) / k for name, _ in STAGES + } + overhead = total - stage_ms["generate"] + payload_mb = self.cloud.payload_bytes / 1e6 + + return { + "n_points": f"{self.cloud.n_points:,}", + "n_points_pending": self.pending_points, + "payload_mb": f"{payload_mb:.1f}", + "ups": f"{1000 / total:.1f}", + "frame_ms": f"{total:.1f}", + "overhead_ms": f"{overhead:.1f}", + "overhead_mbs": f"{payload_mb / (overhead / 1000):.0f}" + if overhead > 0 + else "-", + "stages": [ + { + "name": name, + "help": help, + "ms": f"{stage_ms[name]:.2f}", + "pct": f"{100 * stage_ms[name] / total:.0f}", + } + for name, help in STAGES + ], + "levels": [ + lv.row() + for lv in sorted(self.levels, key=lambda lv: (lv.textures, lv.n_points)) + if lv.count + ], + "fit": fit_capacity( + [lv for lv in self.levels if lv.textures == self.texture_count] + ), + } + + # UI ----------------------------------------------------------------------------- + + def _build_ui(self): + self.state.update( + { + "running": True, + "n_points": f"{self.cloud.n_points:,}", + "n_points_pending": None, + "payload_mb": "-", + "ups": "-", + "frame_ms": "-", + "overhead_ms": "-", + "overhead_mbs": "-", + "stages": [], + "levels": [], + "textures": self.texture_count, + "fit": None, + } + ) + # Frame stats from the canvas, once a second: only the panel needs them, not the server. + self.state.client_only("stats") + self.state.stats = {"fps": 0, "frames": 0, "info": ""} + + with DivLayout(self.server) as self.ui: + self.ui.root.style = "height:100vh;position:relative;" + client.Style(CSS) + + with html.Div(style="position:absolute;inset:0;display:flex;"): + with html.Div(style="flex:1;min-width:0;position:relative;"): + view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + stats="stats = $event", + ) + self.view_handler = view.create_electron_handler( + self.render_window, + target_fps=self.target_fps, + texture_count=self.texture_count, + before_render=self.before_render, + ) + if self.second_render_window: + with html.Div( + style="flex:1;min-width:0;position:relative;border-left:1px solid #333;" + ): + second_view = rca.RemoteControlledArea( + display="shared-texture", + event_throttle_ms=10, + ) + self.second_view_handler = second_view.create_electron_handler( + self.second_render_window, + target_fps=self.target_fps, + texture_count=self.texture_count, + before_render=self.before_render_second, + ) + + with html.Div(classes="bench", style=PANEL_STYLE): + with html.Div(style="margin-bottom:0.5rem;"): + html.Button( + "{{ running ? 'Pause' : 'Resume' }}", + click=self.toggle_running, + ) + html.Button(f"−{self.step_size:,}", click=self.remove_points) + html.Button(f"+{self.step_size:,}", click=self.add_points) + html.Span("textures:", classes="muted", style="margin:0 6px;") + for n in range(1, 5): + html.Button( + str(n), + click=f"textures = {n}", + style=( + "{ fontWeight: textures === %d ? 'bold' : 'normal' }" + % n, + ), + ) + html.Span( + "→ {{ n_points_pending.toLocaleString() }} on next update", + v_if="n_points_pending", + classes="muted", + ) + + with html.Div(style="display:flex;gap:1.5rem;margin-bottom:0.5rem;"): + with html.Div(): + html.Div("{{ stats.fps }} fps", classes="big") + html.Div("client render rate", classes="muted") + with html.Div(): + html.Div("{{ ups }} updates/s", classes="big") + html.Div("{{ frame_ms }} ms / update", classes="muted") + with html.Div(): + html.Div("{{ n_points }} pts", classes="big") + html.Div("{{ payload_mb }} MB / update", classes="muted") + + html.Div( + "Overhead (update − app work): {{ overhead_ms }} ms " + "→ {{ overhead_mbs }} MB/s" + f", capped at {self.target_fps:g} updates/s" + ) + + with html.Table(): + with html.Tr(): + html.Th("stage") + html.Th("ms") + html.Th("%") + html.Th("", style="width:35%;") + with html.Tr(v_for="s in stages", key="s.name"): + with html.Td(classes="stage"): + html.Span("{{ s.name }} ") + html.Span("({{ s.help }})", classes="muted") + html.Td("{{ s.ms }}") + html.Td("{{ s.pct }}") + with html.Td(): + html.Div(classes="bar", style=("{ width: s.pct + '%' }",)) + + html.Div( + f"Per point count (first {WARMUP_FRAMES} updates skipped)", + classes="muted", + ) + with html.Table(): + with html.Tr(): + html.Th("points") + html.Th("textures") + html.Th("updates") + html.Th("ms") + html.Th("updates/s") + html.Th("MB/s") + with html.Tr( + v_for="l in levels", key="l.points + '/' + l.textures" + ): + html.Td("{{ l.points }}") + html.Td("{{ l.textures }}") + html.Td("{{ l.frames }}") + html.Td("{{ l.ms }}") + html.Td("{{ l.ups }}") + html.Td("{{ l.mbs }}") + + with html.Div(v_if="fit"): + html.Div( + "Fit with {{ textures }} textures: {{ fit.fixed_ms }} ms fixed " + "+ {{ fit.ns_per_point }} ns/point" + ) + html.Div( + "Realtime capacity: {{ fit.at30 }} pts @ 30 updates/s, " + "{{ fit.at60 }} pts @ 60 updates/s", + classes="big", + style="font-size:14px;", + ) + html.Div( + "Add points at least twice to fit the per-point cost", + v_else=True, + classes="muted", + ) + + +if __name__ == "__main__": + PointCloudBenchmark().server.start() diff --git a/examples/07_electron_point_cloud/app_native.py b/examples/07_electron_point_cloud/app_native.py new file mode 100644 index 0000000..c8b102b --- /dev/null +++ b/examples/07_electron_point_cloud/app_native.py @@ -0,0 +1,265 @@ +"""A dynamic point cloud rendered in a native desktop window without trame. + +Every frame moves each point randomly and assigns new random colors, then renders. +The frames are computed ahead, on resize: --frames 0 computes each one live instead. +Prints the updates/s and the time spent per stage to the console once a second, +as a baseline to compare the shared texture path against. + +Keys: + / - add or remove --step points, space pauses or resumes the animation. +""" + +import argparse +import time + +import numpy as np +from vtkmodules.util.numpy_support import numpy_to_vtk, numpy_to_vtkIdTypeArray +from vtkmodules.vtkCommonCore import vtkPoints +from vtkmodules.vtkCommonDataModel import vtkCellArray, vtkPolyData +from vtkmodules.vtkInteractionStyle import vtkInteractorStyleTrackballCamera +from vtkmodules.vtkRenderingCore import ( + vtkActor, + vtkPolyDataMapper, + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, +) + +# Required for vtk factory +import vtkmodules.vtkRenderingOpenGL2 # noqa: F401 + +BYTES_PER_POINT = 3 * 4 + 3 * 1 # float32 xyz + uint8 rgb +DISABLE_SHIFT_SCALE = 0 # vtkOpenGLVertexBufferObject::ShiftScaleMethod +RADII = (2.0, 1.0, 0.5) # standard deviation of the cloud along x, y and z + + +class PointCloud: + """vtkPolyData whose point and color arrays change every frame + + With ``n_frames`` > 1 the frames are computed once, on resize, and each update only + points the polydata at the next frame's arrays: the app work drops to ~0 while every + frame still uploads new data. They play back and forth so the loop has no jump. + With ``n_frames`` = 0 each update computes a new frame, in place. + The points spread out along x, y and z by ``radii``: an ellipsoidal cloud. + """ + + def __init__(self, n_points, n_frames=8, seed=0, radii=RADII): + self.rng = np.random.default_rng(seed) + self.radii = np.asarray(radii, dtype=np.float32) + self.polydata = vtkPolyData() + self.n_points = 0 + self.n_frames = n_frames + self.positions = np.zeros((0, 3), dtype=np.float32) + self.frames = [] + self._t = 0 + self.resize(n_points) + + def resize(self, n_points): + n_old = self.n_points + positions = np.empty((n_points, 3), dtype=np.float32) + positions[: min(n_old, n_points)] = self.positions[:n_points] + if n_points > n_old: + positions[n_old:] = self.rng.standard_normal( + (n_points - n_old, 3), dtype=np.float32 + ) + positions[n_old:] *= self.radii + colors = np.empty((n_points, 3), dtype=np.uint8) + self._noise = np.empty_like(positions) + self.n_points = n_points + + # Keep python references alive: the VTK arrays share numpy memory. + self.frames = [] + for _ in range(max(1, self.n_frames)): + self._walk(positions, colors) + self.frames.append(self._to_vtk(positions, colors)) + positions, colors = positions.copy(), colors.copy() + self.positions = self.frames[-1][0] + self.colors = self.frames[-1][1] + self._t = 0 + + # A single polyvertex cell with every point. The mapper rebuilds its index buffer + # whenever the points change, and that is about twice as fast as from one vertex + # cell per point. + verts = vtkCellArray() + verts.SetData( + numpy_to_vtkIdTypeArray(np.array([0, n_points], dtype=np.int64), deep=True), + numpy_to_vtkIdTypeArray(np.arange(n_points, dtype=np.int64), deep=True), + ) + + self.polydata.SetPoints(vtkPoints()) + self.polydata.SetVerts(verts) + self._show(self.frames[-1]) + self.polydata.Modified() + + def animate(self): + if self.n_frames == 0: + self._walk(self.positions, self.colors) + self._show(self.frames[0]) + return + # Ping-pong: 0, 1, ..., K-1, K-2, ..., 1, 0, 1, ... + k = len(self.frames) + period = max(1, 2 * k - 2) + i = self._t % period + self._t += 1 + self._show(self.frames[i if i < k else period - i]) + + def _walk(self, positions, colors, step=0.02, pull=0.995): + # Random walk with a weak pull toward the origin so the cloud stays put. The steps + # scale with the radii too, so the walk keeps the cloud's shape. + self.rng.standard_normal(out=self._noise, dtype=np.float32) + self._noise *= step * self.radii + positions *= pull + positions += self._noise + colors.reshape(-1)[:] = np.frombuffer(self.rng.bytes(colors.size), np.uint8) + + @staticmethod + def _to_vtk(positions, colors): + vtk_positions = numpy_to_vtk(positions, deep=False) + vtk_colors = numpy_to_vtk(colors, deep=False) + vtk_colors.SetName("colors") + return positions, colors, vtk_positions, vtk_colors + + def _show(self, frame): + _, _, vtk_positions, vtk_colors = frame + # Modified() also covers a single frame, or the live mode, reusing the arrays. + vtk_positions.Modified() + vtk_colors.Modified() + points = self.polydata.GetPoints() + points.SetData(vtk_positions) + points.Modified() + self.polydata.point_data.SetScalars(vtk_colors) + + @property + def payload_bytes(self): + return self.n_points * BYTES_PER_POINT + + +def create_vtk_pipeline(polydata): + mapper = vtkPolyDataMapper() + mapper.SetInputData(polydata) + mapper.SetColorModeToDirectScalars() + mapper.SetScalarModeToUsePointData() + # The cloud stays around the origin, so float32 coordinates need no shift and scale: + # skipping it saves a pass over the points on every upload. + mapper.SetVBOShiftScaleMethod(DISABLE_SHIFT_SCALE) + + actor = vtkActor(mapper=mapper) + actor.property.point_size = 2 + + renderer = vtkRenderer(background=(0.1, 0.1, 0.12)) + renderer.AddActor(actor) + renderer.ResetCamera() + + render_window = vtkRenderWindow(size=(1024, 768)) + # MSAA costs more than drawing the points themselves, ~6 ms per frame at 1080p. + render_window.SetMultiSamples(0) + render_window.SetWindowName("Dynamic point cloud native example") + render_window.AddRenderer(renderer) + interactor = vtkRenderWindowInteractor( + render_window=render_window, + interactor_style=vtkInteractorStyleTrackballCamera(), + ) + return render_window, interactor + + +class Stats: + """Accumulates per-frame stage timings and prints their mean once a second""" + + def __init__(self): + self.reset() + + def reset(self): + self.frames = 0 + self.generate_ms = 0.0 + self.render_ms = 0.0 + self.start = time.perf_counter() + + def add(self, generate_ms, render_ms, cloud): + self.frames += 1 + self.generate_ms += generate_ms + self.render_ms += render_ms + elapsed = time.perf_counter() - self.start + if elapsed < 1.0: + return + n = self.frames + frame_ms = 1000 * elapsed / n + render_ms = self.render_ms / n + payload_mb = cloud.payload_bytes / 1e6 + print( + f"points={cloud.n_points:>12,} updates/s={n / elapsed:>7.1f} " + f"update={frame_ms:>7.1f} ms generate={self.generate_ms / n:>7.2f} ms " + f"render={render_ms:>7.2f} ms ({payload_mb:.1f} MB/update" + + (f", {payload_mb / (render_ms / 1000):.0f} MB/s" if render_ms else "") + + ")", + flush=True, + ) + self.reset() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--points", type=int, default=100_000, help="Initial points") + parser.add_argument( + "--step", type=int, default=100_000, help="Points per + or - key press" + ) + parser.add_argument( + "--frames", + type=int, + default=8, + help="Frames computed ahead on resize, 0 to compute each one live", + ) + parser.add_argument( + "--radii", + type=float, + nargs=3, + default=RADII, + metavar=("X", "Y", "Z"), + help="Spread of the ellipsoidal cloud along x, y and z", + ) + parser.add_argument( + "--no-vsync", action="store_true", help="Don't cap the fps at the display rate" + ) + args = parser.parse_args() + + cloud = PointCloud(args.points, args.frames, radii=args.radii) + render_window, interactor = create_vtk_pipeline(cloud.polydata) + interactor.Initialize() + if args.no_vsync: + # Needs a created window, so after Initialize(). + render_window.SetSwapControl(0) + + stats = Stats() + running = True + + def step(*_): + if not running: + return + t0 = time.perf_counter() + cloud.animate() + t1 = time.perf_counter() + render_window.Render() + t2 = time.perf_counter() + stats.add(1000 * (t1 - t0), 1000 * (t2 - t1), cloud) + + def on_key(obj, _): + nonlocal running + key = obj.GetKeySym() + if key in ("plus", "equal", "KP_Add"): + cloud.resize(cloud.n_points + args.step) + elif key in ("minus", "KP_Subtract"): + cloud.resize(max(1, cloud.n_points - args.step)) + elif key == "space": + running = not running + else: + return + stats.reset() + render_window.Render() + + interactor.AddObserver("KeyPressEvent", on_key) + interactor.AddObserver("TimerEvent", step) + interactor.CreateRepeatingTimer(1) + render_window.Render() + interactor.Start() + + +if __name__ == "__main__": + main() diff --git a/examples/07_electron_point_cloud/electron/.gitignore b/examples/07_electron_point_cloud/electron/.gitignore new file mode 100644 index 0000000..504afef --- /dev/null +++ b/examples/07_electron_point_cloud/electron/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/examples/07_electron_point_cloud/electron/main.js b/examples/07_electron_point_cloud/electron/main.js new file mode 100644 index 0000000..05ae456 --- /dev/null +++ b/examples/07_electron_point_cloud/electron/main.js @@ -0,0 +1,26 @@ +import { app, BrowserWindow } from 'electron/main'; +import { fileURLToPath } from 'node:url'; +import { launchTrame, webPreferences } from '@kitware/trame-rca-electron'; + +// Run ../app.py with the python of the active environment, unless PYTHON says otherwise. +const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); + +// Forward the arguments after `npm start --` to app.py, e.g. --points 500000. +const appArgs = process.argv.slice(app.isPackaged ? 1 : 2); + +app.whenReady().then(() => { + const win = new BrowserWindow({ + width: 1920, + height: 1080, + useContentSize: true, + webPreferences: webPreferences(), + }); + launchTrame(win, { + command: python, + args: [fileURLToPath(new URL('../app.py', import.meta.url)), ...appArgs], + onStats: (s) => console.log(`[renderer] ${s.view} frames=${s.frames} fps=${s.fps} ${s.info}`), + onExit: () => app.quit(), + }); +}); + +app.on('window-all-closed', () => app.quit()); diff --git a/examples/07_electron_point_cloud/electron/package.json b/examples/07_electron_point_cloud/electron/package.json new file mode 100644 index 0000000..cc16118 --- /dev/null +++ b/examples/07_electron_point_cloud/electron/package.json @@ -0,0 +1,18 @@ +{ + "name": "trame-rca-electron-point-cloud-example", + "private": true, + "version": "0.0.0", + "description": "Electron shell for the shared texture dynamic point cloud trame-rca example", + "license": "Apache-2.0", + "type": "module", + "main": "main.js", + "scripts": { + "start": "electron ." + }, + "dependencies": { + "@kitware/trame-rca-electron": "file:../../../electron" + }, + "devDependencies": { + "electron": "^44.7.0" + } +} diff --git a/js-components/src/style.css b/js-components/src/style.css index c2ba355..69f0940 100644 --- a/js-components/src/style.css +++ b/js-components/src/style.css @@ -42,6 +42,16 @@ position: relative; } +.shared-texture-display-area { + display: block; + margin: 0; + outline: none; + padding: 0; + position: relative; + width: 100%; + height: 100%; +} + .image-region { margin: 0; padding: 0; diff --git a/pyproject.toml b/pyproject.toml index c2ccc07..eb32b24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = [ "packaging", "pillow", ] -requires-python = ">=3.9" +requires-python = ">=3.10" readme = "README.rst" license = {text = "Apache License 2.0"} keywords = ["Python", "Interactive", "Web", "Application", "Framework"] @@ -51,6 +51,9 @@ vtkstreaming = [ "vtk", "vtk-streaming>=0.4.0", ] +electron = [ + "vtk", # Shared textures with trame-rca-electron +] diff --git a/src/trame_rca/electron/__init__.py b/src/trame_rca/electron/__init__.py new file mode 100644 index 0000000..e427587 --- /dev/null +++ b/src/trame_rca/electron/__init__.py @@ -0,0 +1,50 @@ +"""Render VTK into a texture shared with an Electron app, instead of streaming images. + +The Electron side lives in the trame-rca-electron npm package: it spawns the trame +server with ``--emp-port`` and shows the shared texture in the page. + +Call ``trame_rca.widgets.init_vtk_for_shared_textures()`` before creating any VTK render +window, then use +``RemoteControlledArea(display="shared-texture").create_electron_handler(...)``. +""" + +import argparse + +from .comm import PROTOCOL_VERSION + +__all__ = [ + "EMP_PORT_ARG", + "PROTOCOL_VERSION", + "emp_port", + "server_url", +] + +# The CLI argument that trame-rca-electron passes the port of its `texture` channel. +EMP_PORT_ARG = "--emp-port" + + +def emp_port(server): + """The port of the Electron main process's `texture` channel, from the CLI. + + Raises RuntimeError when the server wasn't started by trame-rca-electron. + """ + try: + server.cli.add_argument( + EMP_PORT_ARG, + type=int, + help="Port of the Electron main process's texture channel", + ) + except argparse.ArgumentError: + pass # Another view added it already. + args, _ = server.cli.parse_known_args() + if args.emp_port is None: + raise RuntimeError( + f"{EMP_PORT_ARG} is missing: shared-texture views only work when " + "the trame server is started by trame-rca-electron" + ) + return args.emp_port + + +def server_url(server): + """The URL of the page trame serves, once the server is ready.""" + return f"http://{server.server_options.host}:{server.port}/" diff --git a/src/trame_rca/electron/comm.py b/src/trame_rca/electron/comm.py new file mode 100644 index 0000000..fe80cdf --- /dev/null +++ b/src/trame_rca/electron/comm.py @@ -0,0 +1,195 @@ +""" +Talks newline-delimited JSON to the Electron main process (EMP) on 127.0.0.1:. + +-> means python to EMP +<- means EMP to Python + + <- {"type":"hello", "protocol": N, "passport": name} at start, the texture passport to use + -> {"type":"server_ready", "protocol": N, "url": url} once, when trame serves its page + -> {"type":"texture", "view": v, "slot": i, "slots": k, ...} + at start, after every resize and + change of texture count, one per slot + -> {"type":"frame", "view": v, "seq": n, "slot": i, "timestamp_us": t} + after every render into slot i + (finished on the GPU) + <- {"type":"frame_done", "view": v, "seq": n} when Electron released the texture + <- {"type":"frame_done", "view": v, "seq": n, "dropped": true} + when the frame didn't reach the ERP + +Every shared-texture view of the server talks over the same connection: `view` is the name of +its RemoteControlledArea, and `seq` and `slot` count per view. +The k textures of a view form a ring: several frames can be in flight, one per slot. Slots past +k are gone. + +Everything else (size, input, app state) goes from the renderer process to trame over its websocket. + +PROTOCOL_VERSION must match the one of the trame-rca-electron npm package. +""" + +import asyncio +import json +import logging +import socket +from typing import Callable + +logger = logging.getLogger(__name__) + +PROTOCOL_VERSION = 1 + + +class ProtocolMismatchError(RuntimeError): + pass + + +def check_protocol(msg): + """Raise ProtocolMismatchError unless the EMP's hello speaks PROTOCOL_VERSION.""" + version = msg.get("protocol") + if version != PROTOCOL_VERSION: + raise ProtocolMismatchError( + f"Electron speaks shared texture protocol {version}, trame-rca speaks " + f"{PROTOCOL_VERSION}: update trame-rca or trame-rca-electron so they match" + ) + + +class LineConnection: + def __init__(self, reader, writer): + self.reader = reader + self.writer = writer + # Several views send on the same connection. + self._send_lock = asyncio.Lock() + + @classmethod + async def open(cls, port, host="127.0.0.1"): + reader, writer = await asyncio.open_connection(host, port) + # Small request/response messages: don't let Nagle wait for delayed ACKs. + writer.get_extra_info("socket").setsockopt( + socket.IPPROTO_TCP, socket.TCP_NODELAY, 1 + ) + return cls(reader, writer) + + async def send(self, msg): + async with self._send_lock: + self.writer.write((json.dumps(msg) + "\n").encode()) + await self.writer.drain() + + async def recv(self): + """Return the next message. Raises EOFError on close.""" + line = await self.reader.readline() + if not line: + raise EOFError + return json.loads(line) + + def close(self): + self.writer.close() + + +class EmpClient: + """The connection to the EMP, shared by all the shared-texture views of the server. + + The EMP takes a single connection per window, so views register here by name and get + the frame_done messages of their own frames. + """ + + # The open clients, by EMP port. + _clients: dict[int, "EmpClient"] = {} + + @classmethod + def for_port(cls, port: int) -> "EmpClient": + """The client connected (or connecting) to the EMP on ``port``.""" + client = cls._clients.get(port) + if client is None: + client = cls._clients[port] = cls(port) + return client + + def __init__(self, port: int): + self.port = port + self._conn: LineConnection | None = None + self._connect_task: asyncio.Task | None = None + self._read_task: asyncio.Task | None = None + # view name: (on_frame_done(msg), on_disconnect()) + self._views: dict[str, tuple[Callable[[dict], None], Callable[[], None]]] = {} + self._server_ready_sent = False + self._closed = False + + def add_view( + self, + view: str, + on_frame_done: Callable[[dict], None], + on_disconnect: Callable[[], None], + ): + if view in self._views: + raise ValueError(f"a shared-texture view named {view!r} already exists") + self._views[view] = (on_frame_done, on_disconnect) + + def remove_view(self, view: str): + """Forget ``view``. The connection closes with the last view.""" + self._views.pop(view, None) + if not self._views: + self.close() + + async def connect(self) -> str: + """Connect to the EMP once, and return the texture passport name of its hello.""" + if self._connect_task is None: + self._connect_task = asyncio.ensure_future(self._connect()) + # shield: one view giving up must not cancel the connection of the others. + return await asyncio.shield(self._connect_task) + + async def _connect(self): + try: + self._conn = await LineConnection.open(self.port) + logger.info("connected to the Electron main process on port %d", self.port) + try: + msg = await self._conn.recv() + except EOFError: + raise ConnectionError("disconnected before the hello message") from None + if msg.get("type") != "hello": + raise ConnectionError(f"expected a hello message, got {msg!r}") + check_protocol(msg) + except Exception: + self.close() + raise + self._read_task = asyncio.ensure_future(self._read()) + return msg["passport"] + + async def server_ready(self, url: str | None): + """Tell the EMP which page to load. Only the first call sends it.""" + if self._server_ready_sent: + return + self._server_ready_sent = True + await self.send( + {"type": "server_ready", "protocol": PROTOCOL_VERSION, "url": url} + ) + + async def send(self, msg): + assert self._conn is not None, "Connection to EMP is uninitialized!" + await self._conn.send(msg) + + async def _read(self): + assert self._conn is not None, "Connection to EMP is uninitialized!" + try: + while True: + msg = await self._conn.recv() + if msg["type"] == "frame_done": + callbacks = self._views.get(msg.get("view")) + if callbacks: + callbacks[0](msg) + except (EOFError, ConnectionError): + logger.info("the Electron main process disconnected") + self.close() + + def close(self): + """Close the connection and tell every view it is gone.""" + if self._closed: + return + self._closed = True + if EmpClient._clients.get(self.port) is self: + del EmpClient._clients[self.port] + current = asyncio.current_task() + for task in (self._read_task, self._connect_task): + if task and task is not current: + task.cancel() + if self._conn: + self._conn.close() + views, self._views = self._views, {} + for _, on_disconnect in views.values(): + on_disconnect() diff --git a/src/trame_rca/electron/passports/__init__.py b/src/trame_rca/electron/passports/__init__.py new file mode 100644 index 0000000..a31eefa --- /dev/null +++ b/src/trame_rca/electron/passports/__init__.py @@ -0,0 +1,29 @@ +"""Platform specific ways to share a VTK texture with Electron.""" + +from .base import TexturePassport + +__all__ = [ + "TexturePassport", + "texture_passport_class", +] + + +def texture_passport_class(name): + """The name of the passport class that the EMP wants to use.""" + if not isinstance(name, str): + raise TypeError("Passport name must be a string") + if name == "iosurface": + from .iosurface import IOSurfaceTexturePassport + + return IOSurfaceTexturePassport + if name == "nthandle": + from .nthandle import NTHandleTexturePassport + + return NTHandleTexturePassport + if name == "wayland-dmabuf": + from .wayland_dmabuf import WaylandDMABUFTexturePassport + + return WaylandDMABUFTexturePassport + if name == "x11-dmabuf": + raise NotImplementedError("x11-dmabuf passport is not supported yet") + raise ValueError(f"Passport name '{name}' is not recognized") diff --git a/src/trame_rca/electron/passports/base.py b/src/trame_rca/electron/passports/base.py new file mode 100644 index 0000000..e8a77b4 --- /dev/null +++ b/src/trame_rca/electron/passports/base.py @@ -0,0 +1,42 @@ +GL_FRAMEBUFFER = 0x8D40 + + +class TexturePassport: + """A cross-process abstraction for letting textures travel from Python to Electron. + The passport has everything needed to access VTK render window's texture from another process. + """ + + # The vtkTextureObject VTK resolves its frames into, set by the subclasses. + vtk_texture = None + + def __init__(self, render_window, width, height): + self.render_window = render_window + self.width = width + self.height = height + + def to_json(self): + """The ``texture`` message sent to the Electron main process.""" + return {} + + def attach(self): + """Make this texture the one the render window resolves its next frames into. + + A passport attaches its texture when created: only needed when several take turns. + """ + if self.vtk_texture is None: + return + self.render_window.MakeCurrent() + display_fbo = self.render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, self.vtk_texture) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the shared texture") + + def release(self): + """Free the texture and any OS handle to it.""" + + async def flush_graphics_commands(self): + """Wait until the GPU finished rendering into the texture.""" + self.render_window.WaitForCompletion() diff --git a/src/trame_rca/electron/passports/iosurface.py b/src/trame_rca/electron/passports/iosurface.py new file mode 100644 index 0000000..5451147 --- /dev/null +++ b/src/trame_rca/electron/passports/iosurface.py @@ -0,0 +1,176 @@ +import ctypes +import logging +import os + +from .base import TexturePassport +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +vp = ctypes.c_void_p + +# --- CoreFoundation / IOSurface ---------------------------------------------- +cf = ctypes.CDLL("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation") +cf.CFNumberCreate.restype = vp +cf.CFNumberCreate.argtypes = [vp, ctypes.c_long, vp] +cf.CFDictionaryCreateMutable.restype = vp +cf.CFDictionaryCreateMutable.argtypes = [vp, ctypes.c_long, vp, vp] +cf.CFDictionarySetValue.argtypes = [vp, vp, vp] +cf.CFRelease.argtypes = [vp] +kCFNumberSInt32Type = 3 +kCFBooleanTrue = vp.in_dll(cf, "kCFBooleanTrue") + +iosurface = ctypes.CDLL("/System/Library/Frameworks/IOSurface.framework/IOSurface") +iosurface.IOSurfaceCreate.restype = vp +iosurface.IOSurfaceCreate.argtypes = [vp] +iosurface.IOSurfaceGetID.restype = ctypes.c_uint32 +iosurface.IOSurfaceGetID.argtypes = [vp] +iosurface.IOSurfaceGetBytesPerRow.restype = ctypes.c_size_t +iosurface.IOSurfaceGetBytesPerRow.argtypes = [vp] + +# --- CGL / GL ------------------------------------------------------------------ +cgl = ctypes.CDLL("/System/Library/Frameworks/OpenGL.framework/OpenGL") +cgl.CGLGetCurrentContext.restype = vp +cgl.CGLErrorString.restype = ctypes.c_char_p +cgl.CGLErrorString.argtypes = [ctypes.c_int] +cgl.CGLTexImageIOSurface2D.argtypes = [ + vp, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_int32, + ctypes.c_int32, + ctypes.c_uint32, + ctypes.c_uint32, + vp, + ctypes.c_uint32, +] +cgl.glGenTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +cgl.glDeleteTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +cgl.glBindTexture.argtypes = [ctypes.c_uint32, ctypes.c_uint32] +cgl.glGetString.restype = ctypes.c_char_p +cgl.glGetString.argtypes = [ctypes.c_uint32] + +GL_VERSION = 0x1F02 +GL_RENDERER = 0x1F01 +GL_TEXTURE_RECTANGLE = 0x84F5 +GL_RGBA = 0x1908 +GL_BGRA = 0x80E1 +GL_UNSIGNED_INT_8_8_8_8_REV = 0x8367 +GL_FRAMEBUFFER = 0x8D40 + +# IOSurface pixel format 'BGRA' -> electron pixelFormat +PIXEL_FORMAT_BGRA = 0x42475241 + + +def check(err, what): + if err != 0: + raise RuntimeError(f"{what} failed: {cgl.CGLErrorString(err).decode()} ({err})") + + +def _iosurface_key(name): + return vp.in_dll(iosurface, name) + + +def _create_surface(width, height): + """A BGRA IOSurface that other processes can open with IOSurfaceLookup(id).""" + props = cf.CFDictionaryCreateMutable( + None, + 0, + ctypes.addressof(ctypes.c_char.in_dll(cf, "kCFTypeDictionaryKeyCallBacks")), + ctypes.addressof(ctypes.c_char.in_dll(cf, "kCFTypeDictionaryValueCallBacks")), + ) + for key, value in ( + ("kIOSurfaceWidth", width), + ("kIOSurfaceHeight", height), + ("kIOSurfaceBytesPerElement", 4), + ("kIOSurfacePixelFormat", PIXEL_FORMAT_BGRA), + ): + v = ctypes.c_int32(value) + number = cf.CFNumberCreate(None, kCFNumberSInt32Type, ctypes.byref(v)) + cf.CFDictionarySetValue(props, _iosurface_key(key), number) + cf.CFRelease(number) + # Deprecated, but the only way to share an IOSurface without passing a Mach port. + cf.CFDictionarySetValue(props, _iosurface_key("kIOSurfaceIsGlobal"), kCFBooleanTrue) + surface = iosurface.IOSurfaceCreate(props) + cf.CFRelease(props) + if not surface: + raise RuntimeError(f"IOSurfaceCreate failed for {width}x{height}") + return surface + + +def _attach_surface(render_window, surface, width, height): + """Make the IOSurface the color attachment 0 of the window's DisplayFramebuffer.""" + render_window.MakeCurrent() + ctx = cgl.CGLGetCurrentContext() + + # CGL only binds IOSurfaces to rectangle textures. + handle = ctypes.c_uint32() + cgl.glGenTextures(1, ctypes.byref(handle)) + cgl.glBindTexture(GL_TEXTURE_RECTANGLE, handle.value) + check( + cgl.CGLTexImageIOSurface2D( + ctx, + GL_TEXTURE_RECTANGLE, + GL_RGBA, + width, + height, + GL_BGRA, + GL_UNSIGNED_INT_8_8_8_8_REV, + surface, + 0, + ), + "CGLTexImageIOSurface2D", + ) + cgl.glBindTexture(GL_TEXTURE_RECTANGLE, 0) + + tex = vtkTextureObject() + tex.SetContext(render_window) + tex.AssignToExistingTexture(handle.value, GL_TEXTURE_RECTANGLE) + # Only records the size: an assigned texture has no dimensions, so Resize() never + # calls glTexImage2D (which would detach the IOSurface). The DisplayFramebuffer + # resizes its attachments on every Start(), and this keeps that a no-op too. + tex.Resize(width, height) + + display_fbo = render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError( + "DisplayFramebuffer incomplete with the IOSurface attachment" + ) + return tex, handle + + +class IOSurfaceTexturePassport(TexturePassport): + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + self.surface = _create_surface(width, height) + self.vtk_texture, self.handle = _attach_surface( + render_window, self.surface, width, height + ) + self.surface_id = iosurface.IOSurfaceGetID(self.surface) + logger.info( + f"IOSurface id={self.surface_id} {width}x{height} BGRA " + f"stride={iosurface.IOSurfaceGetBytesPerRow(self.surface)}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "pixelFormat": "bgra", + "ioSurfaceId": self.surface_id, + } + + def release(self): + self.render_window.MakeCurrent() + cgl.glDeleteTextures(1, ctypes.byref(self.handle)) + cf.CFRelease(self.surface) + + async def flush_graphics_commands(self): + # no need to call superclass method since we already call glFinish. + cgl.glFinish() diff --git a/src/trame_rca/electron/passports/nthandle.py b/src/trame_rca/electron/passports/nthandle.py new file mode 100644 index 0000000..c4da960 --- /dev/null +++ b/src/trame_rca/electron/passports/nthandle.py @@ -0,0 +1,493 @@ +"""Render into a D3D11 texture (WGL_NV_DX_interop2) and share it with Electron by NT handle (Windows). + +The vtkOpenGLRenderWindow resolves its RenderFramebuffer into the DisplayFramebuffer in Frame(). +We swap the DisplayFramebuffer's color attachment for a GL texture that aliases a D3D11 texture +through WGL_NV_DX_interop2. Each frame is then copied into a texture shared by NT handle. +""" + +import asyncio +import ctypes +import logging +import os +from ctypes import wintypes + +from .base import TexturePassport +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +vp = ctypes.c_void_p +HRESULT = ctypes.c_long + + +def check_hr(hr, what): + if hr < 0: + raise RuntimeError(f"{what} failed: HRESULT 0x{hr & 0xFFFFFFFF:08x}") + + +class GUID(ctypes.Structure): + _fields_ = [ + ("Data1", ctypes.c_uint32), + ("Data2", ctypes.c_uint16), + ("Data3", ctypes.c_uint16), + ("Data4", ctypes.c_ubyte * 8), + ] + + def __init__(self, s): + a, b, c, d, e = s.split("-") + super().__init__( + int(a, 16), + int(b, 16), + int(c, 16), + (ctypes.c_ubyte * 8)(*bytes.fromhex(d + e)), + ) + + +IID_IDXGIFactory1 = GUID("770aae78-f26f-4dba-a829-253c83d1b387") +IID_IDXGIResource1 = GUID("30961379-4609-4a41-998e-54fe567ee0c1") +IID_ID3D11Device5 = GUID("8ffde202-a0e7-45df-9e01-e837801b5ea0") +IID_ID3D11DeviceContext4 = GUID("917600da-f58c-4c33-98d8-3e15b390fa24") +IID_ID3D11Fence = GUID("affde9d1-1df7-4bb7-8a34-0f46251dab80") + + +def com_call(obj, index, restype, *args): + """Call method `index` of a COM object's vtable (indices from the Windows SDK headers).""" + vtbl = ctypes.cast(obj, ctypes.POINTER(ctypes.POINTER(vp))).contents + fn = ctypes.WINFUNCTYPE( + restype, + vp, + *(type(a) if isinstance(a, ctypes._SimpleCData) else vp for a in args), + )(vtbl[index]) + return fn(obj, *args) + + +def com_release(obj): + com_call(obj, 2, ctypes.c_ulong) # IUnknown::Release + + +# --- D3D11 / DXGI ---------------------------------------------------------------- +dxgi = ctypes.WinDLL("dxgi") +d3d11 = ctypes.WinDLL("d3d11") +kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) +kernel32.CloseHandle.argtypes = [wintypes.HANDLE] +kernel32.CreateEventW.restype = wintypes.HANDLE +kernel32.CreateEventW.argtypes = [vp, wintypes.BOOL, wintypes.BOOL, wintypes.LPCWSTR] +kernel32.WaitForSingleObject.restype = wintypes.DWORD +kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] + + +class LUID(ctypes.Structure): + _fields_ = [("LowPart", wintypes.DWORD), ("HighPart", wintypes.LONG)] + + +class DXGI_ADAPTER_DESC1(ctypes.Structure): + _fields_ = [ + ("Description", ctypes.c_wchar * 128), + ("VendorId", ctypes.c_uint), + ("DeviceId", ctypes.c_uint), + ("SubSysId", ctypes.c_uint), + ("Revision", ctypes.c_uint), + ("DedicatedVideoMemory", ctypes.c_size_t), + ("DedicatedSystemMemory", ctypes.c_size_t), + ("SharedSystemMemory", ctypes.c_size_t), + ("AdapterLuid", LUID), + ("Flags", ctypes.c_uint), + ] + + +class D3D11_TEXTURE2D_DESC(ctypes.Structure): + _fields_ = [ + ("Width", ctypes.c_uint), + ("Height", ctypes.c_uint), + ("MipLevels", ctypes.c_uint), + ("ArraySize", ctypes.c_uint), + ("Format", ctypes.c_uint), + ("SampleCount", ctypes.c_uint), + ("SampleQuality", ctypes.c_uint), + ("Usage", ctypes.c_uint), + ("BindFlags", ctypes.c_uint), + ("CPUAccessFlags", ctypes.c_uint), + ("MiscFlags", ctypes.c_uint), + ] + + +DXGI_FORMAT_B8G8R8A8_UNORM = 87 +D3D11_BIND_SHADER_RESOURCE = 0x8 +D3D11_BIND_RENDER_TARGET = 0x20 +D3D11_RESOURCE_MISC_SHARED = 0x2 +D3D11_RESOURCE_MISC_SHARED_NTHANDLE = 0x800 +D3D11_CREATE_DEVICE_BGRA_SUPPORT = 0x20 +D3D11_SDK_VERSION = 7 +D3D_DRIVER_TYPE_UNKNOWN = 0 +D3D11_FENCE_FLAG_NONE = 0 +WAIT_OBJECT_0 = 0 +DXGI_SHARED_RESOURCE_READ = 0x80000000 +DXGI_SHARED_RESOURCE_WRITE = 1 + +# vtable indices +IDXGIFactory1_EnumAdapters1 = 12 +IDXGIAdapter1_GetDesc1 = 10 +ID3D11Device_CreateTexture2D = 5 +ID3D11Device5_CreateFence = 68 +ID3D11DeviceContext_CopyResource = 47 +ID3D11DeviceContext_Flush = 111 +ID3D11DeviceContext4_Signal = 147 +ID3D11Fence_SetEventOnCompletion = 9 +IDXGIResource1_CreateSharedHandle = 13 + + +def _create_texture(device, width, height, misc_flags=0): + desc = D3D11_TEXTURE2D_DESC( + Width=width, + Height=height, + MipLevels=1, + ArraySize=1, + Format=DXGI_FORMAT_B8G8R8A8_UNORM, + SampleCount=1, + BindFlags=D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE, + MiscFlags=misc_flags, + ) + texture = vp() + check_hr( + com_call( + device, + ID3D11Device_CreateTexture2D, + HRESULT, + ctypes.byref(desc), + None, + ctypes.byref(texture), + ), + "CreateTexture2D", + ) + return texture + + +def _create_shared_texture(device, width, height): + """A BGRA texture with an NT handle that Chromium can open.""" + texture = _create_texture( + device, + width, + height, + D3D11_RESOURCE_MISC_SHARED | D3D11_RESOURCE_MISC_SHARED_NTHANDLE, + ) + resource = vp() + check_hr( + com_call( + texture, + 0, + HRESULT, + ctypes.byref(IID_IDXGIResource1), + ctypes.byref(resource), + ), + "QueryInterface(IDXGIResource1)", + ) + handle = wintypes.HANDLE() + check_hr( + com_call( + resource, + IDXGIResource1_CreateSharedHandle, + HRESULT, + None, + wintypes.DWORD(DXGI_SHARED_RESOURCE_READ | DXGI_SHARED_RESOURCE_WRITE), + None, + ctypes.byref(handle), + ), + "CreateSharedHandle", + ) + com_release(resource) + return texture, handle.value + + +# --- WGL / GL -------------------------------------------------------------------- +# VTK owns the WGL context; this only registers the D3D11 texture with it. +# Everything framebuffer-related goes through VTK so its vtkOpenGLState cache stays in sync. +opengl32 = ctypes.WinDLL("opengl32") +opengl32.wglGetProcAddress.restype = vp +opengl32.wglGetProcAddress.argtypes = [ctypes.c_char_p] +opengl32.glGenTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] +opengl32.glDeleteTextures.argtypes = [ctypes.c_int32, ctypes.POINTER(ctypes.c_uint32)] + +GL_TEXTURE_2D = 0x0DE1 +GL_FRAMEBUFFER = 0x8D40 +WGL_ACCESS_WRITE_DISCARD_NV = 0x0002 + + +def _wgl_func(name, restype, *argtypes): + addr = opengl32.wglGetProcAddress(name.encode()) + if not addr: + raise RuntimeError( + f"{name} missing: the GL driver doesn't support WGL_NV_DX_interop2" + ) + return ctypes.WINFUNCTYPE(restype, *argtypes)(addr) + + +class DXInterop: + """WGL_NV_DX_interop2 entry points; needs a current WGL context to resolve.""" + + def __init__(self): + H = wintypes.HANDLE + self.open_device = _wgl_func("wglDXOpenDeviceNV", H, vp) + self.close_device = _wgl_func("wglDXCloseDeviceNV", wintypes.BOOL, H) + self.register = _wgl_func( + "wglDXRegisterObjectNV", + H, + H, + vp, + ctypes.c_uint32, + ctypes.c_uint32, + ctypes.c_uint32, + ) + self.unregister = _wgl_func("wglDXUnregisterObjectNV", wintypes.BOOL, H, H) + self.lock = _wgl_func( + "wglDXLockObjectsNV", wintypes.BOOL, H, ctypes.c_int32, ctypes.POINTER(H) + ) + self.unlock = _wgl_func( + "wglDXUnlockObjectsNV", wintypes.BOOL, H, ctypes.c_int32, ctypes.POINTER(H) + ) + + +class D3DDevice: + """The process's D3D11 device, shared by every passport, and its interop with VTK's GL context.""" + + def __init__(self, adapter_index): + factory = vp() + check_hr( + dxgi.CreateDXGIFactory1( + ctypes.byref(IID_IDXGIFactory1), ctypes.byref(factory) + ), + "CreateDXGIFactory1", + ) + adapter = vp() + check_hr( + com_call( + factory, + IDXGIFactory1_EnumAdapters1, + HRESULT, + ctypes.c_uint(adapter_index), + ctypes.byref(adapter), + ), + f"EnumAdapters1({adapter_index})", + ) + desc = DXGI_ADAPTER_DESC1() + check_hr( + com_call(adapter, IDXGIAdapter1_GetDesc1, HRESULT, ctypes.byref(desc)), + "GetDesc1", + ) + luid = (desc.AdapterLuid.HighPart << 32) | desc.AdapterLuid.LowPart + logger.info( + f"D3D11 adapter {adapter_index}: {desc.Description} (luid 0x{luid:x})" + ) + + self.device, self.context = vp(), vp() + check_hr( + d3d11.D3D11CreateDevice( + adapter, + D3D_DRIVER_TYPE_UNKNOWN, + None, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + None, + 0, + D3D11_SDK_VERSION, + ctypes.byref(self.device), + None, + ctypes.byref(self.context), + ), + "D3D11CreateDevice", + ) + com_release(adapter) + com_release(factory) + + # A fence signals an event once the GPU ran the commands before it, so a thread can + # wait for the copy without spinning or blocking the event loop. + device5, self.context4 = vp(), vp() + check_hr( + com_call( + self.device, + 0, + HRESULT, + ctypes.byref(IID_ID3D11Device5), + ctypes.byref(device5), + ), + "QueryInterface(ID3D11Device5)", + ) + check_hr( + com_call( + self.context, + 0, + HRESULT, + ctypes.byref(IID_ID3D11DeviceContext4), + ctypes.byref(self.context4), + ), + "QueryInterface(ID3D11DeviceContext4)", + ) + self.fence = vp() + check_hr( + com_call( + device5, + ID3D11Device5_CreateFence, + HRESULT, + ctypes.c_uint64(0), + ctypes.c_uint(D3D11_FENCE_FLAG_NONE), + ctypes.byref(IID_ID3D11Fence), + ctypes.byref(self.fence), + ), + "CreateFence", + ) + com_release(device5) + self.fence_value = 0 + # Auto-reset: SetEventOnCompletion arms it again for every frame. + self.fence_event = kernel32.CreateEventW(None, False, False, None) + if not self.fence_event: + raise RuntimeError(f"CreateEventW failed: {ctypes.get_last_error()}") + # Opened with the first passport: resolving the interop needs VTK's context. + self.interop = None + self.gl_device = None + + def open_interop(self, render_window): + if self.interop: + return + render_window.MakeCurrent() + self.interop = DXInterop() + self.gl_device = self.interop.open_device(self.device) + if not self.gl_device: + raise RuntimeError( + f"wglDXOpenDeviceNV failed: {ctypes.GetLastError()} " + "(GL and D3D11 on different GPUs? try D3D_ADAPTER)" + ) + + def copy(self, dst, src): + com_call(self.context, ID3D11DeviceContext_CopyResource, None, dst, src) + + async def wait_idle(self, timeout_ms=1000): + """Wait until the GPU ran every command submitted so far, without blocking the event loop.""" + self.fence_value += 1 + check_hr( + com_call( + self.context4, + ID3D11DeviceContext4_Signal, + HRESULT, + self.fence, + ctypes.c_uint64(self.fence_value), + ), + "Signal", + ) + check_hr( + com_call( + self.fence, + ID3D11Fence_SetEventOnCompletion, + HRESULT, + ctypes.c_uint64(self.fence_value), + wintypes.HANDLE(self.fence_event), + ), + "SetEventOnCompletion", + ) + # The signal only reaches the GPU once the context's commands are submitted. + com_call(self.context, ID3D11DeviceContext_Flush, None) + # Only the event is touched off the event loop's thread: the context isn't thread safe. + result = await asyncio.to_thread( + kernel32.WaitForSingleObject, self.fence_event, timeout_ms + ) + if result != WAIT_OBJECT_0: + raise RuntimeError( + f"the D3D11 copy didn't finish within {timeout_ms}ms (wait returned {result})" + ) + + +_d3d = None + + +def init_device(adapter_index=0): + """Create the D3D11 device on the DXGI adapter Chromium uses (0 unless told otherwise).""" + global _d3d + if _d3d is None: + _d3d = D3DDevice(adapter_index) + return _d3d + + +def _attach_texture(render_window, d3d, d3d_texture, width, height): + """Make the D3D11 texture the color attachment 0 of the window's DisplayFramebuffer. Leaves it locked.""" + render_window.MakeCurrent() + handle = ctypes.c_uint32() + opengl32.glGenTextures(1, ctypes.byref(handle)) + obj = d3d.interop.register( + d3d.gl_device, + d3d_texture, + handle.value, + GL_TEXTURE_2D, + WGL_ACCESS_WRITE_DISCARD_NV, + ) + if not obj: + raise RuntimeError(f"wglDXRegisterObjectNV failed: {ctypes.GetLastError()}") + objs = (wintypes.HANDLE * 1)(obj) + + tex = vtkTextureObject() + tex.SetContext(render_window) + tex.AssignToExistingTexture(handle.value, GL_TEXTURE_2D) + # Only records the size: an assigned texture has no dimensions, so Resize() never + # calls glTexImage2D (which would detach the D3D11 storage). The DisplayFramebuffer + # resizes its attachments on every Start(), and this keeps that a no-op too. + tex.Resize(width, height) + + display_fbo = render_window.GetDisplayFramebuffer() + d3d.interop.lock(d3d.gl_device, 1, objs) + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the D3D11 texture") + return tex, handle, objs + + +class NTHandleTexturePassport(TexturePassport): + """VTK's D3D11 render target and the texture shared with Electron, at one size. + + GL owns the render target (it stays locked) except while a frame is copied out in + flush_graphics_commands(), so VTK can render into it at any time. + """ + + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + self.d3d = init_device() + self.d3d.open_interop(render_window) + # NVIDIA's interop can't register textures created with SHARED_NTHANDLE (ERROR_OPEN_FAILED), + # so VTK renders into a private texture that gets copied into the shared one on the GPU. + self.d3d_render_texture = _create_texture(self.d3d.device, width, height) + self.d3d_shared_texture, self.nt_handle = _create_shared_texture( + self.d3d.device, width, height + ) + self.vtk_texture, self.d3d_texture_gl_handle, self.objs = _attach_texture( + render_window, self.d3d, self.d3d_render_texture, width, height + ) + logger.info( + f"D3D11 texture {width}x{height} BGRA NT handle=0x{self.nt_handle:x}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "pixelFormat": "bgra", + "ntHandle": self.nt_handle, + } + + def release(self): + self.render_window.MakeCurrent() + self.d3d.interop.unlock(self.d3d.gl_device, 1, self.objs) + self.d3d.interop.unregister(self.d3d.gl_device, self.objs[0]) + opengl32.glDeleteTextures(1, ctypes.byref(self.d3d_texture_gl_handle)) + kernel32.CloseHandle(self.nt_handle) + com_release(self.d3d_shared_texture) + com_release(self.d3d_render_texture) + + async def flush_graphics_commands(self): + # Unlocking hands the texture back to D3D11, ordered after VTK's GL commands. + self.d3d.interop.unlock(self.d3d.gl_device, 1, self.objs) + try: + self.d3d.copy(dst=self.d3d_shared_texture, src=self.d3d_render_texture) + # The shared texture has no keyed mutex, so wait until the GPU is done before telling Electron. + await self.d3d.wait_idle() + finally: + self.d3d.interop.lock(self.d3d.gl_device, 1, self.objs) diff --git a/src/trame_rca/electron/passports/wayland_dmabuf.py b/src/trame_rca/electron/passports/wayland_dmabuf.py new file mode 100644 index 0000000..fb8b190 --- /dev/null +++ b/src/trame_rca/electron/passports/wayland_dmabuf.py @@ -0,0 +1,226 @@ +import ctypes +import logging +import glob +import os + +from .base import TexturePassport +from vtkmodules.vtkCommonCore import VTK_UNSIGNED_CHAR +from vtkmodules.vtkRenderingOpenGL2 import vtkTextureObject + +logger = logging.getLogger(__name__) + +# --- EGL ----------------------------------------------------------------------- +EGL_NONE = 0x3038 +EGL_EXTENSIONS = 0x3055 +EGL_DRM_RENDER_NODE_FILE_EXT = 0x3377 +EGL_GL_TEXTURE_2D_KHR = 0x30B1 +EGL_GL_TEXTURE_LEVEL_KHR = 0x30BC +GL_FRAMEBUFFER = 0x8D40 + +EGLint = ctypes.c_int32 +EGLBoolean = ctypes.c_uint32 +EGLenum = ctypes.c_uint32 +EGLuint64KHR = ctypes.c_uint64 +vp = ctypes.c_void_p + +egl = ctypes.CDLL("libEGL.so.1") +egl.eglGetProcAddress.restype = vp +egl.eglGetProcAddress.argtypes = [ctypes.c_char_p] +egl.eglGetError.restype = EGLint +egl.eglQueryString.argtypes = [vp, EGLint] +egl.eglQueryString.restype = ctypes.c_char_p +egl.eglGetCurrentDisplay.restype = vp +egl.eglGetCurrentContext.restype = vp + + +def egl_ext(name, restype, *argtypes): + addr = egl.eglGetProcAddress(name.encode()) + if not addr: + raise RuntimeError(f"missing EGL function {name}") + return ctypes.CFUNCTYPE(restype, *argtypes)(addr) + + +eglQueryDevicesEXT = egl_ext( + "eglQueryDevicesEXT", EGLBoolean, EGLint, ctypes.POINTER(vp), ctypes.POINTER(EGLint) +) +eglQueryDeviceStringEXT = egl_ext( + "eglQueryDeviceStringEXT", ctypes.c_char_p, vp, EGLint +) +eglCreateImageKHR = egl_ext( + "eglCreateImageKHR", vp, vp, vp, EGLenum, vp, ctypes.POINTER(EGLint) +) +eglDestroyImageKHR = egl_ext("eglDestroyImageKHR", EGLBoolean, vp, vp) +eglExportDMABUFImageQueryMESA = egl_ext( + "eglExportDMABUFImageQueryMESA", + EGLBoolean, + vp, + vp, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(EGLuint64KHR), +) +eglExportDMABUFImageMESA = egl_ext( + "eglExportDMABUFImageMESA", + EGLBoolean, + vp, + vp, + ctypes.POINTER(ctypes.c_int), + ctypes.POINTER(EGLint), + ctypes.POINTER(EGLint), +) + + +def attribs(*values): + return (EGLint * (len(values) + 1))(*values, EGL_NONE) + + +def check(ok, what): + if not ok: + raise RuntimeError(f"{what} failed: EGL error 0x{egl.eglGetError():04x}") + return ok + + +def fourcc_str(code): + return "".join(chr((code >> (8 * i)) & 0xFF) for i in range(4)) + + +# DRM fourcc -> electron pixelFormat +PIXEL_FORMATS = {"AB24": "rgba", "AR24": "bgra"} + + +def default_render_node(): + """Return (render node, reason). Prefers the GPU driving the display (boot_vga=1). + + Chromium can only import the dma-buf if it was allocated on the same GPU + that it's compositor runs on. In a multi-GPU box that isn't necessarily /dev/dri/renderD128. + """ + for boot_vga in sorted(glob.glob("/sys/class/drm/card*/device/boot_vga")): + try: + with open(boot_vga) as f: + if f.read().strip() != "1": + continue + except OSError: + continue + nodes = sorted( + glob.glob(os.path.join(os.path.dirname(boot_vga), "drm", "renderD*")) + ) + if nodes: + return f"/dev/dri/{os.path.basename(nodes[0])}", "boot_vga" + return "/dev/dri/renderD128", "fallback" + + +def device_index(render_node): + """The index of render_node among the EGL devices, just the way vtkEGLRenderWindow counts them.""" + devices = (vp * 16)() + count = EGLint() + check(eglQueryDevicesEXT(16, devices, ctypes.byref(count)), "eglQueryDevicesEXT") + for i in range(count.value): + node = eglQueryDeviceStringEXT(devices[i], EGL_DRM_RENDER_NODE_FILE_EXT) + node = node.decode() if node else None + logger.info(f"EGL device {i}: render node {node}") + if node == render_node: + return i + raise RuntimeError(f"no EGL device with render node {render_node}") + + +def attach_texture(render_window, width, height): + """Make a fresh RGBA8 texture the color attachment 0 of the window's DisplayFramebuffer.""" + tex = vtkTextureObject() + tex.SetContext(render_window) + if not tex.Allocate2D(width, height, 4, VTK_UNSIGNED_CHAR): + raise RuntimeError("allocating the RGBA8 texture failed") + + display_fbo = render_window.GetDisplayFramebuffer() + display_fbo.AddColorAttachment(0, tex) + display_fbo.Bind() + complete = display_fbo.CheckFrameBufferStatus(GL_FRAMEBUFFER) + display_fbo.UnBind() + if not complete: + raise RuntimeError("DisplayFramebuffer incomplete with the exported texture") + return tex + + +def export_dmabuf(render_window, tex, height): + render_window.MakeCurrent() + dpy, ctx = egl.eglGetCurrentDisplay(), egl.eglGetCurrentContext() + exts = egl.eglQueryString(dpy, EGL_EXTENSIONS).decode().split() + for ext in ("EGL_MESA_image_dma_buf_export", "EGL_KHR_gl_texture_2D_image"): + if ext not in exts: + raise RuntimeError(f"display lacks {ext}") + + image = check( + eglCreateImageKHR( + dpy, + ctx, + EGL_GL_TEXTURE_2D_KHR, + vp(tex.GetHandle()), + attribs(EGL_GL_TEXTURE_LEVEL_KHR, 0), + ), + "eglCreateImageKHR", + ) + fourcc, num_planes = ctypes.c_int(), ctypes.c_int() + modifiers = (EGLuint64KHR * 4)() + check( + eglExportDMABUFImageQueryMESA( + dpy, image, ctypes.byref(fourcc), ctypes.byref(num_planes), modifiers + ), + "eglExportDMABUFImageQueryMESA", + ) + fourcc_strings = fourcc_str(fourcc.value) + if fourcc_strings not in PIXEL_FORMATS: + raise RuntimeError(f"unsupported fourcc {fourcc_strings}") + fds = (ctypes.c_int * 4)(-1, -1, -1, -1) + strides, offsets = (EGLint * 4)(), (EGLint * 4)() + check( + eglExportDMABUFImageMESA(dpy, image, fds, strides, offsets), + "eglExportDMABUFImageMESA", + ) + eglDestroyImageKHR(dpy, image) + render_window.WaitForCompletion() + + planes = [] + for i in range(num_planes.value): + # The dma-buf's real size; tiled layouts can be larger than stride*height. + try: + size = os.lseek(fds[i], 0, os.SEEK_END) - offsets[i] + except OSError: + size = strides[i] * height + planes.append( + {"fd": fds[i], "stride": strides[i], "offset": offsets[i], "size": size} + ) + + return fourcc_strings, modifiers[0], planes + + +class WaylandDMABUFTexturePassport(TexturePassport): + """The texture VTK resolves into, exported as a dma-buf, at one size.""" + + def __init__(self, render_window, width, height): + super().__init__(render_window, width, height) + render_window.MakeCurrent() + self.vtk_texture = attach_texture(render_window, width, height) + self.fourcc, self.modifier, self.planes = export_dmabuf( + render_window, self.vtk_texture, height + ) + logger.info( + f"exported {width}x{height} fourcc={self.fourcc} modifier=0x{self.modifier:016x} " + f"planes={self.planes}" + ) + + def to_json(self): + return { + "type": "texture", + "pid": os.getpid(), + "width": self.width, + "height": self.height, + "fourcc": self.fourcc, + "pixelFormat": PIXEL_FORMATS[self.fourcc], + "modifier": str(self.modifier), + "planes": self.planes, + } + + def release(self): + for p in self.planes: + os.close(p["fd"]) + self.render_window.MakeCurrent() + self.vtk_texture.ReleaseGraphicsResources(self.render_window) diff --git a/src/trame_rca/rca/vtk_rca.py b/src/trame_rca/rca/vtk_rca.py index 06360e3..ff4b007 100644 --- a/src/trame_rca/rca/vtk_rca.py +++ b/src/trame_rca/rca/vtk_rca.py @@ -32,7 +32,7 @@ def __init__(self, vtk_render_window: vtkRenderWindow): @property def img_cols_rows(self): - self._render() + self.render() with self._timer_capture: self._window_to_image.Modified() self._window_to_image.Update() @@ -50,10 +50,9 @@ def img_cols_rows(self): @property def render_window(self): - self._render() return self._vtk_render_window - def _render(self): + def render(self): with self._timer_render: self._vtk_render_window.Render() diff --git a/src/trame_rca/schedulers/__init__.py b/src/trame_rca/schedulers/__init__.py index 3accf67..0f0f0a4 100644 --- a/src/trame_rca/schedulers/__init__.py +++ b/src/trame_rca/schedulers/__init__.py @@ -8,10 +8,15 @@ from .video_scheduler import RcaVideoRenderScheduler except (ModuleNotFoundError, ImportError) as e: logger.info(e.msg) +try: + from .shared_texture_scheduler import RcaSharedTextureRenderScheduler +except (ModuleNotFoundError, ImportError) as e: + logger.info(e.msg) __all__ = [ "RcaImageRenderScheduler", "RcaRenderSchedulerProtocol", + "RcaSharedTextureRenderScheduler", "RcaVideoRenderScheduler", ] diff --git a/src/trame_rca/schedulers/shared_texture_scheduler.py b/src/trame_rca/schedulers/shared_texture_scheduler.py new file mode 100644 index 0000000..c6b6d7a --- /dev/null +++ b/src/trame_rca/schedulers/shared_texture_scheduler.py @@ -0,0 +1,320 @@ +"""A render scheduler that renders into a texture shared with the Electron main process (EMP). + +OS handles do not go through trame's websocket because that is wired to the Electron renderer +process (ERP), which does not have access to the OS API needed to import a texture. + +Instead the scheduler tells the EMP about the shared textures and each finished frame over +trame_rca.electron.comm, and the EMP hands them to the renderer process. All the schedulers of +a server share one connection to the EMP, and tag their messages with their view's name. Size and input still +come from the RemoteControlledArea through the RcaViewAdapter. + +The textures form a ring: VTK renders the next frame into the next texture while Electron +still shows the previous ones, and only waits when the texture it's about to reuse is still +held by Electron. + +Renders run on trame's event loop. +""" + +from __future__ import annotations +import asyncio +import logging +import time +from typing import TYPE_CHECKING, Callable + +if TYPE_CHECKING: + from vtkmodules.vtkRenderingCore import vtkRenderWindow + +from trame.app import asynchronous +from trame_common.utils import profiler + +from trame_rca.electron.comm import EmpClient +from trame_rca.electron.passports import texture_passport_class, TexturePassport +from trame_rca.rca import ( + RemoteControlledAreaProtocol, + VtkRemoteControlledArea, + window_wrapper, +) + +logger = logging.getLogger(__name__) + +FRAME_DONE_TIMEOUT_S = 1.0 + + +def _check_texture_count(value): + if int(value) != value or value < 1: + raise ValueError(f"texture_count must be a positive integer, not {value!r}") + return int(value) + + +def _report_failure(task): + # Otherwise asyncio only reports it once the task is garbage collected. + if not task.cancelled() and task.exception(): + logger.error("render task failed", exc_info=task.exception()) + + +class RcaSharedTextureRenderScheduler: + """Implements RcaRenderSchedulerProtocol by sharing the render window's texture with Electron.""" + + def __init__( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + emp_port: int, + *, + target_fps: float = 60.0, + on_disconnect: Callable[[], None] | None = None, + before_render: Callable[[], None] | None = None, + texture_count: int = 2, + view: str = "default", + ): + """``view`` is the name of the RemoteControlledArea that shows the frames.""" + self._timer_render = profiler.Timer("rca.shared_texture.render") + self._timer_flush = profiler.Timer("rca.shared_texture.flush") + self._rca = window_wrapper(window) + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + self._emp_port = emp_port + self._view = view + self._target_fps = target_fps + self._on_disconnect = on_disconnect + # Called before every frame, e.g. to animate the scene. It may schedule_render() again. + self._before_render = before_render + self._push_callback = None + + self._client: EmpClient | None = None + self._texture_count = _check_texture_count(texture_count) + # The ring of shared textures + self._passports = [] + # the frame_done of the last frame per shared texture. + self._slot_frames = [] + self._next_slot = 0 + # `seq`:`frame_done` future map valid until Electron fulfils with the "type":"frame_done" message. + self._frames_in_flight = {} + self._seq = 0 + # Renders asked for before start() wait in here. + self._request_event = asyncio.Event() + self._tasks = [] + self._is_closing = False + + # Render into the FBOs only: Frame() then stops after resolving into the DisplayFramebuffer. + self._rca.render_window.SetUseOffScreenBuffers(True) + self._rca.render_window.SetDoubleBuffer(False) + self._rca.render_window.SetShowWindow(False) + self._rca.render_window.GetInteractor().Initialize() + + # Wait for the client to report it's canvas size before rendering to avoid + # sending a not-so-useful 300x300 default sized frame. + self._sized = False + process_resize_event = self._rca.process_resize_event + + def on_resize(width, height): + process_resize_event(width, height) + self._sized = True + self.schedule_render() # to ensure self._request_event is set + + self._rca.process_resize_event = on_resize + + @property + def rca(self): + return self._rca + + @property + def view(self): + return self._view + + @property + def target_fps(self): + return self._target_fps + + @target_fps.setter + def target_fps(self, value): + self._target_fps = value + + @property + def texture_count(self): + """How many shared textures VTK renders into in turn. 1 waits for Electron after every frame.""" + return self._texture_count + + @texture_count.setter + def texture_count(self, value): + value = _check_texture_count(value) + if value != self._texture_count: + self._texture_count = value + # The ring is rebuilt before the next frame. + self.schedule_render() + + def set_push_callback(self, callback): + # Frames go to the EMP, never through the push callback. + self._push_callback = callback + + def reset(self): + """Nothing to re-initialize.""" + + def schedule_render(self): + self._request_event.set() + + def start(self, url: str | None = None, **_): + """Connect to the EMP and start rendering. Call once trame's event loop runs. + + ``url`` is the page the EMP should load: the one trame serves. Only the first + scheduler to connect tells the EMP about it. + """ + self._client = EmpClient.for_port(self._emp_port) + self._client.add_view(self._view, self._on_frame_done, self._on_client_closed) + self._start_task(self._run(url)) + + def _start_task(self, coro): + task = asynchronous.create_task(coro) + task.add_done_callback(_report_failure) + self._tasks.append(task) + + async def close(self): + if self._is_closing: + return + self._is_closing = True + for task in self._tasks: + if task is not asyncio.current_task(): + task.cancel() + if self._client: + self._client.remove_view(self._view) + for passport in self._passports: + passport.release() + self._passports = [] + + async def _run(self, url): + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + assert isinstance(self._client, EmpClient), "start() did not run!" + try: + passport_name = await self._client.connect() + except Exception: + await self._disconnect() + raise + await self._client.server_ready(url) + next_frame = 0 + period = 1.0 / self._target_fps + + while not self._is_closing: + await self._request_event.wait() + self._request_event.clear() + # Only render when the ERP has booted up ( is sized) + if not self._sized: + continue + + # Do not touch a texture in the ring until Electron is done reading it. + # Wait until a slot frees up in the ring. + await self._wait_for_slot(self._next_slot) + await self._update_textures(passport_name) + slot = self._next_slot + passport = self._passports[slot] + if (passport.width, passport.height) != tuple( + self._rca.render_window.GetSize() + ): + # Resized while waiting: rendering now would reallocate the shared texture. + self.schedule_render() + continue + self._next_slot = (slot + 1) % len(self._passports) + + passport.attach() + if self._before_render: + self._before_render() + with self._timer_render: + self._rca.render_window.Render() + with self._timer_flush: + self._rca.render_window.MakeCurrent() + await passport.flush_graphics_commands() + + seq = self._seq + self._seq += 1 + frame_done = asyncio.get_running_loop().create_future() + self._slot_frames[slot] = frame_done + self._frames_in_flight[seq] = frame_done + await self._client.send( + { + "type": "frame", + "view": self._view, + "seq": seq, + "slot": slot, + "timestamp_us": int(time.time() * 1e6), + } + ) + # Pace against fixed frame clock held in `next_frame` + next_frame = max(next_frame + period, time.monotonic()) + await asyncio.sleep(max(0.0, next_frame - time.monotonic())) + + async def _wait_for_slot(self, slot): + """Wait until Electron released the last frame rendered into the texture at ``slot``.""" + frame_done = self._slot_frames[slot] if slot < len(self._slot_frames) else None + if frame_done is None or frame_done.done(): + return + try: + # shield: a frame_done arriving late must still find its future. + await asyncio.wait_for(asyncio.shield(frame_done), FRAME_DONE_TIMEOUT_S) + except asyncio.TimeoutError: + logger.warning( + "texture %d: no frame_done within %ss, continuing", + slot, + FRAME_DONE_TIMEOUT_S, + ) + + async def _update_textures(self, passport_name): + """(Re)allocate the shared textures when the render window's size or their count changed.""" + assert isinstance(self._rca, VtkRemoteControlledArea), ( + "SharedTexture scheduler only works with VTK render windows!" + ) + assert isinstance(self._client, EmpClient), "start() did not run!" + width, height = self._rca.render_window.GetSize() + count = self._texture_count + if len(self._passports) == count and all( + (p.width, p.height) == (width, height) for p in self._passports + ): + return + if not self._passports: + # The first render will create the context and the framebuffers. + self._rca.render_window.Render() + self._rca.render_window.MakeCurrent() + passport_class = texture_passport_class(passport_name) + old, self._passports = ( + self._passports, + [ + passport_class(self._rca.render_window, width, height) + for _ in range(count) + ], + ) + for passport in old: + passport.release() + self._slot_frames = [None] * count + self._next_slot = 0 + for slot, passport in enumerate(self._passports): + assert isinstance(passport, TexturePassport) + await self._client.send( + { + **passport.to_json(), + "view": self._view, + "slot": slot, + "slots": count, + } + ) + + def _on_frame_done(self, msg): + frame_done = self._frames_in_flight.pop(msg["seq"], None) + if frame_done and not frame_done.done(): + frame_done.set_result(None) + if msg.get("dropped"): + # The page never processed it: e.g. it was still loading or busy.. + logger.info( + "view %s: frame %d dropped, rendering again", self._view, msg["seq"] + ) + self.schedule_render() + + def _on_client_closed(self): + if not self._is_closing: + self._start_task(self._disconnect()) + + async def _disconnect(self): + if self._is_closing: + return + await self.close() + if self._on_disconnect: + self._on_disconnect() diff --git a/src/trame_rca/schedulers/video_scheduler.py b/src/trame_rca/schedulers/video_scheduler.py index a2bf15d..ee16037 100644 --- a/src/trame_rca/schedulers/video_scheduler.py +++ b/src/trame_rca/schedulers/video_scheduler.py @@ -45,6 +45,8 @@ def __init__( self._rca: VtkRemoteControlledArea = window_wrapper(window) self._push_callback = push_callback self.on_codec_changed = on_codec_changed + # The encoder needs the render window's GL context, which the first render creates. + self._rca.render() self._rca_encoder = RcaVideoEncoder( self._rca.render_window, push_callback=self._push, defer=negotiate ) @@ -115,6 +117,7 @@ async def _render(self): self._request_event.clear() started = monotonic() + self._rca.render() self._rca_encoder.encode(self._rca.render_window) # Sleep only for what is left of the _target_period_s. await sleep(max(0.0, self._target_period_s - (monotonic() - started))) @@ -124,4 +127,5 @@ def _push(self, content: bytes, meta: dict, _m_time: int): self._push_callback(content, meta) def reset(self): + self._rca.render() self._rca_encoder.reset(self._rca.render_window) diff --git a/src/trame_rca/view_adapter.py b/src/trame_rca/view_adapter.py index 0d960b8..a53893d 100644 --- a/src/trame_rca/view_adapter.py +++ b/src/trame_rca/view_adapter.py @@ -76,6 +76,15 @@ def target_fps(self) -> float: def target_fps(self, value) -> None: self._scheduler.target_fps = value + @property + def texture_count(self) -> int: + """How many shared textures the scheduler renders into in turn (shared-texture only)""" + return self._scheduler.texture_count + + @texture_count.setter + def texture_count(self, value) -> None: + self._scheduler.texture_count = value + @property def max_pixel_count(self): """Use 0 to disable capping of pixel count""" diff --git a/src/trame_rca/widgets/__init__.py b/src/trame_rca/widgets/__init__.py index e69de29..be9d04f 100644 --- a/src/trame_rca/widgets/__init__.py +++ b/src/trame_rca/widgets/__init__.py @@ -0,0 +1,5 @@ +from .rca import init_vtk_for_shared_textures + +__all__ = [ + "init_vtk_for_shared_textures", +] diff --git a/src/trame_rca/widgets/rca.py b/src/trame_rca/widgets/rca.py index c671aaa..cbe08b6 100644 --- a/src/trame_rca/widgets/rca.py +++ b/src/trame_rca/widgets/rca.py @@ -2,6 +2,9 @@ from __future__ import annotations +import logging +import os +import sys import warnings from typing import TYPE_CHECKING, Callable from weakref import WeakKeyDictionary, WeakValueDictionary @@ -18,7 +21,10 @@ from .. import module +logger = logging.getLogger(__name__) + __all__ = [ + "init_vtk_for_shared_textures", "RemoteControlledArea", "DisplayArea", "StatisticsDisplay", @@ -26,6 +32,7 @@ "MediaSourceDisplayArea", "VideoDecoderDisplayArea", "RawImageDisplayArea", + "SharedTextureDisplayArea", "ImageStream", "ImageRegion", ] @@ -38,6 +45,53 @@ def __init__(self, _elem_name, children=None, **kwargs): self.server.enable_module(module) +_shared_textures_initialized = False + + +def init_vtk_for_shared_textures(): + """Make VTK render on the GPU Electron composites with. + + The shared texture can only be imported on the GPU Chromium uses, and VTK reads + some of these settings when its render window is constructed: call this before + creating any. It can be called more than once. + + Linux: ``RENDER_NODE`` overrides the render node (default: the GPU driving the + display). + Windows: ``D3D_ADAPTER`` overrides the DXGI adapter index (default 0). + MacOS: Calling this method is unnecessary. + """ + global _shared_textures_initialized + if _shared_textures_initialized: + return + _shared_textures_initialized = True + + if sys.platform == "linux": + from trame_rca.electron.passports import wayland_dmabuf + + if os.environ.get("RENDER_NODE"): + render_node, reason = os.environ["RENDER_NODE"], "env" + else: + render_node, reason = wayland_dmabuf.default_render_node() + logger.info("using render node %s (%s)", render_node, reason) + # vtkEGLRenderWindow::SetDeviceIndex doesn't reach the EGL display setup; the + # constructor only reads the device from this variable. + os.environ["VTK_EGL_DEVICE_INDEX"] = str( + wayland_dmabuf.device_index(render_node) + ) + os.environ["VTK_DEFAULT_OPENGL_WINDOW"] = "vtkEGLRenderWindow" + elif sys.platform == "win32": + from trame_rca.electron.passports import nthandle + + # The shared texture must live on the GPU Chromium uses, which is DXGI adapter 0 + # unless told otherwise. VTK's WGL context has to be on that GPU too. + nthandle.init_device(int(os.environ.get("D3D_ADAPTER", "0"))) + # Ask for a 1ms resolution in the Python process. Otherwise, asyncio.sleep + # rounds short waits to ~15.6ms + import ctypes + + ctypes.WinDLL("winmm").timeBeginPeriod(1) + + # Expose your vue component(s) class RemoteControlledArea(HtmlElement): _next_id = 0 @@ -136,6 +190,66 @@ def create_view_handler( self.add_view_handler(view_handler) return view_handler + def create_electron_handler( + self, + window: RemoteControlledAreaProtocol | vtkRenderWindow, + target_fps: float = 60.0, + before_render: Callable[[], None] | None = None, + on_disconnect: Callable[[], None] | None = None, + texture_count: int = 2, + ): + """ + Render ``render_window`` into a texture shared with the Electron app that + started this server (see @Kitware/trame-rca-electron), for ``display="shared-texture"``. + + Call ``init_vtk_for_shared_textures()`` before creating ``render_window``. + ``before_render`` is called before every frame and may schedule another render, + e.g. to animate. ``on_disconnect`` is called when Electron goes away and + defaults to stopping the server. ``texture_count`` is how many shared textures + VTK renders into in turn, so it can render the next frame while Electron shows the + previous ones. The returned handler's ``texture_count`` changes it at runtime. + + Each shared-texture view of a page needs its own ``name``. + """ + from trame.app import asynchronous + + from trame_rca.electron import emp_port, server_url + from trame_rca.schedulers import RcaSharedTextureRenderScheduler + + if self.display != "shared-texture": + raise ValueError( + "create_electron_handler needs a RemoteControlledArea with " + f'display="shared-texture", not "{self.display}"' + ) + + if on_disconnect is None: + + def on_disconnect(): + asynchronous.create_task(self.server.stop()) + + rca_protocol = window_wrapper(window) + scheduler = RcaSharedTextureRenderScheduler( + rca_protocol, + emp_port(self.server), + target_fps=target_fps, + on_disconnect=on_disconnect, + before_render=before_render, + texture_count=texture_count, + view=self.name, + ) + + def start(**_): + scheduler.start(url=server_url(self.server)) + + if self.server.running: + start() + else: + self.ctrl.on_server_ready.add(start) + + view_handler = RcaViewAdapter(scheduler.rca, self.name, scheduler=scheduler) + self.add_view_handler(view_handler) + return view_handler + def create_vtk_handler( self, render_window: vtkRenderWindow, @@ -252,6 +366,22 @@ def __init__(self, **kwargs): ] +class SharedTextureDisplayArea(HtmlElement): + """The canvas trame-rca-electron draws the shared texture of view ``name`` into, once a frame.""" + + def __init__(self, **kwargs): + super().__init__( + "shared-texture-display-area", + **kwargs, + ) + self._attr_names += [ + "name", + ] + self._event_names += [ + "stats", + ] + + class ImageStream(HtmlElement): ID = 0 NAMES = WeakKeyDictionary() diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py index 9f67be4..f1a710e 100644 --- a/tests/test_benchmark.py +++ b/tests/test_benchmark.py @@ -54,7 +54,7 @@ def full_hd_scene(): interactor.SetRenderWindow(render_window) rca = VtkRemoteControlledArea(render_window) - rca_render = rca._render + rca_render = rca.render def render(): rca_render() @@ -68,10 +68,10 @@ def recording_render(): rendered.append(True) render() - rca._render = recording_render + rca.render = recording_render rca.img_cols_rows - assert rendered, "img_cols_rows no longer renders through _render" - rca._render = lambda: None + assert rendered, "img_cols_rows no longer renders through render()" + rca.render = lambda: None yield rca, render, renderer.GetActiveCamera() render_window.Finalize() diff --git a/tests/test_rca_electron.py b/tests/test_rca_electron.py new file mode 100644 index 0000000..7357943 --- /dev/null +++ b/tests/test_rca_electron.py @@ -0,0 +1,322 @@ +import asyncio +import json + +import pytest + +from trame_rca.electron import PROTOCOL_VERSION +from trame_rca.electron.comm import ProtocolMismatchError, check_protocol +from trame_rca.electron.passports import TexturePassport, texture_passport_class +from trame_rca.schedulers import RcaSharedTextureRenderScheduler + +TIMEOUT_S = 10 + + +class FakePassport(TexturePassport): + released = [] + + def to_json(self): + return {"type": "texture", "width": self.width, "height": self.height} + + def release(self): + FakePassport.released.append((self.width, self.height)) + + +class FakeEmp: + """The Electron main process's end of the texture channel.""" + + def __init__(self, hello): + self.hello = hello + self.connected = asyncio.Event() + self.reader = self.writer = None + + async def start(self): + self.server = await asyncio.start_server(self._on_connect, "127.0.0.1", 0) + return self.server.sockets[0].getsockname()[1] + + async def _on_connect(self, reader, writer): + self.reader, self.writer = reader, writer + await self.send(self.hello) + self.connected.set() + + async def send(self, msg): + self.writer.write((json.dumps(msg) + "\n").encode()) + await self.writer.drain() + + async def recv(self): + await asyncio.wait_for(self.connected.wait(), TIMEOUT_S) + line = await asyncio.wait_for(self.reader.readline(), TIMEOUT_S) + return json.loads(line) + + def close(self): + if self.writer: + self.writer.close() + self.server.close() + + +@pytest.fixture +def fake_passport(monkeypatch): + FakePassport.released = [] + monkeypatch.setattr( + "trame_rca.schedulers.shared_texture_scheduler.texture_passport_class", + lambda name: FakePassport, + ) + + +def test_texture_passport_class(): + with pytest.raises(TypeError): + texture_passport_class(1) + with pytest.raises(NotImplementedError): + texture_passport_class("x11-dmabuf") + with pytest.raises(ValueError): + texture_passport_class("nope") + + +def test_check_protocol(): + check_protocol({"protocol": PROTOCOL_VERSION}) + with pytest.raises(ProtocolMismatchError): + check_protocol({"protocol": PROTOCOL_VERSION + 1}) + with pytest.raises(ProtocolMismatchError): + check_protocol({}) + + +@pytest.mark.asyncio +async def test_scheduler_shares_frames(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + disconnected = asyncio.Event() + scheduler = RcaSharedTextureRenderScheduler( + a_render_window, port, on_disconnect=disconnected.set + ) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert await emp.recv() == { + "type": "server_ready", + "protocol": PROTOCOL_VERSION, + "url": "http://127.0.0.1:1234/", + } + + # Nothing to render into until the page reported its size. + scheduler.schedule_render() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + + # As the RcaViewAdapter does when the canvas reports its size. + scheduler.rca.process_resize_event(300, 300) + scheduler.schedule_render() + # One texture per slot of the ring, two by default. + for slot in range(2): + assert await emp.recv() == { + "type": "texture", + "width": 300, + "height": 300, + "view": "default", + "slot": slot, + "slots": 2, + } + frame = await emp.recv() + assert frame["type"] == "frame" and frame["seq"] == 0 and frame["slot"] == 0 + assert frame["view"] == "default" + await emp.send({"type": "frame_done", "view": "default", "seq": 0}) + + # A resize makes new textures and releases the old ones. + scheduler.rca.process_resize_event(200, 100) + scheduler.schedule_render() + for slot in range(2): + assert (await emp.recv())["width"] == 200 + assert FakePassport.released == [(300, 300)] * 2 + frame = await emp.recv() + assert frame["seq"] == 1 and frame["slot"] == 0 + + # Electron going away closes the scheduler. + emp.close() + await asyncio.wait_for(disconnected.wait(), TIMEOUT_S) + assert FakePassport.released == [(300, 300)] * 2 + [(200, 100)] * 2 + finally: + await scheduler.close() + emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_rejects_other_protocol(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION + 1, "passport": "x"}) + port = await emp.start() + disconnected = asyncio.Event() + scheduler = RcaSharedTextureRenderScheduler( + a_render_window, port, on_disconnect=disconnected.set + ) + try: + scheduler.start(url="http://127.0.0.1:1234/") + await asyncio.wait_for(disconnected.wait(), TIMEOUT_S) + # Never told Electron the page is ready. + await asyncio.wait_for(emp.connected.wait(), TIMEOUT_S) + assert await emp.reader.readline() == b"" + finally: + await scheduler.close() + emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_renders_dropped_frames_again(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + scheduler = RcaSharedTextureRenderScheduler(a_render_window, port) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert (await emp.recv())["type"] == "server_ready" + scheduler.rca.process_resize_event(300, 300) + scheduler.schedule_render() + assert (await emp.recv())["type"] == "texture" + assert (await emp.recv())["type"] == "texture" + assert (await emp.recv())["seq"] == 0 + + # e.g. sendSharedTexture timed out while the page was loading. + await emp.send( + {"type": "frame_done", "view": "default", "seq": 0, "dropped": True} + ) + frame = await emp.recv() + assert frame["type"] == "frame" and frame["seq"] == 1 + finally: + await scheduler.close() + emp.close() + + +@pytest.mark.asyncio +async def test_scheduler_texture_ring(a_render_window, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + port = await emp.start() + scheduler = RcaSharedTextureRenderScheduler(a_render_window, port, texture_count=3) + try: + scheduler.start(url="http://127.0.0.1:1234/") + assert (await emp.recv())["type"] == "server_ready" + scheduler.rca.process_resize_event(300, 300) + for _ in range(3): + assert (await emp.recv())["slots"] == 3 + + # Up to one frame in flight per texture, without waiting for frame_done. + for seq in range(3): + scheduler.schedule_render() + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (seq, seq) + + # Back to slot 0: waits until Electron released its frame. + scheduler.schedule_render() + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + await emp.send({"type": "frame_done", "view": "default", "seq": 0}) + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (3, 0) + for seq in range(1, 4): + await emp.send({"type": "frame_done", "view": "default", "seq": seq}) + + # Changing the count at runtime rebuilds the ring before the next frame. + scheduler.texture_count = 1 + assert await emp.recv() == { + "type": "texture", + "width": 300, + "height": 300, + "view": "default", + "slot": 0, + "slots": 1, + } + assert len(FakePassport.released) == 3 + frame = await emp.recv() + assert (frame["seq"], frame["slot"]) == (4, 0) + + with pytest.raises(ValueError): + scheduler.texture_count = 0 + finally: + await scheduler.close() + emp.close() + + +@pytest.fixture +def two_render_windows(): + from vtkmodules.vtkRenderingCore import ( + vtkRenderer, + vtkRenderWindow, + vtkRenderWindowInteractor, + ) + + windows = [] + for _ in range(2): + window = vtkRenderWindow() + window.AddRenderer(vtkRenderer()) + window.ShowWindowOff() + vtkRenderWindowInteractor().SetRenderWindow(window) + windows.append(window) + yield windows + for window in windows: + window.Finalize() + + +@pytest.mark.asyncio +async def test_schedulers_share_the_connection(two_render_windows, fake_passport): + emp = FakeEmp({"type": "hello", "protocol": PROTOCOL_VERSION, "passport": "fake"}) + connections = [] + on_connect = emp._on_connect + + async def count_connections(reader, writer): + connections.append(writer) + if len(connections) == 1: + await on_connect(reader, writer) + + emp._on_connect = count_connections + port = await emp.start() + disconnected = {"a": asyncio.Event(), "b": asyncio.Event()} + schedulers = { + view: RcaSharedTextureRenderScheduler( + window, port, view=view, on_disconnect=disconnected[view].set + ) + for view, window in zip(("a", "b"), two_render_windows) + } + try: + for scheduler in schedulers.values(): + scheduler.start(url="http://127.0.0.1:1234/") + # The page is loaded once, for both views. + assert (await emp.recv())["type"] == "server_ready" + + schedulers["a"].rca.process_resize_event(300, 300) + schedulers["b"].rca.process_resize_event(200, 100) + textures = {} + frames = {} + while len(frames) < 2: + msg = await emp.recv() + assert msg["type"] in ("texture", "frame") + if msg["type"] == "texture": + textures.setdefault(msg["view"], []).append(msg) + else: + frames[msg["view"]] = msg + assert len(connections) == 1 + assert [t["width"] for t in textures["a"]] == [300, 300] + assert [t["width"] for t in textures["b"]] == [200, 200] + # seq counts per view. + assert frames["a"]["seq"] == frames["b"]["seq"] == 0 + + # A frame_done only reaches its own view. + await emp.send({"type": "frame_done", "view": "b", "seq": 0, "dropped": True}) + frame = await emp.recv() + assert (frame["view"], frame["seq"]) == ("b", 1) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(emp.reader.readline(), 0.2) + + # Two views can't share a name. + with pytest.raises(ValueError): + RcaSharedTextureRenderScheduler( + two_render_windows[0], port, view="a" + ).start() + + # Closing one view keeps the connection of the other. + await schedulers["a"].close() + schedulers["b"].schedule_render() + frame = await emp.recv() + assert frame["view"] == "b" + assert not disconnected["b"].is_set() + + # Electron going away disconnects the remaining views. + emp.close() + await asyncio.wait_for(disconnected["b"].wait(), TIMEOUT_S) + assert not disconnected["a"].is_set() + finally: + for scheduler in schedulers.values(): + await scheduler.close() + emp.close() diff --git a/vue-components/src/components/DisplayArea.js b/vue-components/src/components/DisplayArea.js index ad8783d..5692d86 100644 --- a/vue-components/src/components/DisplayArea.js +++ b/vue-components/src/components/DisplayArea.js @@ -2,6 +2,7 @@ import ImageDisplayArea from './ImageDisplayArea.js'; import MediaSourceDisplayArea from './MediaSourceDisplayArea.js'; import VideoDecoderDisplayArea from './VideoDecoderDisplayArea.js'; import RawImageDisplayArea from './RawImageDisplayArea.js'; +import SharedTextureDisplayArea from './SharedTextureDisplayArea.js'; export default { components: { @@ -9,6 +10,7 @@ export default { MediaSourceDisplayArea, VideoDecoderDisplayArea, RawImageDisplayArea, + SharedTextureDisplayArea, }, props: { name: { @@ -38,6 +40,7 @@ export default { + `, }; diff --git a/vue-components/src/components/SharedTextureDisplayArea.js b/vue-components/src/components/SharedTextureDisplayArea.js new file mode 100644 index 0000000..bb91d78 --- /dev/null +++ b/vue-components/src/components/SharedTextureDisplayArea.js @@ -0,0 +1,50 @@ +const { ref, onBeforeUnmount, watchEffect } = window.Vue; + +// Shows the frames the trame-rca-electron preload draws from the texture VTK shares with +// Electron, for the view `name`. Only works in a page loaded by trame-rca-electron: the preload +// defines the bridge. +export default { + props: { + name: { + type: String, + default: 'default', + }, + }, + emits: ['stats'], + setup(props, { emit }) { + const el = ref(null); + const attached = []; + + function detachAll() { + while (attached.length) { + const [n, c] = attached.pop(); + window.trameRcaSharedTexture.detach(n, c); + } + } + onBeforeUnmount(detachAll); + + if (!window.trameRcaSharedTexture) { + console.warn( + 'shared-texture-display-area: no trameRcaSharedTexture bridge, not running in trame-rca-electron?' + ); + } + + function emitStats(stats) { + emit('stats', stats); + } + + watchEffect(() => { + const name = props.name; + const container = el.value; + detachAll(); + if (!container) return; + attached.push([name, container]); + window.trameRcaSharedTexture.attach(name, container, emitStats); + }); + + return { el }; + }, + template: ` + + `, +}; diff --git a/vue-components/src/components/index.js b/vue-components/src/components/index.js index f6445b8..a8d27f7 100644 --- a/vue-components/src/components/index.js +++ b/vue-components/src/components/index.js @@ -5,6 +5,7 @@ import RemoteControlledArea from './RemoteControlledArea.js'; import MediaSourceDisplayArea from './MediaSourceDisplayArea.js'; import VideoDecoderDisplayArea from './VideoDecoderDisplayArea.js'; import RawImageDisplayArea from './RawImageDisplayArea.js'; +import SharedTextureDisplayArea from './SharedTextureDisplayArea.js'; import ImageStream from './ImageStream.js'; import ImageRegion from './ImageRegion.js'; @@ -16,6 +17,7 @@ export default { MediaSourceDisplayArea, VideoDecoderDisplayArea, RawImageDisplayArea, + SharedTextureDisplayArea, ImageStream, ImageRegion, };