Add keyboard SE3/SE2 device: schema, tracker, plugin, retargeters

examples/teleop/python/keyboard_printer_example.py

@@ -0,0 +1,185 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Keyboard Printer Example.
+
+Prints every currently-held key (not just the SE3-relevant subset) each frame,
+via KeyboardSource's "keyboard_all_keys" bitmap output. The keyboard plugin
+self-discovers its device and is auto-launched by TeleopSession -- no external
+process to start manually.
+"""
+
+import sys
+import time
+from pathlib import Path
+
+from isaaccapture.cloudxr import CloudXRLauncher
+from isaaccapture.retargeting_engine.deviceio_source_nodes import KeyboardSource
+from isaaccapture.teleop_session_manager import (
+    TeleopSession,
+    TeleopSessionConfig,
+    PluginConfig,
+)
+
+
+PLUGIN_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent / "plugins"
+PLUGIN_NAME = "keyboard"
+PLUGIN_ROOT_ID = "keyboard"
+
+# Evdev key codes (linux/input-event-codes.h) -> display name, for the keys on a
+# standard PC keyboard. Codes not listed here still show up (as "code<N>") --
+# this table is for readability, not a completeness gate.
+KEY_NAMES = {
+    1: "ESC",
+    14: "BACKSPACE",
+    15: "TAB",
+    28: "ENTER",
+    29: "LCTRL",
+    42: "LSHIFT",
+    54: "RSHIFT",
+    56: "LALT",
+    57: "SPACE",
+    58: "CAPSLOCK",
+    69: "NUMLOCK",
+    70: "SCROLLLOCK",
+    97: "RCTRL",
+    100: "RALT",
+    102: "HOME",
+    103: "UP",
+    104: "PAGEUP",
+    105: "LEFT",
+    106: "RIGHT",
+    107: "END",
+    108: "DOWN",
+    109: "PAGEDOWN",
+    110: "INSERT",
+    111: "DELETE",
+    **{2 + i: str((i + 1) % 10) for i in range(10)},  # KEY_1..KEY_0 -> "1".."9","0"
+    **{59 + i: f"F{i + 1}" for i in range(10)},  # KEY_F1..KEY_F10
+    87: "F11",
+    88: "F12",
+    16: "Q",
+    17: "W",
+    18: "E",
+    19: "R",
+    20: "T",
+    21: "Y",
+    22: "U",
+    23: "I",
+    24: "O",
+    25: "P",
+    30: "A",
+    31: "S",
+    32: "D",
+    33: "F",
+    34: "G",
+    35: "H",
+    36: "J",
+    37: "K",
+    38: "L",
+    44: "Z",
+    45: "X",
+    46: "C",
+    47: "V",
+    48: "B",
+    49: "N",
+    50: "M",
+}
+
+ALL_KEYS_BITMAP_SIZE = 256
+
+
+def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    CloudXRLauncher.add_launcher_arguments(parser)
+    args = parser.parse_args()
+
+    print("\n" + "=" * 80)
+    print("  Keyboard Printer Example")
+    print("=" * 80)
+    print("Press any key on the machine running this example.")
+    print("=" * 80 + "\n")
+
+    # ==================================================================
+    # Setup: Create keyboard source
+    # ==================================================================
+    keyboard_source = KeyboardSource(name="keyboard")
+
+    # ==================================================================
+    # Configure Plugins
+    # ==================================================================
+
+    plugins = []
+    if PLUGIN_ROOT_DIR.exists():
+        plugins.append(
+            PluginConfig(
+                plugin_name=PLUGIN_NAME,
+                plugin_root_id=PLUGIN_ROOT_ID,
+                search_paths=[PLUGIN_ROOT_DIR],
+            )
+        )
+
+    # ==================================================================
+    # Create and run TeleopSession
+    # ==================================================================
+
+    session_config = TeleopSessionConfig(
+        app_name="KeyboardPrinterExample",
+        trackers=[],
+        pipeline=keyboard_source,
+        plugins=plugins,
+    )
+
+    with CloudXRLauncher.launch_context(args):
+        with TeleopSession(session_config) as session:
+            start_time = time.time()
+            prev_pressed: set[int] = set()
+
+            while time.time() - start_time < 30.0:
+                result = session.step()
+                bitmap_group = result["keyboard_all_keys"]
+
+                elapsed = session.get_elapsed_time()
+                if bitmap_group.is_none:
+                    print(
+                        f"[{elapsed:5.1f}s] (no keyboard data yet)",
+                        end="\r",
+                        flush=True,
+                    )
+                    time.sleep(0.01)
+                    continue
+
+                bitmap = bitmap_group[0]
+                pressed = {code for code in range(ALL_KEYS_BITMAP_SIZE) if bitmap[code]}
+                names = [KEY_NAMES.get(code, f"code{code}") for code in sorted(pressed)]
+
+                # Live status line (overwritten each frame).
+                print(
+                    f"[{elapsed:5.1f}s] Held: {' '.join(names) or '-'}" + " " * 20,
+                    end="\r",
+                    flush=True,
+                )
+
+                # Permanent, scrollable log of every press/release transition -- a
+                # quick tap can flash by on the status line above before you notice
+                # it, but every transition is logged here.
+                for code in sorted(pressed - prev_pressed):
+                    print(
+                        f"[{elapsed:5.1f}s] {KEY_NAMES.get(code, f'code{code}')} down"
+                    )
+                for code in sorted(prev_pressed - pressed):
+                    print(f"[{elapsed:5.1f}s] {KEY_NAMES.get(code, f'code{code}')} up")
+                prev_pressed = pressed
+
+                time.sleep(0.01)  # ~100 FPS
+
+            print("\nTime limit reached.")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/core/deviceio_trackers/trackers.toml

@@ -43,6 +43,12 @@ traits = "PedalRecordingTraits"
 max_flatbuffer_size = 256
 python_accessor = "get_pedal_data"
 
+[[tracker]]
+name = "keyboard"
+table = "KeyboardOutput"
+max_flatbuffer_size = 2048
+python_accessor = "get_keyboard_data"
+
 [[tracker]]
 name = "haptic_command"
 direction = "push"

src/core/schema/fbs/keyboard.fbs

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+include "timestamp.fbs";
+
+namespace core;
+
+// Raw press state of a keyboard: the set of currently-held Linux evdev key codes
+// (see linux/input-event-codes.h, e.g. KEY_W = 17). Carries no semantic mapping --
+// which keys mean what (an axis, a toggle, a mode switch) is entirely up to the
+// consuming retargeter.
+//
+// All fields are always present whenever this table itself is present.
+table KeyboardOutput {
+    // Evdev key codes currently held down, in no particular order.
+    pressed_keys: [ushort] (id: 0);
+
+    // Whether the tracker has emitted at least one sample.
+    is_valid: bool (id: 1);
+}
+
+// MCAP recording wrapper for KeyboardOutput.
+table KeyboardOutputRecord {
+    data: KeyboardOutput (id: 0);
+    timestamp: DeviceDataTimestamp (id: 1);
+}
+
+root_type KeyboardOutputRecord;

src/core/schema/python/CMakeLists.txt

@@ -10,6 +10,7 @@ pybind11_add_module(schema_py
     head_bindings.h
     joint_se3_pose_bindings.h
     joint_state_bindings.h
+    keyboard_bindings.h
     message_channel_bindings.h
     oglo_tactile_bindings.h
     pedals_bindings.h

src/core/schema/python/keyboard_bindings.h

@@ -0,0 +1,58 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// Python bindings for the Keyboard FlatBuffer schema.
+// Types: KeyboardOutput (table), exposed as an encoded view.
+
+#pragma once
+
+#include "schema_serialized.h"
+
+#include <pybind11/pybind11.h>
+#include <schema/keyboard_generated.h>
+
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace py = pybind11;
+
+namespace core
+{
+
+inline void bind_keyboard(py::module& m)
+{
+    serialized_class<KeyboardOutput>(m, "KeyboardOutput", "Encoded raw keyboard press state.")
+        .def(py::init(
+                 [](std::vector<uint16_t> pressed_keys, bool is_valid)
+                 {
+                     KeyboardOutputT native;
+                     native.pressed_keys = std::move(pressed_keys);
+                     native.is_valid = is_valid;
+                     return pack<KeyboardOutput>(native);
+                 }),
+             py::arg("pressed_keys"), py::arg("is_valid"), "Encode a keyboard press-state snapshot.")
+        .def_property_readonly("pressed_keys", vector_field(&KeyboardOutput::pressed_keys))
+        .def_property_readonly("is_valid", field(&KeyboardOutput::is_valid))
+        .def("__repr__",
+             [](const Serialized<KeyboardOutput>& self)
+             {
+                 std::string result = "KeyboardOutput(pressed_keys=[";
+                 const auto* keys = self->pressed_keys();
+                 if (keys != nullptr)
+                 {
+                     for (size_t i = 0; i < keys->size(); ++i)
+                     {
+                         if (i > 0)
+                             result += ", ";
+                         result += std::to_string((*keys)[i]);
+                     }
+                 }
+                 result += "], is_valid=" + std::to_string(self->is_valid()) + ")";
+                 return result;
+             });
+
+    bind_record<KeyboardOutputRecord, KeyboardOutput>(m, "KeyboardOutputRecord", "KeyboardOutput");
+}
+
+} // namespace core

src/core/schema/python/schema_module.cpp

@@ -13,6 +13,7 @@
 #include "head_bindings.h"
 #include "joint_se3_pose_bindings.h"
 #include "joint_state_bindings.h"
+#include "keyboard_bindings.h"
 #include "message_channel_bindings.h"
 #include "oak_bindings.h"
 #include "oglo_tactile_bindings.h"
@@ -46,6 +47,9 @@ PYBIND11_MODULE(_schema, m)
     // Bind pedals types (Generic3AxisPedalOutput table).
     core::bind_pedals(m);
 
+    // Bind keyboard types (KeyboardOutput table) for raw keyboard press state.
+    core::bind_keyboard(m);
+
     // Bind OGLO tactile glove types (OgloGloveSample table).
     core::bind_oglo_tactile(m);
 

src/plugins/CMakeLists.txt

@@ -17,6 +17,7 @@ add_subdirectory(controller_se3_tracker)
 add_subdirectory(vive_se3_tracker)
 add_subdirectory(controller_synthetic_hands)
 add_subdirectory(generic_3axis_pedal)
+add_subdirectory(keyboard)
 add_subdirectory(xsens_full_body)
 add_subdirectory(so101_leader)
 add_subdirectory(rebot_devarm_leader)

src/plugins/keyboard/CMakeLists.txt

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    message(STATUS "Skipping keyboard plugin (Linux only)")
+    add_custom_target(keyboard_plugin
+        COMMAND ${CMAKE_COMMAND} -E echo "Skipping keyboard: Linux only")
+    return()
+endif()
+
+add_executable(keyboard_plugin
+    main.cpp
+    keyboard_plugin.cpp
+)
+
+target_link_libraries(keyboard_plugin PRIVATE
+    pusherio::pusherio
+    oxr::oxr_core
+    isaacteleop_schema
+)
+
+install(TARGETS keyboard_plugin RUNTIME DESTINATION plugins/keyboard)
+install(FILES plugin.yaml README.md DESTINATION plugins/keyboard)

src/plugins/keyboard/README.md

@@ -0,0 +1,42 @@
+<!--
+SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Keyboard Plugin
+
+Reads a keyboard from `/dev/input/event*` and pushes `KeyboardOutput` via OpenXR. Use with `KeyboardTracker` with the same `collection_id`.
+
+Reports raw press state only, with no semantic mapping to axes or commands -- that mapping belongs
+in a retargeter (e.g. `KeyboardToSe3RelRetargeter`) consuming this tracker's output.
+
+Self-discovers its device (the first `*-event-kbd` entry under `/dev/input/by-path/`), so it needs
+no arguments to run and can be auto-launched by `PluginManager` via `PluginConfig` -- no manual
+process to start.
+
+## Usage
+
+Auto-launched (recommended -- matches how `PluginManager` invokes plugins):
+
+```bash
+./keyboard_plugin --plugin-root-id=keyboard
+```
+
+Manual / standalone, with an explicit device:
+
+```bash
+./keyboard_plugin [device_path] [--plugin-root-id=<collection_id>]
+```
+
+- **device_path**: Optional. Defaults to the first `*-event-kbd` entry under `/dev/input/by-path/`.
+  Identify a specific keyboard event device with `cat /proc/bus/input/devices` (look for a
+  `Handlers=... eventN` line under a keyboard entry) or `evtest`. Reading `/dev/input/event*`
+  typically requires membership in the `input` group.
+- **collection_id**: Default `keyboard`. Match this when creating `KeyboardTracker`.
+
+## Key mapping
+
+Reports the current press state of: `W`, `A`, `S`, `D`, `Q`, `E`, `Z`, `X`, `T`, `G`, `C`, `V`, `K`.
+Autorepeat events are ignored (press state is already true).
+
+Linux only.